Skip to content

Commit 9ec0a9d

Browse files
author
Jason Munro
authored
Only run MPDataDoc factory once per chunk (#665)
* Only run MPDataDoc factory once per chunk * Only post-process if data exists
1 parent 22501cd commit 9ec0a9d

1 file changed

Lines changed: 65 additions & 57 deletions

File tree

‎mp_api/client/core/client.py‎

Lines changed: 65 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -685,11 +685,26 @@ def _submit_request_and_process(
685685
# other sub-urls may use different document models
686686
# the client does not handle this in a particularly smart way currently
687687
if self.document_model and use_document_model:
688+
688689
raw_doc_list = [self.document_model.parse_obj(d) for d in data["data"]] # type: ignore
689690

690-
# Temporarily removed until user-testing completed
691-
data["data"] = self._generate_returned_model(raw_doc_list)
692-
# data["data"] = raw_doc_list
691+
if len(raw_doc_list) > 0:
692+
data_model, set_fields, _ = self._generate_returned_model(
693+
raw_doc_list[0]
694+
)
695+
696+
data["data"] = [
697+
data_model(
698+
**{
699+
field: value
700+
for field, value in raw_doc.dict().items()
701+
if field in set_fields
702+
}
703+
)
704+
for raw_doc in raw_doc_list
705+
]
706+
707+
# data["data"] = raw_doc_list
693708

694709
meta_total_doc_num = data.get("meta", {}).get("total_doc", 1)
695710

@@ -715,70 +730,63 @@ def _submit_request_and_process(
715730
f"on URL {response.url} with message:\n{message}"
716731
)
717732

718-
def _generate_returned_model(self, data):
733+
def _generate_returned_model(self, doc):
719734

720-
new_data = []
735+
set_fields = [
736+
field for field, _ in doc if field in doc.dict(exclude_unset=True)
737+
]
738+
unset_fields = [field for field in doc.__fields__ if field not in set_fields]
721739

722-
for doc in data:
723-
set_data = {
724-
field: value
725-
for field, value in doc
726-
if field in doc.dict(exclude_unset=True)
727-
}
728-
unset_fields = [field for field in doc.__fields__ if field not in set_data]
740+
data_model = create_model(
741+
"MPDataDoc",
742+
fields_not_requested=unset_fields,
743+
__base__=self.document_model,
744+
)
729745

730-
data_model = create_model(
731-
"MPDataDoc",
732-
fields_not_requested=unset_fields,
733-
__base__=self.document_model,
746+
data_model.__fields__ = {
747+
**{
748+
name: description
749+
for name, description in data_model.__fields__.items()
750+
if name in set_fields
751+
},
752+
"fields_not_requested": data_model.__fields__["fields_not_requested"],
753+
}
754+
755+
def new_repr(self) -> str:
756+
extra = ",\n".join(
757+
f"\033[1m{n}\033[0;0m={getattr(self, n)!r}"
758+
for n in data_model.__fields__
734759
)
735760

736-
data_model.__fields__ = {
737-
**{
738-
name: description
739-
for name, description in data_model.__fields__.items()
740-
if name in set_data
741-
},
742-
"fields_not_requested": data_model.__fields__["fields_not_requested"],
743-
}
744-
745-
def new_repr(self) -> str:
746-
extra = ",\n".join(
747-
f"\033[1m{n}\033[0;0m={getattr(self, n)!r}"
748-
for n in data_model.__fields__
749-
)
750-
751-
s = f"\033[4m\033[1m{self.__class__.__name__}<{self.__class__.__base__.__name__}>\033[0;0m\033[0;0m(\n{extra}\n)" # noqa: E501
752-
return s
761+
s = f"\033[4m\033[1m{self.__class__.__name__}<{self.__class__.__base__.__name__}>\033[0;0m\033[0;0m(\n{extra}\n)" # noqa: E501
762+
return s
753763

754-
def new_str(self) -> str:
755-
extra = ",\n".join(
756-
f"\033[1m{n}\033[0;0m={getattr(self, n)!r}"
757-
for n in data_model.__fields__
758-
if n != "fields_not_requested"
759-
)
764+
def new_str(self) -> str:
765+
extra = ",\n".join(
766+
f"\033[1m{n}\033[0;0m={getattr(self, n)!r}"
767+
for n in data_model.__fields__
768+
if n != "fields_not_requested"
769+
)
760770

761-
s = f"\033[4m\033[1m{self.__class__.__name__}<{self.__class__.__base__.__name__}>\033[0;0m\033[0;0m\n{extra}\n\n\033[1mFields not requested:\033[0;0m\n{unset_fields}" # noqa: E501
762-
return s
771+
s = f"\033[4m\033[1m{self.__class__.__name__}<{self.__class__.__base__.__name__}>\033[0;0m\033[0;0m\n{extra}\n\n\033[1mFields not requested:\033[0;0m\n{unset_fields}" # noqa: E501
772+
return s
763773

764-
def new_getattr(self, attr) -> str:
765-
if attr in unset_fields:
766-
raise AttributeError(
767-
f"'{attr}' data is available but has not been requested in 'fields'."
768-
" A full list of unrequested fields can be found in `fields_not_requested`."
769-
)
770-
else:
771-
raise AttributeError(
772-
f"{self.__class__.__name__!r} object has no attribute {attr!r}"
773-
)
774-
775-
data_model.__repr__ = new_repr
776-
data_model.__str__ = new_str
777-
data_model.__getattr__ = new_getattr
774+
def new_getattr(self, attr) -> str:
775+
if attr in unset_fields:
776+
raise AttributeError(
777+
f"'{attr}' data is available but has not been requested in 'fields'."
778+
" A full list of unrequested fields can be found in `fields_not_requested`."
779+
)
780+
else:
781+
raise AttributeError(
782+
f"{self.__class__.__name__!r} object has no attribute {attr!r}"
783+
)
778784

779-
new_data.append(data_model(**set_data))
785+
data_model.__repr__ = new_repr
786+
data_model.__str__ = new_str
787+
data_model.__getattr__ = new_getattr
780788

781-
return new_data
789+
return data_model, set_fields, unset_fields
782790

783791
def _query_resource_data(
784792
self,

0 commit comments

Comments
 (0)