From fb1e77f9d09dae48070f6415abe882ead1c42dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rud=C3=A1=20Porto=20Filgueiras?= Date: Fri, 26 Jan 2018 15:32:54 +0100 Subject: [PATCH 1/3] Avoiding full table scan when looking for inclusions. --- flask_restless/views/base.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/flask_restless/views/base.py b/flask_restless/views/base.py index 0b300377..5fc15cf0 100644 --- a/flask_restless/views/base.py +++ b/flask_restless/views/base.py @@ -1388,7 +1388,7 @@ def _handle_validation_exception(self, exception): current_app.logger.exception(str(exception)) return errors_response(400, errors) - def get_all_inclusions(self, instance_or_instances): + def get_all_inclusions(self, instance_or_instances, page_size=None): """Returns a list of all the requested included resources associated with the given instance or instances of a SQLAlchemy model. @@ -1407,13 +1407,18 @@ def get_all_inclusions(self, instance_or_instances): contains a list of the :exc:`SerializationException` objects that caused it. + ``page_size`` is an integer with the size of the result set. + this is to avoid full table scan to find data to include. """ # If `instance_or_instances` is actually just a single instance # of a SQLAlchemy model, get the resources to include for that # one instance. Otherwise, collect the resources to include for # each instance in `instances`. if isinstance(instance_or_instances, Query): - instances = instance_or_instances + if page_size and isinstance(page_size, int): + instances = instance_or_instances.limit(page_size) + else: + instances = instance_or_instances to_include = set(chain(map(self.resources_to_include, instances))) else: instance = instance_or_instances @@ -1577,10 +1582,10 @@ def _get_resource_helper(self, resource, primary_resource=None, # includer = Includer(resource) # includes = includer.generate_includes(resource) # result['includes'] = includes - # Include any requested resources in a compound document. try: - included = self.get_all_inclusions(resource) + page_size = len(result.get('data', [])) + included = self.get_all_inclusions(resource, page_size=page_size) except MultipleExceptions as e: # By the way we defined `get_all_inclusions()`, we are # guaranteed that each of the underlying exceptions is a @@ -1729,7 +1734,8 @@ def _get_collection_helper(self, resource=None, relation_name=None, instances = search_items # Include any requested resources in a compound document. try: - included = self.get_all_inclusions(instances) + page_size = len(paginated.items) + included = self.get_all_inclusions(instances, page_size=page_size) except MultipleExceptions as e: # By the way we defined `get_all_inclusions()`, we are # guaranteed that each of the underlying exceptions is a From a32373566008113eab755f6a106b76df554d63f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rud=C3=A1=20Porto=20Filgueiras?= Date: Fri, 26 Jan 2018 16:03:02 +0100 Subject: [PATCH 2/3] Checking if there is a paginated instance with items attribute. --- flask_restless/views/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flask_restless/views/base.py b/flask_restless/views/base.py index 5fc15cf0..f96bd412 100644 --- a/flask_restless/views/base.py +++ b/flask_restless/views/base.py @@ -1734,7 +1734,10 @@ def _get_collection_helper(self, resource=None, relation_name=None, instances = search_items # Include any requested resources in a compound document. try: - page_size = len(paginated.items) + if paginated and hasattr(paginated, 'items'): + page_size = len(paginated.items) + else: + page_size = None included = self.get_all_inclusions(instances, page_size=page_size) except MultipleExceptions as e: # By the way we defined `get_all_inclusions()`, we are From 298ad3548b89ba0aa02ff314ad9251c381e09e16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rud=C3=A1=20Porto=20Filgueiras?= Date: Fri, 26 Jan 2018 16:20:32 +0100 Subject: [PATCH 3/3] Fix tests. --- flask_restless/views/base.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/flask_restless/views/base.py b/flask_restless/views/base.py index f96bd412..274485cf 100644 --- a/flask_restless/views/base.py +++ b/flask_restless/views/base.py @@ -1584,7 +1584,9 @@ def _get_resource_helper(self, resource, primary_resource=None, # result['includes'] = includes # Include any requested resources in a compound document. try: - page_size = len(result.get('data', [])) + page_size = None + if result and result.get('data', None): + page_size = len(result.get('data')) included = self.get_all_inclusions(resource, page_size=page_size) except MultipleExceptions as e: # By the way we defined `get_all_inclusions()`, we are @@ -1734,10 +1736,12 @@ def _get_collection_helper(self, resource=None, relation_name=None, instances = search_items # Include any requested resources in a compound document. try: - if paginated and hasattr(paginated, 'items'): - page_size = len(paginated.items) - else: - page_size = None + page_size = None + try: + if hasattr(paginated, 'items'): + page_size = len(paginated.items) + except (TypeError, UnboundLocalError): + pass included = self.get_all_inclusions(instances, page_size=page_size) except MultipleExceptions as e: # By the way we defined `get_all_inclusions()`, we are