From aa962c12bbe2579c95d15bbcb055a55dbdcf6af3 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Wed, 19 Jul 2017 13:59:14 -0400 Subject: [PATCH 01/16] Missing dependancy --- requirements-development.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-development.txt b/requirements-development.txt index 252ecedd..01255bb2 100644 --- a/requirements-development.txt +++ b/requirements-development.txt @@ -10,3 +10,4 @@ recommonmark Sphinx sphinx_rtd_theme tox +packaging==16.8 \ No newline at end of file From da0c94c7271f72fd8f9e5781f5240757db01101f Mon Sep 17 00:00:00 2001 From: Aidan Lister Date: Thu, 20 Jul 2017 21:51:38 +1000 Subject: [PATCH 02/16] Add Django Debug Toolbar, and add a failing test setting out our performance expectations (also flake8) --- .gitignore | 3 + .../{factories/__init__.py => factories.py} | 0 example/models.py | 18 ++++++ example/settings/dev.py | 15 ++++- example/tests/test_performance.py | 55 +++++++++++++++++++ example/tests/test_views.py | 3 +- example/urls.py | 8 +++ example/utils.py | 20 +++++++ example/views.py | 3 +- requirements-development.txt | 2 + rest_framework_json_api/relations.py | 9 ++- rest_framework_json_api/serializers.py | 1 - 12 files changed, 128 insertions(+), 9 deletions(-) rename example/{factories/__init__.py => factories.py} (100%) create mode 100644 example/tests/test_performance.py create mode 100644 example/utils.py diff --git a/.gitignore b/.gitignore index ff0958f1..ac41fdd8 100644 --- a/.gitignore +++ b/.gitignore @@ -40,4 +40,7 @@ pip-delete-this-directory.txt # VirtualEnv .venv/ +# Developers *.sw* +manage.py +.DS_Store diff --git a/example/factories/__init__.py b/example/factories.py similarity index 100% rename from example/factories/__init__.py rename to example/factories.py diff --git a/example/models.py b/example/models.py index 6c1c6078..5cc70b47 100644 --- a/example/models.py +++ b/example/models.py @@ -28,6 +28,9 @@ class TaggedItem(BaseModel): def __str__(self): return self.tag + class Meta: + ordering = ('id',) + @python_2_unicode_compatible class Blog(BaseModel): @@ -38,6 +41,9 @@ class Blog(BaseModel): def __str__(self): return self.name + class Meta: + ordering = ('id',) + @python_2_unicode_compatible class Author(BaseModel): @@ -47,6 +53,9 @@ class Author(BaseModel): def __str__(self): return self.name + class Meta: + ordering = ('id',) + @python_2_unicode_compatible class AuthorBio(BaseModel): @@ -56,6 +65,9 @@ class AuthorBio(BaseModel): def __str__(self): return self.author.name + class Meta: + ordering = ('id',) + @python_2_unicode_compatible class Entry(BaseModel): @@ -73,6 +85,9 @@ class Entry(BaseModel): def __str__(self): return self.headline + class Meta: + ordering = ('id',) + @python_2_unicode_compatible class Comment(BaseModel): @@ -87,6 +102,9 @@ class Comment(BaseModel): def __str__(self): return self.body + class Meta: + ordering = ('id',) + class Project(PolymorphicModel): topic = models.CharField(max_length=30) diff --git a/example/settings/dev.py b/example/settings/dev.py index c5a1f742..fb969c45 100644 --- a/example/settings/dev.py +++ b/example/settings/dev.py @@ -25,6 +25,7 @@ 'rest_framework', 'polymorphic', 'example', + 'debug_toolbar', ] TEMPLATES = [ @@ -58,7 +59,11 @@ PASSWORD_HASHERS = ('django.contrib.auth.hashers.UnsaltedMD5PasswordHasher', ) -MIDDLEWARE_CLASSES = () +MIDDLEWARE_CLASSES = ( + 'debug_toolbar.middleware.DebugToolbarMiddleware', +) + +INTERNAL_IPS = ('127.0.0.1', ) JSON_API_FORMAT_KEYS = 'camelize' JSON_API_FORMAT_TYPES = 'camelize' @@ -74,7 +79,13 @@ ), 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework_json_api.renderers.JSONRenderer', - 'rest_framework.renderers.BrowsableAPIRenderer', + + # If you're performance testing, you will want to use the browseable API + # without forms, as the forms can generate their own queries. + # If performance testing, enable: + 'example.utils.BrowsableAPIRendererWithoutForms', + # Otherwise, to play around with the browseable API, enable: + #'rest_framework.renderers.BrowsableAPIRenderer', ), 'DEFAULT_METADATA_CLASS': 'rest_framework_json_api.metadata.JSONAPIMetadata', } diff --git a/example/tests/test_performance.py b/example/tests/test_performance.py new file mode 100644 index 00000000..c81c4d75 --- /dev/null +++ b/example/tests/test_performance.py @@ -0,0 +1,55 @@ +from django.utils import timezone +from rest_framework.test import APITestCase + +from example.factories import CommentFactory +from example.models import Author, Blog, Comment, Entry + + +class PerformanceTestCase(APITestCase): + def setUp(self): + self.author = Author.objects.create(name='Super powerful superhero', email='i.am@lost.com') + self.blog = Blog.objects.create(name='Some Blog', tagline="It's a blog") + self.other_blog = Blog.objects.create(name='Other blog', tagline="It's another blog") + self.first_entry = Entry.objects.create( + blog=self.blog, + headline='headline one', + body_text='body_text two', + pub_date=timezone.now(), + mod_date=timezone.now(), + n_comments=0, + n_pingbacks=0, + rating=3 + ) + self.second_entry = Entry.objects.create( + blog=self.blog, + headline='headline two', + body_text='body_text one', + pub_date=timezone.now(), + mod_date=timezone.now(), + n_comments=0, + n_pingbacks=0, + rating=1 + ) + self.comment = Comment.objects.create(entry=self.first_entry) + CommentFactory.create_batch(50) + + def test_query_count_no_includes(self): + """ We expect a simple list view to issue only two queries. + + 1. The number of results in the set (e.g. a COUNT query), only necessary because we're using PageNumberPagination + 2. The SELECT query for the set + """ + with self.assertNumQueries(2): + response = self.client.get('/comments?page_size=25') + self.assertEqual(len(response.data['results']), 25) + + def test_query_count_include_author(self): + """ We expect a list view with an include have three queries: + + 1. Primary resource COUNT query + 2. Primary resource SELECT + 3. Author's prefetched + """ + with self.assertNumQueries(3): + response = self.client.get('/comments?include=author&page_size=25') + self.assertEqual(len(response.data['results']), 25) diff --git a/example/tests/test_views.py b/example/tests/test_views.py index e8c11ff8..9d73a136 100644 --- a/example/tests/test_views.py +++ b/example/tests/test_views.py @@ -4,11 +4,10 @@ from django.utils import timezone from rest_framework.reverse import reverse from rest_framework.test import APITestCase, force_authenticate - from rest_framework_json_api.utils import format_resource_type from . import TestBase -from .. import views +from .. import factories, views from example.models import Author, Blog, Comment, Entry diff --git a/example/urls.py b/example/urls.py index 9c789274..d6b58f3d 100644 --- a/example/urls.py +++ b/example/urls.py @@ -1,3 +1,4 @@ +from django.conf import settings from django.conf.urls import include, url from rest_framework import routers @@ -22,3 +23,10 @@ urlpatterns = [ url(r'^', include(router.urls)), ] + + +if settings.DEBUG: + import debug_toolbar + urlpatterns = [ + url(r'^__debug__/', include(debug_toolbar.urls)), + ] + urlpatterns diff --git a/example/utils.py b/example/utils.py new file mode 100644 index 00000000..65403038 --- /dev/null +++ b/example/utils.py @@ -0,0 +1,20 @@ +from rest_framework.renderers import BrowsableAPIRenderer + + +class BrowsableAPIRendererWithoutForms(BrowsableAPIRenderer): + """Renders the browsable api, but excludes the forms.""" + + def get_context(self, *args, **kwargs): + ctx = super().get_context(*args, **kwargs) + ctx['display_edit_forms'] = False + return ctx + + def show_form_for_method(self, view, method, request, obj): + """We never want to do this! So just return False.""" + return False + + def get_rendered_html_form(self, data, view, method, request): + """Why render _any_ forms at all. This method should return + rendered HTML, so let's simply return an empty string. + """ + return "" diff --git a/example/views.py b/example/views.py index b65b96cf..ca87ca14 100644 --- a/example/views.py +++ b/example/views.py @@ -1,10 +1,9 @@ import rest_framework.parsers import rest_framework.renderers -from rest_framework import exceptions - import rest_framework_json_api.metadata import rest_framework_json_api.parsers import rest_framework_json_api.renderers +from rest_framework import exceptions from rest_framework_json_api.utils import format_drf_errors from rest_framework_json_api.views import ModelViewSet, RelationshipView diff --git a/requirements-development.txt b/requirements-development.txt index 252ecedd..2184b669 100644 --- a/requirements-development.txt +++ b/requirements-development.txt @@ -10,3 +10,5 @@ recommonmark Sphinx sphinx_rtd_theme tox +mock +django-debug-toolbar diff --git a/rest_framework_json_api/relations.py b/rest_framework_json_api/relations.py index 1b4e82ee..6c4d3c54 100644 --- a/rest_framework_json_api/relations.py +++ b/rest_framework_json_api/relations.py @@ -1,12 +1,17 @@ import collections import json +from collections import OrderedDict + +import six import inflection +from django.core.exceptions import ImproperlyConfigured +from django.core.urlresolvers import NoReverseMatch from django.utils.translation import ugettext_lazy as _ from rest_framework.fields import MISSING_ERROR_MESSAGE -from rest_framework.relations import * # noqa: F403 +from rest_framework.relations import MANY_RELATION_KWARGS, PrimaryKeyRelatedField +from rest_framework.reverse import reverse from rest_framework.serializers import Serializer - from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.utils import ( Hyperlink, diff --git a/rest_framework_json_api/serializers.py b/rest_framework_json_api/serializers.py index 66d6add7..98c04189 100644 --- a/rest_framework_json_api/serializers.py +++ b/rest_framework_json_api/serializers.py @@ -4,7 +4,6 @@ from django.utils.translation import ugettext_lazy as _ from rest_framework.exceptions import ParseError from rest_framework.serializers import * # noqa: F403 - from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.relations import ResourceRelatedField from rest_framework_json_api.utils import ( From 1d84c80d8c5b5e1d5d13db814daa3ed356c198bd Mon Sep 17 00:00:00 2001 From: Aidan Lister Date: Thu, 20 Jul 2017 22:29:33 +1000 Subject: [PATCH 03/16] Add a helper for specifying a prefetch_for_related attribute on your viewset to help with prefetching includes. --- rest_framework_json_api/views.py | 40 +++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/rest_framework_json_api/views.py b/rest_framework_json_api/views.py index b8e81e6a..c99ddad4 100644 --- a/rest_framework_json_api/views.py +++ b/rest_framework_json_api/views.py @@ -8,7 +8,6 @@ from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.serializers import Serializer - from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.serializers import ResourceIdentifierObjectSerializer from rest_framework_json_api.utils import ( @@ -39,9 +38,40 @@ ) -class ModelViewSet(viewsets.ModelViewSet): +class PrefetchForIncludesHelperMixin(object): + def get_queryset(self): + """ This viewset provides a helper attribute to prefetch related models + based on the include specified in the URL. + + __all__ can be used to specify a prefetch which should be done regardless of the include + + @example + # When MyViewSet is called with ?include=author it will prefetch author and authorbio + class MyViewSet(viewsets.ModelViewSet): + queryset = Book.objects.all() + prefetch_for_includes = { + '__all__': [], + 'author': ['author', 'author__authorbio'] + 'category.section': ['category'] + } + """ + qs = super(PrefetchForIncludesHelperMixin, self).get_queryset() + if not hasattr(self, 'prefetch_for_includes'): + return qs + + includes = self.request.GET.get('include', '').split(',') + for inc in includes + ['__all__']: + prefetches = self.prefetch_for_includes.get(inc) + if prefetches: + qs = qs.prefetch_related(*prefetches) + + return qs + + +class AutoPrefetchMixin(object): def get_queryset(self, *args, **kwargs): - qs = super(ModelViewSet, self).get_queryset(*args, **kwargs) + """ This mixin adds automatic prefetching for OneToOne and ManyToMany fields. """ + qs = super(AutoPrefetchMixin, self).get_queryset(*args, **kwargs) included_resources = get_included_resources(self.request) for included in included_resources: @@ -84,6 +114,10 @@ def get_queryset(self, *args, **kwargs): return qs +class ModelViewSet(AutoPrefetchMixin, PrefetchForIncludesHelperMixin, viewsets.ModelViewSet): + pass + + class RelationshipView(generics.GenericAPIView): serializer_class = ResourceIdentifierObjectSerializer self_link_view_name = None From cacca2e31fc049e75580855dce46da394f199a31 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 13:27:09 -0400 Subject: [PATCH 04/16] Trying to make isort happy --- rest_framework_json_api/relations.py | 4 ++-- rest_framework_json_api/serializers.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rest_framework_json_api/relations.py b/rest_framework_json_api/relations.py index 6c4d3c54..b5b36382 100644 --- a/rest_framework_json_api/relations.py +++ b/rest_framework_json_api/relations.py @@ -2,9 +2,8 @@ import json from collections import OrderedDict -import six - import inflection +import six from django.core.exceptions import ImproperlyConfigured from django.core.urlresolvers import NoReverseMatch from django.utils.translation import ugettext_lazy as _ @@ -12,6 +11,7 @@ from rest_framework.relations import MANY_RELATION_KWARGS, PrimaryKeyRelatedField from rest_framework.reverse import reverse from rest_framework.serializers import Serializer + from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.utils import ( Hyperlink, diff --git a/rest_framework_json_api/serializers.py b/rest_framework_json_api/serializers.py index 98c04189..1cd878d7 100644 --- a/rest_framework_json_api/serializers.py +++ b/rest_framework_json_api/serializers.py @@ -1,9 +1,9 @@ import inflection from django.db.models.query import QuerySet -from django.utils import six from django.utils.translation import ugettext_lazy as _ from rest_framework.exceptions import ParseError from rest_framework.serializers import * # noqa: F403 + from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.relations import ResourceRelatedField from rest_framework_json_api.utils import ( From e96fe6cb274754e5f978ce32d0f84e512768abd0 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:41:55 -0400 Subject: [PATCH 05/16] Adding `select_related` and `prefetch_for_includes` to `CommentViewSet` --- example/tests/test_performance.py | 5 +++-- example/views.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/example/tests/test_performance.py b/example/tests/test_performance.py index c81c4d75..2c4acf4e 100644 --- a/example/tests/test_performance.py +++ b/example/tests/test_performance.py @@ -48,8 +48,9 @@ def test_query_count_include_author(self): 1. Primary resource COUNT query 2. Primary resource SELECT - 3. Author's prefetched + 3. Authors prefetched + 3. Entries prefetched """ - with self.assertNumQueries(3): + with self.assertNumQueries(4): response = self.client.get('/comments?include=author&page_size=25') self.assertEqual(len(response.data['results']), 25) diff --git a/example/views.py b/example/views.py index ca87ca14..555ec567 100644 --- a/example/views.py +++ b/example/views.py @@ -73,8 +73,13 @@ class AuthorViewSet(ModelViewSet): class CommentViewSet(ModelViewSet): - queryset = Comment.objects.all() + queryset = Comment.objects.select_related('author', 'entry') serializer_class = CommentSerializer + prefetch_for_includes = { + '__all__': [], + 'author': ['author', 'author__bio', 'author__entries'], + 'entry': ['author', 'author__bio', 'author__entries'] + } class CompanyViewset(ModelViewSet): From 8fdbc0a36f58457e9dd85296a0919a9af3b36386 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:48:09 -0400 Subject: [PATCH 06/16] Fixes ImportError on TravisCI --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 016d3df0..207bc8ca 100755 --- a/setup.py +++ b/setup.py @@ -109,6 +109,7 @@ def get_package_data(package): 'pytest>=2.8,<3', 'django-polymorphic', 'packaging', + 'debug_toolbar' ] + mock, zip_safe=False, ) From 46c3809fe1baed4b92dc701229c245bf94446788 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:51:10 -0400 Subject: [PATCH 07/16] E265 block comment should start with '# ' --- example/settings/dev.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/settings/dev.py b/example/settings/dev.py index fb969c45..c2ef8a7d 100644 --- a/example/settings/dev.py +++ b/example/settings/dev.py @@ -85,7 +85,7 @@ # If performance testing, enable: 'example.utils.BrowsableAPIRendererWithoutForms', # Otherwise, to play around with the browseable API, enable: - #'rest_framework.renderers.BrowsableAPIRenderer', + # 'rest_framework.renderers.BrowsableAPIRenderer', ), 'DEFAULT_METADATA_CLASS': 'rest_framework_json_api.metadata.JSONAPIMetadata', } From 3c240930816a5f80576302a411f77918a0c9e611 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:53:09 -0400 Subject: [PATCH 08/16] E501 line too long (121 > 100 characters) --- example/tests/test_performance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example/tests/test_performance.py b/example/tests/test_performance.py index 2c4acf4e..3ec2f676 100644 --- a/example/tests/test_performance.py +++ b/example/tests/test_performance.py @@ -36,7 +36,8 @@ def setUp(self): def test_query_count_no_includes(self): """ We expect a simple list view to issue only two queries. - 1. The number of results in the set (e.g. a COUNT query), only necessary because we're using PageNumberPagination + 1. The number of results in the set (e.g. a COUNT query), + only necessary because we're using PageNumberPagination 2. The SELECT query for the set """ with self.assertNumQueries(2): From 151bdb9ba4c84cc7aadc46b28be44b474ceddf45 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:53:48 -0400 Subject: [PATCH 09/16] F401 '..factories' imported but unused --- example/tests/test_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/tests/test_views.py b/example/tests/test_views.py index 9d73a136..a164e6e9 100644 --- a/example/tests/test_views.py +++ b/example/tests/test_views.py @@ -7,7 +7,7 @@ from rest_framework_json_api.utils import format_resource_type from . import TestBase -from .. import factories, views +from .. import views from example.models import Author, Blog, Comment, Entry From 844e88efc95fa4fbee75f72c60d551b125b88a7e Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 14:59:45 -0400 Subject: [PATCH 10/16] Imports are (now) correctly sorted as per isort. --- example/factories.py | 12 ++++++++++-- example/tests/test_views.py | 1 + example/views.py | 1 + rest_framework_json_api/views.py | 1 + 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/example/factories.py b/example/factories.py index a7485500..012fb85e 100644 --- a/example/factories.py +++ b/example/factories.py @@ -2,8 +2,17 @@ import factory from faker import Factory as FakerFactory + from example.models import ( - Blog, Author, AuthorBio, Entry, Comment, TaggedItem, ArtProject, ResearchProject, Company + ArtProject, + Author, + AuthorBio, + Blog, + Comment, + Company, + Entry, + ResearchProject, + TaggedItem ) faker = FakerFactory.create() @@ -64,7 +73,6 @@ class Meta: class TaggedItemFactory(factory.django.DjangoModelFactory): - class Meta: model = TaggedItem diff --git a/example/tests/test_views.py b/example/tests/test_views.py index a164e6e9..e8c11ff8 100644 --- a/example/tests/test_views.py +++ b/example/tests/test_views.py @@ -4,6 +4,7 @@ from django.utils import timezone from rest_framework.reverse import reverse from rest_framework.test import APITestCase, force_authenticate + from rest_framework_json_api.utils import format_resource_type from . import TestBase diff --git a/example/views.py b/example/views.py index 555ec567..1fbc3a1e 100644 --- a/example/views.py +++ b/example/views.py @@ -4,6 +4,7 @@ import rest_framework_json_api.parsers import rest_framework_json_api.renderers from rest_framework import exceptions + from rest_framework_json_api.utils import format_drf_errors from rest_framework_json_api.views import ModelViewSet, RelationshipView diff --git a/rest_framework_json_api/views.py b/rest_framework_json_api/views.py index c99ddad4..efb27df7 100644 --- a/rest_framework_json_api/views.py +++ b/rest_framework_json_api/views.py @@ -8,6 +8,7 @@ from rest_framework.response import Response from rest_framework.reverse import reverse from rest_framework.serializers import Serializer + from rest_framework_json_api.exceptions import Conflict from rest_framework_json_api.serializers import ResourceIdentifierObjectSerializer from rest_framework_json_api.utils import ( From 07f67a3dee92d0f2046a31be56b4ca0b3b12290f Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 15:34:21 -0400 Subject: [PATCH 11/16] Package name, not module name... woops! --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 207bc8ca..e91ec5e5 100755 --- a/setup.py +++ b/setup.py @@ -109,7 +109,7 @@ def get_package_data(package): 'pytest>=2.8,<3', 'django-polymorphic', 'packaging', - 'debug_toolbar' + 'django-debug-toolbar' ] + mock, zip_safe=False, ) From f924ac8f7c65223b3fa6258a0291295958588341 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Fri, 21 Jul 2017 15:37:15 -0400 Subject: [PATCH 12/16] More of this isort madness --- example/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/views.py b/example/views.py index 1fbc3a1e..0e1f8bb4 100644 --- a/example/views.py +++ b/example/views.py @@ -1,10 +1,10 @@ import rest_framework.parsers import rest_framework.renderers +from rest_framework import exceptions + import rest_framework_json_api.metadata import rest_framework_json_api.parsers import rest_framework_json_api.renderers -from rest_framework import exceptions - from rest_framework_json_api.utils import format_drf_errors from rest_framework_json_api.views import ModelViewSet, RelationshipView From 97b6dd38cbbc4addafdd5c8b51f96b21ebc79588 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Sat, 22 Jul 2017 00:02:29 -0400 Subject: [PATCH 13/16] Use forms in the browsable API --- example/settings/dev.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/example/settings/dev.py b/example/settings/dev.py index c2ef8a7d..94c9628e 100644 --- a/example/settings/dev.py +++ b/example/settings/dev.py @@ -83,9 +83,9 @@ # If you're performance testing, you will want to use the browseable API # without forms, as the forms can generate their own queries. # If performance testing, enable: - 'example.utils.BrowsableAPIRendererWithoutForms', + # 'example.utils.BrowsableAPIRendererWithoutForms', # Otherwise, to play around with the browseable API, enable: - # 'rest_framework.renderers.BrowsableAPIRenderer', + 'rest_framework.renderers.BrowsableAPIRenderer', ), 'DEFAULT_METADATA_CLASS': 'rest_framework_json_api.metadata.JSONAPIMetadata', } From f0bda693ad3a343928352875726461cd0c8914e9 Mon Sep 17 00:00:00 2001 From: Jonathan Senecal Date: Sat, 22 Jul 2017 00:46:59 -0400 Subject: [PATCH 14/16] Updating docs to reflect the added ModelViewSet helper --- docs/usage.md | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index f4faeea8..19d29e62 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -23,7 +23,12 @@ REST_FRAMEWORK = { ), 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework_json_api.renderers.JSONRenderer', - 'rest_framework.renderers.BrowsableAPIRenderer', + # If you're performance testing, you will want to use the browseable API + # without forms, as the forms can generate their own queries. + # If performance testing, enable: + # 'example.utils.BrowsableAPIRendererWithoutForms', + # Otherwise, to play around with the browseable API, enable: + 'rest_framework.renderers.BrowsableAPIRenderer' ), 'DEFAULT_METADATA_CLASS': 'rest_framework_json_api.metadata.JSONAPIMetadata', } @@ -36,6 +41,12 @@ retrieve the page can be customized by subclassing `PageNumberPagination` and overriding the `page_query_param`. Page size can be controlled per request via the `PAGINATE_BY_PARAM` query parameter (`page_size` by default). +#### Performance Testing + +If you are trying to see if your viewsets are configured properly to optimize performance, +it is preferable to use `example.utils.BrowsableAPIRendererWithoutForms` instead of the default `BrowsableAPIRenderer` +to remove queries introduced by the forms themselves. + ### Serializers It is recommended to import the base serializer classes from this package @@ -558,6 +569,39 @@ class QuestSerializer(serializers.ModelSerializer): `included_resources` informs DJA of **what** you would like to include. `included_serializers` tells DJA **how** you want to include it. +#### Performance improvements + +Be aware that using included resources without any form of prefetching **WILL HURT PERFORMANCE** as it will introduce m*(n+1) queries. + +A viewset helper was designed to allow for greater flexibility and it is automatically available when subclassing +`views.ModelViewSet` +``` + # When MyViewSet is called with ?include=author it will dynamically prefetch author and author.bio + class MyViewSet(viewsets.ModelViewSet): + queryset = Book.objects.all() + prefetch_for_includes = { + '__all__': [], + 'author': ['author', 'author__bio'] + 'category.section': ['category'] +} +``` + +The special keyword `__all__` can be used to specify a prefetch which should be done regardless of the include, similar to making the prefetch yourself on the QuerySet. + +Using the helper instead of prefetching/selecting everything manually will prevent django from trying to load what could be a significant amount of data in memory for every single request. + +> If you have a single model, e.g. Book, which has four relations e.g. Author, Publisher, CopyrightHolder, Category. +> +> To display 25 books in DRF without any includes, I would need a single query: SELECT * FROM book. +> +> To display 25 books DRF-JSONAPI without any includes, I would need either: +> a) 1 query ala SELECT * FROM books LEFT JOIN author LEFT JOIN publisher LEFT JOIN CopyrightHolder LEFT JOIN Category +> b) 4 queries with prefetches. +> +> Let's say I have 1M books, 50k authors, 10k categories, 10k copyrightholders. In the select_related scenario, you've just created a in-memory table with 1e18 rows ... do this a few times per second and you have melted your database. All to display 25 rows, with no included relationships. So select_related is only going to work if you have a small dataset or a small volume of traffic. +> +> -- Aidan Lister in issue [#337](https://github.com/django-json-api/django-rest-framework-json-api/issues/337#issuecomment-297335342) +