I have an Angular frontend that uses Apollo Graphql client to interface with a Django Graphql backend (using graphene). It all works well, but the API is very slow, especially when there are concurrent users. I am trying a few things to speed up the API server. One of the things that I am seriously considering is to use server-side caching.
I'm aware that there are some caching solutions that exist for Django out of the box, but these seem to be for caching the views. Is it possible to use these methods to cache Graphql API requests?
What is the most effective caching solution for caching the responses to graphql queries in Django?
I'm posting some of my project's code here so that you get a sense of how my API works.
This is a sample of the Query class containing the various graphql endpints that serve data when queries are received from the client. And I suspect this is where we'll need most of the optimization and caching applied:-
class Query(ObjectType):
# Public Queries
user_by_username = graphene.Field(PublicUserType, username=graphene.String())
public_users = graphene.Field(
PublicUsers, searchField=graphene.String(), membership_status_not=graphene.List(graphene.String), membership_status_is=graphene.List(graphene.String), roles=graphene.List(graphene.String), limit=graphene.Int(), offset=graphene.Int())
public_institution = graphene.Field(PublicInstitutionType, code=graphene.String())
public_institutions = graphene.Field(PublicInstitutions, searchField=graphene.String(), limit = graphene.Int(), offset = graphene.Int())
public_announcement = graphene.Field(AnnouncementType, id=graphene.ID())
public_announcements = graphene.List(
AnnouncementType, searchField=graphene.String(), limit=graphene.Int(), offset=graphene.Int())
def resolve_public_institution(root, info, code, **kwargs):
institution = Institution.objects.get(code=code, active=True)
if institution is not None:
public_institution = generate_public_institution(institution)
return public_institution
else:
return None
def resolve_public_institutions(root, info, searchField=None, limit=None, offset=None, **kwargs):
qs = Institution.objects.all().filter(public=True, active=True).order_by('-id')
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower())
)
qs = qs.filter(filter)
total = len(qs)
if offset is not None:
qs = qs[offset:]
if limit is not None:
qs = qs[:limit]
public_institutions = []
for institution in qs:
public_institution = generate_public_institution(institution)
public_institutions.append(public_institution)
public_institutions.sort(key=lambda x: x.score, reverse=True) # Sorting the results by score before proceeding with pagination
results = PublicInstitutions(records=public_institutions, total=total)
return results
#login_required
def resolve_institution_by_invitecode(root, info, invitecode, **kwargs):
institution_instance = Institution.objects.get(
invitecode=invitecode, active=True)
if institution_instance is not None:
return institution_instance
else:
return None
#login_required
#user_passes_test(lambda user: has_access(user, RESOURCES['INSTITUTION'], ACTIONS['GET']))
def resolve_institution(root, info, id, **kwargs):
current_user = info.context.user
institution_instance = Institution.objects.get(pk=id, active=True)
allow_access = is_record_accessible(current_user, RESOURCES['INSTITUTION'], institution_instance)
if allow_access != True:
institution_instance = None
return institution_instance
#login_required
#user_passes_test(lambda user: has_access(user, RESOURCES['INSTITUTION'], ACTIONS['LIST']))
def resolve_institutions(root, info, searchField=None, limit=None, offset=None, **kwargs):
current_user = info.context.user
qs = rows_accessible(current_user, RESOURCES['INSTITUTION'])
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower())
)
qs = qs.filter(filter)
total = len(qs)
if offset is not None:
qs = qs[offset:]
if limit is not None:
qs = qs[:limit]
results = Institutions(records=qs, total=total)
return results
#login_required
def resolve_user(root, info, id, **kwargs):
user_instance = User.objects.get(pk=id, active=True)
if user_instance is not None:
user_instance = redact_user(root, info, user_instance)
return user_instance
else:
return None
def resolve_user_by_username(root, info, username, **kwargs):
user = None
try:
user = User.objects.get(username=username, active=True)
except:
raise GraphQLError('User does not exist!')
courses = Report.objects.filter(active=True, participant_id=user.id)
if user is not None:
user = redact_user(root, info, user)
title = user.title if user.title else user.role.name
new_user = PublicUserType(id=user.id, username=user.username, name=user.name, title=title, bio=user.bio, avatar=user.avatar,institution=user.institution, courses=courses)
return new_user
else:
return None
def process_users(root, info, searchField=None, all_institutions=False, membership_status_not=[], membership_status_is=[], roles=[], unpaginated = False, limit=None, offset=None, **kwargs):
current_user = info.context.user
admin_user = is_admin_user(current_user)
qs = rows_accessible(current_user, RESOURCES['MEMBER'], {'all_institutions': all_institutions})
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower()) | Q(username__icontains=searchField.lower()) | Q(email__icontains=searchField.lower())
)
qs = qs.filter(filter)
if membership_status_not:
qs = qs.exclude(membership_status__in=membership_status_not)
if membership_status_is:
qs = qs.filter(membership_status__in=membership_status_is)
if roles:
qs = qs.filter(role__in=roles)
redacted_qs = []
if admin_user:
redacted_qs = qs
else:
# Replacing the user avatar if the requesting user is not of the same institution and is not a super admin
for user in qs:
user = redact_user(root, info, user)
redacted_qs.append(user)
pending = []
uninitialized = []
others = []
for user in redacted_qs:
if user.membership_status == User.StatusChoices.PENDINIG:
pending.append(user)
elif user.membership_status == User.StatusChoices.UNINITIALIZED:
uninitialized.append(user)
else:
others.append(user)
sorted_qs = pending + uninitialized + others
total = len(sorted_qs)
if unpaginated == True:
results = Users(records=sorted_qs, total=total)
return results
if offset is not None:
sorted_qs = sorted_qs[offset:]
if limit is not None:
sorted_qs = sorted_qs[:limit]
results = Users(records=sorted_qs, total=total)
return results
#login_required
def resolve_users(root, info, searchField=None, membership_status_not=[], membership_status_is=[], roles=[], limit=None, offset=None, **kwargs):
all_institutions=False
unpaginated = False
qs = Query.process_users(root, info, searchField, all_institutions, membership_status_not, membership_status_is, roles, unpaginated, limit, offset, **kwargs)
return qs
def resolve_public_users(root, info, searchField=None, membership_status_not=[], membership_status_is=[], roles=[], limit=None, offset=None, **kwargs):
all_institutions=True
unpaginated = True
results = Query.process_users(root, info, searchField, all_institutions, membership_status_not, membership_status_is, roles, unpaginated, limit, offset, **kwargs)
records = results.records
total = results.total
public_users = []
# This is to limit the fields in the User model that we are exposing in this GraphQL query
for user in records:
courses = Report.objects.filter(active=True, participant_id=user.id)
score = 0
for course in courses:
score += course.completed * course.percentage
new_user = PublicUserType(id=user.id, username=user.username, name=user.name, title=user.title, bio=user.bio, avatar=user.avatar,institution=user.institution, score=score)
public_users.append(new_user)
public_users.sort(key=lambda x: x.score, reverse=True) # Sorting the results by score before proceeding with pagination
if offset is not None:
public_users = public_users[offset:]
if limit is not None:
public_users = public_users[:limit]
results = PublicUsers(records=public_users, total=total)
return results
I'm not sure what other code you'll need to see to get a sense, because the rest of the setup is typical of a Django application.
Is it someway to filter querysets with multiple optional parameters in Django more efficiently?
For ex. I have product list and user can filter it by using multiple GET params. 6 params in this case. Thanks.
class ProductList(ListAPIView):
permission_classes = (IsAdminUser,)
serializer_class = ProductSerializer
def get_queryset(self):
queryset = Product.objects.order_by('-created_at')
category_id = self.request.GET.get('category_id')
color = self.request.GET.get('color')
size = self.request.GET.get('size')
status = self.request.GET.get('status')
date_from = self.request.GET.get('date_from')
date_to = self.request.GET.get('date_to')
if category_id:
queryset = queryset.filter(category_id=category_id)
if color:
queryset = queryset.filter(color=color)
if size:
queryset = queryset.filter(size=size)
if status:
queryset = queryset.filter(status=sistatusze)
if date_from:
queryset = queryset.filter(created_at__gte=date_from)
if date_to:
queryset = queryset.filter(created_at__lte=date_to)
return queryset
You can make a utility function that will not filter the conditions where the value is None:
def filter_if_not_none(qs, **kwargs):
return qs.filter(**{k: v for k, v in kwargs.items() if v is not None})
then we can use this utility as:
class ProductList(ListAPIView):
permission_classes = (IsAdminUser,)
serializer_class = ProblemSerializer
def get_queryset(self):
queryset = Product.objects.order_by('-created_at')
return filter_qs_if_not_none(
queryset,
category_id=self.request.GET.get('category_id')
color=self.request.GET.get('color')
size=self.request.GET.get('size')
status=self.request.GET.get('status')
created_at__gte=self.request.GET.get('date_from')
created_at__lte=self.request.GET.get('date_to')
)
How To Provide OR Search for Filters Introduced by User
def teachers_list(request):
qs = Teacher.objects.all()
if request.GET.get('fname'):
qs = qs.filter(first_name=request.GET.get('fname'))
if request.GET.get('lname'):
qs = qs.filter(last_name=request.GET.get('lname'))
if request.GET.get('email'):
qs = qs.filter(email=request.GET.get('email'))
result = '<br>'.join(
str(teacher)
for teacher in qs
)
# return HttpResponse(result)
return render(
request=request,
template_name='teachers_list.html',
context={'teachers_list': result}
You can use the Q object, using it you or operation can be applied in queries
def teachers_list(request):
qs = Teacher.objects.all()
search = request.GET.get('search','')
if search:
qs = qs.filter(Q(first_name=search) | Q(last_name=search) | Q(email=search))
result = '<br>'.join(
str(teacher)
for teacher in qs
)
# return HttpResponse(result)
return render(
request=request,
template_name='teachers_list.html',
context={'teachers_list': result}
It will be better if you send the searching keyword using search instead of using 3 different keywords.
You can learn more about Q object from the doc
I want to make some tournament matches in a DetailView. But I can't figure out how to make a query of all registrations context['regs'] and use the queryset in my function create_matches().
class KategorieDetail(DetailView):
model = Kategorie
context_object_name = 'kategorie'
def create_matches(regs):
red_corner = []
blue_corner = []
matches = []
# Separate the regs into both corners
i = 1
for reg in regs:
if i%2 == 1:
red_corner.append(reg)
else:
blue_corner.append(reg)
i += 1
# Create Match-Ups
while blue_corner:
match = {'red': red_corner.pop(), 'blue': blue_corner.pop()}
matches.append(match)
return matches
def get_context_data(self, **kwargs):
context = super(KategorieDetail, self).get_context_data(**kwargs)
kid = context['kategorie'].id
context['regs'] = Registrierung.objects.filter(kategorie=context['kategorie'].id)
context['regs_count'] = context['regs'].count()
context['matches'] = create_matches(context['regs'].values(), kid)
return context
In my HTML-View I can't display the matches. If I say {{matches}}, I get:
HttpResponseRedirect status_code=302, "text/html; charset=utf-8", url="/events/"
I also don't get why I have to give the Kategorie_ID to the create_matches(regs) function.
I am attempting to make my API get return a maximum of 10 per page. This helps me with infinite loading. The API url will be I am trying looks like this:
www.mysite.com/api/test/?user=5&page=1
However, this does not work.
I've followed the official docs here without success.
I have only modified two files, settings.py & rest_views.py.
settings.py-
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination'
}
rest_views.py-
from django.core.paginator import Paginator
...
wardrobematch = {
'user': lambda x: ('user__pk', x)
}
class WardrobeListView(APIView):
renderer_classes = (JSONRenderer, )
paginate_by = 10
paginate_by_param = 'page_size'
max_paginate_by = 100
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
.values('product_id', 'product__name', 'product__brand', 'product__store__store_name', 'product__variation__image__image', 'product__variation__price__price',)
)
return Response(qset)
When using regular ApiView, you should call the pagination API yourself, it will not perform pagination automatically.
I have created a pagination and a serializer mixim. I'm not sure it is best method, but it worked for me.
class SerializerMixim(object):
def serialize_object(self,obj):
"""Serilize only needed fields"""
return NotImplementedError
class PaginationMixim(object):
_serializer = None
def paginate(self,queryset,num=10):
page = self.request.GET.get('page')
paginator = Paginator(queryset, num)
try:
queryset = paginator.page(page)
except PageNotAnInteger:
queryset = paginator.page(1)
except EmptyPage:
queryset = paginator.page(paginator.num_pages)
count = paginator.count
previous = None if not queryset.has_previous() else queryset.previous_page_number()
next = None if not queryset.has_next() else queryset.next_page_number()
if self._serializer:
objects = self._serializer(queryset.object_list,many=True).data
else:
objects = [self.serialize_object(i) for i in queryset.object_list]
data = {'count':count,'previous':previous,
'next':next,'object_list':objects}
return Response(data)
def serialize_object(self,obj):
return {'id':obj.pk}
class WardrobeListView(APIView,PaginationMixim,SerializerMixim):
renderer_classes = (JSONRenderer, )
#_serializer = AnalyticSerializer
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
return self.paginate(qset)
def serialize_object(self,obj):
return obj.serilized
then you need to create a propery for Analytic model like,
class Analytic(models.Model):
.....
#property
def serilized(self):
summary = {
'id':self.product.id,
'image':self.product.name,
.......
}
return summary
this will also work with django rest serializers
I got your first example working- to me it was clearer and more basic. All I did was add ".object_list" to stop the "is not JSON serializable" error.
This is your answer with my tiny tweak:
class WardrobeListView(APIView):
renderer_classes = (JSONRenderer, )
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
.values('product_id', 'product__name', 'product__brand', 'product__store__store_name', 'product__variation__image__image', 'product__variation__price__price',)
)
paginator = Paginator(qset, 2) # Show 25 items per page
page = request.GET.get('page')
try:
qset = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
qset = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
qset = paginator.page(paginator.num_pages)
return Response(qset.object_list)