Django Rest Framework Paging - django

I am attempting to make my API get return a maximum of 10 per page. This helps me with infinite loading. The API url will be I am trying looks like this:
www.mysite.com/api/test/?user=5&page=1
However, this does not work.
I've followed the official docs here without success.
I have only modified two files, settings.py & rest_views.py.
settings.py-
REST_FRAMEWORK = {
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination'
}
rest_views.py-
from django.core.paginator import Paginator
...
wardrobematch = {
'user': lambda x: ('user__pk', x)
}
class WardrobeListView(APIView):
renderer_classes = (JSONRenderer, )
paginate_by = 10
paginate_by_param = 'page_size'
max_paginate_by = 100
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
.values('product_id', 'product__name', 'product__brand', 'product__store__store_name', 'product__variation__image__image', 'product__variation__price__price',)
)
return Response(qset)

When using regular ApiView, you should call the pagination API yourself, it will not perform pagination automatically.
I have created a pagination and a serializer mixim. I'm not sure it is best method, but it worked for me.
class SerializerMixim(object):
def serialize_object(self,obj):
"""Serilize only needed fields"""
return NotImplementedError
class PaginationMixim(object):
_serializer = None
def paginate(self,queryset,num=10):
page = self.request.GET.get('page')
paginator = Paginator(queryset, num)
try:
queryset = paginator.page(page)
except PageNotAnInteger:
queryset = paginator.page(1)
except EmptyPage:
queryset = paginator.page(paginator.num_pages)
count = paginator.count
previous = None if not queryset.has_previous() else queryset.previous_page_number()
next = None if not queryset.has_next() else queryset.next_page_number()
if self._serializer:
objects = self._serializer(queryset.object_list,many=True).data
else:
objects = [self.serialize_object(i) for i in queryset.object_list]
data = {'count':count,'previous':previous,
'next':next,'object_list':objects}
return Response(data)
def serialize_object(self,obj):
return {'id':obj.pk}
class WardrobeListView(APIView,PaginationMixim,SerializerMixim):
renderer_classes = (JSONRenderer, )
#_serializer = AnalyticSerializer
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
return self.paginate(qset)
def serialize_object(self,obj):
return obj.serilized
then you need to create a propery for Analytic model like,
class Analytic(models.Model):
.....
#property
def serilized(self):
summary = {
'id':self.product.id,
'image':self.product.name,
.......
}
return summary
this will also work with django rest serializers

I got your first example working- to me it was clearer and more basic. All I did was add ".object_list" to stop the "is not JSON serializable" error.
This is your answer with my tiny tweak:
class WardrobeListView(APIView):
renderer_classes = (JSONRenderer, )
def get(self, request, *args, **kwargs):
filters = {}
for key, value in request.GET.items():
key = key.lower()
if key in wardrobematch:
lookup, val = wardrobematch[key](value.lower())
filters[lookup] = val
qset = (
Analytic.objects
.filter(like=True,**filters)
.order_by('-updated',)
# .distinct('product_id',)
.values('product_id', 'product__name', 'product__brand', 'product__store__store_name', 'product__variation__image__image', 'product__variation__price__price',)
)
paginator = Paginator(qset, 2) # Show 25 items per page
page = request.GET.get('page')
try:
qset = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
qset = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
qset = paginator.page(paginator.num_pages)
return Response(qset.object_list)

Related

Best approach to implement server-side caching on a Django GraphQL API?

I have an Angular frontend that uses Apollo Graphql client to interface with a Django Graphql backend (using graphene). It all works well, but the API is very slow, especially when there are concurrent users. I am trying a few things to speed up the API server. One of the things that I am seriously considering is to use server-side caching.
I'm aware that there are some caching solutions that exist for Django out of the box, but these seem to be for caching the views. Is it possible to use these methods to cache Graphql API requests?
What is the most effective caching solution for caching the responses to graphql queries in Django?
I'm posting some of my project's code here so that you get a sense of how my API works.
This is a sample of the Query class containing the various graphql endpints that serve data when queries are received from the client. And I suspect this is where we'll need most of the optimization and caching applied:-
class Query(ObjectType):
# Public Queries
user_by_username = graphene.Field(PublicUserType, username=graphene.String())
public_users = graphene.Field(
PublicUsers, searchField=graphene.String(), membership_status_not=graphene.List(graphene.String), membership_status_is=graphene.List(graphene.String), roles=graphene.List(graphene.String), limit=graphene.Int(), offset=graphene.Int())
public_institution = graphene.Field(PublicInstitutionType, code=graphene.String())
public_institutions = graphene.Field(PublicInstitutions, searchField=graphene.String(), limit = graphene.Int(), offset = graphene.Int())
public_announcement = graphene.Field(AnnouncementType, id=graphene.ID())
public_announcements = graphene.List(
AnnouncementType, searchField=graphene.String(), limit=graphene.Int(), offset=graphene.Int())
def resolve_public_institution(root, info, code, **kwargs):
institution = Institution.objects.get(code=code, active=True)
if institution is not None:
public_institution = generate_public_institution(institution)
return public_institution
else:
return None
def resolve_public_institutions(root, info, searchField=None, limit=None, offset=None, **kwargs):
qs = Institution.objects.all().filter(public=True, active=True).order_by('-id')
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower())
)
qs = qs.filter(filter)
total = len(qs)
if offset is not None:
qs = qs[offset:]
if limit is not None:
qs = qs[:limit]
public_institutions = []
for institution in qs:
public_institution = generate_public_institution(institution)
public_institutions.append(public_institution)
public_institutions.sort(key=lambda x: x.score, reverse=True) # Sorting the results by score before proceeding with pagination
results = PublicInstitutions(records=public_institutions, total=total)
return results
#login_required
def resolve_institution_by_invitecode(root, info, invitecode, **kwargs):
institution_instance = Institution.objects.get(
invitecode=invitecode, active=True)
if institution_instance is not None:
return institution_instance
else:
return None
#login_required
#user_passes_test(lambda user: has_access(user, RESOURCES['INSTITUTION'], ACTIONS['GET']))
def resolve_institution(root, info, id, **kwargs):
current_user = info.context.user
institution_instance = Institution.objects.get(pk=id, active=True)
allow_access = is_record_accessible(current_user, RESOURCES['INSTITUTION'], institution_instance)
if allow_access != True:
institution_instance = None
return institution_instance
#login_required
#user_passes_test(lambda user: has_access(user, RESOURCES['INSTITUTION'], ACTIONS['LIST']))
def resolve_institutions(root, info, searchField=None, limit=None, offset=None, **kwargs):
current_user = info.context.user
qs = rows_accessible(current_user, RESOURCES['INSTITUTION'])
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower())
)
qs = qs.filter(filter)
total = len(qs)
if offset is not None:
qs = qs[offset:]
if limit is not None:
qs = qs[:limit]
results = Institutions(records=qs, total=total)
return results
#login_required
def resolve_user(root, info, id, **kwargs):
user_instance = User.objects.get(pk=id, active=True)
if user_instance is not None:
user_instance = redact_user(root, info, user_instance)
return user_instance
else:
return None
def resolve_user_by_username(root, info, username, **kwargs):
user = None
try:
user = User.objects.get(username=username, active=True)
except:
raise GraphQLError('User does not exist!')
courses = Report.objects.filter(active=True, participant_id=user.id)
if user is not None:
user = redact_user(root, info, user)
title = user.title if user.title else user.role.name
new_user = PublicUserType(id=user.id, username=user.username, name=user.name, title=title, bio=user.bio, avatar=user.avatar,institution=user.institution, courses=courses)
return new_user
else:
return None
def process_users(root, info, searchField=None, all_institutions=False, membership_status_not=[], membership_status_is=[], roles=[], unpaginated = False, limit=None, offset=None, **kwargs):
current_user = info.context.user
admin_user = is_admin_user(current_user)
qs = rows_accessible(current_user, RESOURCES['MEMBER'], {'all_institutions': all_institutions})
if searchField is not None:
filter = (
Q(searchField__icontains=searchField.lower()) | Q(username__icontains=searchField.lower()) | Q(email__icontains=searchField.lower())
)
qs = qs.filter(filter)
if membership_status_not:
qs = qs.exclude(membership_status__in=membership_status_not)
if membership_status_is:
qs = qs.filter(membership_status__in=membership_status_is)
if roles:
qs = qs.filter(role__in=roles)
redacted_qs = []
if admin_user:
redacted_qs = qs
else:
# Replacing the user avatar if the requesting user is not of the same institution and is not a super admin
for user in qs:
user = redact_user(root, info, user)
redacted_qs.append(user)
pending = []
uninitialized = []
others = []
for user in redacted_qs:
if user.membership_status == User.StatusChoices.PENDINIG:
pending.append(user)
elif user.membership_status == User.StatusChoices.UNINITIALIZED:
uninitialized.append(user)
else:
others.append(user)
sorted_qs = pending + uninitialized + others
total = len(sorted_qs)
if unpaginated == True:
results = Users(records=sorted_qs, total=total)
return results
if offset is not None:
sorted_qs = sorted_qs[offset:]
if limit is not None:
sorted_qs = sorted_qs[:limit]
results = Users(records=sorted_qs, total=total)
return results
#login_required
def resolve_users(root, info, searchField=None, membership_status_not=[], membership_status_is=[], roles=[], limit=None, offset=None, **kwargs):
all_institutions=False
unpaginated = False
qs = Query.process_users(root, info, searchField, all_institutions, membership_status_not, membership_status_is, roles, unpaginated, limit, offset, **kwargs)
return qs
def resolve_public_users(root, info, searchField=None, membership_status_not=[], membership_status_is=[], roles=[], limit=None, offset=None, **kwargs):
all_institutions=True
unpaginated = True
results = Query.process_users(root, info, searchField, all_institutions, membership_status_not, membership_status_is, roles, unpaginated, limit, offset, **kwargs)
records = results.records
total = results.total
public_users = []
# This is to limit the fields in the User model that we are exposing in this GraphQL query
for user in records:
courses = Report.objects.filter(active=True, participant_id=user.id)
score = 0
for course in courses:
score += course.completed * course.percentage
new_user = PublicUserType(id=user.id, username=user.username, name=user.name, title=user.title, bio=user.bio, avatar=user.avatar,institution=user.institution, score=score)
public_users.append(new_user)
public_users.sort(key=lambda x: x.score, reverse=True) # Sorting the results by score before proceeding with pagination
if offset is not None:
public_users = public_users[offset:]
if limit is not None:
public_users = public_users[:limit]
results = PublicUsers(records=public_users, total=total)
return results
I'm not sure what other code you'll need to see to get a sense, because the rest of the setup is typical of a Django application.

Possible to get queryset from list of queryset -Django

I wanted to take queryset from multiple models. I am trying to achieve multiple search with and condition.
views.py,
def list(self, request, *args, **kwargs):
search_query = self.request.query_params.get('search')
split_query = search_query.split()
employment = None
employee1 = []
employment1 = []
for query in split_query:
print("hi", query)
# query = self.request.query_params.get('search')
employee = PositionFulfillment.objects.filter(
Q(employment__employee__code__icontains=query) |
Q(employment__employee__person__name__icontains=query) |
Q(employment__employee__person__surname__icontains=query)
)
# emp = list(chain(employee))
employee1.append(employee)
print("employee", employee1)
active_employee = PositionFulfillment.objects.filter(primary_flag=True, thru_date=None)
if active_employee:
employment = active_employee.filter(
Q(position__position_type__name__icontains=query) |
Q(employment__organization__name__icontains=query) |
Q(employment__status__status__employment_status__icontains=query)
)
employment1.append(employment)
all_results = list(chain(map(lambda x: x, employee1), map(lambda y: y, employment1)))
# all_results = list(chain(employee, employment))
print("all_results", all_results)
serializer = EmployeeSearchSerializer(all_results)
return Response(serializer.data, status=status.HTTP_200_OK)
I have got output like below,
all_results,
[<QuerySet [<PositionFulfillment: 27>, <PositionFulfillment: 29>, <PositionFulfillment: 30>]>, <QuerySet []>, <QuerySet []>, <QuerySet [<PositionFulfillment: 28>]>]
Expected output,
[<PositionFulfillment: 27>, <PositionFulfillment: 29>, <PositionFulfillment: 30>]> ,<QuerySet [<PositionFulfillment: 28>]>]
How can i achieve this???
You can add conditions to the query instead of creating new queries:
def list(self, request, *args, **kwargs):
search_query = self.request.query_params.get('search')
split_query = search_query.split()
q = None # this is filter
for query in split_query:
print("hi", query)
if q is None:
q = Q(employment__employee__code__icontains=query)
else:
q.add(Q(employment__employee__code__icontains=query), Q.OR)
q.add(Q(employment__employee__person__name__icontains=query), Q.OR)
q.add(Q(employment__employee__person__surname__icontains=query), Q.OR)
# same for active_employee
# make only one query based on all conditions in q
all_results = PositionFulfillment.objects.filter(q)
print("all_results", all_results)
serializer = EmployeeSearchSerializer(all_results)
return Response(serializer.data, status=status.HTTP_200_OK)

How to implement paginations in function base views in django

i know how to implement pagination in class base views using ListView
class ProductListView(ListView):
model = product
template_name = 'products/product_list.html'
context_object_name = 'products'
ordering = ['-pub_date']
paginate_by = 5
but i don't know how to implement paginations in function base views. i read that we have to import Paginator for django.core.paginator and use it in functions base view like this paginator = Paginator(qs, 5) but it's not working.
def Productlistview(request):
qs = product.objects.all()
location_search = request.GET.get('location')
print(location_search)
categories = request.GET.get('categories')
price = request.GET.get('price')
ordering = ['-pub_date']
paginator = Paginator(qs, 5)
if location_search != "" and location_search is not None:
qs = qs.filter(location__contains=location_search)
context = {
'posts': qs
}
return render(request, "products/product_list.html", context)
This is covered in the Pagination section of the documentation. You indeed use a paginator, but you still need to pass the page that you want to show. This is often done through a GET parameter, like for example page. Furthermore you should do the filtering and ordering before paginating.
def product_list_view(request):
qs = product.objects.all()
location_search = request.GET.get('location')
print(location_search)
categories = request.GET.get('categories')
price = request.GET.get('price')
if location_search:
qs = qs.filter(location__contains=location_search)
qs = qs.order_by('-pub_date')
paginator = Paginator(qs, 5)
page = p.page(request.GET.get('page'))
context = {
'posts': page
}
return render(request, "products/product_list.html", context)

Override page_size & ordering of CursorPagination in Django Rest Framework

I using CursorPagination of Django Rest Framework, I want to override default page_size(10) and ordering('timestamp') in a single viewset. How can I do this?
I tried with my viewset but it's not success:
from rest_framework.pagination import CursorPagination
class ListAPIView(ListAPIView):
queryset = Cake.objects.all()
permission_classes = [AllowAny]
serializer_class = ListSerializer
pagination_class = CursorPagination
filter_backends = (OrderingFilter, DjangoFilterBackend)
filter_class = CakeListFilter
filterset_fields = ('cake_type', 'user__username')
ordering = '-date'
page_size = 5
You may create a new class inheriting from CursorPagination class in order to set custom page_size and/or max_page_size like so:
class CustomPageSizeCursorPagination(CursorPagination):
page_size = 5
max_page_size = 100
And then use this class as the pagination_class field of your viewset
WARNING: The following code is untested
Another option is to write a custom paginator class that gets the page size from the viewset. For example:
class PageSizeInViewSetCursorPagination(CursorPagination):
def get_page_size(self, request, viewset_page_size):
if self.page_size_query_param:
try:
return _positive_int(
request.query_params[self.page_size_query_param],
strict=True,
cutoff=self.max_page_size
)
except (KeyError, ValueError):
pass
return viewset_page_size or self.page_size
def paginate_queryset(self, queryset, request, view=None):
# Get the page_size from the viewset and then decide which page_size to use
viewset_page_size = getattr(view, 'page_size', None)
page_size = self.get_page_size(request, viewset_page_size)
# What follows is copy/paste of the code from CursorPagination paginate_queryset method
if not self.page_size:
return None
self.base_url = request.build_absolute_uri()
self.ordering = self.get_ordering(request, queryset, view)
self.cursor = self.decode_cursor(request)
if self.cursor is None:
(offset, reverse, current_position) = (0, False, None)
else:
(offset, reverse, current_position) = self.cursor
# Cursor pagination always enforces an ordering.
if reverse:
queryset = queryset.order_by(*_reverse_ordering(self.ordering))
else:
queryset = queryset.order_by(*self.ordering)
# If we have a cursor with a fixed position then filter by that.
if current_position is not None:
order = self.ordering[0]
is_reversed = order.startswith('-')
order_attr = order.lstrip('-')
# Test for: (cursor reversed) XOR (queryset reversed)
if self.cursor.reverse != is_reversed:
kwargs = {order_attr + '__lt': current_position}
else:
kwargs = {order_attr + '__gt': current_position}
queryset = queryset.filter(**kwargs)
# If we have an offset cursor then offset the entire page by that amount.
# We also always fetch an extra item in order to determine if there is a
# page following on from this one.
results = list(queryset[offset:offset + self.page_size + 1])
self.page = list(results[:self.page_size])
# Determine the position of the final item following the page.
if len(results) > len(self.page):
has_following_position = True
following_position = self._get_position_from_instance(results[-1], self.ordering)
else:
has_following_position = False
following_position = None
# If we have a reverse queryset, then the query ordering was in reverse
# so we need to reverse the items again before returning them to the user.
if reverse:
self.page = list(reversed(self.page))
if reverse:
# Determine next and previous positions for reverse cursors.
self.has_next = (current_position is not None) or (offset > 0)
self.has_previous = has_following_position
if self.has_next:
self.next_position = current_position
if self.has_previous:
self.previous_position = following_position
else:
# Determine next and previous positions for forward cursors.
self.has_next = has_following_position
self.has_previous = (current_position is not None) or (offset > 0)
if self.has_next:
self.next_position = following_position
if self.has_previous:
self.previous_position = current_position
# Display page controls in the browsable API if there is more
# than one page.
if (self.has_previous or self.has_next) and self.template is not None:
self.display_page_controls = True
return self.page
Note that in the above example the page_size from the request always takes precedence over whatever you have set up in your code. Then the viewset_page_size is the second in line and lastly the deafult page_size from the Pagination class.
Here is a custom pagination class that extends CursorPagination. It checks for ordering and page_size attributes defined in the viewset and if they exist use them. If not, fallback to original settings defined in the pagination class itself.
class NewsCursorPaginator(CursorPagination):
ordering = 'title'
page_size = 5
# get_page_size do not have view attribute, so we have our custom one
def get_custom_page_size(self, request, view):
viewset_page_size = getattr(view, 'page_size', None)
if viewset_page_size:
self.page_size = viewset_page_size
return super(NewsCursorPaginator, self).get_page_size(request)
def get_ordering(self, request, queryset, view):
viewset_ordering = getattr(view, 'ordering', None)
if viewset_ordering:
self.ordering = viewset_ordering
return super(NewsCursorPaginator, self).get_ordering(request, queryset, view)
def paginate_queryset(self, queryset, request, view=None):
self.page_size = self.get_custom_page_size(request, view)
return super(NewsCursorPaginator, self).paginate_queryset(queryset, request, view)
This implementation takes "limit" (page_size) as an optional querystring parameter.
class CursorPagination(pagination.CursorPagination):
page_size = settings.REST_FRAMEWORK["PAGE_SIZE"]
def get_custom_page_size(self, request, view):
try:
self.page_size = int(request.GET.get("limit"))
except (ValueError, TypeError):
pass
return super().get_page_size(request)
def paginate_queryset(self, queryset, request, view=None):
self.page_size = self.get_custom_page_size(request, view)
return super().paginate_queryset(queryset, request, view)

ZeroDivisionError on GET parameter django

middleware.py
def get_perpage(self):
try:
self.session['perpage'] = int(self.REQUEST['perpage'])
return self.session['perpage']
except (KeyError, ValueError, TypeError):
pass
try:
return int(self.session['perpage'])
except (KeyError, ValueError, TypeError):
return DEFAULT_PAGINATION
I have a problem: when i want to turn zero into URL as GET parameter (?perpage=0), it shows me ZeroDivisionError float division by zero. I need to take ALL objects on page without pagination when perpage=0. How can I do this? What is must be in view.py?
def render(self, context):
key = self.queryset_var.var
value = self.queryset_var.resolve(context)
if (self.paginate_by == None):
paginate_by = int(context['request'].perpage)
else:
paginate_by = self.paginate_by.resolve(context)
if (paginate_by == 0): #HERE
context['page_obj'] = value # IS
return u'' #SOLUTION
print (paginate_by)
paginator = Paginator(value, paginate_by, self.orphans)
try:
page_obj = paginator.page(context['request'].page)
except InvalidPage:
if INVALID_PAGE_RAISES_404:
raise Http404('Invalid page requested. If DEBUG were set to ' +
'False, an HTTP 404 page would have been shown instead.')
context[key] = []
context['invalid_page'] = True
return u''
if self.context_var is not None:
context[self.context_var] = page_obj.object_list
else:
context[key] = page_obj.object_list
context['paginator'] = paginator
context['page_obj'] = page_obj
return u''
when my view gets perpage=0, it returns pure object list (value) without Pagination to template