Merge queryset from two databases in Django Rest - django

I have two routes in my api looking like that :
http://mywebsite/websites/website_1/annonces/
http://mywebsite/websites/website_2/annonces/
I need to make an ListAPIView merging these 2 routes but each route call its own database.
The two databases are made with the same django Model. I made two databases because it suits my architecture better.
The problem is that I have no column in my databases which indicates the website the records are from. The records are only differentiated by the names of their database.
I want to get all record in a single route but also being able to tell from which database they are from in the json response.
class Annonce(models.Model):
annonce_type = models.CharField(max_length=200, blank=True, null=True)
annonce_id = models.CharField(unique=True, max_length=200, blank=True, null=True)
url = models.TextField(blank=True, null=True)
region = models.TextField(blank=True, null=True)
class AnnoncesList(generics.ListAPIView):
authentication_classes = ()
permission_classes = ()
serializer_class = AnnonceListSerializer
pagination_class = LargeResultsSetPagination
filter_backends = (DjangoFilterBackend,)
filterset_fields = ('advert_type', 'asset_type', 'sales_type', 'price', 'area', 'department', 'department_id', 'city', 'postal_code')
def get_queryset(self):
queryset = Annonce.objects.using(self.kwargs["website_name"]).all()
return queryset

Make the queryset for each database, then use annotate() to add a column website_name for each record on queryset. Concatenate the querysets into a list (check this) (will hit all items on database), make sure the querysets have already been properly filtered.
from itertools import chain
from rest_framework.generics import ListAPIView
from rest_framework.response import Response
from django.db.models import Value, CharField
class AnnonceMergedList(ListAPIView):
serializer_class = AnnonceMergedListSerializer
queryset = Annonce.objects.all()
def list(self, request, **kwargs):
# Make the querysets for each database
q1 = self.get_queryset().using('website_1').annotate(website_name=Value('website_1', CharField()))
q2 = self.get_queryset().using('website_2').annotate(website_name=Value('website_2', CharField()))
# Filtering the querysets
q1 = self.filter_queryset(q1)
q2 = self.filter_queryset(q2)
# Merge (hit items on database)
data = list(chain(q1, q2))
serializer = self.get_serializer(data, many=True)
return Response(serializer.data)
The serializer for this view must receive the website_name to display the website the records are from
class AnnonceMergedListSerializer(serializers.ModelSerializer):
website_name = serializers.CharField(read_only=True) # Field from annotation
class Meta:
model = Annonce
fields = ('website_name', ...)

Related

how to query filter self foreign key

query = Services.objects.filter(parent_id__isnull=True,sub_service__type=0)
When im filtering sub_service__type=1 It returns correct output. i.e. type 1 for sub_service, But when i change it to sub_service__type=0 filters doesnot work. Instead it returns me every output. i.e. type 0,1 Instead of type 0
Heres Code:
# MODELS
class Services(models.Model):
type_ = ((1, 'INCLUSIVE'),
(0, 'EXCLUSIVE'))
service_id = models.AutoField(primary_key=True)
parent_id = models.ForeignKey('self', on_delete=models.SET_NULL, null=True, blank=True, related_name='sub_service')
type = models.SmallIntegerField(blank=True, null=True, choices=type_)
# VIEWS
#action(detail=False, methods=['get'], permission_classes=(AllowAny,))
def service_list(self, request):
query = Services.objects.filter(parent_id__isnull=True,sub_service__type=0)
serializer = ServiceSerializer(query , many=True).data
return Response(serializer)
# SERIALIZER
class SubServiceSerializer(serializers.ModelSerializer):
class Meta:
model = Services
fields = "__all__"
class ServiceSerializer(serializers.ModelSerializer):
sub_service = SubServiceSerializer(many=True)
class Meta:
model = Services
fields = "__all__"
If you filter with sub_service__type=1 you retrieve all Services that have at least one related Service with type=1. But it is thus allowed that there are other related sub-Services with a different type. The .sub_service manager will furthermore not filter the queryset of related objects.
You can make use of a Prefetch object [Django-doc] to filter the relation as well:
from django.db.models import Prefetch
query = Services.objects.filter(
parent_id__isnull=True,sub_service__type=0
).prefetch_related(Prefetch('sub_service', Service.objects.filter(type=0)))

How to filter records in complex DB by the fields in related models in DRF?

We want to let possibility to find in all tables the records by the fields in related tables. Actually we would like to search through all of them.
For example the code:
/models.py
class Tourney(models.Model):
tourney = models.CharField()
tourney_1 = models.CharField()
tourney_2 = models.CharField()
class Stage(models.Model):
tourney = models.ForeignKey(Tourney, on_delete=models.CASCADE)
stage = models.CharField()
stage_1 = models.CharField()
stage_2 = models.CharField()
class Group(models.Model):
stage = models.ForeignKey(Stage, on_delete=models.CASCADE)
group = models.CharField()
group_1 = models.CharField()
group_2 = models.CharField()
Group has relation on Stage which has relation on Tourney.
So now we want to set API for them. Imagine we have simple serializers for them which include all their fields and called TourneySerializer, StageSerializer and GroupSerializer.
Now let's try to filter the Group model.
/views.py
class GroupViewSet(viewsets.ModelViewSet):
queryset = Group.objects.all()
serializer_class = serializers.GroupSerializer
def list(self, request, *args, **kwargs):
queryset = self.get_queryset()
if 'tourney' in request.GET:
queryset = queryset.filter(stage__tourney__tourney=request.GET['tourney'])
if 'tourney_1' in request.GET:
queryset = queryset.filter(stage__tourney__tourney_1=request.GET['tourney_1'])
if 'tourney_2' in request.GET:
queryset = queryset.filter(stage__tourney__tourney_2=request.GET['tourney_2'])
if 'stage' in request.GET:
queryset = queryset.filter(stage__stage=request.GET['stage'])
if 'stage_1' in request.GET:
queryset = queryset.filter(stage__stage_1=request.GET['stage_1'])
if 'stage_2' in request.GET:
queryset = queryset.filter(stage__stage_2=request.GET['stage_2'])
if 'group' in request.GET:
queryset = queryset.filter(group=request.GET['group'])
if 'group_1' in request.GET:
queryset = queryset.filter(group_1=request.GET['group_1'])
if 'group_2' in request.GET:
queryset = queryset.filter(group_2=request.GET['group_2'])
serializer = self.get_serializer_class()(
queryset,
many=True)
return Response(serializer.data)
Here we have one ViewSet with a bunch of obvious code and there will be more if there are more tables and more fields in the tables. I have up to 20 tables and the last table in this related chain can be able to filter though about 40 fields. So it's possible to have about 400 filter rules on all models so it's about 800 lines of stupid code just for one last model. Not good at all.
So is there any right known way to do this? This problems looks common so maybe are there embedded solutions from Django or any libraries of it?
The fastest way it is change the parameters name you pass to that view GET['tourney_2'] should available as GET['stage__tourney__tourney_2'].
Than the whole filtering will be:
queryset = self.get_queryset().filter(**request.GET)
Of course you should check parameters name to avoid SQL injection like request.GET['deleted'] = True
In real project you can not use **request.GET. You should cast it to normal Python dict filters = dict(request.GET) and check filters against sql injection attacks.
queryset = self.get_queryset().filter(**filters)
Please look how to build query dynamically.
How to dynamically provide lookup field name in Django query?

Inter dependent filter api in django rest framework

I want to expose the sample dataset through a single generic HTTP API endpoint, which is capable of filtering, grouping and sorting.
In this I want API to be interdependent. For eg. If I have to show the number of impressions and clicks that occurred before the 1st of June 2017, broken down by channel and country, sorted by clicks in descending order.
I know how to perform singular operation in django filter by using django ORM and django rest_framework_filters but I'm completely unaware of this problem statement. Any sort of help will be appreciated.
This is my view:
class filter_dataset(ListCreateAPIView):
serializer_class = AdjustDatasetSerializers
filter_backends = (filters.DjangoFilterBackend,)
filterset_fields = ('revenue', 'spend', 'OS', 'date')
filterset_class = ProductFilter
def get_queryset(self):
dataset = Dataset.objects.all()
return dataset
def get(self, request):
dataset = self.get_queryset()
serializer = self.serializer_class(dataset, many=True)
return Response(serializer.data, status=status.HTTP_200_OK)
This is my serializer class:
class AdjustDatasetSerializers(serializers.ModelSerializer):
class Meta:
model = Dataset
fields = (
'date',
'channel',
'OS',
'impressions',
'clicks',
'installs',
'spend',
'revenue',
)
And this is my models.py:
class Dataset(models.Model):
date = models.DateTimeField(max_length=255,auto_now_add=True)
channel = models.CharField(max_length=255,null=True, blank=True)
OS = models.CharField(max_length=255,null=True, blank=True)
impressions = models.CharField(max_length=255,null=True, blank=True)
clicks = models.CharField(max_length=255,null=True, blank=True)
installs = models.CharField(max_length=255,null=True, blank=True)
spend = models.CharField(max_length=255,null=True, blank=True)
revenue = models.CharField(max_length=255,null=True, blank=True)

django orm latest item group by each foreign key

We have following models:
class Publisher(models.Model):
name = models.CharField(max_length=32)
local_name = models.CharField(max_length=32)
url_pattern = models.CharField(max_length=128)
enabled = models.BooleanField(default=True)
home_page = models.BooleanField(default=False)
category = models.ForeignKey(NewspaperCategory, null=True)
class Newspaper(models.Model):
class Meta:
unique_together = ("publisher", "date")
ordering = ("-date",)
publisher = models.ForeignKey(Publisher)
image = models.ImageField(upload_to=newspaper_upload_to)
thumbnail = models.ImageField(upload_to=newspaper_thumbnail_upload_to)
date = models.DateField()
I have a APIView (Django rest framework) and there are several different query parameters which filter the output of API. I'd like to have a "latest" query parameter which lists only the latest version of each publisher. Also, I need to be able to do further filtering and slicing on that QuerySet before evaluating it.
But the result of query should be Newspaper instances not dict so I can feed them to my serializer.
My view looks like this at the moment :
class Newspapers(APIView):
def get(self, request):
queryset = models.Newspaper.objects.filter(deleted=False).prefetch_related('publisher')
if "latest" in request.GET:
# doing something here with queryset
if "publisher_id" in request.GET:
queryset = queryset.filter(publisher_id=request.GET['publisher_id'])
if "category" in request.GET:
queryset = queryset.filter(publisher__category_id=request.GET['category'])
if "before" in request.GET:
queryset = queryset.filter(date__lt=request.GET['before'])
if "after" in request.GET:
queryset = queryset.filter(date__gte=request.GET['after'])
if "home_page" in request.GET:
queryset = queryset.filter(publisher__home_page=request.GET['home_page'].lower() == 'true')
queryset = queryset.order_by('-date', 'publisher__order')
return PagingResponse(
request,
serializers.NewspaperVersionSerializer,
query=queryset,
identifier_name='date'
)
We can do this with two queries:
if "latest" in request.GET:
latest_ids = list( # <-- first query
queryset.order_by('id')
.values('publisher_id')
.annotate(id=Max('id'))
.values_list('id', flat=True)
)
queryset = queryset.filter(id__in=latest_ids) # <-- second query
the first query gets list of latest newspaper group by publisher, and the second query gets a queryset of those newspaper for further filtering, slicing and ...

Django Rest queryset filter by url parameter

I need to filter queryset in ListAPIView depending on url parameter. Basically, application lists all attendees (User) of an event with a call to API like /events/:id/attendees/
You can find serializers, models, urls and views below. I also wonder any other practices to do such end-point implementation in Django REST
**serializers.py**
class AttendeeSerializer(serializers.ModelSerializer):
user = UserSerializer()
class Meta:
model = Attendee
fields = ('user', 'status')
**views.py**
class EventAttendeeList(generics.ListAPIView):
queryset = Attendee.objects.all()
serializer_class = AttendeeSerializer
#permission_classes = (permissions.IsAuthenticatedOrReadOnly,)
def get_queryset(self):
# It should filter attendees by event_id posted in URL
return Attendee.objects.all()
**urls.py**
url(r'^events/(?P<pk>[0-9]+)/attendees/$', views.EventAttendeeList.as_view()),
**models.py**
class Event(models.Model):
name = models.CharField(max_length=500, blank=True)
attendees = models.ManyToManyField(settings.AUTH_USER_MODEL, through='Attendee', related_name='attendees_event')
class Attendee(models.Model):
event = models.ForeignKey(Event, related_name="a_event")
user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name="a_user")
requested_on = models.DateTimeField(default=timezone.now)
You can access the url parameter in your view with self.kwargs['parameter_name'] ( http://www.django-rest-framework.org/api-guide/filtering#filtering-against-the-url ). So the simplest solution would be
def get_queryset(self):
return Attendee.objects.filter(event=self.kwargs['pk'])