Reduce the number of SQL queries in a serializer with SerializerMethodField - django

I need to optimize a serializer with SerializerMethodField that contains a query.
The query is used to retrieve information from another object (Event) which is linked by a primarykey to Sensor object.
class SiteSensorSerializer(serializers.ModelSerializer):
issue = serializers.SerializerMethodField()
class Meta:
model = Sensor
fields = ('id', 'label', 'view', 'issue',)
def get_issue(self, obj):
return (
Event.objects.filter(sensor=obj, date_end__isnull=True)
.order_by('-date_start')
.exists()
)
class SiteDeviceSerializer(serializers.ModelSerializer):
label = DeviceLabelSerializer()
sensors = SiteSensorSerializer(many=True, read_only=True)
class Meta:
model = Device
fields = ('id', 'name', 'label', 'sensors')
My issue is for each sensor, the query in get_issue method is executed.
How to reduce the number of query ?
My viewset:
class SiteViewSet(viewsets.ReadOnlyModelViewSet):
permission_classes = (permissions.IsAuthenticated,)
serializer_class = SiteSerializer
filter_backends = [
DjangoFilterBackend,
filters.SearchFilter,
filters.OrderingFilter,
]
filter_fields = [
"name",
]
search_fields = ["name"]
ordering = ["name"]
def get_queryset(self):
if self.request.user.is_superuser:
return (
Site.objects.all()
.prefetch_related("devices")
.prefetch_related("devices__label")
.prefetch_related("devices__sensors")
)
else:
return (
Site.objects.filter(devices__users=self.request.user)
.prefetch_related(
Prefetch(
"devices",
Device.objects.filter(users=self.request.user).select_related(
"label"
),
)
)
.distinct()
.prefetch_related(
Prefetch(
"devices__sensors",
Sensor.objects.filter(device__users=self.request.user),
)
)
.distinct()
)

A minor optimization would be to remove the order_by('-date_start') from get_issue() since you are calling exists() which only returns a boolean based on whether the QuerySet has more than 0 objects within it, so there is no point in sorting the QuerySet.
get_issue() would look like the following:
def get_issue(self, obj):
return (
Event.objects.filter(sensor=obj, date_end__isnull=True)
.exists()
)

Related

How to filter query by multiple values using DRF, djagno-filters and HyperlinkedIdentityField values

Main target
is to get query set based on multiple values in query.
Business logic is to get all contracts for multiple drivers.
Example:
request url:
/api/contract/?driver=http://localhost:8000/api/driver/1,http://localhost:8000/api/driver/2
Response should be all contracts for these two drivers.
Driver Serializer:
class DriverSerializer(serializers.HyperlinkedModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='driver-detail',
read_only=True
)
class Meta:
model = Driver
fields = [
'url',
'id',
'first_name',
'last_name',
]
Contract serializer:
class ContractSerializer(serializers.HyperlinkedModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='contract-detail',
read_only=True
)
driver = serializers.StringRelatedField(many=False)
class Meta:
model = Contract
fields = [
'url',
'id',
'contract_detail_fields',
'driver',
]
Contract View
class ContractViewSet(viewsets.ModelViewSet):
serializer_class = serializers.ContractSerializer
queryset = Contract.objects.all()
permission_classes = (IsAuthenticated,)
filter_backends = [DjangoFilterBackend]
filterset_class = ContractFilter
ContractFilter:
class ContractFilter(FilterSet):
driver = CustomHyperlinkedIdentityFilterList('driver')
What I have tried is to make custom filterField based on answer by Sherpa
class CustomHyperlinkedIdentityFilterList(django_filters.BaseCSVFilter,
django_filters.CharFilter):
def filter(self, qs, value):
values = value or []
for value in values:
qs = super(CustomHyperlinkedIdentityFilterList,
self).filter(qs, value)
return qs
Answer is ValueError: Field 'id' expected a number but got 'http://localhost:8000/api/drivers/driver/3/'.
Then I am trying to modify to filter by id not urlField and changing this line
qs = super(CustomHyperlinkedIdentityFilterList, self).filter(qs, value)
to this:
qs = super(CustomHyperlinkedIdentityFilterList, self).filter(qs, get_id_from_url(value))
where get_id_from_url is:
def get_id_from_url(url):
return int(resolve(urlparse(unquote(url)).path).kwargs.get('pk'))
But it return me only contracts for last driver, not for both.
Then I also tried configurations based on answer by Slava
class ContractFilter(FilterSet):
class Meta:
model = Contract
fields = ['driver']
by using this solutions response is Bad request
{"driver":["Select a valid choice. That choice is not one of the available choices."]}
I hope there is very simple solutions which I have missed.

Django REST framework, serializer performance degradation

I have a simple list API view which is using serializer:
class ListCreateDeploymentView(
generics.ListCreateAPIView
):
permission_classes = (IsAuthenticated,)
renderer_classes = [JSONRenderer]
content_negotiation_class = IgnoreClientContentNegotiation
def get_queryset(self):
queryset = Deployment.objects.all()
return queryset
def list(self, request, version):
queryset = self.get_queryset()
serializer = DeploymentListSerializer(queryset, many=True)
data = serializer.data
return Response(data)
Serializer is simple:
class DeploymentListSerializer(serializers.ModelSerializer):
class Meta:
model = Deployment
fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
read_only_fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
Then I do a local load test with 10 users and delay 1s each execution so target rps is 10 req/s and see this picture with a clear performance degradation after few minutes
What means If I open 10 tabs in browser with ajax request every second to this endpoint the server will get unresponsive in a minute:
Then I used recommendations from here and used read-only regular serializer:
class DeploymentListSerializer(serializers.ModelSerializer):
# commands = CommandListSerializer(read_only=True, many=True)
# clients = ClientSimpleSerializer(read_only=True, many=True)
id = serializers.IntegerField(read_only=True)
query = serializers.CharField(read_only=True)
config = serializers.CharField(read_only=True)
started_at = serializers.DateTimeField(read_only=True)
finished_at = serializers.DateTimeField(read_only=True)
status = serializers.IntegerField(read_only=True)
project = serializers.CharField(read_only=True)
class Meta:
model = Deployment
fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
The situation became even worse:
Finally, if I remove serialization:
def list(self, request, version):
queryset = self.get_queryset()
data = queryset.values(
'id', 'query', 'config', 'started_at', 'finished_at',
'status', 'project'
)
return Response(data)
And do same test again the performance getting much better(expected), but also stable:
The problem is I need serialization because the task is a bit more complicated and I need to return nested objects, but it`s already falling on such an easy example.
What do I wrong?
UPD: same bad picture if I use the function-based view:
#api_view(['GET'])
def get_deployments(request, version):
queryset = Deployment.objects.all()
serializer = DeploymentCreateSerializer(queryset, many=True)
data = serializer.data
return Response(data)

How to filter by date range in django-rest?

I'd like to filter my data by date range typed in browser, all other filtering are working.
views.py
class BookView(generics.ListAPIView):
queryset = Book.objects.all()
serializer_class = BookSerializers
filter_backends = [filters.SearchFilter]
search_fields = ['title', 'language', 'authors', 'date']
You need create a new filter:
class StatementItemFilter(filters.FilterSet):
date_between = filters.DateFromToRangeFilter(field_name="MODEL_FIELD_NAME", label="Date (Between)")
class Meta:
model = StatementItem
fields = [
...
"date_between"
]
and use in your viewset
class MODELItemViewSet(viewsets.ReadOnlyModelViewSet):
filter_backends = (DjangoFilterBackend,)
filterset_class = StatementItemFilter
...
Move your list ['title', 'language', 'authors', 'date'] to your new filterset class

Django REST Framework serializer - turn id field into title while keeping the POST method

I am creating a cart with a list of products. I am trying to turn my product ids into titles while still being able to use my POST/ PUT methods on the products to be able to add/ remove the products.
Previously my products in the cart was displayed as IDs, this was my serializer:
class CartSerializer(serializers.ModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='cart-api:cart-detail',
read_only=True,
lookup_field='id',
)
class Meta:
model = Cart
fields = [
"id",
"url",
"products",
"sub_total",
"shipping",
"total",
]
After making some changes, I was able to turn the product ids into titles and several other fields related to each product. Here is my serializer code after the changes:
But my products disappeared from my put method and I am no longer able to add/ remove products.
(snippet 2)
How can we keep the the POST/ PUT methods on my product while displaying them as the way I want in snippet 2? Thanks so much in advance!
class MyPaintingSerializer(serializers.ModelSerializer):
painting_url = serializers.HyperlinkedIdentityField(
view_name='paintings-api:detail',
read_only=True,
lookup_field='slug'
)
class Meta:
model = Painting
fields = [
'id',
'title',
'painting_url',
'price',
]
class CartSerializer(serializers.ModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='cart-api:cart-detail',
read_only=True,
lookup_field='id',
)
products = MyPaintingSerializer(many=True, read_only=True)
class Meta:
model = Cart
fields = [
"id",
"url",
"products",
"sub_total",
"shipping",
"total",
]
Here is my code for views.py
class CartListCreateAPIView(generics.ListCreateAPIView):
queryset = Cart.objects.all()
serializer_class = CartSerializer
permission_classes = [permissions.AllowAny]
def perform_create(self, serializer):
serializer.save()
def get_queryset(self):
queryList = Cart.objects.all()
cart_id = self.request.session.get("cart_id", None)
return queryList
class CartDetailAPIView(generics.RetrieveUpdateDestroyAPIView):
queryset = Cart.objects.all()
serializer_class = CartSerializer
lookup_field = 'id'
permission_classes = [permissions.AllowAny]

Django rest api - searching a method field with the search filter

im trying to filter search a rest api page and want to use a method field as one of the search fields, however when I do this I get an error stating the field is not valid and it then lists the field in my model as the only valid source
serialiser:
class SubnetDetailsSerializer(QueryFieldsMixin, serializers.HyperlinkedModelSerializer):
subnet = serializers.SerializerMethodField()
device = serializers.ReadOnlyField(
source='device.hostname',
)
circuit_name = serializers.ReadOnlyField(
source='circuit.name',
)
subnet_name = serializers.ReadOnlyField(
source='subnet.description',
)
safe_subnet = serializers.SerializerMethodField()
def get_safe_subnet(self, obj):
return '{}{}'.format(obj.subnet.subnet, obj.subnet.mask.replace('/','_'))
def get_subnet(self, obj):
return '{}{}'.format(obj.subnet.subnet, obj.subnet.mask)
class Meta:
model = DeviceCircuitSubnets
fields = ('id','device_id','subnet_id','circuit_id','subnet','safe_subnet','subnet_name','device','circuit_name')
views:
class SubnetDetailsSet(viewsets.ReadOnlyModelViewSet):
queryset = DeviceCircuitSubnets.objects.all().select_related('circuit','subnet','device')
serializer_class = SubnetDetailsSerializer
permission_classes = (IsAdminUser,)
filter_class = DeviceCircuitSubnets
filter_backends = (filters.SearchFilter,)
search_fields = (
'device__hostname',
'circuit__name',
'subnet__subnet',
'safe_subnet'
)
how can include the safe_subnet in the search fields?
Thanks
EDIT
This is the code now
views.py
class SubnetDetailsSet(viewsets.ReadOnlyModelViewSet):
queryset = DeviceCircuitSubnets.objects.all()
serializer_class = SubnetDetailsSerializer
permission_classes = (IsAdminUser,)
filter_class = DeviceCircuitSubnets
filter_backends = (filters.SearchFilter,)
search_fields = (
'device__hostname',
'circuit__name',
'subnet__subnet',
'safe_subnet'
)
def get_queryset(self):
return (
super().get_queryset()
.select_related('circuit','subnet','device')
.annotate(
safe_subnet=Concat(
F('subnet__subnet'),
Replace(F('subnet__mask'), V('/'), V('_')),
output_field=CharField()
)
)
)
serializer.py
class SubnetDetailsSerializer(QueryFieldsMixin, serializers.HyperlinkedModelSerializer):
subnet = serializers.SerializerMethodField()
device = serializers.ReadOnlyField(
source='device.hostname',
)
circuit_name = serializers.ReadOnlyField(
source='circuit.name',
)
subnet_name = serializers.ReadOnlyField(
source='subnet.description',
)
def get_safe_subnet(self, obj):
return getattr(obj, 'safe_subnet', None)
def get_subnet(self, obj):
return '{}{}'.format(obj.subnet.subnet, obj.subnet.mask)
class Meta:
model = DeviceCircuitSubnets
fields = ('id','device_id','subnet_id','circuit_id','subnet','safe_subnet','subnet_name','device','circuit_name')
Model:
class DeviceCircuitSubnets(models.Model):
device = models.ForeignKey(Device, on_delete=models.CASCADE)
circuit = models.ForeignKey(Circuit, on_delete=models.CASCADE, blank=True, null=True)
subnet = models.ForeignKey(Subnet, on_delete=models.CASCADE)
active_link = models.BooleanField(default=False, verbose_name="Active Link?")
active_link_timestamp = models.DateTimeField(auto_now=True, blank=True, null=True)
Error:
Exception Type: ImproperlyConfigured at /api/subnets/
Exception Value: Field name `safe_subnet` is not valid for model `DeviceCircuitSubnets`.
You need to annotate your queryset with the safe_subnet attribute so it becomes searchable.
from django.db.models import F, Value as V
from django.db.models.functions import Concat, Replace
class SubnetDetailsSet(viewsets.ReadOnlyModelViewSet):
queryset = DeviceCircuitSubnets.objects.all()
serializer_class = SubnetDetailsSerializer
permission_classes = (IsAdminUser,)
filter_class = DeviceCircuitSubnets
filter_backends = (filters.SearchFilter,)
search_fields = (
'device__hostname',
'circuit__name',
'subnet__subnet',
'safe_subnet'
)
def get_queryset(self):
return (
super().get_queryset()
.select_related('circuit','subnet','device')
.annotate(
safe_subnet=Concat(
F('subnet__subnet'),
Replace(F('subnet__mask'), V('/'), V('_')),
output_field=CharField()
)
)
)
Then in your serializer you can use the following.
def get_safe_subnet(self, obj):
return obj.safe_subnet
Previous answer with annotate is a really good start:
from .rest_filters import DeviceCircuitSubnetsFilter
class SubnetDetailsSet(viewsets.ReadOnlyModelViewSet):
queryset = DeviceCircuitSubnets.objects.all()
serializer_class = SubnetDetailsSerializer
permission_classes = (IsAdminUser,)
# That's where hint lays
filter_class = DeviceCircuitSubnetsFilter
#filter_backends = (filters.SearchFilter,)
search_fields = (
'device__hostname',
'circuit__name',
'subnet__subnet',
'safe_subnet'
)
#No need to override your queryset
Now in rest_filters.py
from django_filters import rest_framework as filters
from django.db.models import F, Value as V
from django.db.models.functions import Concat, Replace
#.... import models
class DeviceCircuitSubnets(filters.FilterSet):
safe_subnet = filters.CharFilter(
name='safe_subnet',
method='safe_subnet_filter')
def safe_subnet_filter(self, queryset, name, value):
"""
Those line will make ?safe_subnet=your_pk available
"""
return queryset.annotate(
safe_subnet=Concat(
F('subnet__subnet'),
Replace(F('subnet__mask'), V('/'), V('_')),
output_field=CharField()
)
).filter(safe_subnet=value)
)
class Meta:
model = DeviceCircuitSubnets
# See https://django-filter.readthedocs.io/en/master/guide/usage.html#generating-filters-with-meta-fields
# This pattern is definitely a killer!
fields = {
'device': ['exact', 'in'],
'circuit': ['exact', 'in'],
'subnet': ['exact', 'in'],
'active_link': ['exact'],
'active_link_timestamp': ['lte', 'gte']
}
Please note: I'm annotating safe_subnet within the filer, depending on how much you use this, you might want to set this up in your model's manager!
Going in a completely different direction from the other (excellent) answers. Since you want to be able to filter frequently on the safe_subnet field, why not just let it be an actual database field in your model? You could calculate and populate/update the value during one of your save methods and then just let django-filters do it's thing. This also has the advantage of allowing the filtering to be done directly through SQL which would theoretically provide better performance.