Django REST framework, serializer performance degradation - django

I have a simple list API view which is using serializer:
class ListCreateDeploymentView(
generics.ListCreateAPIView
):
permission_classes = (IsAuthenticated,)
renderer_classes = [JSONRenderer]
content_negotiation_class = IgnoreClientContentNegotiation
def get_queryset(self):
queryset = Deployment.objects.all()
return queryset
def list(self, request, version):
queryset = self.get_queryset()
serializer = DeploymentListSerializer(queryset, many=True)
data = serializer.data
return Response(data)
Serializer is simple:
class DeploymentListSerializer(serializers.ModelSerializer):
class Meta:
model = Deployment
fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
read_only_fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
Then I do a local load test with 10 users and delay 1s each execution so target rps is 10 req/s and see this picture with a clear performance degradation after few minutes
What means If I open 10 tabs in browser with ajax request every second to this endpoint the server will get unresponsive in a minute:
Then I used recommendations from here and used read-only regular serializer:
class DeploymentListSerializer(serializers.ModelSerializer):
# commands = CommandListSerializer(read_only=True, many=True)
# clients = ClientSimpleSerializer(read_only=True, many=True)
id = serializers.IntegerField(read_only=True)
query = serializers.CharField(read_only=True)
config = serializers.CharField(read_only=True)
started_at = serializers.DateTimeField(read_only=True)
finished_at = serializers.DateTimeField(read_only=True)
status = serializers.IntegerField(read_only=True)
project = serializers.CharField(read_only=True)
class Meta:
model = Deployment
fields = (
'id',
'query',
'config',
'started_at',
'finished_at',
'status',
'project',
)
The situation became even worse:
Finally, if I remove serialization:
def list(self, request, version):
queryset = self.get_queryset()
data = queryset.values(
'id', 'query', 'config', 'started_at', 'finished_at',
'status', 'project'
)
return Response(data)
And do same test again the performance getting much better(expected), but also stable:
The problem is I need serialization because the task is a bit more complicated and I need to return nested objects, but it`s already falling on such an easy example.
What do I wrong?
UPD: same bad picture if I use the function-based view:
#api_view(['GET'])
def get_deployments(request, version):
queryset = Deployment.objects.all()
serializer = DeploymentCreateSerializer(queryset, many=True)
data = serializer.data
return Response(data)

Related

How to filter query by multiple values using DRF, djagno-filters and HyperlinkedIdentityField values

Main target
is to get query set based on multiple values in query.
Business logic is to get all contracts for multiple drivers.
Example:
request url:
/api/contract/?driver=http://localhost:8000/api/driver/1,http://localhost:8000/api/driver/2
Response should be all contracts for these two drivers.
Driver Serializer:
class DriverSerializer(serializers.HyperlinkedModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='driver-detail',
read_only=True
)
class Meta:
model = Driver
fields = [
'url',
'id',
'first_name',
'last_name',
]
Contract serializer:
class ContractSerializer(serializers.HyperlinkedModelSerializer):
url = serializers.HyperlinkedIdentityField(
view_name='contract-detail',
read_only=True
)
driver = serializers.StringRelatedField(many=False)
class Meta:
model = Contract
fields = [
'url',
'id',
'contract_detail_fields',
'driver',
]
Contract View
class ContractViewSet(viewsets.ModelViewSet):
serializer_class = serializers.ContractSerializer
queryset = Contract.objects.all()
permission_classes = (IsAuthenticated,)
filter_backends = [DjangoFilterBackend]
filterset_class = ContractFilter
ContractFilter:
class ContractFilter(FilterSet):
driver = CustomHyperlinkedIdentityFilterList('driver')
What I have tried is to make custom filterField based on answer by Sherpa
class CustomHyperlinkedIdentityFilterList(django_filters.BaseCSVFilter,
django_filters.CharFilter):
def filter(self, qs, value):
values = value or []
for value in values:
qs = super(CustomHyperlinkedIdentityFilterList,
self).filter(qs, value)
return qs
Answer is ValueError: Field 'id' expected a number but got 'http://localhost:8000/api/drivers/driver/3/'.
Then I am trying to modify to filter by id not urlField and changing this line
qs = super(CustomHyperlinkedIdentityFilterList, self).filter(qs, value)
to this:
qs = super(CustomHyperlinkedIdentityFilterList, self).filter(qs, get_id_from_url(value))
where get_id_from_url is:
def get_id_from_url(url):
return int(resolve(urlparse(unquote(url)).path).kwargs.get('pk'))
But it return me only contracts for last driver, not for both.
Then I also tried configurations based on answer by Slava
class ContractFilter(FilterSet):
class Meta:
model = Contract
fields = ['driver']
by using this solutions response is Bad request
{"driver":["Select a valid choice. That choice is not one of the available choices."]}
I hope there is very simple solutions which I have missed.

Update field without rest API

I have a model.py
class UserPaymentInformation(models.Model):
...
awaiting_confirmation = models.BooleanField(default=False)
I want to make awaiting_confirmation = True in code. But forbid awaiting_confirmation update via RestAPI call.
views.py
class UserPaymentInformationUpdateAPIView(generics.UpdateAPIView):
permission_classes = (IsAuthenticatedDriver,)
serializer_class = UserPaymentInformationUpdateSerializer
queryset = serializer_class.Meta.model.objects.all()
def update(self, request, *args, **kwargs):
partial = kwargs.pop("partial", False)
instance = self.serializer_class.Meta.model.objects.get(
user=self.request.user
)
self.mark_user_as_new()
# awaiting_confirmation = True # I WANT SOMETHING LIKE THIS
serializer = self.get_serializer(
instance, data=request.data, partial=partial
)
serializer.is_valid(raise_exception=True)
self.perform_update(serializer)
return Response({"result": serializer.data})
serializers.py
class UserPaymentInformationUpdateSerializer(serializers.ModelSerializer):
class Meta:
model = UserPaymentInformation
fields = ("id", "full_name", "card_number", "account_number", "bik", "awaiting_confirmation")
How can I fix update method?
In your UserPaymentInformationUpdateSerializer you could set the read_only_fields = ('awaiting_confirmation',), so your serializer would become:
class UserPaymentInformationUpdateSerializer(serializers.ModelSerializer):
class Meta:
model = UserPaymentInformation
fields = ("id", "full_name", "card_number", "account_number", "bik", "awaiting_confirmation")
read_only_fields = ("awaiting_confirmation",)
This would mean it'd still be returned in the serializer data but it would not be possible to update it through an API request.

How to properly serialize an object whose model has a foreign key which also takes an ImageField?

# models.py
class Post(models.Model):
content = models.TextField(blank=True, default='')
created_by = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
class PostImage(models.Model):
image = models.ImageField(upload_to=unique_upload)
post = models.ForeignKey(
Post, related_name='images', on_delete=models.CASCADE)
This is my model set up for a basic scenario where a user can enter content or upload images as their posts.
I want to bundle my logic to handle creating a post with either content or images or both.
I first started playing around with GenericViewSet and CreateViewSet but was images was never being passed to my serializer.
# views.py
class CreatePostViewSet(generics.CreateAPIView /* viewsets.GenericViewSet */):
permission_classes = (IsAuthenticated,)
queryset = Post.objects.order_by('id')
serializer_class = CreatePostSerializer
def create(self, request, *args, **kwargs):
data = {}
print(request.data)
serializer = self.get_serializer(data=request.data)
serializer.is_valid(raise_exception=True)
serializer.save(created_by=request.user)
# post = serializer.instance
# print(post)
# for im in post.images.all():
# im.save(post=post)
# print(post.images.all())
return Response(data,
status=status.HTTP_201_CREATED,
headers=self.get_success_headers(serializer.data))
# serializers.py
class PostImageSerializer(serializers.ModelSerializer):
class Meta:
model = PostImage
fields = ('id', 'url', 'image', 'post',)
read_only_fields = ('post',)
depth = 1
class CreatePostSerializer(serializers.ModelSerializer):
images = PostImageSerializer(many=True, required=False)
class Meta:
model = Post
fields = ('id', 'url', 'content', 'images',)
read_only_fields = ('created_by',)
depth = 1
def create(self, validated_data):
# validated_data['images'] is always []
print(validated_data)
raise
images is always [] when I pass it to a serializer, but it does exist in request.data['images'] as [<TemporaryUploadedFile: 1 - 5H5hHgY.png (image/png)>, ...
I was hoping to use ModelSerializer to help auto-resolve the ImageField.
# CreatePostSerializer serializers breaks down to
CreatePostSerializer():
id = UUIDField(read_only=True)
url = HyperlinkedIdentityField(view_name='post-detail')
content = CharField(allow_blank=True, required=False, style={'base_template': 'textarea.html'})
images = PostImageSerializer(many=True, required=False):
id = UUIDField(read_only=True)
url = HyperlinkedIdentityField(view_name='postimage-detail')
image = ImageField(max_length=100)
post = NestedSerializer(read_only=True):
id = UUIDField(read_only=True)
content = CharField(allow_blank=True, required=False, style={'base_template': 'textarea.html'})
created_by = PrimaryKeyRelatedField(queryset=User.objects.all())
It think request.data['images'] will need to be changed slightly because your PostImageSerializer will be expecting an object containing the "image" key, whereas you are passing the list of TemporaryUploadedFile.
Given request.data['images'] you could do something like the following in your view before you pass the data to the serializer:
images_list: List[TemporaryUploadedFile] = request.data.pop("images")
images = []
for image in images_list:
images.append({
"image": image,
})
request.data["images"] = images
So we are transforming your list of TemporaryUploadedFiles into a list of objects with the image key.
:edit: So you don't want to transform your data at the view to be compatible with the serializer? Then you can change the serializer to be compatible with the data, this involves customizing the create and update methods, I'm just going to show you how to override the create method for now.
class CreatePostSerializer(serializers.ModelSerializer):
images = serializers.ImageField(many=True)
class Meta:
model = Post
fields = ('id', 'url', 'content', 'images',)
read_only_fields = ('created_by',)
depth = 1
def create(self, validated_data):
images = validated_data.pop("images")
post = super().create(validated_data)
for image in images:
serializer = PostImageSerializer(data={"image": image, "post": post.pk}, context=self.context)
serializer.is_valid()
serializer.save()
return post
So you don't want to override the data in the request and you don't want to customize the serializers create method? Change how the serializer converts your initial data into validated data with the validate method (I think this works for nested serializers but its untested):
class CreatePostSerializer(serializers.ModelSerializer):
images = PostImageSerializer(many=True, required=False)
class Meta:
model = Post
fields = ('id', 'url', 'content', 'images',)
read_only_fields = ('created_by',)
depth = 1
def validate(self, attrs):
images_list = attrs.pop("images")
images = []
for image in images_list:
images.append({
"image": image,
})
attrs["images"] = images
return attrs
So, I was able to get it to work with #ARJMP's suggestion.
# views.py
class CreatePostViewSet(generics.CreateAPIView):
# authentication_classes = (TokenAuthentication,)
permission_classes = (IsAuthenticated,)
queryset = Post.objects.order_by('id')
serializer_class = CreatePostSerializer
def create(self, request, *args, **kwargs):
data = {}
print(request.data)
images = [{'image': i} for i in request.data.pop('images', [])]
serializer = self.get_serializer(
data={'content': request.data['content'], 'images': images})
serializer.is_valid(raise_exception=True)
post = serializer.save(created_by=request.user)
# self.perform_create(serializer)
data['post'] = serializer.data
return Response(data,
status=status.HTTP_201_CREATED,
headers=self.get_success_headers(serializer.data))
# serializers.py
class CreatePostSerializer(serializers.ModelSerializer):
images = PostImageSerializer(many=True, required=False)
class Meta:
model = Post
fields = ('id', 'content', 'images',
'is_private', 'created_by',)
read_only_fields = ('view_count', 'created',)
depth = 1
def create(self, validated_data):
images = validated_data.pop('images', [])
p = Post.objects.create(**validated_data)
for im in images:
pi = PostImage.objects.create(image=im['image'], post=p)
return p
My thing is this seems rather convoluted to get it to work. A lot of manipulating it myself. I was really hoping to leverage more of the "magic" stuff that gets done with ModelSerializer and CreateAPIView.
Are there better approaches to doing this?

Django Rest Framework SerializerMethodField only on GET Request

Running into a little snag here with my DRF backend.
I am populating fields with choices on certain models.
I have a foreign key requirement on one model. When I create the model I want to save it under the foreign id.
When I request the models, I want the model with whatever the choice field maps to.
I was able to do this with SerializerMethodField, however when I try to create a model, I get a 400 error because the block is not valid. If I remove the SerializerMethodField, I can save, but get the number stored in the db from the request.
Any help would be appreciated.
class BlockViewSet(ModelViewSet):
model = apps.get_model('backend', 'Block')
queryset = model.objects.all()
serializer_class = serializers.BlockSerializer
permissions = ('All',)
def create(self, request, format=None):
data = request.data
data['user'] = request.user.id
data['goal'] = WorkoutGoal.objects.get(goal=data['goal']).id
block = serializers.BlockSerializer(data=data, context={'request': request})
if block.is_valid():
new_block = block.save()
return Response({'block': {'name': new_block.name, 'id': new_block.id}}, status=status.HTTP_201_CREATED)
else:
return Response(block.errors, status=status.HTTP_400_BAD_REQUEST)
class WorkoutGoalSerializer(serializers.ModelSerializer):
class Meta:
model = apps.get_model('backend', 'WorkoutGoal')
fields = ('goal',)
goal = serializers.SerializerMethodField(read_only=True, source='get_goal')
def get_goal(self, obj):
return dict(WorkoutGoal.GOALS).get(obj.goal)
class BlockSerializer(serializers.ModelSerializer):
workout_count = serializers.IntegerField(required=False)
completed_workouts = serializers.IntegerField(required=False)
goal = WorkoutGoalSerializer()
class Meta:
model = apps.get_model('backend', 'Block')
read_only_fields = ('workout_count', 'completed_workouts')
fields = read_only_fields + ('id', 'name', 'user', 'created', 'goal')
The above code returns the correct choice, but I can't save under it. Remove the goal = WorkoutGoalSerializer() and it saves but doesn't return the mapped choice.
I think this will work like a charm,
class WorkoutGoalSerializer(serializers.ModelSerializer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if 'request' in self.context and self.context['request'].method == 'GET':
self.fields['goal'] = serializers.SerializerMethodField(read_only=True, source='get_goal')
class Meta:
model = apps.get_model('backend', 'WorkoutGoal')
fields = ('goal',)
goal = serializers.SerializerMethodField(read_only=True, source='get_goal') # remove this line
def get_goal(self, obj):
return dict(WorkoutGoal.GOALS).get(obj.goal)
How this Work?
It will re-initiate the goal field with SerializerMethodField, if the reuested method is GET.
Remember one thing, you should remove the line,
goal = serializers.SerializerMethodField(read_only=True, source='get_goal')
serializers.py
class BlockCreateSerializer(serializers.ModelSerializer):
workout_count = serializers.IntegerField(required=False)
completed_workouts = serializers.IntegerField(required=False)
class Meta:
model = apps.get_model('backend', 'Block')
read_only_fields = ('workout_count', 'completed_workouts')
fields = read_only_fields + ('id', 'name', 'user', 'created', 'goal')
class BlockSerializer(serializers.ModelSerializer):
workout_count = serializers.IntegerField(required=False)
completed_workouts = serializers.IntegerField(required=False)
goal = WorkoutGoalSerializer()
class Meta:
model = apps.get_model('backend', 'Block')
read_only_fields = ('workout_count', 'completed_workouts')
fields = read_only_fields + ('id', 'name', 'user', 'created', 'goal')
views.py
class BlockViewSet(ModelViewSet):
model = apps.get_model('backend', 'Block')
queryset = model.objects.all()
serializer_class = serializers.BlockSerializer
permissions = ('All',)
def get_serializer_class(self):
if self.action == 'create':
return serializers.BlockCreateSerializer
else:
return self.serializer_class
def create(self, request, format=None):
data = request.data
data['user'] = request.user.id
data['goal'] = WorkoutGoal.objects.get(goal=data['goal']).id
block = self.get_serializer(data=data)
if block.is_valid():
new_block = block.save()
return Response({'block': {'name': new_block.name, 'id': new_block.id}}, status=status.HTTP_201_CREATED)
else:
return Response(block.errors, status=status.HTTP_400_BAD_REQUEST)
override get_serializer_class to return different serializer_class for create and other action(list\retrieve\update\partial_update)

Test Django REST framework API response with nested resources

I'm writing an REST APIs for items and the menu endpoint is returning a JSON with items groups inside it the items inside the items any extra the user can add to the item.
serializers.py
====================
class ItemExtraSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = ItemExtra
fields = ('id', 'name', 'price')
class ItemSerializer(serializers.HyperlinkedModelSerializer):
extras = ItemExtraSerializer(many=True, read_only=True)
class Meta:
model = Item
fields = ('id', 'url', 'name', 'description', 'image', 'code', 'price', 'extras')
class ItemGroupSerializer(serializers.HyperlinkedModelSerializer):
items = ItemSerializer(many=True, read_only=True)
class Meta:
model = ItemGroup
fields = ('id', 'url', 'name', 'items')
views.py
=========================
class MenuView(ListAPIView):
serializer_class = ItemGroupSerializer
def get_queryset(self):
"""
Return the items inside their groups for search query
Filtering is one Group name and Item name
:return:
"""
queryset = ItemGroup.objects.all()
search_terms = self.request.query_params.get('q', None)
if search_terms:
queryset = Item.objects.search(search_terms)
return queryset
tests.py
========================
class ItemTestCases(APITestCase):
def setUp(self):
self.sandwich_group, created = ItemGroup.objects.get_or_create(name='Sandwiches')
self.meal_group, created = ItemGroup.objects.get_or_create(name='Meals')
self.shawarma_sandwich, created = Item.objects.get_or_create(name='Shawarma Sandwich',
description='Meat Shawarma Sandwich',
price=1.250,
code='SW-01',
group=self.sandwich_group)
self.burger_sandwich, created = Item.objects.get_or_create(name='Cheese Burger',
description='Single cheese burger Sandwich',
price=1.000,
code='SW-02',
group=self.sandwich_group)
self.burger_sandwich_extra, created = ItemExtra.objects.get_or_create(name='Extra cheese',
price=0.100,
item=self.burger_sandwich)
self.sharawma_meal, created = Item.objects.get_or_create(name='Shawarma Meal',
description='Shawarma Sandwich with fries and drink',
price=2.000,
code='ME-01',
group=self.meal_group)
self.burger_meal, created = Item.objects.get_or_create(name='Burger Meal',
description='Single cheese burger Sandwich',
price=2.250,
code='ME-02',
group=self.meal_group)
self.cheese_meal_extra, created = ItemExtra.objects.get_or_create(name='Extra cheese',
price=0.100,
item=self.burger_meal)
self.factory = APIRequestFactory()
def test_menu_GET_request(self):
item_list = reverse('menu')
response = self.client.get(item_list)
self.assertEqual(response.status_code, status.HTTP_200_OK)
groups = ItemGroup.objects.all()
expected = ItemGroupSerializer(groups)
self.assertContains(response, expected.data)
The output of the test is:
AssertionError: HyperlinkedIdentityField requires the request in the serializer context. Add context={'request': request} when instantiating the serializer.
How to render the serlizer to JSON in order to compare it with the API endpoint JSON?
Update 1:
I figure it out but I think there is a cleaner solution
I've created helper function
def render_many_serializer_as_json(serializer, request, instance):
serializer_data = serializer(instance=instance, many=True, context={'request': request}).data
return JSONRenderer().render(serializer_data)
And rewrite my test
def test_menu_GET_request(self):
item_list = reverse('menu')
request = self.factory.get(item_list, format='json')
response = self.client.get(item_list)
groups = ItemGroup.objects.all()
expected = render_many_serializer_as_json(ItemGroupSerializer, request, groups)
self.assertEqual(response.content, expected)
Why do something complex ?
Just explicitly write down the expected JSON output:
expected = {
'id': 1,
'name': ...,
...,
'items': [{
'id': ..,
'url': ...,
}]
}