How to create a dataframe from a model (Django) - django

Question

I am developing an app in Django.
I have this model
class my_model(models.Model):
Field_A = models.CharField(max_length=256, blank=True, null=True)
Field_B = models.CharField(max_length=25, blank=True, null=True)
Field_C = models.TextField(blank=True, null=True)
I want to create a dataframe with columns names equal to model fields names,
containing in every row the model objects, and in every column the object fields values.
How can I do it? Is there a single command to do it? Or do I have to iterate?
EDIT: Here is the raw and unelegant solution that I found until now:
import pandas as pd
entries = my_model.objects.all()
# this generates an array containing the names of the model fields
columns_names = [field.name for field in my_model._meta.get_fields()]
L_GI = len(entries)
# generate empty dataframe
GI = pd.DataFrame(columns = columns_names)
for element in entries:
new_entry = {"Field_A":element.Field_A, "Field_B":element.Field_B, "Field_C":element.Field_C}
GI = GI.append(new_entry, ignore_index=True)
I bet there is a faster way that avoids iteration. Any suggestions?

Answer 1

Nice question
I think you are forced to iterate them.
I have implemented it in 3 different ways so that you can choose your favorite one
import time
import pandas as pd
from django.core import serializers
class PandasModelMixin(models.Model):
class Meta:
abstract = True
#classmethod
def as_dataframe(cls, queryset=None, field_list=None):
t1 = time.time()
if queryset is None:
queryset = cls.objects.all()
if field_list is None:
field_list = [_field.name for _field in cls._meta._get_fields(reverse=False)]
data = []
[data.append([obj.serializable_value(column) for column in field_list]) for obj in queryset]
columns = field_list
df = pd.DataFrame(data, columns=columns)
print("Execution time without serialization: %s" % time.time()-t1)
return df
#classmethod
def as_dataframe_using_django_serializer(cls, queryset=None):
t1 = time.time()
if queryset is None:
queryset = cls.objects.all()
if queryset.exists():
serialized_models = serializers.serialize(format='python', queryset=queryset)
serialized_objects = [s['fields'] for s in serialized_models]
data = [x.values() for x in serialized_objects]
columns = serialized_objects[0].keys()
df = pd.DataFrame(data, columns=columns)
df = pd.DataFrame()
print("Execution time using Django serializer: %s" % time.time()-t1)
return df
#classmethod
def as_dataframe_using_drf_serializer(cls, queryset=None, drf_serializer=None, field_list=None):
from rest_framework import serializers
t1 = time.time()
if queryset is None:
queryset = cls.objects.all()
if drf_serializer is None:
class CustomModelSerializer(serializers.ModelSerializer):
class Meta:
model = cls
fields = field_list or '__all__'
drf_serializer = CustomModelSerializer
serialized_objects = drf_serializer(queryset, many=True).data
data = [x.values() for x in serialized_objects]
columns = drf_serializer().get_fields().keys()
df = pd.DataFrame(data, columns=columns)
print("Execution time using DjangoRestFramework serializer: %s" % time.time()-t1)
return df
So inherit your Model in this way:
class MyModel(PandasModelMixin):
field_a = models.CharField(max_length=256, blank=True, null=True)
field_b = models.CharField(max_length=25, blank=True, null=True)
field_c = models.TextField(blank=True, null=True)
and try the code in this way:
>> MyModel.as_dataframe()
>> MyModel.as_dataframe_using_django_serializer()
>> MyModel.as_dataframe_using_drf_serializer()
I have tried my code using a Model with 450 instances and 15 columns and I had these results:
Execution time without serialization: 0.07040905952453613
Execution time using Django serializer: 0.07644820213317871
Execution time using DjangoRestFramework serializer: 0.12314629554748535
N.B.
I'm using Django 2.2 and Python 3.6.5