Django ORM equivalent for this SQL..calculated field derived from related table - django

I have the following model structure below:
class Master(models.Model):
name = models.CharField(max_length=50)
mounting_height = models.DecimalField(max_digits=10,decimal_places=2)
class MLog(models.Model):
date = models.DateField(db_index=True)
time = models.TimeField(db_index=True)
sensor_reading = models.IntegerField()
m_master = models.ForeignKey(Master)
The goal is to produce a queryset that returns all the fields from MLog plus a calculated field (item_height) based on the related data in Master
using Django's raw sql:
querySet = MLog.objects.raw('''
SELECT a.id,
date,
time,
sensor_reading,
mounting_height,
(sensor_reading - mounting_height) as item_height
FROM db_mlog a JOIN db_master b
ON a.m_master_id = b.id
''')
How do I code this using Django's ORM?

I can think of two ways to go about this without relying on raw(). The first is pretty much the same as what #tylerl suggested. Something like this:
class Master(models.Model):
name = models.CharField(max_length=50)
mounting_height = models.DecimalField(max_digits=10,decimal_places=2)
class MLog(models.Model):
date = models.DateField(db_index=True)
time = models.TimeField(db_index=True)
sensor_reading = models.IntegerField()
m_master = models.ForeignKey(Master)
def _get_item_height(self):
return self.sensor_reading - self.m_master.mounting_height
item_height = property(_get_item_height)
In this case I am defining a custom (derived) property for MLog called item_height. This property is calculated as the difference of the sensor_reading of an instance and the mounting_height of its related master instance. More on property here.
You can then do something like this:
In [4]: q = MLog.objects.all()
In [5]: q[0]
Out[5]: <MLog: 2010-09-11 8>
In [6]: q[0].item_height
Out[6]: Decimal('-2.00')
The second way to do this is to use the extra() method and have the database do the calculation for you.
In [14]: q = MLog.objects.select_related().extra(select =
{'item_height': 'sensor_reading - mounting_height'})
In [16]: q[0]
Out[16]: <MLog: 2010-09-11 8>
In [17]: q[0].item_height
Out[17]: Decimal('-2.00')
You'll note the use of select_related(). Without this the Master table will not be joined with the query and you will get an error.

I always do the calculations in the app rather than in the DB.
class Thing(models.Model):
foo = models.IntegerField()
bar = models.IntegerField()
#Property
def diff():
def fget(self):
return self.foo - self.bar
def fset(self,value):
self.bar = self.foo - value
Then you can manipulate it just as you would any other field, and it does whatever you defined with the underlying data. For example:
obj = Thing.objects.all()[0]
print(obj.diff) # prints .foo - .bar
obj.diff = 4 # sets .bar to .foo - 4
Property, by the way, is just a standard property decorator, in this case coded as follows (I don't remember where it came from):
def Property(function):
keys = 'fget', 'fset', 'fdel'
func_locals = {'doc':function.__doc__}
def probeFunc(frame, event, arg):
if event == 'return':
locals = frame.f_locals
func_locals.update(dict((k,locals.get(k)) for k in keys))
sys.settrace(None)
return probeFunc
sys.settrace(probeFunc)
function()
return property(**func_locals)

Related

how to access to property in view(django)

I'm beginner and I need some solution
First, I have Racket and Detail class.
class Racket(models.Model):
name = models.CharField(max_length=20)
class RacketDetail(models.Model):
racket = models.OneToOneField(Racket, related_name='detail', on_delete=models.CASCADE)
adminReview = models.TextField()
adminPower = models.FloatField(default=0)
adminSpin = models.FloatField(default=0)
adminManeuverability = models.FloatField(default=0)
adminStability = models.FloatField(default=0)
adminComfort = models.FloatField(default=0)
#property
def adminAvgScore(self):
scoreAvg = (
self.adminPower +
self.adminSpin +
self.adminManeuverability +
self.adminStability +
self.adminComfort
) / 5
return round(scoreAvg, 2)
Second, I want to rander list using the #property(adminAvgScore), so I made view like this.
def racketMain(request: HttpRequest):
getRacket = Racket.objects.all().order_by('detail__adminAvgScore')
return render(request, "racket/racketMain.html", {'racketItems': getRacket, })
Unfortunally when I use 'RacketDetail' class's column I can access all column except 'adminAvgScore' using by "order_by('detail__".
If I use like "order_by('detail__adminAvgScore')" then Django show to me error "Unsupported lookup 'adminAvgScore' for BigAutoField or join on the field not permitted."
How can I solve it? Or should I think in a different way?
You cannot use property with Query as Property is Function. You can use annotate and aggregate combination to get the result as your property function but inside a query.Something like this will do the trick.
from django.db.models import F, Sum, FloatField, Avg
Model.objects.filter(...)\
.values('id')\
.annotate(subtotal=Sum(...math here..., output_field=FloatField()))\
.aggregate(total=Avg(F('subtotal')))

django - improve performance of __in queryset in M2M filtering

I have a models that has a M2M relationship to another model.
These are my models:
class Catalogue(models.Model):
city = models.CharField(db_index=True,max_length=100, null=True)
district = models.CharField(db_index=True,max_length=100, null=True)
type = models.ManyToManyField(Type, db_index=True)
datetime = models.CharField(db_index=True, max_length=100, null=True)
class Type(models.Model):
name = models.CharField(max_length=100)
def __str__(self):
return self.name
And this is views.py:
class all_ads(generic.ListView):
paginate_by = 12
template_name = 'new_list_view_grid-card.html'
def get_queryset(self):
city_district = self.request.GET.getlist('city_district')
usage = self.request.GET.get('usage')
status = self.request.GET.get('status')
last2week = datetime.datetime.now() - datetime.timedelta(days=14)
status = status.split(',')
if usage:
usage = usage.split(',')
else:
usage = ['1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']
intersections = list(set(status).intersection(usage))
type_q = (Q(type__in=intersections) & Q(type__isnull=False))
result = models.Catalogue.objects.filter(
Q(datetime__gte=last2week) &
type_q &
((reduce(operator.or_, (Q(city__contains=x) for x in city_district)) & Q(city__isnull=False)) |
(reduce(operator.or_, (Q(district__contains=x) for x in city_district)) & Q(district__isnull=False)))
).distinct().order_by('-datetime').prefetch_related('type')
return result
I want to filter MySQL db with some queries and return result in a listview.
It works good on a small database, but with large database it takes over 10 seconds to return results. If I delete type_q query, It takes 2 seconds (reduce 10 second!).
How can I improve performance of __in queryset?
It looks like type_q itself is not really the culprit, but acts as a multiplier, since now we make a LEFT OUTER JOIN, and thus the __contains runs over all combinations. This is thus more a peculiarity of two filters that work together
We can omit this with:
cat_ids = list(Catalogue.objects.filter(
Q(*[Q(city__contains=x) for x in city_district], _connector=Q.OR) |
Q(*[Q(district__contains=x) for x in city_district], _connector=Q.OR)
).values_list('pk', flat=True))
result = models.Catalogue.objects.filter(
Q(datetime__gte=last2week),
type_q,
pk__in=cat_ids
).distinct().order_by('-datetime').prefetch_related('type')
Some database (MySQL is known to not optimize a subquery very well), can even do that with a subquery with. So here we do not materialize the list, but let Django work with a subquery:
cat_ids = Catalogue.objects.filter(
Q(*[Q(city__contains=x) for x in city_district], _connector=Q.OR) |
Q(*[Q(district__contains=x) for x in city_district], _connector=Q.OR)
).values_list('pk', flat=True)
result = models.Catalogue.objects.filter(
Q(datetime__gte=last2week),
type_q,
pk__in=cat_ids
).distinct().order_by('-datetime').prefetch_related('type')

How do I filter by occurrences count in another table in Django?

Here are my models:
class Zoo(TimeStampedModel):
id = models.AutoField(primary_key=True)
class Animal(models.Model):
id = models.AutoField(primary_key=True)
zoo = models.ForeignKey(Zoo, on_delete=models.PROTECT, related_name='diffbot_results')
I would like to run a query like this:
Zoo.objects.filter("WHERE zoo.id IN (select zoo_id from animal_table having count(*) > 10 group by zoo_id)")
One way is to use a raw queryset:
>>> from testapp.models import Zoo, Animal
>>> z1, z2 = Zoo(), Zoo()
>>> z1.save(), z2.save()
(None, None)
>>> z1_animals = [Animal(zoo=z1) for ii in range(5)]
>>> z2_animals = [Animal(zoo=z2) for ii in range(15)]
>>> x = [a.save() for a in z1_animals+z2_animals]
>>> qs = Zoo.objects.raw("select * from testapp_zoo zoo WHERE zoo.id IN (select zoo_id from testapp_animal group by zoo_id having count(1) > 10)")
>>> list(qs)
[<Zoo: Zoo object (2)>]
In theory, per these docs, it should be possible to pass in a queryset to a regular .filter(id__in=<queryset>), but the queryset must only select one column, and I can't find a way of adding the HAVING clause without also causing the queryset to select a num_animals column, preventing it from being used with an __in filter expression.

django FilterSet for end of day

How to make the query take correct date from start date and end end of the day for FilterSet.
Now if you type start_filter_date = 2018.05.23 and end_filter_date = 2018.05.25. Then start_filter_date=2018.05.23T00:00 and end_filter_date=2018.05.25T00:00. Time is taken 00:00, but need to be 23:59?
My class is following:
class TaskFilterSet(django_filters.rest_framework.FilterSet):
id = django_filters.NumberFilter(name="pk")
start_filter_date = django_filters.DateFilter(name="date_added", lookup_expr="gte")
end_filter_date = django_filters.DateFilter(name="date_added", lookup_expr="lte")
i think the best way is use gte with lt but send end date date_added + timedelta(1), i don't use the django filters but based on the tips.html#solution-1-magic-values you can try:
class EndFilter(django_filters.DateFilter):
def filter(self, qs, value):
if value:
value = value + timdelta(1)
return super(EndFilter, self).filter(qs, value)
class TaskFilterSet(django_filters.rest_framework.FilterSet):
id = django_filters.NumberFilter(name="pk")
start_filter_date = django_filters.DateFilter(name="date_added", lookup_expr="gte")
end_filter_date = EndFilter(name="date_added", lookup_expr="lt")

Aggregation on 'one to many' for matrix view in Django

I have two tables like below.
These are in 'one(History.testinfoid) to many(Result.testinfoid)' relationship.
(Result table is external database)
class History(models.Model): # default database
idx = models.AutoField(primary_key=True)
scenario_id = models.ForeignKey(Scenario)
executor = models.CharField(max_length=255)
createdate = models.DateTimeField()
testinfoid = models.IntegerField(unique=True)
class Result(models.Model): # external (Result.objects.using('external'))
idx = models.AutoField(primary_key=True)
testinfoid = models.ForeignKey(History, to_field='testinfoid', related_name='result')
testresult = models.CharField(max_length=10)
class Meta:
unique_together = (('idx', 'testinfoid'),)
So, I want to express the Count by 'testresult' field in Result table.
It has some condition such as 'Pass' or 'Fail'.
I want to express a count query set for each condition. like this.
[{'idx': 1, 'pass_count': 10, 'fail_count': 5, 'executor': 'someone', ...} ...
...
{'idx': 10, 'pass_count': 1, 'fail_count': 10, 'executor': 'someone', ...}]
Is it possible?
It is a two level aggregation where the second level should be displayed as table columns - "matrix view".
A) Solution with Python loop to create columns with annotations by the second level ("testresult").
from django.db.models import Count
from collections import OrderedDict
qs = (History.objects
.values('pk', 'executor', 'testinfoid',... 'result__testresult')
.annotate(result_count=Count('pk'))
)
qs = qs.filter(...).order_by(...)
data = OrderedDict()
count_columns = ('pass_count', 'fail_count', 'error_count',
'expected_failure_count', 'unexpected_success_count')
for row in qs:
data.setdefault(row.pk, dict.fromkeys(count_columns, 0)).update(
{(k if k != result_count else row['result__testresult'] + '_count'): v
for k, v in row_items()
if k != 'result__testresult'
}
)
out = list(data.values())
The class OrderedDict is used to preserve order_by().
B) Solution with Subquery in Django 1.11+ (if the result should be a queryset. e.g. to be sorted or filtered finally in an Admin view by clicking, and if a more complicated query is acceptable and number of columns *_count is very low.). I can write a solution with subquery, but I'm not sure if the query will be fast enough with different database backends. Maybe someone other answers.