I have two tables like below.
These are in 'one(History.testinfoid) to many(Result.testinfoid)' relationship.
(Result table is external database)
class History(models.Model): # default database
idx = models.AutoField(primary_key=True)
scenario_id = models.ForeignKey(Scenario)
executor = models.CharField(max_length=255)
createdate = models.DateTimeField()
testinfoid = models.IntegerField(unique=True)
class Result(models.Model): # external (Result.objects.using('external'))
idx = models.AutoField(primary_key=True)
testinfoid = models.ForeignKey(History, to_field='testinfoid', related_name='result')
testresult = models.CharField(max_length=10)
class Meta:
unique_together = (('idx', 'testinfoid'),)
So, I want to express the Count by 'testresult' field in Result table.
It has some condition such as 'Pass' or 'Fail'.
I want to express a count query set for each condition. like this.
[{'idx': 1, 'pass_count': 10, 'fail_count': 5, 'executor': 'someone', ...} ...
...
{'idx': 10, 'pass_count': 1, 'fail_count': 10, 'executor': 'someone', ...}]
Is it possible?
It is a two level aggregation where the second level should be displayed as table columns - "matrix view".
A) Solution with Python loop to create columns with annotations by the second level ("testresult").
from django.db.models import Count
from collections import OrderedDict
qs = (History.objects
.values('pk', 'executor', 'testinfoid',... 'result__testresult')
.annotate(result_count=Count('pk'))
)
qs = qs.filter(...).order_by(...)
data = OrderedDict()
count_columns = ('pass_count', 'fail_count', 'error_count',
'expected_failure_count', 'unexpected_success_count')
for row in qs:
data.setdefault(row.pk, dict.fromkeys(count_columns, 0)).update(
{(k if k != result_count else row['result__testresult'] + '_count'): v
for k, v in row_items()
if k != 'result__testresult'
}
)
out = list(data.values())
The class OrderedDict is used to preserve order_by().
B) Solution with Subquery in Django 1.11+ (if the result should be a queryset. e.g. to be sorted or filtered finally in an Admin view by clicking, and if a more complicated query is acceptable and number of columns *_count is very low.). I can write a solution with subquery, but I'm not sure if the query will be fast enough with different database backends. Maybe someone other answers.
Related
I have three models, Accrual and Member, and the common field of these two models is register_no, but this field is not a foreign key
class Accrual(models.Model):
register_no = models.PositiveIntegerField(verbose_name=_('Register No'))
amount=models.DecimalField(decimal_places=2, max_digits=17, verbose_name=_('Total Amount'))
class Member(models.Model):
register_no = models.PositiveIntegerField(unique=True, verbose_name=_('Register No'))
class Driver(models.Model):
register_no = models.PositiveIntegerField(unique=True, verbose_name=_('Register No'))
I want to list the debt each member/driver has. It can be done with #property;
class Member(models.Model):
register_no = models.PositiveIntegerField(unique=True, verbose_name=_('Register No'))
#property
def debt(self):
ret_val = Accrual.objects.filter(register_no=self.register_no).aggregate(
debt=Sum('amount'))
debt = ret_val.get('debt', 0)
return debt if debt else 0
but I can't use order_by this way. I want to sort each member by debt. How can I solve this problem?
You can use a subquery to add all the related accruals based on the register_no:
from django.db.models import OuterRef, Subquery, Sum
accruals = Accrual.objects.filter(
register_no=OuterRef('register_no')
).values('register_no').annotate(debt=Sum('amount'))
Member.objects.annotate(
debt=Subquery(accruals.values('debt'))
).order_by('debt')
which produces this sql:
SELECT "id", "register_no", (
SELECT SUM(U0."amount") AS "debt"
FROM "accrual" U0
WHERE U0."register_no" = ("register_no")
GROUP BY U0."register_no"
) AS "debt"
FROM "member"
ORDER BY "debt" ASC
I have an random model:
class Model(models.Model):
other_field1 = models.SomeField(...)
m2m_field1 = models.ManyToManyField(...)
other_field2 = models.SomeField(...)
m2m_field2 = models.ManyToManyField(...)
other_field3 = models.SomeField(...)
I want to know the count of fields that correspond to the relation many to many and the count of other fields.
In the example above, I have 2 fields with a many-to-many relationship and 3 other fields.
EDIT
How to calculate the number of reverse relationships?
class OtherModel1(models.Model):
field = models.ForeginKey(Model)
class OtherModel2(models.Model):
field = models.ForeginKey(Model)
You can work with the ._meta option, and thus determine the number of items with:
from django.db.models import ManyToManyField
from django.db.models.fields.reverse_related import ForeignObjectRel
number_of_m2m_fields = sum(
isinstance(m, ManyToManyField) for m in Model._meta.get_fields()
)
number_of_other_fields = sum(
not isinstance(m, ManyToManyField) for m in Model._meta.get_fields()
)
number_of_reverse_relations = sum(
isinstance(mto, ForeignObjectRel) for mto in Model._meta.get_fields()
)
I have the next model:
class Departments(Document):
_id = fields.ObjectIdField()
name = fields.StringField(blank=True, null=True)
department_id = fields.StringField(blank=True, null=True) # Added
list_of_users = fields.ListField(blank=True, null=True)
list_of_workstations = fields.ListField(blank=True, null=True)
As you can see list_of_users and list_of_workstations are lists of items.
I wrote a code in Python, which takes all data from DB, put it into dict and then sorts as I need, but it works too slow.
How can I sort Departments right in the DB by the length of list_of_users or list_of_workstations or by ratio of list_of_users/list_of_workstations, something like:
departments = DepartmentStats.objects.order_by(len(list_of_users)).dsc
or
departments = DepartmentStats.objects.order_by(len(list_of_users)/len(list_of_workstations)).dsc
?
For your first request, use annotation like Umut Gunebakan told you in his comment. But I'm know sure about Count() on ListField
departments = DepartmentStats.objects.all().annotate(num_list_users=Count('list_of_users')).order_by('-num_list_users')
For a desc order by, you just need to add the sign '-' (minus).
https://docs.djangoproject.com/en/1.10/ref/models/querysets/#order-by
The second request will be :
departments = DepartmentStats.objects.all().annotate(user_per_workstation=(Count('list_of_users')/Count('list_of_workstations')).order_by('-user_per_workstation')
UPDATE: (Mongoengine used)
With mongoengine you need to get item frequencies and sorted the result :
Check this part of documentation - futher aggregation
list_user_freqs = DepartmentStats.objects.item_frequencies('list_of_users', normalize=True)
from operator import itemgetter
list_user_freqs_sroted = sorted(list_user_freqs.items(), key=itemgetter(1), reverse=True)
If someone needs raw query:
departments = DepartmentStats._get_collection().aggregate([
{"$project": {
"department_id": 1,
"name": 1,
"list_of_users": 1,
}},
{"$sort": {"list_of_users": -1}},
])
and the case, when the result must be sorted by the ratio list_of_users/list_of_workstations
departments = DepartmentStats._get_collection().aggregate([
{"$project": {
"department_id": 1,
"name": 1,
"list_of_users": 1,
"len_list_of_items": {"$divide": [{"$size": "$list_of_users"},
{"$size": "$list_of_workstations"}]}
}},
{"$sort": {"len_list_of_items": -1}},
])
I have a model that has a type and value, based on the type I want to change the sign of the number to negative (Without changing it in the database)
class Foo(models.Model):
EARNING_CATEGORY = 'E'
DEDUCTION_CATEGORY = 'D'
CATEGORY_CHOICES = (
(EARNING_CATEGORY, 'Earning'),
(DEDUCTION_CATEGORY, 'Deduction'),
)
name = models.CharField(max_length=50)
category = models.CharField(max_length=1, choices=CATEGORY_CHOICES)
value = models.FloatField()
Now in order to aggregate over Foo I need to consider that values with category = D should be negative so I get the real sum.
What I tried so far is to change the sign during the save() method, but I don't want to show it to the user as negative in the application. So the only way I came up with is to calculate in a method using a for loop.
class Account(models.Model):
# feilds
def sum(self):
items = self.foo_set.all()
sum = 0
for i in items:
if i.category == Foo.DEDUCTION_CATEGORY:
sum -= i.value
else:
sum += i.value
return sum
You can annotate the current sign during your query like this:
from django.db.models import Case, Value as V, F, FloatField, When
items = self.foo_set.annotate(fixed_value=Case(
When(category=Foo.DEDUCTION_CATEGORY, then=V('-1')*F('value')),
default=F('value'),
output_field=FloatField())
).all()
So during annotation we do a condition check and if our category is equal to 'D', we change sign of value field, and then do your aggregation on fixed_value field.
And here's more info:
https://docs.djangoproject.com/en/1.9/ref/models/conditional-expressions/#case
I have the following model structure below:
class Master(models.Model):
name = models.CharField(max_length=50)
mounting_height = models.DecimalField(max_digits=10,decimal_places=2)
class MLog(models.Model):
date = models.DateField(db_index=True)
time = models.TimeField(db_index=True)
sensor_reading = models.IntegerField()
m_master = models.ForeignKey(Master)
The goal is to produce a queryset that returns all the fields from MLog plus a calculated field (item_height) based on the related data in Master
using Django's raw sql:
querySet = MLog.objects.raw('''
SELECT a.id,
date,
time,
sensor_reading,
mounting_height,
(sensor_reading - mounting_height) as item_height
FROM db_mlog a JOIN db_master b
ON a.m_master_id = b.id
''')
How do I code this using Django's ORM?
I can think of two ways to go about this without relying on raw(). The first is pretty much the same as what #tylerl suggested. Something like this:
class Master(models.Model):
name = models.CharField(max_length=50)
mounting_height = models.DecimalField(max_digits=10,decimal_places=2)
class MLog(models.Model):
date = models.DateField(db_index=True)
time = models.TimeField(db_index=True)
sensor_reading = models.IntegerField()
m_master = models.ForeignKey(Master)
def _get_item_height(self):
return self.sensor_reading - self.m_master.mounting_height
item_height = property(_get_item_height)
In this case I am defining a custom (derived) property for MLog called item_height. This property is calculated as the difference of the sensor_reading of an instance and the mounting_height of its related master instance. More on property here.
You can then do something like this:
In [4]: q = MLog.objects.all()
In [5]: q[0]
Out[5]: <MLog: 2010-09-11 8>
In [6]: q[0].item_height
Out[6]: Decimal('-2.00')
The second way to do this is to use the extra() method and have the database do the calculation for you.
In [14]: q = MLog.objects.select_related().extra(select =
{'item_height': 'sensor_reading - mounting_height'})
In [16]: q[0]
Out[16]: <MLog: 2010-09-11 8>
In [17]: q[0].item_height
Out[17]: Decimal('-2.00')
You'll note the use of select_related(). Without this the Master table will not be joined with the query and you will get an error.
I always do the calculations in the app rather than in the DB.
class Thing(models.Model):
foo = models.IntegerField()
bar = models.IntegerField()
#Property
def diff():
def fget(self):
return self.foo - self.bar
def fset(self,value):
self.bar = self.foo - value
Then you can manipulate it just as you would any other field, and it does whatever you defined with the underlying data. For example:
obj = Thing.objects.all()[0]
print(obj.diff) # prints .foo - .bar
obj.diff = 4 # sets .bar to .foo - 4
Property, by the way, is just a standard property decorator, in this case coded as follows (I don't remember where it came from):
def Property(function):
keys = 'fget', 'fset', 'fdel'
func_locals = {'doc':function.__doc__}
def probeFunc(frame, event, arg):
if event == 'return':
locals = frame.f_locals
func_locals.update(dict((k,locals.get(k)) for k in keys))
sys.settrace(None)
return probeFunc
sys.settrace(probeFunc)
function()
return property(**func_locals)