Django server cache select statement in Postgres DB - django

I have a select Django statement like this:
import datetime
from django.db.models import Count, Q
from mobileapp.models import *
import math
from functools import partial
from django.utils import timezone
from mobileapp.decorators import logging__time
date=timezone.now()
first = date.replace(day=1)
last_month = first - datetime.timedelta(days=1)
subsidiaries = Subsidiary.objects.select_related('frequency').select_related('last_call').filter(user=user).annotate(number_of_calls=Count('calls', filter=Q(calls__start__gte=last_month, calls__start__date__lte=date)))
Even though today date is 6.30. the query in the logs look like this:
SELECT
"mobileapp_subsidiary"."id",
"mobileapp_subsidiary"."subsidiary_ref",
"mobileapp_subsidiary"."name",
"mobileapp_subsidiary"."address",
"mobileapp_subsidiary"."city",
"mobileapp_subsidiary"."coordinates_x",
"mobileapp_subsidiary"."coordinates_y",
"mobileapp_subsidiary"."phone_number",
"mobileapp_subsidiary"."frequency_id",
"mobileapp_subsidiary"."channel",
"mobileapp_subsidiary"."subchannel",
"mobileapp_subsidiary"."user_id",
"mobileapp_subsidiary"."day_planned",
"mobileapp_subsidiary"."customer_id",
"mobileapp_subsidiary"."last_call_id",
COUNT("mobileapp_calls"."id") FILTER (
WHERE (("mobileapp_calls"."start" AT TIME ZONE 'Europe/Ljubljana')::date <= '2020-06-11'::date AND "mobileapp_calls"."start" >= '2020-05-31T08:01:06.244114+00:00'::timestamptz)) AS "number_of_calls", "mobileapp_frequency"."name", "mobileapp_frequency"."calls_per_month", T5."id", T5."start", T5."end", T5."notes", T5."type", T5."user_id", T5."subsidiary_id", T5."order_id" FROM "mobileapp_subsidiary" LEFT OUTER JOIN "mobileapp_calls" ON ("mobileapp_subsidiary"."id" = "mobileapp_calls"."subsidiary_id") LEFT OUTER JOIN "mobileapp_frequency" ON ("mobileapp_subsidiary"."frequency_id" = "mobileapp_frequency"."name") LEFT OUTER JOIN "mobileapp_calls" T5 ON ("mobileapp_subsidiary"."last_call_id" = T5."id") WHERE "mobileapp_subsidiary"."user_id" = 2 GROUP BY "mobileapp_subsidiary"."id", "mobileapp_frequency"."name", T5."id"; args=(datetime.date(2020, 6, 11), datetime.datetime(2020, 5, 31, 8, 1, 6, 244114, tzinfo=<UTC>), 2);
I check the later logs and it is always query for 6.11 (even for previous days).
It looks like the query is cached somehow and called the same query every time, even that date is changed...
if I use Django-extensions shell and I run the function from there, it queries the right result.
Any idea?
EDIT:
Model for subsidiary looks like this:
class Subsidiary(models.Model):
subsidiary_ref = models.CharField(max_length=50)
name = models.CharField(null=False, max_length=80)
address = models.TextField(null=True, blank=True)
city = models.CharField(null=True, max_length=50)
coordinates_x = models.DecimalField(null=True, decimal_places=5, max_digits=9)
coordinates_y = models.DecimalField(null=True, decimal_places=5, max_digits=9)
phone_number = models.CharField(null=True, max_length=50)
frequency = models.ForeignKey(Frequency, on_delete=models.SET_NULL, null=True, db_index=True)
channel = models.CharField(null=True, blank=True, max_length=50)
subchannel = models.CharField(null=True, blank=True, max_length=50)
user = models.ForeignKey(User, related_name='subsidiaries', on_delete=models.SET_NULL, null=True, db_index=True)
day_planned = models.BooleanField(default=False)
customer = models.ForeignKey(Customer, on_delete=models.CASCADE, related_name='subsidiaries')
last_call = models.ForeignKey('Calls', related_name="subsidiary_reference", on_delete=models.DO_NOTHING, null=True,blank=True, db_index=True)

Related

When I open the model, one field value is replaced automatically

Have a stupid issue: When I open the employee model (in django admin), one field (employee_type) value is replaced automatically, idk why...
Example: I create employee, define employee type as manager, save. In DB value is manager. After that, I open employee and see employee type as sewer and if I save, that value will saved in DB.
I created text choices for this field, and In DB field is defined as enum.
I tried to research the issue, value isn't used from DB, always first value from text choices is used.
By the way, I created same fields (enum in DB and text choices in the model) in other model. And all works properly.
How can I fix it???
Models.py with the issue:
class Employee(models.Model):
class EmployeeType(models.TextChoices):
SEWER = 'SEWER', _('Sewer')
MANAGER = 'MANAGER', _('Manager')
UNDEFINED = 'UNDEFINED', _('Undefined')
user = models.OneToOneField(User,
models.CASCADE,
db_column='user',
verbose_name=_('User'),
primary_key=True)
employee_type = models.CharField(db_column='Employee type',
verbose_name=_('Employee type'),
max_length=9,
choices=EmployeeType.choices,
default=EmployeeType.UNDEFINED)
phone = models.CharField(db_column='Phone',
verbose_name=_('Phone'),
max_length=255)
work_xp = models.IntegerField(db_column='Work XP',
verbose_name=_('Work XP'),
blank=True,
null=True)
Another one models.py. With same fields but without issues:
class Order(models.Model):
class OrderStatus(models.TextChoices):
CREATED = 'Created', _('Created')
CANCELLED = 'Cancelled', _('Cancelled')
IN_PROGRESS = 'In progress', _('In progress')
COMPLETED = 'Completed', _('Completed')
PASSED_TO_CLIENT = 'Passed to the client', _('Passed to the client')
RETURNED_FOR_REWORK = 'Returned for rework', _('Returned for rework')
class Urgency(models.TextChoices):
LOW = 'Low', _('Low urgency')
MEDIUM = 'Medium', _('Medium urgency')
HIGH = 'High', _('High urgency')
VERY_HIGH = 'Very high', _('Very high urgency')
class LabourIntensity(models.TextChoices):
LOW = 'Low', _('1-3 days')
MEDIUM = 'Medium', _('4-6 days')
HIGH = 'High', _('7-9 days')
VERY_HIGH = 'Very high', _('10+ days')
class PaymentStatus(models.TextChoices):
PENDING = 'Pending payment', _('Pending payment')
PREPAYMENT_MADE = 'Prepayment made', _('Prepayment made')
PAID = 'Paid', _('Paid')
id_service = models.ForeignKey(Service,
models.SET_NULL,
db_column='id_Service',
verbose_name=_('Service'),
blank=True,
null=True)
status = models.CharField(db_column='Status',
verbose_name=_('Status'),
max_length=20,
choices=OrderStatus.choices,
default=OrderStatus.CREATED)
payment_status = models.CharField(db_column='Payment status',
verbose_name=_('Payment status'),
max_length=15,
choices=PaymentStatus.choices,
blank=True,
null=True)
prepayment = models.DecimalField(db_column='Prepayment',
verbose_name=_('Prepayment'),
max_digits=19,
decimal_places=2,
blank=True,
null=True)
cost = models.DecimalField(db_column='Cost',
verbose_name=_('Cost'),
max_digits=19,
decimal_places=2,
blank=True,
null=True)
start_date = models.DateTimeField(db_column='Start date',
verbose_name=_('Start date'),
blank=True,
null=True)
end_date = models.DateTimeField(db_column='End date',
verbose_name=_('End date'),
blank=True,
null=True)
id_client = models.ForeignKey(Client,
models.SET_NULL,
db_column='id_Client',
verbose_name=_('Client'),
blank=True,
null=True)
id_employee = models.ForeignKey(Employee,
models.SET_NULL,
db_column='id_Employee',
verbose_name=_('Employee'),
blank=True,
null=True)
labour_intensity = models.CharField(db_column='Labour intensity',
verbose_name=_('Labour intensity'),
max_length=9,
choices=LabourIntensity.choices,
default=LabourIntensity.LOW)
urgency = models.CharField(db_column='Urgency',
verbose_name=_('Urgency'),
max_length=9,
choices=Urgency.choices,
default=Urgency.LOW)
materials = models.ManyToManyField(Material, through='OrderMaterials')
comment = models.TextField(db_column='Comment',
verbose_name=_('Comment'),
blank=True,
null=True)
I solved this issue.
In DB a type of field "employee_type" was defined as:
ENUM('Undefined','Sewer','Manager')
But the model has EmployeeType choices with uppercase.
Solution: I changed lowercase to uppercase of the values in the field type in DB:
ENUM('UNDEFINED','SEWER','MANAGER')
Now everything works fine.

Need help translating raw sql query to ORM Django

Django Models that used in projects:
class LeadStatus(models.Model):
status_id = models.PositiveIntegerField(verbose_name='ID статуса', unique=True)
name = models.CharField(max_length=100, verbose_name='Имя', null=True, default='Неизвестный')
class Lead(models.Model):
lead_id = models.PositiveIntegerField(verbose_name="ID сделки", null=True, unique=True)
created_date_time = models.DateTimeField(verbose_name="Время и дата создания сделки", null=True)
price = models.IntegerField(verbose_name='Бюджет сделки')
current_status = models.ForeignKey(LeadStatus, on_delete=models.SET_NULL, verbose_name='Текущий статус', null=True)
class LeadsNote(models.Model):
note_id = models.CharField(verbose_name="ID примечания", null=True, unique=True, max_length=64)
lead = models.ForeignKey(Lead, on_delete=models.CASCADE)
old_status = models.ForeignKey(LeadStatus, related_name='old_status', null=True, blank=True,
on_delete=models.CASCADE)
new_status = models.ForeignKey(LeadStatus, null=True, related_name='new_status', blank=True,
on_delete=models.CASCADE)
crossing_status_date_time = models.DateTimeField(verbose_name="Время и дата пересечения статуса", null=True)
I am trying to execute the following SQL query using Django ORM tools. He does what i need
SELECT leads.id,
notes.crossing_status_date_time,
old_status.name as old_status,
new_status.name as new_status,
(CASE WHEN leads.id in (
select leads.id
from core_leads
where old_status.status_id not in (13829322, 14286379) and new_status.status_id in (13829322, 14286379))
THEN true ELSE false END) as _is_profit
FROM core_leadsnote AS notes
LEFT JOIN core_lead AS leads ON notes.lead_id=leads.id
LEFT JOIN core_leadstatus AS old_status ON notes.old_status_id=old_status.id
LEFT JOIN core_leadstatus AS new_status ON notes.new_status_id=new_status.id
Using Django subqueries I came up with the following ORM query.
But it doesn't work as well as raw sql
from django.db.models import OuterRef, Q, Subquery, Case, When, BooleanField
from .models import LeadsNote, Lead, CompanyConfig, AmoCompany
company = AmoCompany.objects.first()
status_config = CompanyConfig.objects.get(company=company)
accepted_statuses = [status.status_id for status in status_config.accept_statuses.all()]
subquery = (Lead
.objects.select_related('current_status', 'company')
.filter(leadsnote=OuterRef('pk'))
.filter(~Q(leadsnote__old_status__status_id__in=accepted_statuses) &
Q(leadsnote__new_status__status_id__in=accepted_statuses)))
query = (LeadsNote
.objects
.annotate(
_is_profit=Case(
When(lead__id__in=Subquery(subquery.values('id')), then=True),
default=False, output_field=BooleanField())))
Need any help finding the right solution

Traslate this queryset?

My query is made in MYSQL I would like to know how to translate it to the ORM django tnhanks you
SELECT *,(SELECT count(*)
FROM tesisnueva.donador_desactivar ds
where ds.motivo=1 and
ds.donador_id= dd.id and
date_add(ds.fecha_desactivar, INTERVAL 90 DAY)<now()) as cantidad
FROM tesisnueva.donador_donador dd
where dd.genero='F';
MY MODELS
class Donador(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
hospital = models.ForeignKey(Hospital, null=True, blank=True, on_delete=models.SET_NULL)
fecha_nacimiento = models.DateField(validators=[check_date])
direccion = models.CharField(max_length=50,null=False,blank=False)
genero = models.CharField( max_length=1 , choices = GENERO, default='M')
grupo_sanguineo = models.CharField(max_length=10,choices=GRUPO_SANGRE, default='A')
factor_RH = models.CharField(max_length=2, choices=FACTOR_SANGRE, default='+')
telefono = models.CharField(max_length=12)
activo = models.BooleanField(default=0)
groups = models.ForeignKey(Group, null=True, blank=True, on_delete=models.CASCADE)
class Desactivar(models.Model):
donador = models.ForeignKey(Donador,on_delete=models.CASCADE)
motivo= models.IntegerField()
fecha_desactivar=models.DateField()
You can make use of .annotate(…) [Django-doc] and use a filter=… parameter [Django-doc] to filter the related model accordingly:
from datetime import timedelta
from django.db.models import Q
from django.utils.timezone import now
Donador.objects.filter(genero='F').annotate(
cantidad=Count(
'desactivar',
filter=Q(motivo=1, fecha_desactivar=now()-timedelta(days=90))
)
)
The Donador objects that arise from this queryset have an extra attribute .cantidad that contains the number of related desactivar objects that satisfy the given filter.

How to specify GROUP BY field in Dajngo ORM?

I have the following working SQL statement:
SELECT id FROM ops_kpitarget WHERE (site_id = 1 AND validFrom <= "2019-08-28") GROUP BY kpi_id HAVING validFrom = (MAX(validFrom))
But I cannot get this to work inside Django ORM.
The best I got was the code below, but then the database is complaining that it is missing a GROUP BY clause to make HAVING work.
How can I get the same query with specifying "kpi_id" as the GROUP BY clause using Djangos ORM? Any ideas?
KpiTarget.objects
.filter(validFrom__lte=fromDate)
.values("id", "kpi")
.filter(validFrom=Max("validFrom"))
... which translates to:
SELECT "ops_kpitarget"."id", "ops_kpitarget"."kpi_id" FROM "ops_kpitarget" WHERE "ops_kpitarget"."validFrom" <= 2019-08-14 HAVING "ops_kpitarget"."validFrom" = (MAX("ops_kpitarget"."validFrom"))
I played around with annotate but this is not really giving me what I want...
Update:
Some background: I have 3 tables: Kpi, KpiTarget, and KpiTargetObservation.
Kpi holds all general information regarding the KPI like name, typeetc.
KpiTarget stores target values defined for several different sites. These target values can change over time. Hence, I have included the combination of MAX() and validFrom <= (some date) to determine the latest valid target for any given KPI.
KpiTargetObservation stores the individual observations per defined KPI target. It just holds the link to KpiTarget, the date of the observation, and the observation value.
The final queries I need to build will have to give me something like the following:
give me all known KPIs per given site
tell me the most recent target value for the KPIs you found
get me any known observation that is related to the identified kpi targets
I am struggling with the 2nd query, and specifically how to get this working using Djangos ORM. I could just escape to RAW SQL, but I would prefer to not to, if possible.
The models:
class KpiCategory(models.Model):
name = models.CharField(max_length=255)
def __str__(self):
return self.name
class Kpi(models.Model):
KPI_KIND_CHOICES = [("BOOL", "Boolean"), ("FLOAT", "Float"), ("STRING", "String")]
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
name = models.CharField(max_length=255)
category = models.ForeignKey(KpiCategory, on_delete=models.CASCADE)
kind = models.CharField(max_length=150, choices=KPI_KIND_CHOICES)
def __str__(self):
return self.name
class KpiTarget(models.Model):
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
kpi = models.ForeignKey(Kpi, on_delete=models.CASCADE, related_name="kpiTargetSet")
targetDouble = models.DecimalField(
max_digits=20, decimal_places=15, blank=True, null=True
)
targetBool = models.BooleanField(blank=True, null=True)
targetStr = models.CharField(max_length=255, blank=True)
site = models.ForeignKey(Site, on_delete=models.CASCADE)
validFrom = models.DateField()
def __str__(self):
return str(self.kpi)
class KpiObservation(models.Model):
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
kpiTarget = models.ForeignKey(
KpiTarget, on_delete=models.CASCADE, related_name="kpiObservationSet"
)
observed = models.DateField()
observationDouble = models.DecimalField(
max_digits=20, decimal_places=15, blank=True, null=True
)
observationBool = models.BooleanField(blank=True, null=True)
observationStr = models.CharField(max_length=255, blank=True)
def __str__(self):
return str(self.observed)
KpiTarget.objects.filter(validFrom__lte=fromDate).annotate(validFrom=Max("validFrom")).order_by('kpi__id').values("id", "kpi")

Django ORM with Key/Value Table structure

I'm trying to get the django orm to replicate a call on a database for my current table structure:
Tables:
ServiceItems {Id, name, user, date_created}
ServiceItemsProps {fk_to_ServiceItems Item_id, Id, key, value}
I'm trying to select items from the ServiceItem table with multiple keys from the ServiceItemsProps table as columns.
I can accomplish this with a query like the following:
> select tbl1.value as bouncebacks, tbl2.value as assignees from
> service_items join service_item_props as tbl1 on tbl1.item_id =
> service_items.id join service_item_props as tbl2 on tbl2.item_id =
> service_items.id where service_items.item_type='CARD' and
> tbl1.key='bouncebacks' and tbl2.key='assignees'
But I'm not able to figure out how to reproduce this in Django's ORM. I would like to not inject raw SQL into the statements here, because codebase portability is important.
Section of models.py
class ServiceItems(models.Model):
class Meta:
db_table = 'service_items'
unique_together = ('service', 'item_type', 'item_id')
service = models.ForeignKey(Service, blank=False, db_column='service_id', on_delete=models.CASCADE)
item_type = models.CharField(max_length=255, blank=False)
url = models.TextField(blank=True, null=True)
item_id = models.TextField(blank=True, null=True)
item_creation_user = models.TextField(blank=True, null=True)
item_creation_date = models.DateTimeField(blank=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class ServiceItemProps(models.Model):
class Meta:
db_table = 'service_item_props'
item = models.ForeignKey(ServiceItems, blank=False, db_column='item_id', on_delete=models.CASCADE)
prop_id = models.TextField(blank=True, null=True)
key = models.CharField(max_length=255, blank=False)
value = models.TextField(blank=True, null=True)
# change one line to make it easier to query
item = models.ForeignKey(ServiceItems, blank=False, db_column='item_id', on_delete=models.CASCADE, related_name='item_props')
Query should become:
ServiceItem.objects.filter(Q(item_type='CARD') & (Q(item_props__key='bouncebacks') | Q(item_props__key='assignees'))
==============================================================
I think I misunderstood your query.
I believe this is a good case to use .raw() .
Try this one instead:
qs = ServiceItemProps.objects.raw('''
SELECT sip1.*, sip2.value as other_value
FROM {item_table} as service_items
INNER JOIN {props_table} as sip1 on sip1.item_id = service_items.id
INNER JOIN {props_table} as sip2 on sip2.item_id = service_items.id
WHERE service_items.item_type='CARD' and sip1.key='bouncebacks' and sip2.key='assignees'
'''.format(item_table=ServiceItems._meta.db_table, props_table=ServiceItemProps._meta.db_table)
for itemprop in qs:
print(qs.value, qs.other_value)