Need help translating raw sql query to ORM Django - django

Django Models that used in projects:
class LeadStatus(models.Model):
status_id = models.PositiveIntegerField(verbose_name='ID статуса', unique=True)
name = models.CharField(max_length=100, verbose_name='Имя', null=True, default='Неизвестный')
class Lead(models.Model):
lead_id = models.PositiveIntegerField(verbose_name="ID сделки", null=True, unique=True)
created_date_time = models.DateTimeField(verbose_name="Время и дата создания сделки", null=True)
price = models.IntegerField(verbose_name='Бюджет сделки')
current_status = models.ForeignKey(LeadStatus, on_delete=models.SET_NULL, verbose_name='Текущий статус', null=True)
class LeadsNote(models.Model):
note_id = models.CharField(verbose_name="ID примечания", null=True, unique=True, max_length=64)
lead = models.ForeignKey(Lead, on_delete=models.CASCADE)
old_status = models.ForeignKey(LeadStatus, related_name='old_status', null=True, blank=True,
on_delete=models.CASCADE)
new_status = models.ForeignKey(LeadStatus, null=True, related_name='new_status', blank=True,
on_delete=models.CASCADE)
crossing_status_date_time = models.DateTimeField(verbose_name="Время и дата пересечения статуса", null=True)
I am trying to execute the following SQL query using Django ORM tools. He does what i need
SELECT leads.id,
notes.crossing_status_date_time,
old_status.name as old_status,
new_status.name as new_status,
(CASE WHEN leads.id in (
select leads.id
from core_leads
where old_status.status_id not in (13829322, 14286379) and new_status.status_id in (13829322, 14286379))
THEN true ELSE false END) as _is_profit
FROM core_leadsnote AS notes
LEFT JOIN core_lead AS leads ON notes.lead_id=leads.id
LEFT JOIN core_leadstatus AS old_status ON notes.old_status_id=old_status.id
LEFT JOIN core_leadstatus AS new_status ON notes.new_status_id=new_status.id
Using Django subqueries I came up with the following ORM query.
But it doesn't work as well as raw sql
from django.db.models import OuterRef, Q, Subquery, Case, When, BooleanField
from .models import LeadsNote, Lead, CompanyConfig, AmoCompany
company = AmoCompany.objects.first()
status_config = CompanyConfig.objects.get(company=company)
accepted_statuses = [status.status_id for status in status_config.accept_statuses.all()]
subquery = (Lead
.objects.select_related('current_status', 'company')
.filter(leadsnote=OuterRef('pk'))
.filter(~Q(leadsnote__old_status__status_id__in=accepted_statuses) &
Q(leadsnote__new_status__status_id__in=accepted_statuses)))
query = (LeadsNote
.objects
.annotate(
_is_profit=Case(
When(lead__id__in=Subquery(subquery.values('id')), then=True),
default=False, output_field=BooleanField())))
Need any help finding the right solution

Related

Traslate this queryset?

My query is made in MYSQL I would like to know how to translate it to the ORM django tnhanks you
SELECT *,(SELECT count(*)
FROM tesisnueva.donador_desactivar ds
where ds.motivo=1 and
ds.donador_id= dd.id and
date_add(ds.fecha_desactivar, INTERVAL 90 DAY)<now()) as cantidad
FROM tesisnueva.donador_donador dd
where dd.genero='F';
MY MODELS
class Donador(models.Model):
user = models.OneToOneField(User, on_delete=models.CASCADE)
hospital = models.ForeignKey(Hospital, null=True, blank=True, on_delete=models.SET_NULL)
fecha_nacimiento = models.DateField(validators=[check_date])
direccion = models.CharField(max_length=50,null=False,blank=False)
genero = models.CharField( max_length=1 , choices = GENERO, default='M')
grupo_sanguineo = models.CharField(max_length=10,choices=GRUPO_SANGRE, default='A')
factor_RH = models.CharField(max_length=2, choices=FACTOR_SANGRE, default='+')
telefono = models.CharField(max_length=12)
activo = models.BooleanField(default=0)
groups = models.ForeignKey(Group, null=True, blank=True, on_delete=models.CASCADE)
class Desactivar(models.Model):
donador = models.ForeignKey(Donador,on_delete=models.CASCADE)
motivo= models.IntegerField()
fecha_desactivar=models.DateField()
You can make use of .annotate(…) [Django-doc] and use a filter=… parameter [Django-doc] to filter the related model accordingly:
from datetime import timedelta
from django.db.models import Q
from django.utils.timezone import now
Donador.objects.filter(genero='F').annotate(
cantidad=Count(
'desactivar',
filter=Q(motivo=1, fecha_desactivar=now()-timedelta(days=90))
)
)
The Donador objects that arise from this queryset have an extra attribute .cantidad that contains the number of related desactivar objects that satisfy the given filter.

Django server cache select statement in Postgres DB

I have a select Django statement like this:
import datetime
from django.db.models import Count, Q
from mobileapp.models import *
import math
from functools import partial
from django.utils import timezone
from mobileapp.decorators import logging__time
date=timezone.now()
first = date.replace(day=1)
last_month = first - datetime.timedelta(days=1)
subsidiaries = Subsidiary.objects.select_related('frequency').select_related('last_call').filter(user=user).annotate(number_of_calls=Count('calls', filter=Q(calls__start__gte=last_month, calls__start__date__lte=date)))
Even though today date is 6.30. the query in the logs look like this:
SELECT
"mobileapp_subsidiary"."id",
"mobileapp_subsidiary"."subsidiary_ref",
"mobileapp_subsidiary"."name",
"mobileapp_subsidiary"."address",
"mobileapp_subsidiary"."city",
"mobileapp_subsidiary"."coordinates_x",
"mobileapp_subsidiary"."coordinates_y",
"mobileapp_subsidiary"."phone_number",
"mobileapp_subsidiary"."frequency_id",
"mobileapp_subsidiary"."channel",
"mobileapp_subsidiary"."subchannel",
"mobileapp_subsidiary"."user_id",
"mobileapp_subsidiary"."day_planned",
"mobileapp_subsidiary"."customer_id",
"mobileapp_subsidiary"."last_call_id",
COUNT("mobileapp_calls"."id") FILTER (
WHERE (("mobileapp_calls"."start" AT TIME ZONE 'Europe/Ljubljana')::date <= '2020-06-11'::date AND "mobileapp_calls"."start" >= '2020-05-31T08:01:06.244114+00:00'::timestamptz)) AS "number_of_calls", "mobileapp_frequency"."name", "mobileapp_frequency"."calls_per_month", T5."id", T5."start", T5."end", T5."notes", T5."type", T5."user_id", T5."subsidiary_id", T5."order_id" FROM "mobileapp_subsidiary" LEFT OUTER JOIN "mobileapp_calls" ON ("mobileapp_subsidiary"."id" = "mobileapp_calls"."subsidiary_id") LEFT OUTER JOIN "mobileapp_frequency" ON ("mobileapp_subsidiary"."frequency_id" = "mobileapp_frequency"."name") LEFT OUTER JOIN "mobileapp_calls" T5 ON ("mobileapp_subsidiary"."last_call_id" = T5."id") WHERE "mobileapp_subsidiary"."user_id" = 2 GROUP BY "mobileapp_subsidiary"."id", "mobileapp_frequency"."name", T5."id"; args=(datetime.date(2020, 6, 11), datetime.datetime(2020, 5, 31, 8, 1, 6, 244114, tzinfo=<UTC>), 2);
I check the later logs and it is always query for 6.11 (even for previous days).
It looks like the query is cached somehow and called the same query every time, even that date is changed...
if I use Django-extensions shell and I run the function from there, it queries the right result.
Any idea?
EDIT:
Model for subsidiary looks like this:
class Subsidiary(models.Model):
subsidiary_ref = models.CharField(max_length=50)
name = models.CharField(null=False, max_length=80)
address = models.TextField(null=True, blank=True)
city = models.CharField(null=True, max_length=50)
coordinates_x = models.DecimalField(null=True, decimal_places=5, max_digits=9)
coordinates_y = models.DecimalField(null=True, decimal_places=5, max_digits=9)
phone_number = models.CharField(null=True, max_length=50)
frequency = models.ForeignKey(Frequency, on_delete=models.SET_NULL, null=True, db_index=True)
channel = models.CharField(null=True, blank=True, max_length=50)
subchannel = models.CharField(null=True, blank=True, max_length=50)
user = models.ForeignKey(User, related_name='subsidiaries', on_delete=models.SET_NULL, null=True, db_index=True)
day_planned = models.BooleanField(default=False)
customer = models.ForeignKey(Customer, on_delete=models.CASCADE, related_name='subsidiaries')
last_call = models.ForeignKey('Calls', related_name="subsidiary_reference", on_delete=models.DO_NOTHING, null=True,blank=True, db_index=True)

How to specify GROUP BY field in Dajngo ORM?

I have the following working SQL statement:
SELECT id FROM ops_kpitarget WHERE (site_id = 1 AND validFrom <= "2019-08-28") GROUP BY kpi_id HAVING validFrom = (MAX(validFrom))
But I cannot get this to work inside Django ORM.
The best I got was the code below, but then the database is complaining that it is missing a GROUP BY clause to make HAVING work.
How can I get the same query with specifying "kpi_id" as the GROUP BY clause using Djangos ORM? Any ideas?
KpiTarget.objects
.filter(validFrom__lte=fromDate)
.values("id", "kpi")
.filter(validFrom=Max("validFrom"))
... which translates to:
SELECT "ops_kpitarget"."id", "ops_kpitarget"."kpi_id" FROM "ops_kpitarget" WHERE "ops_kpitarget"."validFrom" <= 2019-08-14 HAVING "ops_kpitarget"."validFrom" = (MAX("ops_kpitarget"."validFrom"))
I played around with annotate but this is not really giving me what I want...
Update:
Some background: I have 3 tables: Kpi, KpiTarget, and KpiTargetObservation.
Kpi holds all general information regarding the KPI like name, typeetc.
KpiTarget stores target values defined for several different sites. These target values can change over time. Hence, I have included the combination of MAX() and validFrom <= (some date) to determine the latest valid target for any given KPI.
KpiTargetObservation stores the individual observations per defined KPI target. It just holds the link to KpiTarget, the date of the observation, and the observation value.
The final queries I need to build will have to give me something like the following:
give me all known KPIs per given site
tell me the most recent target value for the KPIs you found
get me any known observation that is related to the identified kpi targets
I am struggling with the 2nd query, and specifically how to get this working using Djangos ORM. I could just escape to RAW SQL, but I would prefer to not to, if possible.
The models:
class KpiCategory(models.Model):
name = models.CharField(max_length=255)
def __str__(self):
return self.name
class Kpi(models.Model):
KPI_KIND_CHOICES = [("BOOL", "Boolean"), ("FLOAT", "Float"), ("STRING", "String")]
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
name = models.CharField(max_length=255)
category = models.ForeignKey(KpiCategory, on_delete=models.CASCADE)
kind = models.CharField(max_length=150, choices=KPI_KIND_CHOICES)
def __str__(self):
return self.name
class KpiTarget(models.Model):
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
kpi = models.ForeignKey(Kpi, on_delete=models.CASCADE, related_name="kpiTargetSet")
targetDouble = models.DecimalField(
max_digits=20, decimal_places=15, blank=True, null=True
)
targetBool = models.BooleanField(blank=True, null=True)
targetStr = models.CharField(max_length=255, blank=True)
site = models.ForeignKey(Site, on_delete=models.CASCADE)
validFrom = models.DateField()
def __str__(self):
return str(self.kpi)
class KpiObservation(models.Model):
# firstCreated = models.DateField(auto_now_add=True)
# firstCreatedBy = models.OneToOneField(User, on_delete=models.CASCADE)
# lastEdited = models.DateField(auto_now=True)
# lastEditedBy = models.OneToOneField(User, on_delete=models.CASCADE)
kpiTarget = models.ForeignKey(
KpiTarget, on_delete=models.CASCADE, related_name="kpiObservationSet"
)
observed = models.DateField()
observationDouble = models.DecimalField(
max_digits=20, decimal_places=15, blank=True, null=True
)
observationBool = models.BooleanField(blank=True, null=True)
observationStr = models.CharField(max_length=255, blank=True)
def __str__(self):
return str(self.observed)
KpiTarget.objects.filter(validFrom__lte=fromDate).annotate(validFrom=Max("validFrom")).order_by('kpi__id').values("id", "kpi")

Django ORM with Key/Value Table structure

I'm trying to get the django orm to replicate a call on a database for my current table structure:
Tables:
ServiceItems {Id, name, user, date_created}
ServiceItemsProps {fk_to_ServiceItems Item_id, Id, key, value}
I'm trying to select items from the ServiceItem table with multiple keys from the ServiceItemsProps table as columns.
I can accomplish this with a query like the following:
> select tbl1.value as bouncebacks, tbl2.value as assignees from
> service_items join service_item_props as tbl1 on tbl1.item_id =
> service_items.id join service_item_props as tbl2 on tbl2.item_id =
> service_items.id where service_items.item_type='CARD' and
> tbl1.key='bouncebacks' and tbl2.key='assignees'
But I'm not able to figure out how to reproduce this in Django's ORM. I would like to not inject raw SQL into the statements here, because codebase portability is important.
Section of models.py
class ServiceItems(models.Model):
class Meta:
db_table = 'service_items'
unique_together = ('service', 'item_type', 'item_id')
service = models.ForeignKey(Service, blank=False, db_column='service_id', on_delete=models.CASCADE)
item_type = models.CharField(max_length=255, blank=False)
url = models.TextField(blank=True, null=True)
item_id = models.TextField(blank=True, null=True)
item_creation_user = models.TextField(blank=True, null=True)
item_creation_date = models.DateTimeField(blank=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
class ServiceItemProps(models.Model):
class Meta:
db_table = 'service_item_props'
item = models.ForeignKey(ServiceItems, blank=False, db_column='item_id', on_delete=models.CASCADE)
prop_id = models.TextField(blank=True, null=True)
key = models.CharField(max_length=255, blank=False)
value = models.TextField(blank=True, null=True)
# change one line to make it easier to query
item = models.ForeignKey(ServiceItems, blank=False, db_column='item_id', on_delete=models.CASCADE, related_name='item_props')
Query should become:
ServiceItem.objects.filter(Q(item_type='CARD') & (Q(item_props__key='bouncebacks') | Q(item_props__key='assignees'))
==============================================================
I think I misunderstood your query.
I believe this is a good case to use .raw() .
Try this one instead:
qs = ServiceItemProps.objects.raw('''
SELECT sip1.*, sip2.value as other_value
FROM {item_table} as service_items
INNER JOIN {props_table} as sip1 on sip1.item_id = service_items.id
INNER JOIN {props_table} as sip2 on sip2.item_id = service_items.id
WHERE service_items.item_type='CARD' and sip1.key='bouncebacks' and sip2.key='assignees'
'''.format(item_table=ServiceItems._meta.db_table, props_table=ServiceItemProps._meta.db_table)
for itemprop in qs:
print(qs.value, qs.other_value)

Django map join in raw request

I have following code:
class Invoice(models.Model):
customer = models.ForeignKey(User, blank=True, null=True)
customer_name = models.CharField(max_length=50, blank=True, null=True)
email = models.CharField(max_length=100, blank=True, null=True)
#----------------------------------
invoices = Invoice.objects.raw("""
SELECT
`invoices`.`id`,
`invoices`.`customer_id`,
`invoices`.`customer_name`,
`invoices`.`email` AS `inv_email`,
`auth_user`.`username`,
`auth_user`.`email` AS `auth_email`,
COUNT('customer_id') AS `buy_count`
FROM `invoices`
LEFT JOIN `auth_user` ON `auth_user`.id = `invoices`.customer_id
GROUP BY `customer_id`, `invoices`.`email`
""", translations={'inv_email': 'email', 'auth_email': 'customer.email'})
But, when I write invoices[i].customer Django makes SQL-request for each customer. There are way to map JOIN to the Django model in the raw request? Or this SQL-request may be realized using pure Django ORM?