I am working with Django Raw SQL as I want to build a relatively complex query,
As part of it, I have a case statement that says 'if the date on the record is < today then 1 else 0 - this works.
overdue_phases = task.objects.filter(orgid=orgid, taskid=taskidx).raw('''
SELECT '1' as id, case when research_due < "2022-12-01" AND research_status != "done" then 1 else 0 end as count
from app_task where taskid = "''' + taskidx + '''"''')
overdue_phases = task.objects.filter(orgid=orgid, taskid=taskidx).raw('''
SELECT '1' as id, case when research_due < "2022-12-01" AND research_status != "done" then 1 else 0 end as count
from app_task where taskid = "''' + taskidx + '''"''')
However, when I want to swap the hard coded date for TODAY() I can't make it work. I have tried passing a python date variable into the script (td = datetime.date.today()), but it doesn't return the right results!
Can anyone advise me please?
There is no need to do this. You can use condition expressions [Django-doc]:
from django.db.models import Case, Q, Value, When
overdue_phases = task.objects.filter(orgid=orgid, taskid=taskidx).annotate(
count=Case(
When(
~Q(research_status='done'),
research_due__lt='2022-12-01',
then=Value(1),
),
default=Value(0),
)
)
Or if a boolean is sufficient as well:
from django.db.models import Q
overdue_phases = task.objects.filter(orgid=orgid, taskid=taskidx).annotate(
count=~Q(research_status='done') & Q(research_due__lt='2012-12-01')
)
or for older versions of Django with an ExpressionWrapper [Django-doc]:
from django.db.models import BooleanField, ExpressionWrapper, Q
overdue_phases = task.objects.filter(orgid=orgid, taskid=taskidx).annotate(
count=ExpressionWrapper(
~Q(research_status='done') & Q(research_due__lt='2012-12-01'),
output_field=BooleanField(),
)
)
You can replace '2022-12-01' with date.today().
I have a function that takes a class reference to create an SQL string.
I can get the class reference from globals() as long as the class name is unique.
Example call.
module_name_a.models.get_insert_string(globals()['DriverShiftPreference'])
If there are duplicate class names in different modules then I get collisions.
For example, if there is module_name_a.models.Driver and module_name_b.models.Driver then
module_name_a.models.get_insert_string(globals()['Driver'])
returns a result from module_name_b.models.Driver.
What is an effective method to get a Django class reference when there are duplicated class names in different modules?
Found using inspect gets most of the way there.
# in a module called junction
import inspect
def get_insert_string(class_reference):
fields = []
arguments = []
for field in class_reference._meta.fields:
if field.primary_key:
pass
elif field.help_text:
fields.append('"%s"' % field.help_text)
arguments.append('?')
result = 'insert into %s ( %s ) values ( %s )' % ( class_reference.filemaker_table_name, u','.join(fields), u','.join(arguments))
return result
def make_models_map_from_module(module_):
result = {}
for item in inspect.getmembers(module_.models):
key = item[0]
value = item[1]
if key == '__builtins__':
break
else:
result[key] = value
return result
Sample usage:
import junction
import depot_maestro
depot_maestro_model_map = junction.models.make_models_map_from_module(depot_maestro)
junction.models.get_insert_string(depot_maestro_model_map['Driver'])
u'insert into driver ( "DEPOT_ID","uuid","ABN_number","account_name","address_1","advance_payment_days","bank_account","barred_flag","biometric_print","bsb","change_log","comm_fixed_percent","company","contract_end_date","contract_fixed","contract_notice_period","contract_start_date","contract_terms","country","date_activated","DOB","default_roster_template_id","EFT_code","email","external_id","fax","first_name","fuel_pin","fuel_split_percent","gender","is_owner","known_as","last_contacted","middle_name","nationality","Next_of_kin","next_of_kin_mobile","Next_of_kin_phone","next_of_kin_relationship","next_of_kin_work","passport","password","pay_in_choice","phone","pref_age","pref_body_type","pref_name_display","pref_transmission","remit_gst_for_ATO_who_remits","roster_template_id","status","surname","toll_serial_no","tsl_owner","unreliable_flag","www_LastKnown_GPS","www_LastKnown_Status","www_LastKnown_Timestamp","creationAccount","creationTimeStamp","modificationAccount","modificationTimeStamp" ) values ( ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,? )'
Found a much simpler way. Just pass in the model.
Example.
junction.models.get_insert_string(depot_maestro.models.Group)
which returns
insert into groups ( "DEPOT_ID","uuid","Notes","Percentage","abn","address","admin_fee","bank_details","bookings_percent","creationAccount","creationTimeStamp","email","fax","group_account_name","group_colour","group_logo","group_logo_2","group_name","group_number","group_service_mark_as_paid","gst","message","meter","modificationAccount","modificationTimeStamp","omit_from_group_reports","parts_at_cost","phone","remit_all_cash","rollover_amount","roster_group","sort_order","vehicle_fk" ) values ( ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,? )'
I am building a search feature with multiple T/F filter options, like so:
Search: ________________
Filters:
___ Open to Public
___ Parking Available
_x_ Free Entrance
In this case the user is specifically looking for events that are free to get into. I would like my Q object to return keyword matched objects where free_entrance is set to True.
My thought is to define the variables in my search function:
search_public = None
search_parking = None
search_free_entrance = True
and set the Q object up like so:
q_objects.append(
Q(
name__icontains=search_term,
public__icontains=search_public,
parking__icontains=search_parking,
free_entrance=search_free_entrance
)
)
However, I want all objects (True or False) to be returned for the unfilitered variables (instead of only objects set to None). Is there a keyword I can insert, or is there a Q object filter type that I am missing?
UPDATE:
In addition to the posted answer, *args can also be used for complex (OR) Q objects:
From:
http://www.nomadjourney.com/2009/04/dynamic-django-queries-with-kwargs/
args = ( Q( name__icontains = 'search_term' ) | Q( company__icontains = 'search_term' ) )
Use kwargs:
search = {'name__icontains': search_term}
if search_public is not None:
search.update({'public__icontains': search_public})
if search_parking is not None:
search.update({'parking__icontains': search_parking})
if search_free_entrance is not None:
search.update({'pree_entrance__icontains': search_free_entrance})
q_objects.append(Q(**search))
or more complicated example (your question from comment):
search_kwargs = {'name__icontains': search_term}
search_args = tuple()
if search_public is not None:
search_args += (Q(Q(company__icontains=search_public) | Q(other__icontains=search_public)),)
q_objects.append(Q(*search_args, **search_kwargs))
I have a geographic object that is used in proximity searches. That object is keyed to different other objects via various paths.
Examples:
blog.geo_object
user.profile.geo_object
group.event.geo_object
Right now, i do a bounding box search, which strings like this:
radius_queryset = base_queryset.filter(
user__profile__geo__lat__gte = bounding_box.lat_min,
user__profile__geo__lat__lte = bounding_box.lat_max,
user__profile__geo__lon__gte = bounding_box.lon_min,
user__profile__geo__lon__lte = bounding_box.lon_max,
and then on other objects:
radius_queryset = base_queryset.filter(
blog__geo__lat__gte = bounding_box.lat_min,
blog__geo__lat__lte = bounding_box.lat_max,
blog__geo__lon__gte = bounding_box.lon_min,
blog__geo__lon__lte = bounding_box.lon_max,
)
This follows a general format of:
radius_queryset = base_queryset.filter(
[lookup_path]__geo__lat__gte = bounding_box.lat_min,
[lookup_path]__geo__lat__lte = bounding_box.lat_max,
[lookup_path]__geo__lon__gte = bounding_box.lon_min,
[lookup_path]__geo__lon__lte = bounding_box.lon_max,
)
# where lookup_path = "blog" or "user__profile" in the two above examples
I'm writing enough of these (3 so far, more to come) to want to generalize the queries -- encapsulation being a good friend of maintainability and an ally in the fight against typo-bugs.
So, to my question: short of using exec and eval (which just look ugly), is there a way to get the filter parameter name to sub-in a variable? Am I missing something simple here?
**kwargs is your answer!
def generate_qs(lookup_path, bounding_box):
return base_queryset.filter(**{
lookup_path + '__geo__lat__gte' : bounding_box.lat_min,
lookup_path + '__geo__lat__lte' : bounding_box.lat_max,
lookup_path + '__geo__lon__gte' : bounding_box.lon_min,
lookup_path + '__geo__lon__lte' : bounding_box.lon_max,
})
radius_queryset = generate_qs('blog', bounding_box)
I think you can use Python's **kwargs syntax to put this nastiness away in a helper function. Something like:
def helper(lookup_path, bounding_box):
return dict([ ("%s__geo__%s" % (lookup_path, lkup_left, ),
getattr(bounding_box, lkup_right), )
for lkup_left, lkup_right in
(("lat__gte", "lat_min", ),
("lat__lte", "lat_max", ),
("lon__gte", "lon_min", ),
("lon__lte", "lon_max", ),
) ])
qs = base_queryset.filter(**helper(lookup_path, bounding_box))
I want to be able to list the items that either a user has added (they are listed as the creator) or the item has been approved.
So I basically need to select:
item.creator = owner or item.moderated = False
How would I do this in Django? (preferably with a filter or queryset).
There is Q objects that allow to complex lookups. Example:
from django.db.models import Q
Item.objects.filter(Q(creator=owner) | Q(moderated=False))
You can use the | operator to combine querysets directly without needing Q objects:
result = Item.objects.filter(item.creator = owner) | Item.objects.filter(item.moderated = False)
(edit - I was initially unsure if this caused an extra query but #spookylukey pointed out that lazy queryset evaluation takes care of that)
It is worth to note that it's possible to add Q expressions.
For example:
from django.db.models import Q
query = Q(first_name='mark')
query.add(Q(email='mark#test.com'), Q.OR)
query.add(Q(last_name='doe'), Q.AND)
queryset = User.objects.filter(query)
This ends up with a query like :
(first_name = 'mark' or email = 'mark#test.com') and last_name = 'doe'
This way there is no need to deal with or operators, reduce's etc.
You want to make filter dynamic then you have to use Lambda like
from django.db.models import Q
brands = ['ABC','DEF' , 'GHI']
queryset = Product.objects.filter(reduce(lambda x, y: x | y, [Q(brand=item) for item in brands]))
reduce(lambda x, y: x | y, [Q(brand=item) for item in brands]) is equivalent to
Q(brand=brands[0]) | Q(brand=brands[1]) | Q(brand=brands[2]) | .....
Similar to older answers, but a bit simpler, without the lambda...
To filter these two conditions using OR:
Item.objects.filter(Q(field_a=123) | Q(field_b__in=(3, 4, 5, ))
To get the same result programmatically:
filter_kwargs = {
'field_a': 123,
'field_b__in': (3, 4, 5, ),
}
list_of_Q = [Q(**{key: val}) for key, val in filter_kwargs.items()]
Item.objects.filter(reduce(operator.or_, list_of_Q))
operator is in standard library: import operator
From docstring:
or_(a, b) -- Same as a | b.
For Python3, reduce is not a builtin any more but is still in the standard library: from functools import reduce
P.S.
Don't forget to make sure list_of_Q is not empty - reduce() will choke on empty list, it needs at least one element.
Multiple ways to do so.
1. Direct using pipe | operator.
from django.db.models import Q
Items.objects.filter(Q(field1=value) | Q(field2=value))
2. using __or__ method.
Items.objects.filter(Q(field1=value).__or__(field2=value))
3. By changing default operation. (Be careful to reset default behavior)
Q.default = Q.OR # Not recommended (Q.AND is default behaviour)
Items.objects.filter(Q(field1=value, field2=value))
Q.default = Q.AND # Reset after use.
4. By using Q class argument _connector.
logic = Q(field1=value, field2=value, field3=value, _connector=Q.OR)
Item.objects.filter(logic)
Snapshot of Q implementation
class Q(tree.Node):
"""
Encapsulate filters as objects that can then be combined logically (using
`&` and `|`).
"""
# Connection types
AND = 'AND'
OR = 'OR'
default = AND
conditional = True
def __init__(self, *args, _connector=None, _negated=False, **kwargs):
super().__init__(children=[*args, *sorted(kwargs.items())], connector=_connector, negated=_negated)
def _combine(self, other, conn):
if not(isinstance(other, Q) or getattr(other, 'conditional', False) is True):
raise TypeError(other)
if not self:
return other.copy() if hasattr(other, 'copy') else copy.copy(other)
elif isinstance(other, Q) and not other:
_, args, kwargs = self.deconstruct()
return type(self)(*args, **kwargs)
obj = type(self)()
obj.connector = conn
obj.add(self, conn)
obj.add(other, conn)
return obj
def __or__(self, other):
return self._combine(other, self.OR)
def __and__(self, other):
return self._combine(other, self.AND)
.............
Ref. Q implementation
This might be useful https://docs.djangoproject.com/en/dev/topics/db/queries/#spanning-multi-valued-relationships
Basically it sounds like they act as OR
Item.objects.filter(field_name__startswith='yourkeyword')