django listview pagination is too slow on large dataset - django

I have a django listview that shows filtered data from Catalogue. It works fine on small database, but when database is too large, it takes too long time to return results.
views.py:
class all_ads(generic.ListView):
paginate_by = 12
template_name = 'new_list_view_grid-card.html'
def get_queryset(self):
city_district = self.request.GET.getlist('city_district')
usage = self.request.GET.get('usage')
status = self.request.GET.get('status')
last2week = datetime.datetime.now() - datetime.timedelta(days=14)
status = status.split(',')
if usage:
usage = usage.split(',')
else:
usage = ['1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17','18','19','20','21','22','23','24','25','26','27','28','29','30','31']
intersections = list(set(status).intersection(usage))
type_q = (Q(type__in=intersections) & Q(type__isnull=False))
cat_ids = list(models.Catalogue.objects.filter(
Q(*[Q(city__contains=x) for x in city_district], _connector=Q.OR) |
Q(*[Q(district__contains=x) for x in city_district], _connector=Q.OR)
).values_list('pk', flat=True))
result = models.Catalogue.objects.filter(
Q(datetime__gte=last2week),
type_q,
pk__in=cat_ids
).distinct().order_by('-datetime').prefetch_related('type')
return result
models.py:
class Catalogue(models.Model):
city = models.CharField(db_index=True,max_length=100, null=True)
district = models.CharField(db_index=True,max_length=100, null=True)
type = models.ManyToManyField(Type, db_index=True)
class Type(models.Model):
name = models.CharField(max_length=100, db_index=True)
def __str__(self):
return self.name
And this is template.html:
{% for Catalogue in catalogue_list %}
"do something"
{% endfor %}

You can also decide to do it via the front-end, i.e. fetch all the data, send it to the front-end and display it with JavaScript

Related

how to fetch data from a list of dictionary using django

I am trying to develop a dictionary(using django) that contains a voice for pronunciation. everything goes fine but I cannot relate the audio with the word using the following query.
Model.py
class Warehouse(models.Model):
word = models.CharField(max_length=200, blank=True)
type = models.CharField(max_length=200, blank=True)
gender = models.CharField(max_length=200, blank=True)
meaning = models.TextField(max_length=2000, blank=True)
synonyms = models.CharField(max_length=200, blank=True)
antonyms = models.CharField(max_length=200, blank=True)
usage = models.TextField(max_length=2000, blank=True)
class Audio(models.Model):
warehouse = models.ForeignKey(Warehouse, on_delete=models.CASCADE)
audio_file = models.FileField(upload_to='audio')
def __str__(self):
return self.warehouse.word
def delete_media(self):
os.remove(path=MEDIA_ROOT + '/' + str(self.audio_file))
query.py
def get_words_warehouse_matching(search_sting):
try:
dictionary_object = Warehouse.objects.filter(Q(word__icontains = search_sting)|Q(type__icontains=search_sting) | Q(gender__icontains=search_sting)|Q(synonyms__icontains=search_sting)|Q(antonyms__icontains=search_sting)
words_list = []
for words in dictionary_object.iterator():
word_dictionary = {'id':words.id, 'word': words.word, 'meaning': words.meaning, 'synonyms': words.synonyms,'antonyms': words.antonyms}
words_list.append(word_dictionary)
return words_list
views.py
def search(request):
context = {}
warehouse={}
if request.method == 'POST':
context['data'] = get_words_by_matching(request.POST['searchString'])
context['warehouse'] = get_words_warehouse_matching(request.POST['searchString'])
voice = Audio.objects.filter(warehouse_id = warehouse.id)
context['audio'] = voice
return render(request, 'dictionary/word.html', context)
I need to display the related audio file in the template so that I need to equate warehouse_id to Warehouse.id. How can I fetch the id from get_words_warehouse_matching (dictionary) so that I could use
filter(warehouse_id = warehouse.id) in views. Or if there are any other better options, suggestions would be highly appreciated. Thanks in advance.
It seems you just need to query on your Audio model.
q = Q(warehouse__word__icontains = search_sting)|Q(warehouse__type__icontains=search_sting) | Q(warehouse__gender__icontains=search_sting)|Q(warehouse__synonyms__icontains=search_sting)|Q(warehouse__antonyms__icontains=search_sting)
context['audio'] = Audio.objects.filter(q)
And then you can access these data in template like below:
{% for aud in audio %}
{{ aud.warehouse.id }}
{{ aud.warehouse.word }}
{{ aud.warehouse.meaning }}
....
{% endfor %}
I have solved this issue by modifying the query as:
context = {}
if request.method == 'POST':
context['data'] = get_words_by_matching(request.POST['searchString'])
warehouse_val = get_words_warehouse_matching(request.POST['searchString'])
id=warehouse_val[0]['id']
context['warehouse'] = get_words_warehouse_matching(request.POST['searchString'])
context['audio'] = Audio.objects.filter(warehouse_id=id)
return render(request, 'dictionary/word.html', context)
Can someone please suggest to me if this is the better solution or I can do something else so that my query will be more efficient. Thanks

ManytoMany query and template rendering from one model to another

I'm new to Django. I had two different questions.
I can't query between one of my model and another model (ManyToMany). I can do this with the shell, but I couldn't handle it in the template.
I cannot assign a default value from one model to another model's field.
For the first question;
What I want to do is show values for multiple options. For this, I could make a query similar to this in the shell:
room[0].room_type_id.all()
But I can't do this in the template. On the other hand, when I want to show it with display, it returns empty. What I want to do here; returning the room types for each room or or accessing the room_cost of the RoomType class and displaying it in the template, repeated for each room type.
{% for room in rooms %}
<h3 class="card-title pricing-card-title"> {{room.room_type_id_display}} </h3>
{% endfor %}
My second question is;
To set the value from the property of a different model as default in the other model field. That is, to assign the value returned from the total_price of the Booking model to the price field in the Payment model by default.
I would appreciate it if anyone could provide documentation or resources on the subject.
class RoomType(models.Model):
ROOM_CHOICES = (
('1', 'O),
('2','T'),
('3', 'Th'),
('4','F'),
('5','Fi')
)
room_type = models.CharField(max_length=50,choices=ROOM_CHOICES)
room_type_des = models.TextField(blank=True,null=True)
room_cost = models.IntegerField()
def __str__(self):
return str(self.room_type)
class Room(models.Model):
room_number = models.IntegerField()
room_des = models.TextField(blank=True,null=True)
room_availabe = models.BooleanField(default=True)
room_type_id = models.ManyToManyField(RoomType)
def __str__(self):
return str(self.room_number)
class Booking(models.Model):
room_number_id = models.ForeignKey(Room,on_delete=models.DO_NOTHING)
customer_id = models.ManyToManyField(Customer)
check_in = models.DateTimeField(auto_now_add=True)
check_out = models.DateTimeField(auto_now_add=False,auto_now=False,auto_created=False, null=True)
status = models.BooleanField(default=False)
#property
def calculate_day(self):
day = self.check_out - self.check_in
return str(day.days)
#property
def total_price(self):
day = self.check_out - self.check_in
price = self.room_number_id.room_type_id.room_cost
return price*day.days
class Payment(models.Model):
booking_id = models.ForeignKey(Booking,on_delete=models.DO_NOTHING)
ACCEPT_CHOICES = (
('N','N'),
('K','K'),
)
payment_type = models.CharField(max_length=1,choices=ACCEPT_CHOICES)
price = models.IntegerField()
payment_detail = models.TextField()
Here's a small modification: don't use "_id", because it's not an id, it's a real instance of the foreign model.
Then, use "related_name", and think like "if I start from the opposite side, what name should I use?" (it's always plural).
And for your (2), you can't set a default value for a "in-between table": a ManyToMany field create a "join" table to join the two other tables. You can only set a default value for OneToOne and ForeignKey fields.
class RoomType(models.Model):
ROOM_CHOICES = (
('1', 'O),
('2','T'),
('3', 'Th'),
('4','F'),
('5','Fi')
)
room_type = models.CharField(max_length=50,choices=ROOM_CHOICES)
room_type_des = models.TextField(blank=True,null=True)
room_cost = models.IntegerField()
def __str__(self):
return str(self.room_type)
class Room(models.Model):
room_number = models.IntegerField()
room_des = models.TextField(blank=True,null=True)
room_availabe = models.BooleanField(default=True)
room_type = models.ManyToManyField(RoomType, related_name="rooms")
def __str__(self):
return str(self.room_number)
class Booking(models.Model):
room = models.ForeignKey(Room, related_name="bookings", on_delete=models.DO_NOTHING)
customer = models.ManyToManyField(Customer, related_name="bookings")
check_in = models.DateTimeField(auto_now_add=True)
check_out = models.DateTimeField(auto_now_add=False,auto_now=False,auto_created=False, null=True)
status = models.BooleanField(default=False)
#property
def calculate_day(self):
day = self.check_out - self.check_in
return str(day.days)
#property
def total_price(self):
day = self.check_out - self.check_in
price = self.room_number.room_type.room_cost
return price * day.days
class Payment(models.Model):
booking = models.ForeignKey(Booking, related_name="payments", on_delete=models.DO_NOTHING)
ACCEPT_CHOICES = (
('N','N'),
('K','K'),
)
payment_type = models.CharField(max_length=1,choices=ACCEPT_CHOICES)
price = models.IntegerField()
payment_detail = models.TextField()
If you want all your room types, it's:
RoomType.objects.all()
If you want to "send" all types to a template, use get_context_data like this:
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context["room_types"] = RoomType.objects.all()
return context
and in your template:
{% for room_type in room_types %}
{{ room_type }}
{% endfor %}
For your template (and with my models code above), you could do:
{% for room in rooms %}
<h3 class="card-title pricing-card-title"> {{ room.room_type }} </h3>
{% endfor %}
And if you want to show all options in a form, it's another subject, too long for a simple answer here, read the official documentation here.

how to write 3 dependant table join query in Django

I am new in django , I want to create join query in django, my model is as follow
Model.py
Class area_country(models.Model):
country_id = models.AutoField(primary_key=True)
country_name = models.CharField(max_length=100, null=True)
short_name= models.CharField(max_length=20, null=True)
flag_enable = models.SmallIntegerField()
class Meta:
db_table = "area_country"
class area_state(models.Model):
state_id = models.AutoField(primary_key=True)
state_name = models.CharField(max_length=100, null=True)
short_name = models.CharField(max_length=20, null=True)
country_id = models.ForeignKey(area_country, on_delete=models.CASCADE)
flag_enable = models.SmallIntegerField()
class Meta:
db_table = "area_state"
class area_city(models.Model):
city_id = models.AutoField(primary_key=True)
city_name = models.CharField(max_length=100, null=True)
short_name = models.CharField(max_length=20, null=True)
state_id = models.ForeignKey(area_state, on_delete=models.CASCADE)
flag_enable = models.SmallIntegerField()
class Meta:
db_table = "area_city"
i need query like
SELECT "area_country"."country_id",
"area_country"."country_name",
"area_state"."state_id",
"area_state"."state_name",
"area_city"."city_id",
"area_city"."city_name"
FROM "area_country"
LEFT OUTER JOIN "area_state" ON ("area_country"."country_id" = "area_state"."country_id_id")
LEFT OUTER JOIN "area_city" ON ("area_state"."state_id" = "area_city"."state_id_id")
** Try by me **
view.py
result = area_country.objects.all().select_related('area_state').values('country_id', 'country_name', 'area_state__state_id', 'area_state__state_name')
when i print it by using query = result.query
SELECT "area_country"."country_id", "area_country"."country_name", "area_state"."state_id", "area_state"."state_name"
FROM "area_country"
LEFT OUTER JOIN "area_state" ON ("area_country"."country_id" = "area_state"."country_id_id")
this is ok now i try
result = area_country.objects.all().select_related('area_state').select_related('area_city').values('country_id', 'country_name', 'area_state__state_id', 'area_state__state_name','area_city__city_id','area_city__city_name')
it show me error
Cannot resolve keyword 'area_city' into field. Choices are: area_state, country_id, country_name, flag_enable, short_name
please help me for django query.
for information friends, i know how to write raw query in django with cursor = connection.cursor()
finally solution is,
let me clear . i want array for js datatable thats why temp = [] and also i want json response so JsonResponse,
view.py
class get_all_country_state_city(View):
def get(self, request):
citys = area_city.objects.all()
response = []
if citys:
response = []
for value in citys:
temp = []
temp.append(value.city_name)
temp.append(value.state_id.state_name)
temp.append(value.state_id.country_id.country_name)
response.append(temp)
return JsonResponse({'data': response})
if you want to show directly in template then
view.py
citys = area_city.objects.all()
context = {
'citys':citys
}
return render(request, "area.html", context)
template.html
<table >
<tbody>
{% for value in citys %}
<tr><td>{{value.state_id.country_id.country_name}}</td>
<td>{{value.state_id.state_name}}</td>
<td>{{value.city_name}}</td></tr>
{% endfor %}
</tbody>
</table>

Django query is not returning all the records

I am trying to do a simple query but I just don’t understand why I am only getting 1 row when it should be in this example 2:
And I am trying to get all the rows where the externalID = 45454
By doing:
context['report'] = Report.objects.filter(externalID = self.kwargs.get('reportID'))
I can see on the debug I get
SELECT "data_app_report"."id", "data_app_report"."main_contact_id", "data_app_report"."company_id", "data_app_report"."created", "data_app_report"."externalID", "data_app_report"."success", "data_app_report"."amountrequested" FROM "data_app_report" WHERE "data_app_report"."externalID" = '''45454'''
And my table looks like
I guess I am doing something wrong in the query but I just don’t know what, I thought having filter will allow me to get all of the records
This are the 2 models:
class Report(models.Model):
main_contact = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE)
company = models.ForeignKey(CompanyDetail, on_delete=models.CASCADE, related_name='companyreport', default=1)
created = models.DateTimeField(auto_now_add=True)
externalID = models.CharField(default='0000000', max_length=35, unique = False, editable=False) #ONE PERSON CAN HAVE MULTIPLE BANKING IN THE SAME OPP, THEN THE REF IS THE SAME
success = models.BooleanField(default = True)
amountrequested = models.DecimalField(max_digits=8, decimal_places=2, null=True, blank=True)
class ReportURL(models.Model):
report = models.ForeignKey(Report, on_delete=models.CASCADE, related_name='reporturls')
file_type = models.CharField(max_length=5, unique = False, default='html')
report_url = models.URLField(unique=True)
This is my view
class ExtAccView(TemplateView):
template_name = 'data_app/ext_access.html'
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
context['reportID'] = self.kwargs.get('reportID')
context['access'] = self.kwargs.get('access')
#GET THE COMPANY INFORMATION
try:
context['company'] = CompanyDetail.objects.get(companyExternalKey = self.kwargs.get('access'))
except Exception as e:
context['message'] = 'The company doesnt match the values you submitted ' + self.kwargs.get('access')
return context
#GET THE REPORT AND WE NEED TO DOUBLE CHECK IF THE REPORT COMPANY IS THE SAME AS ABOVE
try:
# context['report'] = Report.objects.get(externalID = self.kwargs.get('reportID'))
context['report'] = Report.objects.filter(externalID = self.kwargs.get('reportID'))
except Exception as e:
context['message'] = 'No reports found with that reference: ' + self.kwargs.get('reportID')
return context
return context
This is my template (ext_access.html)
{% for r in reportURL %}
{{ r.report_url }}
{% endfor %}

Queries over ManyToManyField with variable set in a sessions user

Here is my models:
class Clients(models.Model):
client_name = models.CharField(max_lenght=100)
commentaire_clients = models.TextField(blank=False)
date_demande = models.TimeField(auto_now_add=True)
choix = models.ManyToManyField('Agence', blank=True)
def __unicode__(self):
return unicode(self.date_demande)
class Market(models.Model):
nom_market = models.CharField(max_length=200)
code_postal_market = models.ManyToManyField('Lieux', blank=True)
statut_vip = models.BooleanField(default=False)
def __unicode__(self):
return self.nom_market
class Lieux(models.Model):
code_postal = models.CharField(max_length=200)
ville = models.CharField(max_length=200)
region = models.CharField(max_length=200)
departement = models.CharField(max_length=200)
longitude = models.DecimalField(max_digits=9, decimal_places=6)
latitude = models.DecimalField(max_digits=9, decimal_places=6)
pays = models.CharField(max_length=100)
def __unicode__(self):
return unicode(self.code_postal)
Here is my view:
def comparelist(request):
if request.session.get('code_postal'):
poste = request.session.get('code_postal')
else:
poste = "RATE"
market_match = Market.objects.filter(statut_vip=False, code_postal_market = poste)
market_match_vip = Market.objects.filter(statut_vip=True)
#edit bis repetita Market replace Agence models
return render_to_response('compare.html', {
'code_postale': poste,
'bien_immobilier': bien,
'listing_agence' : market_match ,
'listing_vip' : market_match_vip ,
})
What I am trying to do is to make a query that will give me all the market that match:
- statut_vip = False
- code_postal_market = poste (that I obtain from user session from a form on the previous page
then I try to render it in my templates via:
{% for mes_market in listing_vip %}
<br>{{mes_market.nom_market}}
<br>{{mes_market.statut_vip}}
{% endfor %}
edit
here is my template for listing_agence (same as the previous one but with the right reference) sorry for the error.
{% for mes_agences in listing_agence %}
<br>{{mes_agences.nom_market}}
<br>{{mes_agences.statut_vip}}
{% endfor %}
My second queries to list all VIP clients do work but when I try to filter via the postal code given by the user via a form (and keeping the record via sessions)
nothing appears.
Thank you for your help!
I finally made it!
I replaced:
market_match = Market.objects.filter(statut_vip=False, code_postal_market = poste)
by
market_match = Market.objects.filter(statut_vip=False, code_postal_market__code_postal=poste)
code_postal is from the table Lieux