Flask - Query route and object creation - flask

I am quite new to Flask and not sure what would be the proper way to handle that. I am working on a Flask app with a lot of view that retrieves data from MongoDB. I decided to build my own pagination class so I am able to modify the pagination structure based on my view.
I decided to import the Pagination class in one of my views where I am displaying all the customers in a table. The issue I have is my route /clients is handling every GET requests without query parameters (http://127.0.0.1:5000/clients) or with query parameters (http://127.0.0.1:5000/clients?page=3). My pagination object need to know the total number of items in order to be created number_of_items.
Therefore, overtime I am calling the /client route, I am creating a new Pagination object which is not my purpose as I just want to initialize the object on the first time the user reaches /clients and after that I will use my increase(quantity) and decrease(quantity) methods from the Pagination object to change page.
Do you have any idea on where should I avoid to create a Pagination object whenever I am changing page and instead calling methods from the Pagination object (increase and decrease page number)?
customers/views.py
#customers.route('/clients', methods=['GET'])
#login_required
def clients():
""" Client customers list overview page """
page = request.args.get(key='page', default=0)
collection = mongo.db[customers_production]
number_of_items = collection.count()
# just want to create that object once
client_pagination = Pagination(number_of_items, ITEMS_PER_PAGE, 3)
items_to_skip = client_pagination.number_of_previous_items()
cursor = collection.find().skip(items_to_skip).limit(ITEMS_PER_PAGE)
customers = cursor.sort("Last Name", pymongo.ASCENDING)
page_data = client_pagination.dict()
return render_template("clients.html", customers=customers, pagination=page_data)
pagination.py
class Pagination():
def __init__(self, total_items, items_per_page, page_range):
self.total_items = total_items # total number of article to display in total
self.items_per_page = items_per_page # maximum number of article per page
self.page_range = page_range # number of previous and next pages to display
self.current_page = 0
self.number_of_pages = math.ceil(
self.total_items / self.items_per_page)
# if the current_page is 7 and page_range is 3 then self.following will be [8, 9, 10] and self.previous [4, 5 ,6]
self.following = list(
range(self.current_page + 1, self.current_page + self.page_range + 1))
# empty because by default the current page is 0 so no previous
self.previous = list()
def increase(self, quantity):
if self.current_page + quantity > self.number_of_pages:
raise ValueError(
f'The maximum number of pages is {self.number_of_pages}')
else:
self.current_page += quantity
self.following = list()
self.previous = list()
for i in range(1, self.page_range + 1):
new_following_page_number = i + self.current_page
new_previous_page_number = self.current_page - i
self.previous.insert(i, new_previous_page_number)
if new_following_page_number < self.number_of_pages:
self.following.insert(i, new_following_page_number)
def decrease(self, quantity):
if self.current_page - quantity < self.number_of_pages:
raise ValueError(f'The mimimum number of pages is 0')
else:
self.current_page -= quantity
self.following = list()
self.previous = list()
for i in range(1, self.page_range + 1):
new_following_page_number = i + self.current_page
new_previous_page_number = self.current_page - i
self.following.insert(i, new_following_page_number)
if new_previous_page_number > self.number_of_pages:
self.previous.insert(i, new_previous_page_number)
def number_of_previous_items(self):
""" Sum of all items in all previous pages not including the current page """
if self.current_page == 0:
return 0
return self.items_per_page * self.current_page
# method call to display elements on the HTML page
def dict(self):
return {
"current_page": self.current_page,
"next_pages": self.following,
"previous_pages": self.previous,
"number_of_pages": self.number_of_pages
}

Related

Flask wtforms AttributeError: 'HTMLString' object has no attribute 'paginate'

Somebody has helped me with some great code here to show the same form multiple times each with a submit button, it works a treat, But as I will have hundreds of forms I need to paginate the page, I have been able to paginate pages in the past but I dont no how to use that code with a form in a for loop.
here is my code:(with lots of help from Greg)
#bp.route('/stock', methods=['GET', 'POST'])
#bp.route('/stock/stock/', methods=['GET', 'POST'])
#login_required
def stock():
stocks = Stock.query.all()
forms = []
for stock in stocks:
form = AddStockForm()
form.id.default = stock.id
form.image.default = stock.image_url
form.date.default = stock.date
form.description.default = stock.description
form.event.default = stock.event
form.achat.default = stock.achat
form.vente.default = stock.vente
form.sold.default = stock.sold
forms.append(form)
for form in forms:
if form.validate_on_submit():
if form.modify.data:
stock = Stock.query.filter_by(id=form.id.data).one()
stock.date = form.date.data
stock.description = form.description.data
stock.event = form.event.data
stock.achat = form.achat.data
stock.vente = form.vente.data
stock.sold = form.sold.data
db.session.add(stock)
db.session.commit()
elif form.delete.data:
stock = Stock.query.filter_by(id=form.id.data).one()
db.session.delete(stock)
db.session.commit()
return redirect(url_for('stock.stock'))
form.process() # Do this after validate_on_submit or breaks CSRF token
page = request.args.get('page', 1, type=int)
forms = forms[1].id().paginate(
page, current_app.config['ITEMS_PER_PAGE'], False)
next_url = url_for('stock.stock', page=forms.next_num) \
if forms.has_next else None
prev_url = url_for('stock.stock', page=forms.prev_num) \
if forms.has_prev else None
return render_template('stock/stock.html',forms=forms.items, title=Stock, stocks=stocks)
I am trying to use the fact "forms" is a list to paginate the results, I obviously dont understand how to do this, I have looked at flask-paginate but I didnt understand that either!
all help is greatly needed
Warm regards, Paul.
EDIT
I have tried to use flask_pagination, here is my code:
#bp.route('/stock/stock/', methods=['GET', 'POST'])
#login_required
def stock():
search = False
q = request.args.get('q')
if q:
search = True
page = request.args.get(get_page_parameter(), type=int, default=1)
stocks = Stock.query.all()
forms = []
#rest of code here#
pagination = Pagination(page=page, total=stocks.count(), search=search, record_name='forms')
form.process() # Do this after validate_on_submit or breaks CSRF token
return render_template('stock/stock.html',forms=forms, title=Stock, pagination=pagination)
This gives a different error "TypeError: count() takes exactly one argument (0 given)" I also tried with "total=forms.count()" and got the same error!
I hate doing this as it shows a lack of patience at the begining but this answer may help others, I solved my problem in two ways the first was the query which decides the order of display (descending or ascending) this then allowed me to use flask-paginate to display the results on several pages, I realised that I was dealing with a list, and the example by one of the developers link showed me the way, here is my code,
from flask_paginate import Pagination, get_page_args
#bp.route('/stock', methods=['GET', 'POST'])
#bp.route('/stock/stock/', methods=['GET', 'POST'])
#login_required
def stock():
stocks = Stock.query.order_by(Stock.id.desc())# this gives order of results
forms = []
def get_forms(offset=0, per_page=25): #this function sets up the number of
return forms[offset: offset + per_page] #results per page
for stock in stocks:
form = AddStockForm()
form.id.default = stock.id
form.image.default = stock.image_url
form.date.default = stock.date
form.description.default = stock.description
form.event.default = stock.event
form.achat.default = stock.achat
form.vente.default = stock.vente
form.sold.default = stock.sold
forms.append(form)
for form in forms:
if form.validate_on_submit():
if form.modify.data:
stock = Stock.query.filter_by(id=form.id.data).one()
stock.date = form.date.data
stock.description = form.description.data
stock.event = form.event.data
stock.achat = form.achat.data
stock.vente = form.vente.data
stock.sold = form.sold.data
db.session.add(stock)
db.session.commit()
elif form.delete.data:
stock = Stock.query.filter_by(id=form.id.data).one()
db.session.delete(stock)
db.session.commit()
return redirect(url_for('stock.stock'))
#this is the code from the link that I used to paginate
page, per_page, offset = get_page_args(page_parameter='page',
per_page_parameter='per_page')
total = len(forms) # this code counts the resulting list to be displayed
pagination_forms = get_forms(offset=offset, per_page=per_page)
pagination = Pagination(page=page, per_page=per_page, total=total)
form.process() # Do this after validate_on_submit or breaks CSRF token
return render_template('stock/stock.html', title=Stock, stocks=stocks
page=page, forms=pagination_forms, per_page=per_page, pagination=pagination)
#And finally this is the pagination passed to the html
so this for all those numptys like me who struggle with everything but still love it.

How should I be formatting my yield requests?

My scrapy spider is very confused, or I am, but one of us is not working as intended. My spider pulls start url's from a file and is supposed to: Start on an Amazon search page, crawl the page and grab the url's of each search result, follow the link to the items page, crawl the items page for information on the item, once all items have been crawled on the first page follow pagination up to page X, rinse and repeat.
I am using ScraperAPI and Scrapy-user-agent to randomize my middlewares. I have formatted my start_requests with a priority based on their index in the file, so they should be crawled in order. I have checked and ensured that I AM receiving a successful 200 html response with the actual html from the Amazon page. Here is the code for the spider:
class AmazonSpiderSpider(scrapy.Spider):
name = 'amazon_spider'
page_number = 2
current_keyword = 0
keyword_list = []
payload = {'api_key': 'mykey', 'url':'https://httpbin.org/ip'}
r = requests.get('http://api.scraperapi.com', params=payload)
print(r.text)
#/////////////////////////////////////////////////////////////////////
def start_requests(self):
with open("keywords.txt") as f:
for index, line in enumerate(f):
try:
keyword = line.strip()
AmazonSpiderSpider.keyword_list.append(keyword)
formatted_keyword = keyword.replace(' ', '+')
url = "http://api.scraperapi.com/?api_key=mykey&url=https://www.amazon.com/s?k=" + formatted_keyword + "&ref=nb_sb_noss_2"
yield scrapy.Request(url, meta={'priority': index})
except:
continue
#/////////////////////////////////////////////////////////////////////
def parse(self, response):
print("========== starting parse ===========")
for next_page in response.css("h2.a-size-mini a").xpath("#href").extract():
if next_page is not None:
if "https://www.amazon.com" not in next_page:
next_page = "https://www.amazon.com" + next_page
yield scrapy.Request('http://api.scraperapi.com/?api_key=mykey&url=' + next_page, callback=self.parse_dir_contents)
second_page = response.css('li.a-last a').xpath("#href").extract_first()
if second_page is not None and AmazonSpiderSpider.page_number < 3:
AmazonSpiderSpider.page_number += 1
yield scrapy.Request('http://api.scraperapi.com/?api_key=mykey&url=' + second_page, callback=self.parse_pagination)
else:
AmazonSpiderSpider.current_keyword = AmazonSpiderSpider.current_keyword + 1
#/////////////////////////////////////////////////////////////////////
def parse_pagination(self, response):
print("========== starting pagination ===========")
for next_page in response.css("h2.a-size-mini a").xpath("#href").extract():
if next_page is not None:
if "https://www.amazon.com" not in next_page:
next_page = "https://www.amazon.com" + next_page
yield scrapy.Request(
'http://api.scraperapi.com/?api_key=mykey&url=' + next_page,
callback=self.parse_dir_contents)
second_page = response.css('li.a-last a').xpath("#href").extract_first()
if second_page is not None and AmazonSpiderSpider.page_number < 3:
AmazonSpiderSpider.page_number += 1
yield scrapy.Request(
'http://api.scraperapi.com/?api_key=mykey&url=' + second_page,
callback=self.parse_pagination)
else:
AmazonSpiderSpider.current_keyword = AmazonSpiderSpider.current_keyword + 1
#/////////////////////////////////////////////////////////////////////
def parse_dir_contents(self, response):
items = ScrapeAmazonItem()
print("============= parsing page ==============")
temp = response.css('#productTitle::text').extract()
product_name = ''.join(temp)
product_name = product_name.replace('\n', '')
product_name = product_name.strip()
temp = response.css('#priceblock_ourprice::text').extract()
product_price = ''.join(temp)
product_price = product_price.replace('\n', '')
product_price = product_price.strip()
temp = response.css('#SalesRank::text').extract()
product_score = ''.join(temp)
product_score = product_score.strip()
product_score = re.sub(r'\D', '', product_score)
product_ASIN = response.css('li:nth-child(2) .a-text-bold+ span').css('::text').extract()
keyword = AmazonSpiderSpider.keyword_list[AmazonSpiderSpider.current_keyword]
items['product_keyword'] = keyword
items['product_ASIN'] = product_ASIN
items['product_name'] = product_name
items['product_price'] = product_price
items['product_score'] = product_score
yield items
For the FIRST start url, it will crawl three or four items and then it will jump to the SECOND start url. It will skip processing the remaining items and pagination pages, going directly to the second start url. For the second url, it will crawl three or four items, then it again will skip to the THIRD start url. It continues in this way, grabbing three or four items, then skipping to the next URL until it reaches the final start url. It will completely gather all information on this URL. Sometimes the spider COMPLETELY SKIPS the first or second starting url. This happens infrequently, but I have no idea as to what could cause this.
My code for following result item URL's works fine, but I never get the print statement for "starting pagination" so it is not correctly following pages. Also, there is something odd with middlewares. It begins parsing before it has assigned a middleware

Django Paginator: where does paginator modify SQL query adding LIMIT/OFFSET?

I'm trying to hook up to django pagination system and got a low-level question.
I've been digging through the code of Manager, Queryset and Paginator, but I can't find the place, where Paginator makes SQLCompiler set limit and offset on SQL query, forcing it to return only a part of queryset.
Can you find that place? Cause it's not in the code of Paginator itself.
Here's an example of SQL, generated by paginator:
>>> from django.db import connection
>>>
>>> paginator = Paginator(Myobject.objects.filter(foreign_key='URS0000416056'), 1)
>>> for myobject in paginator.page(1):
>>> print myobject
Myobject object
>>>
>>> print connection.queries
[{u'time': u'0.903', u'sql': u'SELECT COUNT(*) AS "__count" FROM "xref" WHERE "xref"."upi" = \'URS0000416056\''}, {u'time': u'0.144', u'sql': u'SELECT "xref"."id", "xref"."dbid", "xref"."ac", "xref"."created", "xref"."last", "xref"."upi", "xref"."version_i", "xref"."deleted", "xref"."timestamp", "xref"."userstamp", "xref"."version", "xref"."taxid" FROM "xref" WHERE "xref"."upi" = \'URS0000416056\' LIMIT 1 OFFSET 1'}]
Thanks for the links to the source, they let me dig up the details.
Paginator uses slice notation on the wrapped queryset when constructing Page instances:
return self._get_page(self.object_list[bottom:top], number, self)
https://github.com/django/django/blob/master/django/core/paginator.py#L57
Here object_list is a queryset despite the name if you used the Paginator on a queryset in the first place.
Querysets, in turn, implement slice notation in their __getitem__ method:
qs.query.set_limits(start, stop)
The slice-specific handling starts at https://github.com/django/django/blob/master/django/db/models/query.py#L260 currently.
The query class turns those into low and high mark attributes on the query in the set_limits method:
def set_limits(self, low=None, high=None):
"""
Adjust the limits on the rows retrieved. Use low/high to set these,
as it makes it more Pythonic to read and write. When the SQL query is
created, convert them to the appropriate offset and limit values.
Apply any limits passed in here to the existing constraints. Add low
to the current low value and clamp both to any existing high value.
"""
if high is not None:
if self.high_mark is not None:
self.high_mark = min(self.high_mark, self.low_mark + high)
else:
self.high_mark = self.low_mark + high
if low is not None:
if self.high_mark is not None:
self.low_mark = min(self.high_mark, self.low_mark + low)
else:
self.low_mark = self.low_mark + low
if self.low_mark == self.high_mark:
self.set_empty()
(That's the current implementation as of September 2017, it may change in the future.)
Low and high marks in turn get turned into LIMIT and OFFSET query params in the SQLCompilers as_sql method:
if with_limits:
if self.query.high_mark is not None:
result.append('LIMIT %d' % (self.query.high_mark - self.query.low_mark))
if self.query.low_mark:
if self.query.high_mark is None:
val = self.connection.ops.no_limit_value()
if val:
result.append('LIMIT %d' % val)
result.append('OFFSET %d' % self.query.low_mark)

Django pagination while objects are being added

I've got a website that shows photos that are always being added and people are seeing duplicates between pages on the home page (last added photos)
I'm not entirely sure how to approach this problem but this is basically whats happening:
Home page displays latest 20 photos [0:20]
User scrolls (meanwhile photos are being added to the db
User loads next page (through ajax)
Page displays photos [20:40]
User sees duplicate photos because the photos added to the top of the list pushed them down into the next page
What is the best way to solve this problem? I think I need to somehow cache the queryset on the users session maybe? I don't know much about caches really so a step-by-step explanation would be invaluable
here is the function that gets a new page of images:
def get_images_paginated(query, origins, page_num):
args = None
queryset = Image.objects.all().exclude(hidden=True).exclude(tags__isnull=True)
per_page = 20
page_num = int(page_num)
if origins:
origins = [Q(origin=origin) for origin in origins]
args = reduce(operator.or_, origins)
queryset = queryset.filter(args)
if query:
images = watson.filter(queryset, query)
else:
images = watson.filter(queryset, query).order_by('-id')
amount = images.count()
images = images.prefetch_related('tags')[(per_page*page_num)-per_page:per_page*page_num]
return images, amount
the view that uses the function:
def get_images_ajax(request):
if not request.is_ajax():
return render(request, 'home.html')
query = request.POST.get('query')
origins = request.POST.getlist('origin')
page_num = request.POST.get('page')
images, amount = get_images_paginated(query, origins, page_num)
pages = int(math.ceil(amount / 20))
if int(page_num) >= pages:
last_page = True;
else:
last_page = False;
context = {
'images':images,
'last_page':last_page,
}
return render(request, '_images.html', context)
One approach you could take is to send the oldest ID that the client currently has (i.e., the ID of the last item in the list currently) in the AJAX request, and then make sure you only query older IDs.
So get_images_paginated is modified as follows:
def get_images_paginated(query, origins, page_num, last_id=None):
args = None
queryset = Image.objects.all().exclude(hidden=True).exclude(tags__isnull=True)
if last_id is not None:
queryset = queryset.filter(id__lt=last_id)
...
You would need to send the last ID in your AJAX request, and pass this from your view function to get_images_paginated:
def get_images_ajax(request):
if not request.is_ajax():
return render(request, 'home.html')
query = request.POST.get('query')
origins = request.POST.getlist('origin')
page_num = request.POST.get('page')
# Get last ID. Note you probably need to do some type casting here.
last_id = request.POST.get('last_id', None)
images, amount = get_images_paginated(query, origins, page_num, last_id)
...
As #doniyor says you should use Django's built in pagination in conjunction with this logic.

Prevent django admin from running SELECT COUNT(*) on the list form

Every time I use Admin to list the entries of a model, the Admin count the rows in the table. Worse yet, it seems to be doing so even when you are filtering your query.
For instance if I want to show only the models whose id is 123, 456, 789 I can do:
/admin/myapp/mymodel/?id__in=123,456,789
But the queries ran (among others) are:
SELECT COUNT(*) FROM `myapp_mymodel` WHERE `myapp_mymodel`.`id` IN (123, 456, 789) # okay
SELECT COUNT(*) FROM `myapp_mymodel` # why???
Which is killing mysql+innodb. It seems that the problem is partially acknowledged in this ticket, but my issue seems more specific since it counts all the rows even if it is not supposed to.
Is there a way to disable that global rows count?
Note: I am using django 1.2.7.
Django 1.8 lets you disable this by setting show_full_result_count = False.
https://docs.djangoproject.com/en/stable/ref/contrib/admin/#django.contrib.admin.ModelAdmin.show_full_result_count
Okay, I think I found a solution. As Peter suggested, the best approach is to work on the count property and it can be done by overriding it with custom query set (as seen in this post) that specialises the count with an approximate equivalent:
from django.db import connections, models
from django.db.models.query import QuerySet
class ApproxCountQuerySet(QuerySet):
"""Counting all rows is very expensive on large Innodb tables. This
is a replacement for QuerySet that returns an approximation if count()
is called with no additional constraints. In all other cases it should
behave exactly as QuerySet.
Only works with MySQL. Behaves normally for all other engines.
"""
def count(self):
# Code from django/db/models/query.py
if self._result_cache is not None and not self._iter:
return len(self._result_cache)
is_mysql = 'mysql' in connections[self.db].client.executable_name.lower()
query = self.query
if (is_mysql and not query.where and
query.high_mark is None and
query.low_mark == 0 and
not query.select and
not query.group_by and
not query.having and
not query.distinct):
# If query has no constraints, we would be simply doing
# "SELECT COUNT(*) FROM foo". Monkey patch so the we
# get an approximation instead.
cursor = connections[self.db].cursor()
cursor.execute("SHOW TABLE STATUS LIKE %s",
(self.model._meta.db_table,))
return cursor.fetchall()[0][4]
else:
return self.query.get_count(using=self.db)
Then in the admin:
class MyAdmin(admin.ModelAdmin):
def queryset(self, request):
qs = super(MyAdmin, self).queryset(request)
return qs._clone(klass=ApproxCountQuerySet)
The approximate function could mess things up on page number 100000, but it is good enough for my case.
I found Nova's answer very helpful, but i use postgres. I modified it slightly to work for postgres with some slight alterations to handle table namespaces, and slightly different "detect postgres" logic.
Here's the pg version.
class ApproxCountPgQuerySet(models.query.QuerySet):
"""approximate unconstrained count(*) with reltuples from pg_class"""
def count(self):
if self._result_cache is not None and not self._iter:
return len(self._result_cache)
if hasattr(connections[self.db].client.connection, 'pg_version'):
query = self.query
if (not query.where and query.high_mark is None and query.low_mark == 0 and
not query.select and not query.group_by and not query.having and not query.distinct):
# If query has no constraints, we would be simply doing
# "SELECT COUNT(*) FROM foo". Monkey patch so the we get an approximation instead.
parts = [p.strip('"') for p in self.model._meta.db_table.split('.')]
cursor = connections[self.db].cursor()
if len(parts) == 1:
cursor.execute("select reltuples::bigint FROM pg_class WHERE relname = %s", parts)
else:
cursor.execute("select reltuples::bigint FROM pg_class c JOIN pg_namespace n on (c.relnamespace = n.oid) WHERE n.nspname = %s AND c.relname = %s", parts)
return cursor.fetchall()[0][0]
return self.query.get_count(using=self.db)
The Nova's solution (ApproxCountQuerySet) works great, however in newer versions of Django queryset method got replaced with get_queryset, so it now should be:
class MyAdmin(admin.ModelAdmin):
def get_queryset(self, request):
qs = super(MyAdmin, self).get_queryset(request)
return qs._clone(klass=ApproxCountQuerySet)
If this is a serious problem you may have to take Drastic Actions™.
Looking at the code for a 1.3.1 install, I see that the admin code is using the paginator returned by get_paginator(). The default paginator class appears to be in django/core/paginator.py. That class has a private value called _count which is set in Paginator._get_count() (line 120 in my copy). This in turn is used to set a property of the Paginator class called count. I think that _get_count() is your target. Now the stage is set.
You have a couple of options:
Directly modify the source. I do not recommend this, but since you seem to be stuck at 1.2.7 you may find that it is the most expedient. Remember to document this change! Future maintainers (including possibly yourself) will thank you for the heads up.
Monkeypatch the class. This is better than direct modification because a) if you don't like the change you just comment out the monkeypatch, and b) it is more likely to work with future versions of Django. I have a monkeypatch going back over 4 years because they still have not fixed a bug in the template variable _resolve_lookup() code that doesn't recognize callables at the top level of evaluation, only at lower levels. Although the patch (which wraps the method of a class) was written against 0.97-pre, it still works at 1.3.1.
I did not spend the time to figure out exactly what changes you would have to make for your problem, but it might be along the lines of adding a _approx_count member to appropriate classes class META and then testing to see if that attr exists. If it does and is None then you do the sql.count() and set it. You might also need to reset it if you are on (or near) the last page of the list. Contact me if you need a little more help on this; my email is in my profile.
It is possible to change the default paginator used by the admin class. Here's one that caches the result for a short period of time: https://gist.github.com/e4c5/6852723
I managed to create a custom paginator that shows the current page numbe, a next button and a show full count link. It allows for the use of the original paginator if needed.
The trick used is to take per_page + 1 elements from db in order to see if we have more elements and then provide a fake count.
Let's say that we want the the third page and the page has 25 elements => We want object_list[50:75]. When calling Paginator.count the queryset will be evaluated for object_list[50:76](note that we take 75+1 elements) and then return either the count as 76 if we got 25+1 elements from db or 50 + the number of elements received if we didn't received 26 elements.
TL;DR:
I've created a mixin for the ModelAdmin:
from django.core.paginator import Paginator
from django.utils.functional import cached_property
class FastCountPaginator(Paginator):
"""A faster paginator implementation than the Paginator. Paginator is slow
mainly because QuerySet.count() is expensive on large queries.
The idea is to use the requested page to generate a 'fake' count. In
order to see if the page is the final one it queries n+1 elements
from db then reports the count as page_number * per_page + received_elements.
"""
use_fast_pagination = True
def __init__(self, page_number, *args, **kwargs):
self.page_number = page_number
super(FastCountPaginator, self).__init__(*args, **kwargs)
#cached_property
def count(self):
# Populate the object list when count is called. As this is a cached property,
# it will be called only once per instance
return self.populate_object_list()
def page(self, page_number):
"""Return a Page object for the given 1-based page number."""
page_number = self.validate_number(page_number)
return self._get_page(self.object_list, page_number, self)
def populate_object_list(self):
# converts queryset object_list to a list and return the number of elements until there
# the trick is to get per_page elements + 1 in order to see if the next page exists.
bottom = self.page_number * self.per_page
# get one more object than needed to see if we should show next page
top = bottom + self.per_page + 1
object_list = list(self.object_list[bottom:top])
# not the last page
if len(object_list) == self.per_page + 1:
object_list = object_list[:-1]
else:
top = bottom + len(object_list)
self.object_list = object_list
return top
class ModelAdminFastPaginationMixin:
show_full_result_count = False # prevents root_queryset.count() call
def changelist_view(self, request, extra_context=None):
# strip count_all query parameter from the request before it is processed
# this allows all links to be generated like this parameter was not present and without raising errors
request.GET = request.GET.copy()
request.GET.paginator_count_all = request.GET.pop('count_all', False)
return super().changelist_view(request, extra_context)
def get_paginator(self, request, queryset, per_page, orphans=0, allow_empty_first_page=True):
# use the normal paginator if we want to count all the ads
if hasattr(request.GET, 'paginator_count_all') and request.GET.paginator_count_all:
return Paginator(queryset, per_page, orphans, allow_empty_first_page)
page = self._validate_page_number(request.GET.get('p', '0'))
return FastCountPaginator(page, queryset, per_page, orphans, allow_empty_first_page)
def _validate_page_number(self, number):
# taken from Paginator.validate_number and adjusted
try:
if isinstance(number, float) and not number.is_integer():
raise ValueError
number = int(number)
except (TypeError, ValueError):
return 0
if number < 1:
number = 0
return number
The pagination.html template:
{% if cl and cl.paginator and cl.paginator.use_fast_pagination %}
{# Fast paginator with only next button and show the total number of results#}
{% load admin_list %}
{% load i18n %}
{% load admin_templatetags %}
<p class="paginator">
{% if pagination_required %}
{% for i in page_range %}
{% if forloop.last %}
{% fast_paginator_number cl i 'Next' %}
{% else %}
{% fast_paginator_number cl i %}
{% endif %}
{% endfor %}
{% endif %}
{% show_count_all_link cl "showall" %}
</p>
{% else %}
{# use the default pagination template if we are not using the FastPaginator #}
{% include "admin/pagination.html" %}
{% endif %}
and templatetags used:
from django import template
from django.contrib.admin.views.main import PAGE_VAR
from django.utils.html import format_html
from django.utils.safestring import mark_safe
register = template.Library()
DOT = '.'
#register.simple_tag
def fast_paginator_number(cl, i, text_display=None):
"""Generate an individual page index link in a paginated list.
Allows to change the link text by setting text_display
"""
if i == DOT:
return '… '
elif i == cl.page_num:
return format_html('<span class="this-page">{}</span> ', i + 1)
else:
return format_html(
'<a href="{}"{}>{}</a> ',
cl.get_query_string({PAGE_VAR: i}),
mark_safe(' class="end"' if i == cl.paginator.num_pages - 1 else ''),
text_display if text_display else i + 1,
)
#register.simple_tag
def show_count_all_link(cl, css_class='', text_display='Show the total number of results'):
"""Generate a button that toggles between FastPaginator and the normal
Paginator."""
return format_html(
'<a href="{}"{}>{}</a> ',
cl.get_query_string({PAGE_VAR: cl.page_num, 'count_all': True}),
mark_safe(f' class="{css_class}"' if css_class else ''),
text_display,
)
You can use it this way:
class MyVeryLargeModelAdmin(ModelAdminFastPaginationMixin, admin.ModelAdmin):
# ...
Or an even simpler version that does not show the Next button and Show the total number of results :
from django.core.paginator import Paginator
from django.utils.functional import cached_property
class FastCountPaginator(Paginator):
"""A faster paginator implementation than the Paginator. Paginator is slow
mainly because QuerySet.count() is expensive on large queries.
The idea is to use the requested page to generate a 'fake' count. In
order to see if the page is the final one it queries n+1 elements
from db then reports the count as page_number * per_page + received_elements.
"""
use_fast_pagination = True
def __init__(self, page_number, *args, **kwargs):
self.page_number = page_number
super(FastCountPaginator, self).__init__(*args, **kwargs)
#cached_property
def count(self):
# Populate the object list when count is called. As this is a cached property,
# it will be called only once per instance
return self.populate_object_list()
def page(self, page_number):
"""Return a Page object for the given 1-based page number."""
page_number = self.validate_number(page_number)
return self._get_page(self.object_list, page_number, self)
def populate_object_list(self):
# converts queryset object_list to a list and return the number of elements until there
# the trick is to get per_page elements + 1 in order to see if the next page exists.
bottom = self.page_number * self.per_page
# get one more object than needed to see if we should show next page
top = bottom + self.per_page + 1
object_list = list(self.object_list[bottom:top])
# not the last page
if len(object_list) == self.per_page + 1:
object_list = object_list[:-1]
else:
top = bottom + len(object_list)
self.object_list = object_list
return top
class ModelAdminFastPaginationMixin:
show_full_result_count = False # prevents root_queryset.count() call
def get_paginator(self, request, queryset, per_page, orphans=0, allow_empty_first_page=True):
page = self._validate_page_number(request.GET.get('p', '0'))
return FastCountPaginator(page, queryset, per_page, orphans, allow_empty_first_page)
def _validate_page_number(self, number):
# taken from Paginator.validate_number and adjusted
try:
if isinstance(number, float) and not number.is_integer():
raise ValueError
number = int(number)
except (TypeError, ValueError):
return 0
if number < 1:
number = 0
return number