Flask / Jinja memoization - flask

I have been trying to use and Flask-Cache's memoize feature to only return cached results of statusTS(), unless a certain condition is met in another request where the cache is then deleted.
It is not being deleted though, and the Jinja template still displays Online when it should infact display Offline because the server has been stopped. So it is returning a cached result when it should not.
#cache.memoize(60)
def statusTS(sid):
try:
server = Server.get_info(sid)
m = Masters.get_info(server.mid)
if not m.maintenance:
tsconn = ts3conn(server.mid)
tsconn.use(str(server.msid))
command = tsconn.send_command('serverinfo')
tsconn.disconnect()
if not command.data[0]['virtualserver_status'] == 'template':
return 'Online'
return 'Unknown'
except:
return 'Unknown'
app.jinja_env.globals.update(statusTS=statusTS)
Jinja template:
{% if statusTS(server.sid) == 'Online' %}
<span class="label label-success">
Online
</span>{% endif %}
This renders the view:
#app.route('/manage/')
def manage():
if g.user:
rl = requests_list(g.user.id)
admin = User.is_admin(g.user.id)
g.servers = get_servers_by_uid(g.user.id)
if 's' in request.args:
s = request.args.get('s')
s = literal_eval(s)
else:
s = None
return render_template('manage.html',
user=g.user,
servers=g.servers,
admin=admin,
globallimit=Config.get_opts('globallimit'),
news=News.get_latest(),
form=Form(),
masters=Masters.get_all(),
treply=rl,
s=s)
else:
return redirect(url_for('login'))
this is what is supposed to delete the entry.
#app.route('/stop/<id>/')
#limiter.limit("3/minute")
def stop(id):
if g.user:
if Server.is_owner(g.user.id, id):
m = Masters.get_info(Server.get_info(id).mid)
if not m.maintenance:
cache.delete_memoized(statusTS, id)
flash(stopTS(id))
return redirect(url_for('manage'))
else:
flash(
'You cannot stop this server while the master is locked for maintenance - please check for further info.')
return redirect(url_for('manage'))
else:
flash(
'You do not have permission to modify this server - please contact support.')
return redirect(url_for('manage'))
else:
return redirect(url_for('login'))

Your code looks correct. It could be a bug in the whichever backend cache you are using.
Toy example that works:
from flask import Flask
# noinspection PyUnresolvedReferences
from flask.ext.cache import Cache
app = Flask(__name__)
# Check Configuring Flask-Cache section for more details
cache = Cache(app, config={'CACHE_TYPE': 'simple'})
#cache.memoize(timeout=10)
def statusTS(sid):
print('Cache Miss: {}.'.format(sid))
return sid
#app.route('/<int:status_id>')
def status(status_id):
return 'Status: ' + str(statusTS(status_id))
#app.route('/delete/<int:status_id>')
def delete(status_id):
print('Deleting Cache')
cache.delete_memoized(statusTS, status_id)
return 'Cache Deleted'
if __name__ == '__main__':
app.debug = True
app.run()
What I think is happening is that the delete is failing. I would step into the
cache.delete_memoized(statusTS, status_id) and see if it actually finds and deletes the cached function result. Check werkzeug.contrib.cache.SimpleCache
if you are using the cache type simple.

Related

How to Output More than Once In frontend In DJango

I am building a web app that makes use of the HackerNews API and I keep running into the same error.
I am trying to Output the 10 requested result from API but everytime i use the return response it only outputs the first result.
I want it to output 10 articles from the API instead of just the One.
This is my code:
from operator import itemgetter
import requests
from django.shortcuts import render
# Make an API call and store the response.
def home(request):
url = 'https://hacker-news.firebaseio.com/v0/topstories.json'
r = requests.get(url)
print(f"Status code: {r.status_code}")
# Process information about each submission.
submission_ids = r.json()
submission_dicts = []
for submission_id in submission_ids[:10]:
# Make a separate API call for each submission.
url = f"https://hacker-news.firebaseio.com/v0/item/{submission_id}.json"
r = requests.get(url)
print(f"id: {submission_id}\tstatus: {r.status_code}")
response_dict = r.json()
# Build a dictionary for each article.
submission_dict = {
'title': response_dict['title'],
'hn_link': f"http://news.ycombinator.com/item?id={submission_id}",
# 'comments': response_dict['descendants'],
}
submission_dicts.append(submission_dict)
# submission_dicts = sorted(submission_dicts, key=itemgetter('comments'),
# reverse=True)
for submission_dict in submission_dicts:
print(f"\nTitle: {submission_dict['title']}")
print(f"Discussion link: {submission_dict['hn_link']}")
# print(f"Comments: {submission_dict['comments']}")
count = 0
if count < 10:
return render(request, "news_api/home.html", submission_dict)
You're only getting one result because your return statement is calling "submission_dict". You probably wanted to call "submission_dict(s)" or something similar but it's hard to tell based on what has been commented out in your code. I think that's your primary problem. Without seeing your template, I don't know what your context object is supposed to look like.
But try this:
# views.py
def hacker_news_api_selector():
url = 'https://hacker-news.firebaseio.com/v0/topstories.json'
r = requests.get(url)
submission_list = r.json()
context = {}
context['objects'] = []
# Process information about each submission.
for submission_id in submission_list[:10]:
# Make a separate API call for each submission.
url = f"https://hacker-news.firebaseio.com/v0/item/{submission_id}.json"
r = requests.get(url)
response_dict = r.json()
context['objects'].append(response_dict)
return context
def home(request):
context = hacker_news_api_selector()
return render(request, "news_api/home.html", context)
And in the template...
# news_api/home.html
<html>
...
{% for x in objects %}
<li>{{ x.title }}</li>
{% endfor %}
</html>

Flask Admin - is there a way to store current url (with custom filters applied) of table view?

I am working on ticketing system in Flask Admin. The Flask Admin enviroment will be the main one for all the users. For creating or editing tickets I go out from Flask-Admin and use wtforms to implement backend logic. After creation or editing the ticket (validate_on_submit) I want to redirect back to Flask Admin, so I use redirect(url_for(ticket.index_view)). It works fine.
Is there a way to redirect to flask admin, but also with specific filters which were applied before user left Flask admin enviroment? (it is basiccaly GET parameters of url - but in FLASK)
I was trying to use:
referrer = request.referrer
get_url()
But I am probably missing something crucial and don´t know how to implement it (where to put it so I can call the arguments)
Thank you so much.
EDIT : adding more context:
I have a flask admin customized to different roles of users. The main ModelView is the one showing the TICKETS : the specifics of the Class are not vital to my current problem but here its how it looks:
class TicketModelView(ModelView):
column_list = ['id', 'title', 'osoba', 'content', 'povod_vmc_kom', 'dateVMC','zodpovedni', 'deadline', 'odpoved', 'solution', 'is_finished']
column_searchable_list = ['osoba']
column_filters = [ 'povod_vmc_kom', 'dateVMC', 'osoba', 'zodpovedni']
column_labels = dict(povod_vmc_kom='VMČ / Komisia', dateVMC='Dátum VMČ / komisie', zodpovedni = "Zodpovední")
column_display_actions = True
column_filters = [
FilterEqual(column=Ticket.povod_vmc_kom, name='Výbor/komisia', options=(('VMČ Juh','VMČ Juh'), ('UM','UM'), ('Kom dopravy','Kom dopravy'))),
'zodpovedni', 'is_finished',
'dateVMC', 'osoba'
]
def is_accessible(self):
#práva pre vedenie mesta - môže len nazerať
if current_user.is_authenticated and current_user.role == 0:
self.can_export=True
self.can_delete = False
self.can_edit = False
self.can_create = False
self._refresh_form_rules_cache()
self._refresh_forms_cache()
return True
#práva pre super admina (ostatné práva sú defaultne zapnuté)
if current_user.is_authenticated and current_user.role == 1:
self.can_export=True
self.can_delete=True
self.form_edit_rules = ('zodpovedni', 'is_finished' )
self.column_editable_list = ['is_finished']
self._refresh_form_rules_cache()
self._refresh_forms_cache()
return True
#práva pre garantov
if current_user.is_authenticated and current_user.role == 2:
self.can_delete = False
self.can_create = False
self.can_edit = False
self.can_export=True
self.column_searchable_list = ['title']
self._refresh_form_rules_cache()
self._refresh_forms_cache()
return True
#práva pre veducich utvarov
if current_user.is_authenticated and current_user.role == 3:
self.can_create = False
self.can_delete = False
self.can_export=True
self.column_searchable_list = ['title']
self.column_editable_list = ['odpoved', 'date_odpoved', 'solution', 'date_solution' ]
self.form_edit_rules = ('odpoved', 'date_odpoved', 'solution', 'date_solution')
self._refresh_form_rules_cache()
self._refresh_forms_cache()
return True
return False
def _solution_formatter(view, context, model, name):
# Format your string here e.g show first 20 characters
# can return any valid HTML e.g. a link to another view to show the detail or a popup window
if model.solution:
return model.solution[:50]
pass
def _content_formatter(view, context, model, name):
# Format your string here e.g show first 20 characters
# can return any valid HTML e.g. a link to another view to show the detail or a popup window
if len(model.content) > 100:
markupstring = "<a href= '%s'>%s</a>" % (url_for('ticket', ticket_id=model.id), "...")
return model.content[:100] + Markup(markupstring)
return model.content
def _user_formatter(view, context, model, name):
if model.id:
markupstring = "<a href= '%s'>%s</a>" % (url_for('ticket', ticket_id=model.id), model.id)
return Markup(markupstring)
else:
return ""
column_formatters = {
'content': _content_formatter,
'solution': _solution_formatter,
'id': _user_formatter
}
When user viewing the TicketView in Flask Admin, he can apply various filters which is vital to the user experience of the whole web app. The filters work fine and they are stored in URL as GET arguments. When he wants to create or edit a ticket, I am not allowing him to do it in Flask Admin (I edited Flask-Admin layout.html template and added a button to navbar which redirects to my new_ticket url with wtforms.) because of backend logic I want to be applied. For example when he edits field "solution" : I want the value in field "date_of_solution" be generated automatically (date.today()). So I am using wtforms and flask routing : example is bellow:
#app.route("/ticket/<int:ticket_id>/solution", methods = ['GET', 'POST'])
#login_required
def solution(ticket_id):
if current_user.role != 3:
flash("Pre zadanie riešenia alebo odpovede musíte byť prihlásený ako vedúci útvaru", "danger")
return redirect(url_for('ticket', ticket_id=ticket_id))
ticket = Ticket.query.get_or_404(ticket_id)
form = AdminPanelForm()
if form.validate_on_submit():
print("1")
if not ticket.date_solution:
print("2")
ticket.date_solution= datetime.now()
if not ticket.date_odpoved:
print("3")
if form.odpoved.data != ticket.odpoved:
print("4")
ticket.date_odpoved= datetime.now()
ticket.solution = form.solution.data
ticket.odpoved = form.odpoved.data
ticket.is_finished = True
db.session.commit()
flash("Ticket bol updatenutý", "success")
**return redirect(url_for('ticketmod.index_view'))**
elif request.method == 'GET':
form.solution.data = ticket.solution
form.odpoved.data = ticket.odpoved
return render_template("admin_ticket.html", form=form, ticket = ticket)
Now you can see that after succesful updating the ticket, user is redirected to Ticket model View where he came from, return redirect(url_for('ticketmod.index_view')) but without filters applied. I am looking for the solution, how can you store the url GET parameters (the filters) and then use them when redirecting back to ModelView. I tried function get_url() or request.referrer but I wasn´t succesful.
As I said in my original post, maybe I am missing something crucial in web architecture - if you have in mind some learning material I shoul be looking at : thanks for any advice.
Within the formatter method you can get a view's url including the applied filters/sorting criteria using the following:
_view_url = view.get_url('.index_view', **request.args)
Now pass this along to route request, either as a parameter or some other means. For example:
class TicketModelView(ModelView):
# blah blah
def _user_formatter(view, context, model, name):
if model.id:
# This is the current url of the view including filters
_view_url = view.get_url('.index_view', **request.args)
# Pass this as a parameter to your route
markupstring = "<a href= '%s'>%s</a>" % (url_for('ticket', ticket_id=model.id, return_url=_view_url), model.id)
return Markup(markupstring)
At the route you can now pull out the return_url from the request arg and add it as a hidden field in the form. Then in the post back retrieve the value from the form and redirect.
#app.route("/ticket/<int:ticket_id>/solution", methods = ['GET', 'POST'])
#login_required
def solution(ticket_id):
# Get the return_url from the request
_return_url = request.args.get('return_url'):
# Add the return_url to the form as a hidden field
form.return_url.data = _return_url
# blah blah
if form.validate_on_submit():
# get return value from form
_return_url = form.return_url.data
return redirect(_return_url) if _return_url else redirect(url_for('ticketmod.index_view'))

Django session loses complex object in production, but not in dev

So I am successfully storing a complex object (non-model) in my session in development. I've tried every session engine and cache type and they are all working in development (Pycharm). However, when I move the code to production, while no error are thrown, the session losses the object.
Here is the method I use to set the session object:
def instantiate_command_object(request):
try:
ssc = request.session['specimen_search_criteria']
logger.debug('found ssc session variable')
except KeyError:
logger.debug('failed to find ssc session variable')
ssc = SpecimenSearchCommand()
return ssc
Then in a method that runs asynchronously via an ajax call I start making changes to the object in the session:
def ajax_add_collection_to_search(request):
ssc = instantiate_command_object(request)
collection_id = request.GET.get('collection')
collection = Collection.objects.get(pk=collection_id)
if collection and collection not in ssc.collections:
ssc.collections.append(collection)
# save change to session
request.session['specimen_search_criteria'] = ssc
# refresh search results
ssc.search()
return render(request, '_search.html')
All this works as far as it goes. However, if I then refresh the browser, the session is lost. Here is a snippet from the template:
{% with criteria=request.session.specimen_search_criteria %}
<div class="search-criteria" id="search-criteria">
<div class="row">
Sesssion:
{{ request.session }}<br/>
Search:
{{ request.session.specimen_search_criteria }}<br/>
Created:
{{ request.session.specimen_search_criteria.key }}<br/>
Collections:
{{ request.session.specimen_search_criteria.collections }}<br/>
Again, in development I can refresh all day and the same object will be returned. In production, it will either create a new object or occasionally will return a previously created copy.
A few relevant items:
The production server is running Apache httpd with mod_wsgi.
I've tried memcached, databasecache, etc. the behavior remains the same. Always works in development, never in production.
I've tried it with
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
and without. I can see the session info in the database and when I unpickle it it just seems to be pointing to a location in memory for the complex object.
I'm guessing this might have something to do with running in a multi-user environment, but again, I'm not using locmem and I've tried all of the caching approaches to no effect.
To be clear, the session itself seems to be fine, I can store a string or other simple item in it and it will stick. It's the complex object within the session that seems to be getting lost.
Edit: I might also point out that if I refresh the browser immediately following the return of the search criteria it will actually return successfully. Anything more than about a second and it will disappear.
Edit (adding code of SpecimenSearchCommand):
class SpecimenSearchCommand:
def __init__(self):
pass
created = datetime.datetime.now()
key = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(6))
jurisdictions = []
taxa = []
strata = []
collections = []
chrons = []
has_images = False
query = None # The active SQL query, not the actual result records
page_size = 50
current_page = 1
sort_order = 'number'
results = [] # Page of results from paginator
def is_empty(self):
if len(self.jurisdictions) == 0 and len(self.taxa) == 0 and len(self.strata) == 0 and \
len(self.collections) == 0 and len(self.chrons) == 0 and self.has_images is False:
return True
else:
return False
def get_results(self):
paginator = Paginator(self.query, self.page_size)
try:
self.results = paginator.page(self.current_page)
except PageNotAnInteger:
self.results = paginator.page(1)
except TypeError:
return []
except EmptyPage:
self.results = paginator.page(paginator.num_pages)
return self.results
def get_results_json(self):
points = []
for s in self.results:
if s.locality.latitude and s.locality.longitude:
points.append({"type": "Feature",
"geometry": {"type": "Point",
"coordinates": [s.locality.longitude, s.locality.latitude]},
"properties": {"specimen_id": s.id,
"sci_name": s.taxon.scientific_name(),
"cat_num": s.specimen_number(),
"jurisdiction": s.locality.jurisdiction.full_name()}
})
return json.dumps({"type": "FeatureCollection", "features": points})
def search(self):
if self.is_empty():
self.query = None
return
query = Specimen.objects.filter().distinct().order_by(self.sort_order)
if len(self.taxa) > 0:
query = query.filter(taxon__in=get_hierarchical_search_elements(self.taxa))
if len(self.jurisdictions) > 0:
query = query.filter(locality__jurisdiction__in=get_hierarchical_search_elements(self.jurisdictions))
if len(self.strata) > 0:
query = query.filter(stratum__in=get_hierarchical_search_elements(self.strata))
if len(self.chrons) > 0:
query = query.filter(chron__in=get_hierarchical_search_elements(self.chrons))
if len(self.collections) > 0:
query = query.filter(collection__in=get_hierarchical_search_elements(self.collections))
if self.has_images:
query = query.filter(images__isnull=False)
self.query = query
return
def get_hierarchical_search_elements(elements):
search_elements = []
for element in elements:
search_elements = set().union(search_elements, element.get_descendants(True))
return search_elements
OK, so as Daniel pointed out, the attributes of the SSC class were class-level instead of instance level. The correct version looks like this now:
self.created = datetime.datetime.now()
self.key = ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(6))
self.jurisdictions = []
self.taxa = []
self.strata = []
self.collections = []
self.chrons = []
self.has_images = False
self.query = None # The active SQL query, not the actual result records
self.page_size = 50
self.current_page = 1
self.sort_order = 'number'
self.results = [] # Page of results from paginator

Celery: check if a task is completed to send an email to

I'm new to celery and an overall python noob. I must have stumbled upon the right solution during my research but I just don't seem to understand what I need to do for what seems to be a simple case scenario.
I followed the following guide to learn about flask+celery.
What I understand:
There seems there is something obvious I'm missing about how to trigger a task after the first one is finished. I tried using callbacks, using loops, even tried using Celery Flower and Celery beat to realise this has nothing with what I'm doing...
Goal:
After filling the form, I want to send an email with attachements (result of the task) or a failure email otherwise. Without having to wonder what my user is doing on the app (no HTTP requests)
My code:
class ClassWithTheTask:
def __init__(self, filename, proxies):
# do stuff until a variable results is created
self.results = 'this contains my result'
#app.route('/', methods=['GET', 'POST'])
#app.route('/index', methods=['GET', 'POST'])
def index():
form = MyForm()
if form.validate_on_submit():
# ...
# the task
my_task = task1.delay(file_path, proxies)
return redirect(url_for('taskstatus', task_id=my_task.id, filename=filename, email=form.email.data))
return render_template('index.html',
form=form)
#celery.task(bind=True)
def task1(self, filepath, proxies):
task = ClassWithTheTask(filepath, proxies)
return results
#celery.task
def send_async_email(msg):
"""Background task to send an email with Flask-Mail."""
with app.app_context():
mail.send(msg)
#app.route('/status/<task_id>/<filename>/<email>')
def taskstatus(task_id, filename, email):
task = task1.AsyncResult(task_id)
if task.state == 'PENDING':
# job did not start yet
response = {
'state': task.state,
'status': 'Pending...'
}
elif task.state != 'FAILURE':
response = {
'state': task.state,
'status': task.info.get('status', '')
}
if 'results' in task.info:
response['results'] = task.info['results']
response['untranslated'] = task.info['untranslated']
msg = Message('Task Complete for %s !' % filename,
recipients=[email])
msg.body = 'blabla'
with app.open_resource(response['results']) as fp:
msg.attach(response['results'], "text/csv", fp.read())
with app.open_resource(response['untranslated']) as fp:
msg.attach(response['untranslated'], "text/csv", fp.read())
# the big problem here is that it will send the email only if the user refreshes the page and get the 'SUCCESS' status.
send_async_email.delay(msg)
flash('task finished. sent an email.')
return redirect(url_for('index'))
else:
# something went wrong in the background job
response = {
'state': task.state,
'status': str(task.info), # this is the exception raised
}
return jsonify(response)
I don't get the goal of your method for status check. Anyway what you are describing can be accomplished this way.
if form.validate_on_submit():
# ...
# the task
my_task = (
task1.s(file_path, proxies).set(link_error=send_error_email.s(filename, error))
| send_async_email.s()
).delay()
return redirect(url_for('taskstatus', task_id=my_task.id, filename=filename, email=form.email.data))
Then your error task will look like this. The normal task can stay the way it is.
#celery.task
def send_error_email(task_id, filename, email):
task = AsyncResult(task_id)
.....
What happens here is that you are using a chain. You are telling Celery to run your task1, if that completes successfully then run send_async_email, if it fails run send_error_email. This should work, but you might need to adapt the parameters, consider it as pseudocode.
This does not seem right at all:
def task1(self, filepath, proxies):
task = ClassWithTheTask(filepath, proxies)
return results
The line my_task = task1.delay(file_path, proxies) earlier in your code suggests you want to return task but you return results which is not defined anywhere. (ClassWithTheTask is also undefined). This code would crash, and your task would never execute.

django-nocaptcha-recaptcha always shows additional verification box

I installed django-nocaptcha-recaptcha and integrated it into my form:
from nocaptcha_recaptcha.fields import NoReCaptchaField
class ClientForm(forms.ModelForm):
captcha = NoReCaptchaField()
It shows up fine on the form, but whenever I click on it an additional dialog pops up asking to enter some text and verify. It happens every time. I tested it from another computer on another network and it still asks for additional verification after clicking the box.
This is what it looks like: additional verification dialog box
Here's how I'm handling the form:
#xframe_options_exempt
def registration(request):
if request.method == 'POST':
clientform = ClientForm(request.POST)
# check whether it's valid:
if clientform.is_valid():
new_client = clientform.save()
...
What am I doing wrong? Is it a problem with django-nocaptcha-recaptcha? Should I use something else?
P.S. I'm using django 1.7.1 with python 3.4
Another alternative: Minimalist and non framework dependant.
This is the code, in case you want to rewrite it.
'''
NO-CAPTCHA VERSION: 1.0
PYTHON VERSION: 3.x
'''
import json
from urllib.request import Request, urlopen
from urllib.parse import urlencode
VERIFY_SERVER = "www.google.com"
class RecaptchaResponse(object):
def __init__(self, is_valid, error_code=None):
self.is_valid = is_valid
self.error_code = error_code
def __repr__(self):
return "Recaptcha response: %s %s" % (
self.is_valid, self.error_code)
def __str__(self):
return self.__repr__()
def displayhtml(site_key, language=''):
"""Gets the HTML to display for reCAPTCHA
site_key -- The site key
language -- The language code for the widget.
"""
return """<script src="https://www.google.com/recaptcha/api.js?hl=%(LanguageCode)s" async="async" defer="defer"></script>
<div class="g-recaptcha" data-sitekey="%(SiteKey)s"></div>
""" % {
'LanguageCode': language,
'SiteKey': site_key,
}
def submit(response,
secret_key,
remote_ip,
verify_server=VERIFY_SERVER):
"""
Submits a reCAPTCHA request for verification. Returns RecaptchaResponse
for the request
response -- The value of response from the form
secret_key -- your reCAPTCHA secret key
remote_ip -- the user's ip address
"""
if not(response and len(response)):
return RecaptchaResponse(is_valid=False, error_code='incorrect-captcha-sol')
def encode_if_necessary(s):
if isinstance(s, str):
return s.encode('utf-8')
return s
params = urlencode({
'secret': encode_if_necessary(secret_key),
'remoteip': encode_if_necessary(remote_ip),
'response': encode_if_necessary(response),
})
params = params.encode('utf-8')
request = Request(
url="https://%s/recaptcha/api/siteverify" % verify_server,
data=params,
headers={
"Content-type": "application/x-www-form-urlencoded",
"User-agent": "reCAPTCHA Python"
}
)
httpresp = urlopen(request)
return_values = json.loads(httpresp.read().decode('utf-8'))
httpresp.close()
return_code = return_values['success']
if return_code:
return RecaptchaResponse(is_valid=True)
else:
return RecaptchaResponse(is_valid=False, error_code=return_values['error-codes'])
Restart the server and don't forget to clear your browser's cache. Hope this helps.