I need to ensure transactionality with flask requests. Is there a way to store a database session per request and use that one?
The easiest way is to create a simple extension for session initialization and commit changes after successful request. Here is an example:
from datetime import datetime
from flask import Flask, current_app, _app_ctx_stack
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base, DeferredReflection
from sqlalchemy import Column, String, Integer
# Base model without connection(see: DBExtension.connect()) and a few demo models
Base = declarative_base(cls=DeferredReflection)
class User(Base):
__tablename__ = 'user'
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String())
class Log(Base):
__tablename__ = 'log'
id = Column(Integer, primary_key=True, autoincrement=True)
text = Column(String())
# extension for getting db session
class DBExtension:
def __init__(self, app: Flask = None):
self.app = app
if app is not None:
self.init_app(app)
def init_app(self, app):
app.teardown_appcontext(self.teardown)
def connect(self):
# init engine and set into metadata
engine = create_engine(current_app.config['DB_CONNECTION'], echo=True)
Session = sessionmaker(bind=engine)
session = Session()
# create_all - just for demo(init structure...)
Base.metadata.create_all(engine)
Base.metadata.bind = engine
Base.prepare(engine)
return session
def teardown(self, exception):
ctx = _app_ctx_stack.top
if hasattr(ctx, 'session'):
ctx.session.close()
#property
def session(self):
# connect or get active connection
ctx = _app_ctx_stack.top
if ctx is not None:
if not hasattr(ctx, 'session'):
ctx.session = self.connect()
return ctx.session
app = Flask(__name__)
app.config.update({'DB_CONNECTION': 'sqlite:///test.db'}) # sqlite for demo
db = DBExtension(app)
#app.teardown_request
def teardown_request_func(error=None):
# request processing completed(before requests functions, endpoints and other logic)
if not error:
# one more record and commit all changes in one transaction
db.session.add(Log(text='{dt}. teardown_request'.format(dt=datetime.utcnow())))
try:
db.session.commit()
except Exception as e:
print(e)
# do something
#app.before_request
def before_request_func():
# add a new record before request
db.session.add(Log(text='{dt}. before request'.format(dt=datetime.utcnow())))
#app.route('/user/<name>')
def user_route(name: str):
# do something with db session
db.session.add(Log(text='saving user {user}'.format(user=name)))
user = User(name=name)
db.session.add(user)
# session.add(newModel) or session.delete(oneMoreModel) etc.
return 'done' # return without commit. see: teardown_request_func
if __name__ == '__main__':
app.run(host='localhost', port=5000, debug=True)
Run server, open http://localhost:5000/user/DavidBoshton and see logs(BEGIN-...-COMMIT).
Related
can anyone tell me, how to make a separate Database just for Flask admin and maybe flask security?
this is how i am working with my PostgreSQL database just to load some tables and perform CRUD:
import flask_admin
from flask import Flask
from sqlalchemy import create_engine
app =Flask(__name__)
engine = create_engine{'postgresql://name:password#localhost/tablesdatabase')
i wish to make a separate database for flask-admin :( this is what i am trying)
admin = flask_admin.Admin(app)
app.config['SQLALCHEMY_BINDS'] = {'admindb' : 'postgresql://name:password#localhost/adminedatabase'}
admin.add_view(ModelView) ? // how can i impelement this ? with a seperate datbase ?
There is no official support, but you can customize Flask-SQLalchemy session to use different connections, here the example for using master slave connections, you can easily add as many connections as you want
from functools import partial
from sqlalchemy import orm
from flask import current_app
from flask_sqlalchemy import SQLAlchemy, get_state
class RoutingSession(orm.Session):
def __init__(self, db, autocommit=False, autoflush=True, **options):
self.app = db.get_app()
self.db = db
self._bind_name = None
orm.Session.__init__(
self, autocommit=autocommit, autoflush=autoflush,
bind=db.engine,
binds=db.get_binds(self.app),
**options,
)
def get_bind(self, mapper=None, clause=None):
try:
state = get_state(self.app)
except (AssertionError, AttributeError, TypeError) as err:
current_app.logger.info(
'cant get configuration. default bind. Error:' + err)
return orm.Session.get_bind(self, mapper, clause)
# If there are no binds configured, use default SQLALCHEMY_DATABASE_URI
if not state or not self.app.config['SQLALCHEMY_BINDS']:
return orm.Session.get_bind(self, mapper, clause)
# if want to user exact bind
if self._bind_name:
return state.db.get_engine(self.app, bind=self._bind_name)
else:
# if no bind is used connect to default
return orm.Session.get_bind(self, mapper, clause)
def using_bind(self, name):
bind_session = RoutingSession(self.db)
vars(bind_session).update(vars(self))
bind_session._bind_name = name
return bind_session
class RouteSQLAlchemy(SQLAlchemy):
def __init__(self, *args, **kwargs):
SQLAlchemy.__init__(self, *args, **kwargs)
self.session.using_bind = lambda s: self.session().using_bind(s)
def create_scoped_session(self, options=None):
if options is None:
options = {}
scopefunc = options.pop('scopefunc', None)
return orm.scoped_session(
partial(RoutingSession, self, **options),
scopefunc=scopefunc,
)
Than the default session will be master, when you want to select from slave you can call it directly, here the examples:
In your app:
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql:///master'
app.config['SQLALCHEMY_BINDS'] = {
'slave': 'postgresql:///slave'
}
db = RouteSQLAlchemy(app)
Select from master
session.query(User).filter_by(id=1).first()
Select from slave
session.using_bind('slave').query(User).filter_by(id=1).first()
I recently updated my project to Django 2 and channels 2. Right now I am trying to rewrite my tests for chat app.
I am facing a problem with tests that depend on django db mark from pytest-django. I tried to create objects in fixtures, setup methods, in test function itself, using async_to_sync on WebsocketCommunicator. However, none of those worked.
If I create a user in a fixture and save it correctly gets an id. However, in my consumer Django does not see that User in the database. And treat it like an anonymous user.
I have a temporary token which I use to authenticate a user on websocket.connect.
#pytest.fixture
def room():
room = generate_room()
room.save()
return room
#pytest.fixture
def room_with_user(room, normal_user):
room.users.add(normal_user)
yield room
room.users.remove(normal_user)
#pytest.fixture
def normal_user():
user = generate_user()
user.save()
return user
#pytest.mark.django_db
class TestConnect:
#pytest.mark.asyncio
async def test_get_connected_client(self, path, room_with_user, temp_token):
assert get_path(room_with_user.id) == path
communicator = QSWebsocketCommunicator(application, path, query_string=get_query_string(temp_token))
connected, subprotocol = await communicator.connect()
assert connected
await communicator.disconnect()
Consumer:
class ChatConsumer(JsonWebsocketConsumer):
def connect(self):
# Called on connection. Either call
self.user = self.scope['user']
self.room_id = self.scope['url_route']['kwargs']['room_id']
group = f'room_{self.room_id}'
users = list(User.objects.all()) # no users here
self.group_name = group
if not (self.user is not None and self.user.is_authenticated):
return self.close({'Error': 'Not authenticated user'})
try:
self.room = Room.objects.get(id=self.room_id, users__id=self.user.id)
except ObjectDoesNotExist:
return self.close({'Error': 'Room does not exists'})
# Send success response
self.accept()
# Save user as active
self.room.active_users.add(self.user)
My authentication Middleware
class OAuthTokenAuthMiddleware:
"""
Custom middleware that takes Authorization header and read OAuth token from it.
"""
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
def __call__(self, scope):
temp_token = self.get_token(scope)
scope['user'] = self.validate_token(temp_token)
return self.inner(scope)
#staticmethod
def get_token(scope) -> str:
return url_parse.parse_qs(scope['query_string'])[b'token'][0].decode("utf-8")
#staticmethod
def validate_token(token):
try:
token = TemporaryToken.objects.select_related('user').get(token=token)
if token.is_active():
token.delete()
return token.user
else:
return AnonymousUser()
except ObjectDoesNotExist:
return AnonymousUser()
And custom WebsocketCommunicator which accepts query_string in order to include my one time token
class QSWebsocketCommunicator(WebsocketCommunicator):
def __init__(self, application, path, headers=None, subprotocols=None,
query_string: Optional[Union[str, bytes]]=None):
if isinstance(query_string, str):
query_string = str.encode(query_string)
self.scope = {
"type": "websocket",
"path": path,
"headers": headers or [],
"subprotocols": subprotocols or [],
"query_string": query_string or ''
}
ApplicationCommunicator.__init__(self, application, self.scope)
My question is how can I create User, Room, etc. objects in tests/fixtures so that I can access them in Django consumer.
Or do you have another idea how can I overcome this?
It's pretty much impossible to reproduce your issue using the code you've provided. Read about How to create a Minimal, Complete, and Verifiable example. However, I suppose that you should use real transactions in your test as the plain pytest.mark.django_db will skip the transactions and not store any data in the database per se. A working example:
# routing.py
from django import http
from django.conf.urls import url
from django.contrib.auth.models import User
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.generic.websocket import JsonWebsocketConsumer
class ChatConsumer(JsonWebsocketConsumer):
def connect(self):
self.user = self.scope['user']
print('user in scope, set by middleware:', self.user)
users = list(User.objects.all()) # no users here
print('all users in chat consumer:', users)
if not (self.user is not None and self.user.is_authenticated):
return self.close({'Error': 'Not authenticated user'})
# Send success response
self.accept()
class OAuthTokenAuthMiddleware:
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
def __call__(self, scope):
token = self.get_token(scope)
print('token in middleware:', token)
scope['user'] = User.objects.get(username=token)
return self.inner(scope)
#staticmethod
def get_token(scope) -> str:
d = http.QueryDict(scope['query_string'])
return d['token']
APP = ProtocolTypeRouter({
'websocket': OAuthTokenAuthMiddleware(URLRouter([url(r'^websocket/$', ChatConsumer)])),
})
Sample fixture that creates a user with username spam:
#pytest.fixture(scope='function', autouse=True)
def create_user():
with transaction.atomic():
User.objects.all().delete()
user = User.objects.create_user(
'spam', 'spam#example.com', password='eggs',
first_name='foo', last_name='bar'
)
return user
Now, I mark the test as transactional one, meaning that each query is actually committed. Now the test user is stored into database and the queries made in middleware/consumer can actually return something meaningful:
#pytest.mark.django_db(transaction=True)
#pytest.mark.asyncio
async def test_get_connected_client():
app = OAuthTokenAuthMiddleware(URLRouter([url(r'^websocket/$', ChatConsumer)]))
communicator = QSWebsocketCommunicator(app, '/websocket/', query_string='token=spam')
connected, subprotocol = await communicator.connect()
assert connected
await communicator.disconnect()
Running test test yields the desired result:
$ pytest -vs
================================== test session starts =================================
platform darwin -- Python 3.6.3, pytest-3.4.0, py-1.5.2, pluggy-0.6.0 -- /Users/hoefling/.virtualenvs/stackoverflow/bin/python
cachedir: .pytest_cache
Django settings: spam.settings (from environment variable)
rootdir: /Users/hoefling/projects/private/stackoverflow/so-49136564/spam, inifile: pytest.ini
plugins: celery-4.1.0, forked-0.2, django-3.1.2, cov-2.5.1, asyncio-0.8.0, xdist-1.22.0, mock-1.6.3, hypothesis-3.44.4
collected 1 item
tests/test_middleware.py::test_get_connected_client Creating test database for alias 'default'...
token in middleware: spam
user in scope: spam
all users in chat consumer: [<User: spam>]
PASSEDDestroying test database for alias 'default'...
=============================== 1 passed in 0.38 seconds ================================
Btw you don't need to hack around the WebsocketCommunicator anymore since it is now able to deal with query strings, see this issue closed.
I have successfully followed the examples in the documentation of Flask-Dance to add Oauth from GitHub, and manage the users with SQLAlchemy. However, I am unable to add more providers. I tried to simply add another blueprint for Twitter and registering it, but I get various errors when trying to login with Twitter. Also, I end up with lots of duplicated code, so this is not a good approach.
Is there a better way to add another provider?
import sys
from flask import Flask, redirect, url_for, flash, render_template
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm.exc import NoResultFound
from flask_dance.contrib.github import make_github_blueprint, github
from flask_dance.consumer.backend.sqla import OAuthConsumerMixin, SQLAlchemyBackend
from flask_dance.consumer import oauth_authorized, oauth_error
from flask_login import (
LoginManager, UserMixin, current_user,
login_required, login_user, logout_user
)
# setup Flask application
app = Flask(__name__)
app.secret_key = "supersekrit"
blueprint = make_github_blueprint(
client_id="my-key-here",
client_secret="my-secret-here",
)
app.register_blueprint(blueprint, url_prefix="/login")
# setup database models
app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///multi.db"
db = SQLAlchemy()
class User(db.Model, UserMixin):
id = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(256), unique=True)
# ... other columns as needed
class OAuth(db.Model, OAuthConsumerMixin):
user_id = db.Column(db.Integer, db.ForeignKey(User.id))
user = db.relationship(User)
# setup login manager
login_manager = LoginManager()
login_manager.login_view = 'github.login'
#login_manager.user_loader
def load_user(user_id):
return User.query.get(int(user_id))
# setup SQLAlchemy backend
blueprint.backend = SQLAlchemyBackend(OAuth, db.session, user=current_user)
# create/login local user on successful OAuth login
#oauth_authorized.connect_via(blueprint)
def github_logged_in(blueprint, token):
if not token:
flash("Failed to log in with {name}".format(name=blueprint.name))
return
# figure out who the user is
resp = blueprint.session.get("/user")
if resp.ok:
username = resp.json()["login"]
query = User.query.filter_by(username=username)
try:
user = query.one()
except NoResultFound:
# create a user
user = User(username=username)
db.session.add(user)
db.session.commit()
login_user(user)
flash("Successfully signed in with GitHub")
else:
msg = "Failed to fetch user info from {name}".format(name=blueprint.name)
flash(msg, category="error")
# notify on OAuth provider error
#oauth_error.connect_via(blueprint)
def github_error(blueprint, error, error_description=None, error_uri=None):
msg = (
"OAuth error from {name}! "
"error={error} description={description} uri={uri}"
).format(
name=blueprint.name,
error=error,
description=error_description,
uri=error_uri,
)
flash(msg, category="error")
#app.route("/logout")
#login_required
def logout():
logout_user()
flash("You have logged out")
return redirect(url_for("index"))
#app.route("/")
def index():
return render_template("home.html")
# hook up extensions to app
db.init_app(app)
login_manager.init_app(app)
if __name__ == "__main__":
if "--setup" in sys.argv:
with app.app_context():
db.create_all()
db.session.commit()
print("Database tables created")
else:
app.run(debug=True)
Here is a sample project that uses multiple providers simultaneously: flask-dance-multi-provider. You can read through the code to see how it works. Does that help?
I am trying to build a simple SMS app using Flask, Twilio, and SqlAlchemy. When someone texts my Twilio number, it populates the database, but I can't query the Postgres database to populate the "to" field. Here's the code. Any help is greatly appreciated.
from flask import Flask, render_template, request
from flask.ext.sqlalchemy import SQLAlchemy
import twilio.twiml
from twilio.rest import TwilioRestClient
from sqlalchemy.sql import select
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://localhost/twilio_sms'
db = SQLAlchemy(app)
account_sid = ""
auth_token = ""
# Database model
class User(db.Model):
__tablename__ = "users"
id = db.Column(db.Integer, primary_key=True)
phone = db.Column(db.String(120), unique=True)
def __init__(self, phone):
self.phone = phone
# The script that processes the incoming phone numbers
#app.route("/", methods=['GET', 'POST'])
def sms():
phone = None
if request.method == 'POST':
phone = request.values.get('From')
if not db.session.query(User).filter(User.phone == phone).count():
reg = User(phone)
db.session.add(reg)
db.session.commit()
resp = twilio.twiml.Response()
with resp.message("Let's dance the night away!") as m:
m.media("http://i.giphy.com/2lxG3ySjtbpBe.gif")
return str(resp)
# Renders the form for message submission
#app.route("/form")
def form():
return render_template('send.html')
Here's the section where I'm having trouble.
# The script that publishes the message
#app.route("/send", methods=['GET', 'POST'])
def send():
phone = None
client = TwilioRestClient(account_sid, auth_token)
for users in db.session.query(User).filter(User.phone == phone):
print users #Inserted to text whether the query returned any value
#message = client.messages.create(to="users", from_="+twilio_number", body=request.form['Message'])
#return render_template('success.html')
if __name__ == '__main__':
app.debug = True
app.run()
I finally figured it out... I had to convert the numbers into a list using query and then pass it as string.
#app.route("/send", methods=['GET', 'POST'])
def send():
users = User.query.order_by(User.phone)
client = TwilioRestClient(account_sid, auth_token)
for user in users:
message = client.messages.create(to=str(user), from_="+12125550433", body=request.form['Message'])
return render_template('success.html')
In your send() function you are assigning phone = None and then filtering your database query on that.
I believe you want to grab the number like you did above in sms() out of the request.
The following test code does not pass even though manually submitting the form on my web interface actually does work.
import os
from flask.ext.testing import TestCase
from flask import url_for
from config import _basedir
from app import app, db
from app.users.models import User
class TestUser(TestCase):
def create_app(self):
"""
Required method. Always implement this so that app is returned with context.
"""
app.config['TESTING'] = True
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(_basedir, 'test.db')
app.config['WTF_CSRF_ENABLED'] = False # This must be disabled for post to succeed during tests
self.client = app.test_client()
ctx = app.app_context()
ctx.push()
return app
def setUp(self):
db.create_all()
#pass
#app.teardown_appcontext
def tearDown(self):
db.session.remove()
db.drop_all()
#pass
def test_admin_home(self):
# url is the admin home page
url = url_for('admin.index')
resp = self.client.get(url)
self.assertTrue(resp.status_code == 200)
def test_admin_registration(self):
url = url_for('admin.register_view')
data = {'username': 'admin', 'email': 'admin#example.com', 'password': 'admin'}
resp = self.client.post(url, data)
self.assertTrue(resp.status_code == 200)
u = User.query.filter_by(username=u'admin').first()
self.assertTrue(u.username == 'admin') # <----- This fails. Why?
After the test client has post to the register_view url and returns a 200 OK response, I fail to retrieve the 'admin' user from the test database. Why is this so?
Here's the view code (this is a flask admin view)
from flask import request
from flask.ext.admin import expose, AdminIndexView, helpers
from app.auth.forms import LoginForm, RegistrationForm
from app.users.models import User
from app import db
class MyAdminIndexView(AdminIndexView):
#expose('/', methods=('GET', 'POST'))
def index(self):
# handle user login
form = LoginForm(request.form)
self._template_args['form'] = form
return super(MyAdminIndexView, self).index()
#expose('/register/', methods=('GET', 'POST'))
def register_view(self):
# handle user registration
form = RegistrationForm(request.form)
if helpers.validate_form_on_submit(form):
user = User()
form.populate_obj(user)
db.session.add(user)
db.session.commit()
self._template_args['form'] = form
return super(MyAdminIndexView, self).index()
Dumbest mistake ever.
The offending line in my test code is
resp = self.client.post(url, data)
It should be
resp = self.client.post(url, data=data)
I managed to track it down by painstakingly walking through the logic and inserting ipdb.set_trace() step by step until I found the bad POST request made by my client.