I had a django 2 app and i used django channels for socket connection.
i just update django to version 3. and now daphne show this error when i try to make a socket connection. i had not any problem with django 2.
[Failure instance: Traceback: <class 'django.core.exceptions.SynchronousOnlyOperation'>: You cannot call this from an async context - use a thread or sync_to_async.
/home/ubuntu/pl_env/lib/python3.6/site-packages/autobahn/websocket/protocol.py:2844:processHandshake
/home/ubuntu/pl_env/lib/python3.6/site-packages/txaio/tx.py:429:as_future
/home/ubuntu/pl_env/lib/python3.6/site-packages/twisted/internet/defer.py:151:maybeDeferred
/home/ubuntu/pl_env/lib/python3.6/site-packages/daphne/ws_protocol.py:83:onConnect
--- <exception caught here> ---
/home/ubuntu/pl_env/lib/python3.6/site-packages/twisted/internet/defer.py:151:maybeDeferred
/home/ubuntu/pl_env/lib/python3.6/site-packages/daphne/server.py:201:create_application
/home/ubuntu/pl_env/lib/python3.6/site-packages/channels/routing.py:54:__call__
/home/ubuntu/pl_env/lib/python3.6/site-packages/channels/security/websocket.py:37:__call__
/home/ubuntu/petroline_django/orders/token_auth.py:25:__call__
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/manager.py:82:manager_method
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/query.py:411:get
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/query.py:258:__len__
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/query.py:1261:_fetch_all
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/query.py:57:__iter__
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/db/models/sql/compiler.py:1142:execute_sql
/home/ubuntu/pl_env/lib/python3.6/site-packages/django/utils/asyncio.py:24:inner
it says the problem is in token_auth.py, line 25. this line is token = Token.objects.get(key=token_key)
this is my token_auth.py that handles token authentication.
from channels.auth import AuthMiddlewareStack
from django.contrib.auth.models import AnonymousUser
from django.db import close_old_connections
from rest_framework.authtoken.models import Token
class TokenAuthMiddleware:
"""
Token authorization middleware for Django Channels 2
see:
https://channels.readthedocs.io/en/latest/topics/authentication.html#custom-authentication
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
headers = dict(scope['headers'])
if b'authorization' in headers:
try:
token_name, token_key = headers[b'authorization'].decode().split()
if token_name == 'Token':
# Close old database connections to prevent usage of timed out connections
close_old_connections()
token = Token.objects.get(key=token_key)
scope['user'] = token.user
except Token.DoesNotExist:
scope['user'] = AnonymousUser()
return self.inner(scope)
TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))
Thanks to #ivissani answers, i fixed my TokenAuthMiddleware with some of SessionMiddleware codes.
I have opened an issue for django channels about updating docs.
#database_sync_to_async
def get_user(token_key):
try:
return Token.objects.get(key=token_key).user
except Token.DoesNotExist:
return AnonymousUser()
class TokenAuthMiddleware:
"""
Token authorization middleware for Django Channels 2
see:
https://channels.readthedocs.io/en/latest/topics/authentication.html#custom-authentication
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
return TokenAuthMiddlewareInstance(scope, self)
class TokenAuthMiddlewareInstance:
def __init__(self, scope, middleware):
self.middleware = middleware
self.scope = dict(scope)
self.inner = self.middleware.inner
async def __call__(self, receive, send):
headers = dict(self.scope['headers'])
if b'authorization' in headers:
token_name, token_key = headers[b'authorization'].decode().split()
if token_name == 'Token':
self.scope['user'] = await get_user(token_key)
inner = self.inner(self.scope)
return await inner(receive, send)
TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))
Fixed by using #database_sync_to_async decorator:
(see https://github.com/MathieuB1/KOREK-backend/commit/ff6a4b542cda583a1d5abbf200a5d57ef328cae0#diff-95e545fb374a9ed7e8af8c31087a3f29)
import jwt, re
import traceback
from channels.auth import AuthMiddlewareStack
from channels.db import database_sync_to_async
from django.contrib.auth.models import AnonymousUser
from django.conf import LazySettings
from jwt import InvalidSignatureError, ExpiredSignatureError, DecodeError
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
settings = LazySettings()
from django.db import close_old_connections
#database_sync_to_async
def close_connections():
close_old_connections()
#database_sync_to_async
def get_user(user_jwt):
try:
return User.objects.get(id=user_jwt)
except User.DoesNotExist:
return AnonymousUser()
class TokenAuthMiddleware:
"""
Token authorization middleware for Django Channels 2
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
# Close old database connections to prevent usage of timed out connections
close_connections()
# Login with JWT
try:
if scope['subprotocols'][0] != 'None':
token = scope['subprotocols'][0]
try:
user_jwt = jwt.decode(
token,
settings.SECRET_KEY,
)
scope['user'] = get_user(user_jwt['user_id'])
return self.inner(scope)
except (InvalidSignatureError, KeyError, ExpiredSignatureError, DecodeError):
traceback.print_exc()
pass
except Exception as e:
traceback.print_exc()
else:
raise
Refer to this part of the documentation. There it is explained that Django 3 will raise such exception if you try to use the ORM from within an async context (which seems to be the case).
As Django Channels documentation explains solution would be to use sync_to_async as follows:
from channels.db import database_sync_to_async
class TokenAuthMiddleware:
# more code here
async def __call__(self, scope):
# and some more code here
token = await database_sync_to_async(Token.objects.get(key=token_key))()
Although please bear in mind that I haven't used this in my life, so it may fail.
Note that in the Django channels documentation it says that you need to write your query in a separate method. So if this fails try doing that.
Related
I'm trying to write a custom authentication middleware for Django channels since I'm using JWT for my app's authentication scheme. I'm using the method that is mentioned in this article which basically gets user's token in the first request that is made to the websockets and then, in the receive method of the consumers.py file, fetches user's data based on that and then pours it in the self.scope['user'] (can't make use of the token_auth.py method because the UI app is separate..). Now since I have NO ACCESS to the request param that is usually being used in the views.py files to get the user's data, is there anyway to get the user's data out of an access token alone??
Hello #Jalal try this.
from channels.auth import AuthMiddlewareStack
from rest_framework.authtoken.models import Token
from django.contrib.auth.models import AnonymousUser
from django.db import close_old_connections
from rest_framework_simplejwt.backends import TokenBackend
class TokenAuthMiddleware:
"""
Token authorization middleware for Django Channels 2
"""
def __init__(self, inner):
self.inner = inner
def __call__(self, scope):
headers = dict(scope['headers'])
if b'authorization' in headers:
try:
token_name, token_key = headers[b'authorization'].decode().split()
if token_name == 'Token':
print(token_key)
valid_data = TokenBackend(algorithm='HS256').decode(token_key,verify=False)
print(valid_data)
scope['user_id'] = valid_data['user_id']
close_old_connections()
except Token.DoesNotExist:
scope['user'] = AnonymousUser()
return self.inner(scope)
TokenAuthMiddlewareStack = lambda inner: TokenAuthMiddleware(AuthMiddlewareStack(inner))
Here valid_data return this:
{'token_type': 'access', 'exp': 1627994463, 'jti':
'192275bd7fd649758838ave1899e1863', 'user_id': 2}
If you want to fetch more data(like:Username,email,etc) through access token then use Customizing token claims:
from rest_framework_simplejwt.serializers import TokenObtainPairSerializer
from rest_framework_simplejwt.views import TokenObtainPairView
class MyTokenObtainPairSerializer(TokenObtainPairSerializer):
#classmethod
def get_token(cls, user):
token = super().get_token(user)
# Add custom claims
token['username'] = user.username
token['email'] = user.email
# ...
return token
class MyTokenObtainPairView(TokenObtainPairView):
serializer_class = MyTokenObtainPairSerializer
I am using a custom JWT Authentication Middleware for verifying of JWT.
import jwt
from urllib.parse import parse_qs
from channels.db import database_sync_to_async
from django.contrib.auth import get_user_model
from django.contrib.auth.models import AnonymousUser
from django.conf import settings
from rest_framework_simplejwt.tokens import AccessToken
#database_sync_to_async
def get_user(user_id):
User = get_user_model()
try:
user = User.objects.get(id=user_id)
return user
except User.DoesNotExist:
return AnonymousUser()
class TokenAuthMiddleware:
"""
Custom middleware (insecure) that takes user IDs from the query string.
"""
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
async def __call__(self, scope, receive, send):
# Look up user from query string (you should also do things like
# checking if it is a valid user ID, or if scope["user"] is already
# populated).
token = parse_qs(scope["query_string"].decode())["token"][0]
AccessToken(token).verify()
payload = jwt.decode(token, settings.SECRET_KEY, algorithms=['HS256'])
scope["user"] = await get_user(int(payload["user_id"]))
return await self.inner(dict(scope), receive, send)
I am getting the error TypeError: object.__init__() takes exactly one argument (the instance to initialize)
I followed the official docs
Can anyone guide me where is exactly the issue?
For anyone facing the same issue, the problem is not actually in the Middleware. For my case, the issue was in asgi.py. I had forgot to call the .as_asgi() method on the consumer.
application = ProtocolTypeRouter(
{
"http": get_asgi_application(),
# Just HTTP for now. (We can add other protocols later.)
"websocket": TokenAuthMiddlewareStack(
URLRouter([
path("chat/", ChatConsumer.as_asgi()),
])
)
}
)
Note the as_asgi() method on ChatConsumer. I missed that which eventually led me to think that the issue was in my custom middleware.
I'm currently trying to write my unittest for a successfull password reset.
TLDR: I can't figure out how to handle the injection of the form data in the post request in the set-password view.
For context, let me give you my code first, since a line of code speaks more than 1000 words.
accounts/api/views.py:
class PasswordResetView(generics.GenericAPIView):
"""(POST) Expects email. Sends a password reset mail to email."""
permission_classes = (permissions.AllowAny,)
serializer_class = PasswordResetSerializer
def post(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
if serializer.is_valid(raise_exception=True):
user = User.objects.get(email=serializer.data['email'])
if user.is_active and user.has_usable_password():
send_password_reset_email(user, request)
return Response(
{"detail": "A password reset email has been send."},
status=status.HTTP_204_NO_CONTENT
)
else:
return Response(
{"detail": "This users password can't be reset."},
status=status.HTTP_406_NOT_ACCEPTABLE
)
accounts/api/urls.py:
url(r'^password_reset/$', views.PasswordResetView.as_view(),
name="password_reset"),
accounts/api/serializers.py:
class PasswordResetSerializer(serializers.Serializer):
email = serializers.EmailField()
def validate_email(self, value):
if User.objects.filter(email__iexact=value).exists():
return value
else:
raise serializers.ValidationError("Email not found.")
accounts/utils.py:
from django.contrib.auth.tokens import PasswordResetTokenGenerator
from django.contrib.sites.shortcuts import get_current_site
from django.utils import six
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode
from templated_email import send_templated_mail
password_reset_token = PasswordResetTokenGenerator()
def build_email_context(user_obj, request):
current_site = get_current_site(request)
context = {
"name": "",
"domain": current_site.domain,
"uid": urlsafe_base64_encode(force_bytes(user_obj.pk)),
'token': password_reset_token.make_token(user_obj)
}
if user_obj.name != "":
context["name"]: " " + user_obj.name
return context
def send_password_reset_email(user_obj, request):
context = build_email_context(user_obj, request)
send_templated_mail(
template_name="password_reset",
from_email="noreply#ordersome.com",
recipient_list=[user_obj.email],
context=context
)
The views that are linked to the route in the email are the standard CBV PasswordResetConfirmView and PasswordResetDoneView from django.contrib.auth.
And here is the test I wrote for this password reset sequence:
api/tests/test_views.py:
from django.contrib.auth import get_user, get_user_model
from django.core import mail
from rest_framework import status
from rest_framework.authtoken.models import Token
from rest_framework.reverse import reverse as api_reverse
from rest_framework.test import APITestCase
User = get_user_model()
class UserAPIViewsTestCase(APITestCase):
def setUp(self):
User.objects.create_user(
email="testuser#test.com", password="test1234")
self.login_url = api_reverse("api:auth:login")
self.register_url = api_reverse("api:auth:register")
self.password_reset_url = api_reverse("api:auth:password_reset")
self.password_change_url = api_reverse("api:auth:password_change")
self.user_delete_url = api_reverse("api:auth:user_delete")
def test_api_auth_password_reset_success(self):
"""Test if a password reset successfully resets password."""
pre_user = User.objects.get(email="testuser#test.com")
self.assertEqual(pre_user.check_password("test1234"), True)
email_data = {
"email": "testuser#test.com",
}
email_response = self.client.post(
self.password_reset_url, email_data, format="json")
self.assertEqual(email_response.status_code,
status.HTTP_204_NO_CONTENT)
password_reset_data = {
"new_password1": "newtest1234",
"new_password2": "newtest1234"
}
password_reset_response = self.client.get(
mail.outbox[0].body[-94:-37], format="json", follow=True)
path = password_reset_response.request["PATH_INFO"]
done_url = "http://testserver" + path
password_reset_done_response = self.client.post(
done_url, password_reset_data, format="json")
post_user = User.objects.get(email="testuser#test.com")
self.assertEqual(post_user.check_password("newtest1234"), True)
The last self.assertEqual fails. But when I test the view manually, it works, so the code should be correct. How should I modify the test?
When I log out done_url it gives me http://testserver/auth/reset/MQ/set-password/, which should be the correct path.
And when I log out password_reset_done_response it says, that it is a TemplateResponse and the status_code is 200 (which should mean successful, but it's not). Also this response still seems to have the path /auth/reset/MQ/set-password/ which should not be the case anymore.
password_reset_data is not complete - there's no info to which user the new password should be applied.
from django.contrib.auth.tokens import default_token_generator
from django.utils.encoding import force_bytes
from django.utils.http import urlsafe_base64_encode
...
password_reset_data = {
'uid': urlsafe_base64_encode(force_bytes(pre_user.pk)),
'token': default_token_generator.make_token(pre_user),
'new_password1': 'newtest1234',
'new_password2': 'newtest1234',
}
Other suggestions
setUp(self) should call super().setUp()
Use self.assertTrue instead of self.assertEqual
post_user is not needed, use pre_user.refresh_from_db() before last assert
I recently updated my project to Django 2 and channels 2. Right now I am trying to rewrite my tests for chat app.
I am facing a problem with tests that depend on django db mark from pytest-django. I tried to create objects in fixtures, setup methods, in test function itself, using async_to_sync on WebsocketCommunicator. However, none of those worked.
If I create a user in a fixture and save it correctly gets an id. However, in my consumer Django does not see that User in the database. And treat it like an anonymous user.
I have a temporary token which I use to authenticate a user on websocket.connect.
#pytest.fixture
def room():
room = generate_room()
room.save()
return room
#pytest.fixture
def room_with_user(room, normal_user):
room.users.add(normal_user)
yield room
room.users.remove(normal_user)
#pytest.fixture
def normal_user():
user = generate_user()
user.save()
return user
#pytest.mark.django_db
class TestConnect:
#pytest.mark.asyncio
async def test_get_connected_client(self, path, room_with_user, temp_token):
assert get_path(room_with_user.id) == path
communicator = QSWebsocketCommunicator(application, path, query_string=get_query_string(temp_token))
connected, subprotocol = await communicator.connect()
assert connected
await communicator.disconnect()
Consumer:
class ChatConsumer(JsonWebsocketConsumer):
def connect(self):
# Called on connection. Either call
self.user = self.scope['user']
self.room_id = self.scope['url_route']['kwargs']['room_id']
group = f'room_{self.room_id}'
users = list(User.objects.all()) # no users here
self.group_name = group
if not (self.user is not None and self.user.is_authenticated):
return self.close({'Error': 'Not authenticated user'})
try:
self.room = Room.objects.get(id=self.room_id, users__id=self.user.id)
except ObjectDoesNotExist:
return self.close({'Error': 'Room does not exists'})
# Send success response
self.accept()
# Save user as active
self.room.active_users.add(self.user)
My authentication Middleware
class OAuthTokenAuthMiddleware:
"""
Custom middleware that takes Authorization header and read OAuth token from it.
"""
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
def __call__(self, scope):
temp_token = self.get_token(scope)
scope['user'] = self.validate_token(temp_token)
return self.inner(scope)
#staticmethod
def get_token(scope) -> str:
return url_parse.parse_qs(scope['query_string'])[b'token'][0].decode("utf-8")
#staticmethod
def validate_token(token):
try:
token = TemporaryToken.objects.select_related('user').get(token=token)
if token.is_active():
token.delete()
return token.user
else:
return AnonymousUser()
except ObjectDoesNotExist:
return AnonymousUser()
And custom WebsocketCommunicator which accepts query_string in order to include my one time token
class QSWebsocketCommunicator(WebsocketCommunicator):
def __init__(self, application, path, headers=None, subprotocols=None,
query_string: Optional[Union[str, bytes]]=None):
if isinstance(query_string, str):
query_string = str.encode(query_string)
self.scope = {
"type": "websocket",
"path": path,
"headers": headers or [],
"subprotocols": subprotocols or [],
"query_string": query_string or ''
}
ApplicationCommunicator.__init__(self, application, self.scope)
My question is how can I create User, Room, etc. objects in tests/fixtures so that I can access them in Django consumer.
Or do you have another idea how can I overcome this?
It's pretty much impossible to reproduce your issue using the code you've provided. Read about How to create a Minimal, Complete, and Verifiable example. However, I suppose that you should use real transactions in your test as the plain pytest.mark.django_db will skip the transactions and not store any data in the database per se. A working example:
# routing.py
from django import http
from django.conf.urls import url
from django.contrib.auth.models import User
from channels.routing import ProtocolTypeRouter, URLRouter
from channels.generic.websocket import JsonWebsocketConsumer
class ChatConsumer(JsonWebsocketConsumer):
def connect(self):
self.user = self.scope['user']
print('user in scope, set by middleware:', self.user)
users = list(User.objects.all()) # no users here
print('all users in chat consumer:', users)
if not (self.user is not None and self.user.is_authenticated):
return self.close({'Error': 'Not authenticated user'})
# Send success response
self.accept()
class OAuthTokenAuthMiddleware:
def __init__(self, inner):
# Store the ASGI application we were passed
self.inner = inner
def __call__(self, scope):
token = self.get_token(scope)
print('token in middleware:', token)
scope['user'] = User.objects.get(username=token)
return self.inner(scope)
#staticmethod
def get_token(scope) -> str:
d = http.QueryDict(scope['query_string'])
return d['token']
APP = ProtocolTypeRouter({
'websocket': OAuthTokenAuthMiddleware(URLRouter([url(r'^websocket/$', ChatConsumer)])),
})
Sample fixture that creates a user with username spam:
#pytest.fixture(scope='function', autouse=True)
def create_user():
with transaction.atomic():
User.objects.all().delete()
user = User.objects.create_user(
'spam', 'spam#example.com', password='eggs',
first_name='foo', last_name='bar'
)
return user
Now, I mark the test as transactional one, meaning that each query is actually committed. Now the test user is stored into database and the queries made in middleware/consumer can actually return something meaningful:
#pytest.mark.django_db(transaction=True)
#pytest.mark.asyncio
async def test_get_connected_client():
app = OAuthTokenAuthMiddleware(URLRouter([url(r'^websocket/$', ChatConsumer)]))
communicator = QSWebsocketCommunicator(app, '/websocket/', query_string='token=spam')
connected, subprotocol = await communicator.connect()
assert connected
await communicator.disconnect()
Running test test yields the desired result:
$ pytest -vs
================================== test session starts =================================
platform darwin -- Python 3.6.3, pytest-3.4.0, py-1.5.2, pluggy-0.6.0 -- /Users/hoefling/.virtualenvs/stackoverflow/bin/python
cachedir: .pytest_cache
Django settings: spam.settings (from environment variable)
rootdir: /Users/hoefling/projects/private/stackoverflow/so-49136564/spam, inifile: pytest.ini
plugins: celery-4.1.0, forked-0.2, django-3.1.2, cov-2.5.1, asyncio-0.8.0, xdist-1.22.0, mock-1.6.3, hypothesis-3.44.4
collected 1 item
tests/test_middleware.py::test_get_connected_client Creating test database for alias 'default'...
token in middleware: spam
user in scope: spam
all users in chat consumer: [<User: spam>]
PASSEDDestroying test database for alias 'default'...
=============================== 1 passed in 0.38 seconds ================================
Btw you don't need to hack around the WebsocketCommunicator anymore since it is now able to deal with query strings, see this issue closed.
How to customize Django rest framework error page. Can't find about it. In my rest action code:
from django.http import Http404
class SomeAction(APIView):
def get(self, *args, **kwargs):
raise Http404()
I need to display custom error page in prod.
from django.http import HttpResponse
from django.shortcuts import render_to_response
from django.template import loader, RequestContext, Context
from apps.settings import DEBUG
from libs.requestprovider.middleware import get_current_request
from rest_framework.views import exception_handler
def custom_exception_handler(exc):
# Call REST framework's default exception handler first,
# to get the standard error response.
response = exception_handler(exc)
# Now add the HTTP status code to the response.
if response is not None:
response.data['status_code'] = response.status_code
if not DEBUG:
#errors can be more generic
if response.data['status_code'] == 404:
t = loader.get_template('404.html')
c = RequestContext(get_current_request(), {})
return HttpResponse(t.render(c), content_type="text/html")
return response
TL;DR
class YourAPIView(APIView):
#staticmethod
def my_exception_handler(exc, context):
response = exception_handler(exc, context)
if if response.status_code >= 400:
return HttpResponse(loader.get_template('your.html').render(context), content_type='text/html')
return response
def get_exception_handler(self):
return self.my_exception_handler
Long story
This link (custom-exception-handling) tells you to set EXCEPTION_HANDLER to yours, and you will notice that the default settings is rest_framework.views.exception_handler
Of course, you can follow the tutorial, but if you don't want to use global settings to control, then see the following.
I suggest you set breakpoints on the function (rest_framework.views.exception_handler) to see what is going on, and then you will know all proceeding.
If you don't want to debug, I list the key points below.
# rest_framework\views.py
class APIView(View):
...
def dispatch(self, request, *args, **kwargs):
...
try:
...
except Exception as exc:
response = self.handle_exception(exc) # <-- focus here
self.response = self.finalize_response(request, response, *args, **kwargs)
return self.response
def handle_exception(self, exc):
...
exception_handler = self.get_exception_handler() # <-- focus here
context = self.get_exception_handler_context()
response = exception_handler(exc, context)
...
return response
def get_exception_handler(self):
return self.settings.EXCEPTION_HANDLER
From the above code, you know you can change get_exception_handler then all down!
here is an example:
# views.py
from rest_framework import generics
from django.http import HttpResponse
from rest_framework import status
from django.template import loader
from rest_framework.views import exception_handler
class YourAPIView(generics.ListAPIView):
...
def get(self, request, *args, **kwargs):
return HttpResponse('ok', content_type='text/html')
#staticmethod
def my_exception_handler(exc, context):
response = exception_handler(exc, context) # <-- this is the default exception_handler
if response.status_code in (status.HTTP_400_BAD_REQUEST, status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN):
my_400_and_403_template = loader.get_template('error.html') # settings.py -> TEMPLATES = [{'DIRS': [ str(Path(BASE_DIR)/Path('templates/your_custom_dir')).replace('\\', '/') ]}] # Where your_custom_dir should contain error.html
return HttpResponse(my_400_and_403_template.render(context), content_type='text/html')
return response
def get_exception_handler(self):
return self.my_exception_handler
# def handle_exception(self, exc):
# return super().handle_exception(exc)