Django unit test - patch multiple requests for external api - django

I need to write 'one' test code with external api, which requires requests 'twice'.
first, I need to check if user is valid.
So I handled this with decorator in ./users/utils.py
import requests
def login_decorator(func):
def wrapper(self, request, *args, **kwargs):
# this access token is issued by external api
access_token = request.headers.get('Authorization', None)
# first requests, which gives me an info about user.
response = requests.get(
'https://kapi.kakao.com/v2/user/me',
headers={'Authorization':f'Bearer {access_token}'}
)
user_email = response.json()['kakao_account']['email']
request.user = User.objects.get(email=user_email)
return func(self, request, *args, **kwargs)
return wrapper
and then, I need to send that user a message with the external api again.
this code is in ./bids/views.py
import requests
class BiddingView(View):
#it uses login_decorator above
#login_decorator
def post(self, request, art_id):
try:
user = request.user
data = json.loads(request.body)
with transaction.atomic():
#handle things with bidding system#
#the external api requires its token to use a message api.
token = request.headers.get('Authorization', None)
#second requests, with post method
response = requests.post(
'https://kapi.kakao.com/v2/api/talk/memo/default/send',
headers = {'Authorization' : f'Bearer {token}'},
data = {"template_object" : json.dumps({'message':'contents'})}
)
return JsonResponse({'MESSAGE' : 'SUCCESS'}, status=200)
except KeyError:
return JsonResponse({'MESSAGE' : 'KEY ERROR'}, status=400)
This is my unit test code about BiddingView so far, which obviously only works for decorator
#patch('users.utils.requests')
def test_kakao_message_success(self, mock_requests):
class MockResponse:
def json(self):
return {'kakao_account' : {'email' : 'test#test.com'}}
mock_requests.get = MagicMock(return_value=MockResponse())
header = {'HTTP_Authorization' : 'ACCESS_TOKEN'}
body = {'offered_price' : 10000}
response = client.post(
'/bidding/1',
json.dumps(body),
content_type='application/json', **header
)
but I need to patch both .users.utils.requests and .bids.views.requests for my mock test.
#patch('users.utils.requests') # + #patch('bids.views.requests')
def test_kakao_message_success(self, mock_requests):
I want to know how to patch two requests at the same time.

I simplified your source code for ease of testing so that we can concentrate on the problem which are the external requests.
./utils.py
import requests
def login_decorator(func):
def wrapper(self, request, *args, **kwargs):
# first requests, which gives me an info about user.
response = requests.get(
'https://kapi.kakao.com/v2/user/me',
headers={'Authorization': 'Bearer access_token'}
)
request.user = response.json()['kakao_account']['email']
return func(self, request, *args, **kwargs)
return wrapper
./views.py
import json
import requests
from utils import login_decorator
class BiddingView:
#it uses login_decorator above
#login_decorator
def post(self, request, art_id):
print(f"The current user is {request.user}")
#second requests, with post method
response = requests.post(
'https://kapi.kakao.com/v2/api/talk/memo/default/send',
headers = {'Authorization' : 'Bearer token'},
data = {"template_object" : json.dumps({'message':'contents'})}
)
return response.text
Solution 1 - Manual patching of requests for each source file
You can stack the unittest.mock.patch decorator, one after the other.
from unittest.mock import MagicMock, patch
from views import BiddingView
class MockLoginResponse:
def json(self):
return {'kakao_account' : {'email' : 'test#test.com'}}
class MockViewResponse:
text = "He alone, who owns the youth, gains the future."
#patch('utils.requests.get', MagicMock(return_value=MockLoginResponse()))
#patch('views.requests.post', MagicMock(return_value=MockViewResponse()))
def test_kakao_message_success():
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
__________________________________________________________________________________ test_kakao_message_success ___________________________________________________________________________________
------------------------------------------------------------------------------------- Captured stdout call --------------------------------------------------------------------------------------
The current user is test#test.com
Response: He alone, who owns the youth, gains the future.
Solution 2.1 - Instead of patching requests per file, patch the exact target request
This requires you to install library https://pypi.org/project/requests-mock/
from unittest.mock import MagicMock
import requests_mock
from views import BiddingView
def test_kakao_message_success_with_library():
with requests_mock.Mocker() as requests_mocker:
# Mock the external requests
requests_mocker.get(
"https://kapi.kakao.com/v2/user/me",
json={'kakao_account' : {'email' : 'test#test.com'}},
)
requests_mocker.post(
"https://kapi.kakao.com/v2/api/talk/memo/default/send",
text="He alone, who owns the youth, gains the future.",
)
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
Same as above
Solution 2.2 - Instead of patching requests per file, patch the exact target request. But now, apply it automatically to any test using pytest's autouse feature.
from unittest.mock import MagicMock
import pytest
import requests_mock
from views import BiddingView
#pytest.fixture(autouse=True)
def setup_external_requests():
with requests_mock.Mocker() as requests_mocker:
# Mock the external requests
requests_mocker.get(
"https://kapi.kakao.com/v2/user/me",
json={'kakao_account' : {'email' : 'test#test.com'}},
)
requests_mocker.post(
"https://kapi.kakao.com/v2/api/talk/memo/default/send",
text="He alone, who owns the youth, gains the future.",
)
# It is required to perform a yield instead of return to not teardown the requests_mocker
yield requests_mocker
def test_kakao_message_success_with_library_2():
response = BiddingView().post(
request=MagicMock(),
art_id="some art"
)
print(f"Response: {response}")
Output:
Same as above

Related

Add existing API into django rest framework

Besides the API I created by using the django rest framework, now I want to add the existing commerial API into the backend, how to do this?
The commerial API include several part(API), for example, /prepare, /upload, /merge,/get_result, etc. They all use the "POST" indeed.
Do I need to call the commerial API directly on the frontend? Or I need to integrate the commerial API in to one backend API? Any suggestions are appreciated.Thanks.
For example:
```
class TestView(APIView):
"""
Add all the commerial API here in order
/prepare(POST)
/upload(POST)
/merge(POST)
/get_result(POST)
return Result
"""
```
Depending on your needs, I suggest making the external API calls on backend.
As a good practice, you should seperate your external API calls from your views. As it can be messy as the project gets bigger.
Check the sample code of how I manage external API calls, by seperating them to a different file, for example api_client.py
My default api_client.py looks something like this.
(You need to install "requests" pip package by pip install requests)
import requests
from django.conf import settings
class MyApiClient(object):
def __init__(self):
self.base_url = settings.EXTERNAL_API_1.get('url')
self.auth_url = "{0}login/".format(self.base_url)
self.username = settings.EXTERNAL_API_1.get('username')
self.password = settings.EXTERNAL_API_1.get('password')
self.session = None
self.access_token = None
self.token_type = None
self.token_expires_in = None
def _request(self, url, data=None, method="POST", as_json=True):
if self.session is None:
self.session = requests.Session()
if not self.access_token:
self.authenticate()
r = requests.Request(method, url=url, data=data, headers={
"Accept": "application/json",
"Content-Type": "application/json",
'Authorization': 'Token {0}'.format(
self.access_token)
})
prepared_req = r.prepare()
res = self.session.send(prepared_req, timeout=60)
if as_json:
json_result = res.json()
return json_result
return res
def _get(self, url):
return self._request(url=url, method="GET")
def _post(self, url, data):
return self._request(url=url, data=data, method="POST")
def authenticate(self):
res = requests.post(self.auth_url, json={'username': self.username,
'password': self.password},
headers={"Content-Type": "application/json"})
if res.status_code != 200:
res.raise_for_status()
json_result = res.json()
self.access_token = json_result.get('response', None).get('token',None)
def prepare(self):
something = 'bla bla'
request_url = "{0}prepare".format(self.base_url)
result = self._post(url=request_url, data=something)
return result
And in your views.py
from api_client import MyApiClient
class TestView(APIView):
api_client = MyApiClient()
def post(request, *args, **kwargs):
res1 = api_client.prepare()
res2 = api_client.your_other_method()
res3 = api_client.your_last_method()
return Response(res3)
Edited! Hope this helps now :)

Django test with APIRequestFactory : how to pass "flat" parameter to a view

Django 2.2
I am writing tests for API using APIRequestFactory. The code that hits
/some_endpoint and /some_endpoint/<item_id> already works, and so does the test that tests /some_endpoint. However the test to test /some_endpoint/<item_id> does not work because I can not find a working way to pass that <item_id> value to the view code. Please not it's not /some_endpoint/<some_keyword>=<item_id> , it's "flat" in my case i.e. there's no keyword. The problem is <item_id> does not make it into the view code (it's always None in the classview in get_queryset method)
I tried to pass it as **kwargs, it does not arrive either ( see here). But that probably would not work anyway without keyword.
I tried to switch to use of Client instead of APIRequestFactory, same result. But I would rather get it working with APIRequestFactory unless it does not work this way in general. Below is the code.
test.py
def test_getByLongId(self) :
factory = APIRequestFactory()
item = Item.active.get(id=1)
print(item.longid)
#it prints correct longid here
request = factory.get("/item/%s" % item.longid)
view = ItemList.as_view()
force_authenticate(request, user=self.user)
response = view(request)
urls.py
urlpatterns = [
...
...
url(item/(?P<item_id>[a-zA-Z0-9-]+)/$', views.ItemList.as_view(), name='item-detail'),
...
...
]
views.py
class ItemList(generics.ListAPIView):
permission_classes = (IsBotOrReadOnly,)
"""
API endpoint that allows users to be viewed or edited.
"""
serializer_class = ItemSerializer
schema = AutoSchema(
manual_fields=[
coreapi.Field("longid"),
]
)
def get_queryset(self):
"""
Optionally restricts the returned SampleSequencing to a given barcode.
"""
longid = self.kwargs.get('item_id', None)
print(longid)
#prints correct longid when executed by the webserver code and prints None when executed by the test
queryset = Item.active.filter(longid=longid)
return queryset
You have to pass item_id into the view():
def test_by_long_id(self) :
factory = APIRequestFactory()
item = Item.active.get(id=1)
print(item.longid)
#it prints correct longid here
request = factory.get("/item/%s" % item.longid)
view = ItemList.as_view()
force_authenticate(request, user=self.user)
response = view(request, item_id=item.longid)
or use APIClient:
from rest_framework.test import APIClient
# ...
#
def test_item_client(self):
item = Item.active.get(id=1)
client = APIClient()
url = '/item/%s/' % item.id
response = client.get(url)

flask_login - current_user

I am using the flask_login extension in my flask app to login users. As you must be knowing, this extension has a variable that stores a current_user. The code is working perfectly, except when it comes to testing it.
When I am testing the code (using unittest), I register a "test user" and log it in. But the current_user variable does not keep the user that logged in.
Here is my app code; the part that adds in a category (current_user gets set when a user logs in) :
def post(self):
# Get the access token from the header
auth_header = request.headers.get('Authorization')
access_token = auth_header.split(" ")[1]
if access_token:
# Attempt to decode the token and get the User ID
user_id = User.decode_token(access_token)
if not isinstance(user_id, str):
# Go ahead and handle the request, the user is authenticated
data = request.get_json()
if data['name']:
category = Category(name = data['name'], user_id = user_id)
db.session.add(category)
db.session.commit()
response = {'id' : category.id,
'category_name' : category.name,
'created_by' : current_user.first_name
}
return response
else:
# user is not legit, so the payload is an error message
message = user_id
response = {
'message': message
}
return response
Here is my code that tests the app:
import unittest
import os
import json
import app
from app import create_app, db
class CategoryTestCase(unittest.TestCase):
"""This class represents the Category test case"""
def setUp(self):
"""setup test variables"""
self.app = create_app(config_name="testing")
self.client = self.app.test_client
self.category_data = {'name' : 'Yummy'}
# binds the app with the current context
with self.app.app_context():
#create all tables
db.session.close()
db.drop_all()
db.create_all()
def register_user(self, first_name='Tester', last_name='Api', username='apitester', email='tester#api.com', password='abc'):
"""This helper method helps register a test user"""
user_data = {
'first_name' : first_name,
'last_name' : last_name,
'username' : username,
'email' : email,
'password' : password
}
return self.client().post('/api/v1.0/register', data=json.dumps(user_data), content_type='application/json')
def login_user(self, email='tester#api.com', password='abc'):
"""this helper method helps log in a test user"""
user_data = {
'email' : email,
'password' : password
}
return self.client().post('/api/v1.0/login', data=json.dumps(user_data), content_type='application/json')
def test_category_creation(self):
"""Test that the Api can create a category"""
self.register_user()
login_result = self.login_user()
token = json.loads(login_result.data)
token = token['access_token']
# Create a category by going to that link
response = self.client().post('/api/v1.0/category', headers=dict(Authorization="Bearer " + token), data=json.dumps(self.category_data), content_type='application/json')
self.assertEquals(response.status_code, 201)
You need to use the same context which you used for logging in. So this is what you need to add in your code:
with self.client() as c:
Then use the c to make get, post, or any other request you want. Here is a complete example:
import unittest
from app import create_app
class CategoryTestCase(unittest.TestCase):
"""This class represents the Category test case"""
def setUp(self):
"""setup test variables"""
self.app = create_app(config_name="testing")
self.client = self.app.test_client
self.category_data = {'name' : 'Yummy'}
def test_category_creation(self):
"""Test that the user can create a category"""
with self.client() as c:
# use the c to make get, post or any other requests
login_response = c.post("""login the user""")
# Create a category by going to that link using the same context i.e c
response = c.post('/api/v1.0/category', self.category_data)

I have a middleware where I a want to log every request/response. How can I access to POST data?

I have this middleware
import logging
request_logger = logging.getLogger('api.request.logger')
class LoggingMiddleware(object):
def process_response(self, request, response):
request_logger.log(logging.DEBUG,
"GET: {}. POST: {} response code: {}. response "
"content: {}".format(request.GET, request.DATA,
response.status_code,
response.content))
return response
The problem is that request in process_response method has no .POST nor .DATA nor .body. I am using django-rest-framework and my requests has Content-Type: application/json
Note, that if I put logging to process_request method - it has .body and everything I need. However, I need both request and response in a single log entry.
Here is complete solution I made
"""
Api middleware module
"""
import logging
request_logger = logging.getLogger('api.request.logger')
class LoggingMiddleware(object):
"""
Provides full logging of requests and responses
"""
_initial_http_body = None
def process_request(self, request):
self._initial_http_body = request.body # this requires because for some reasons there is no way to access request.body in the 'process_response' method.
def process_response(self, request, response):
"""
Adding request and response logging
"""
if request.path.startswith('/api/') and \
(request.method == "POST" and
request.META.get('CONTENT_TYPE') == 'application/json'
or request.method == "GET"):
request_logger.log(logging.DEBUG,
"GET: {}. body: {} response code: {}. "
"response "
"content: {}"
.format(request.GET, self._initial_http_body,
response.status_code,
response.content), extra={
'tags': {
'url': request.build_absolute_uri()
}
})
return response
Note, this
'tags': {
'url': request.build_absolute_uri()
}
will allow you to filter by url in sentry.
Andrey's solution will break on concurrent requests. You'd need to store the body somewhere in the request scope and fetch it in the process_response().
class RequestLoggerMiddleware(object):
def process_request(self, request):
request._body_to_log = request.body
def process_response(self, request, response):
if not hasattr(request, '_body_to_log'):
return response
msg = "method=%s path=%s status=%s request.body=%s response.body=%s"
args = (request.method,
request.path,
response.status_code,
request._body_to_log,
response.content)
request_logger.info(msg, *args)
return response
All answers above have one potential problem -- big request.body passed to the server. In Django request.body is a property. (from framework)
#property
def body(self):
if not hasattr(self, '_body'):
if self._read_started:
raise RawPostDataException("You cannot access body after reading from request's data stream")
try:
self._body = self.read()
except IOError as e:
six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])
self._stream = BytesIO(self._body)
return self._body
Django framework access body directly only in one case. (from framework)
elif self.META.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
As you can see, property body read the entire request into memory. As a result, your server can simply crash. Moreover, it becomes vulnerable to DoS attack.
In this case I would suggest using another method of HttpRequest class. (from framework)
def readlines(self):
return list(iter(self))
So, you no longer need to do this
def process_request(self, request):
request._body_to_log = request.body
you can simply do:
def process_response(self, request, response):
msg = "method=%s path=%s status=%s request.body=%s response.body=%s"
args = (request.method,
request.path,
response.status_code,
request.readlines(),
response.content)
request_logger.info(msg, *args)
return response
EDIT: this approach with request.readlines() has problems. Sometimes it does not log anything.
It's frustrating and surprising that there is no easy-to-use request logging package in Django.
So I created one myself. Check it out: https://github.com/rhumbixsf/django-request-logging.git
Uses the logging system so it is easy to configure. This is what you get with DEBUG level:
GET/POST request url
POST BODY if any
GET/POST request url - response code
Response body
It is like accessing the form data to create a new form.
You must use request.POST for this (perhaps request.FILES is something you'd log as well).
class LoggingMiddleware(object):
def process_response(self, request, response):
request_logger.log(logging.DEBUG,
"GET: {}. POST: {} response code: {}. response "
"content: {}".format(request.GET, request.POST,
response.status_code,
response.content))
return response
See Here for request properties.
You can use like below:
"""
Middleware to log requests and responses.
"""
import socket
import time
import json
import logging
request_logger = logging.getLogger(__name__)
class RequestLogMiddleware:
"""Request Logging Middleware."""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
log_data = {}
# add request payload to log_data
req_body = json.loads(request.body.decode("utf-8")) if request.body else {}
log_data["request_body"] = req_body
# request passes on to controller
response = self.get_response(request)
# add response payload to our log_data
if response and response["content-type"] == "application/json":
response_body = json.loads(response.content.decode("utf-8"))
log_data["response_body"] = response_body
request_logger.info(msg=log_data)
return response
# Log unhandled exceptions as well
def process_exception(self, request, exception):
try:
raise exception
except Exception as e:
request_logger.exception("Unhandled Exception: " + str(e))
return exception
You can also check this out - log requests via middleware explains this
Also note, that response.content returns bytestring and not unicode string so if you need to print unicode, you need to call response.content.decode("utf-8").
You cannot access request.POST (or equivalently request.body) in the process_response part of the middleware. Here is a ticket raising the issue. Though you can have it in the process_request part. The previous answers give a class-based middleware. Django 2.0+ and 3.0+ allow function based middlewares.
from .models import RequestData # Model that stores all the request data
def requestMiddleware(get_response):
# One-time configuration and initialization.
def middleware(request):
# Code to be executed for each request before
# the view (and later middleware) are called.
try : metadata = request.META ;
except : metadata = 'no data'
try : data = request.body ;
except : data = 'no data'
try : u = str(request.user)
except : u = 'nouser'
response = get_response(request)
w = RequestData.objects.create(userdata=u, metadata=metadata,data=data )
w.save()
return response
return middleware
Model RequestData looks as follows -
class RequestData(models.Model):
time = models.DateTimeField(auto_now_add=True)
userdata = models.CharField(max_length=10000, default=' ')
data = models.CharField(max_length=20000, default=' ')
metadata = models.CharField(max_length=20000, default=' ')

Django file upload progress get cache returns None

Django file upload progress process json request getting null response
view.py
def upload_progress(request):
"""
A view to report back on upload progress.
Return JSON object with information about the progress of an upload.
Copied from:
http://djangosnippets.org/snippets/678/
See upload.py for file upload handler.
"""
#import ipdb
#ipdb.set_trace()
progress_id = ''
if 'X-Progress-ID' in request.GET:
progress_id = request.GET['X-Progress-ID']
elif 'X-Progress-ID' in request.META:
progress_id = request.META['X-Progress-ID']
if progress_id:
from django.utils import simplejson
cache_key = "%s_%s" % (request.META['REMOTE_ADDR'], progress_id)
data = cache.get(cache_key)
return HttpResponse(simplejson.dumps(data))
UploadProgressCachedHandler.py
from django.core.files.uploadhandler import FileUploadHandler
from django.core.cache import cache
class UploadProgressCachedHandler(FileUploadHandler):
"""
Tracks progress for file uploads.
The http post request must contain a header or query parameter, 'X-Progress-ID'
which should contain a unique string to identify the upload to be tracked.
Copied from:
http://djangosnippets.org/snippets/678/
See views.py for upload_progress function...
"""
def __init__(self, request=None):
super(UploadProgressCachedHandler, self).__init__(request)
self.progress_id = None
self.cache_key = None
def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
self.content_length = content_length
if 'X-Progress-ID' in self.request.GET :
self.progress_id = self.request.GET['X-Progress-ID']
elif 'X-Progress-ID' in self.request.META:
self.progress_id = self.request.META['X-Progress-ID']
if self.progress_id:
self.cache_key = "%s_%s" % (self.request.META['REMOTE_ADDR'], self.progress_id )
cache.set(self.cache_key, {
'length': self.content_length,
'uploaded' : 0
})
def new_file(self, field_name, file_name, content_type, content_length, charset=None):
pass
def receive_data_chunk(self, raw_data, start):
if self.cache_key:
data = cache.get(self.cache_key)
data['uploaded'] += self.chunk_size
cache.set(self.cache_key, data)
#cache.set(self.cache_key, 5000)
return raw_data
def file_complete(self, file_size):
pass
def upload_complete(self):
if self.cache_key:
cache.delete(self.cache_key)
Iam setting cache with uploadProgressCacheHandler. But when tries to retrieve via json request .The data returning None object.'cache_key' is generating correctly.
Please help.
I guess you are trying to make the JSON request when the flow already has reached upload_complete. At that point cache_key has been deleted from cache (that's why it is empty).
Are you using the Django development server? I think this built-in server only allows to handle one request at time, which in your situation means that first the upload is completed and following the JSON request is processed. You can try with another server (ex: Apache) able to handle multiple requests.