I'm working on a django a project that will serve as the endpoint for a webhook. The webhook will POST some JSON data to my endpoint, which will then parse that data. I'm trying to write unit tests for it, but I'm not sure if I'm sending the JSON properly.
I keep getting "TypeError: string indices must be integers" in pipeline_endpoint
Here's the code:
# tests.py
from django.test import TestCase
from django.test.client import Client
import simplejson
class TestPipeline(TestCase):
def setUp(self):
"""initialize the Django test client"""
self.c = Client()
def test_200(self):
json_string = u'{"1": {"guid": "8a40135230f21bdb0130f21c255c0007", "portalId": 999, "email": "fake#email"}}'
json_data = simplejson.loads(json_string)
self.response = self.c.post('/pipeline-endpoint', json_data, content_type="application/json")
self.assertEqual(self.response.status_code, "200")
and
# views.py
from pipeline.prospect import Prospect
import simplejson
def pipeline_endpoint(request):
#get the data from the json object that came in
prospects_json = simplejson.loads(request.raw_post_data)
for p in prospects_json:
prospect = {
'email' : p['email'],
'hs_id' : p['guid'],
'portal' : p['portalId'],
}
Edit: whole traceback.
======================================================================
ERROR: test_200 (pipeline.tests.TestPipeline)
----------------------------------------------------------------------
Traceback (most recent call last):
File "F:\......\pipeline\tests.py", line 31, in test_200
self.response = self.c.post('/pipeline-endpoint', json_string, content_type="application/json")
File "C:\Python27\lib\site-packages\django\test\client.py", line 455, in post
response = super(Client, self).post(path, data=data, content_type=content_type, **extra)
File "C:\Python27\lib\site-packages\django\test\client.py", line 256, in post
return self.request(**r)
File "C:\Python27\lib\site-packages\django\core\handlers\base.py", line 111, in get_response
response = callback(request, *callback_args, **callback_kwargs)
File "F:\......\pipeline\views.py", line 18, in pipeline_endpoint
'email' : p['email'],
TypeError: string indices must be integers
----------------------------------------------------------------------
Ran 1 test in 0.095s
FAILED (errors=1)
Destroying test database for alias 'default'...
#mrmagooey is right
def test_your_test(self):
python_dict = {
"1": {
"guid": "8a40135230f21bdb0130f21c255c0007",
"portalId": 999,
"email": "fake#email"
}
}
response = self.client.post('/pipeline-endpoint/',
json.dumps(python_dict),
content_type="application/json")
use json.dumps instead of json.loads
Try:
self.client.generic('POST', '/url', json.dumps({'json': 'object'})
rest_framework's APIClient (which is the the default client_class in APITestCase) takes care of dumping dict to JSON and it sets proper content type by passing format='json'.
from rest_framework import status
from rest_framework.test import APIClient, APITestCase
class MyTestCase(APITestCase):
url = '/url'
def post(self, payload, url=None):
"""
Helper to send an HTTP post.
#param (dict) payload: request body
#returns: response
"""
if url is None:
url = self.url
return self.client.post(url, payload, format='json')
def test_my_function(self):
payload = {
'key': 'value'
}
response = self.post(payload)
self.assertEqual(response.status_code, status.HTTP_200_OK)
You can always use the HttpRequest.body which loads the raw request data. This way you can handle your own data processing.
c = Client()
json_str= json.dumps({"data": {"id": 1}})
c.post('/ajax/handler/', data= json_str, content_type='application/json',
HTTP_X_REQUESTED_WITH='XMLHttpRequest')
def index(request):
....
print json.loads(request.body)
Since Django 3.0:
If you provide content_type as application/json, the data is
serialized using json.dumps() if it’s a dict, list, or tuple.
Serialization is performed with DjangoJSONEncoder by default, and can
be overridden by providing a json_encoder argument to Client. This
serialization also happens for put(), patch(), and delete() requests.
response = client.post(
f'/customer/{customer.id}/edit',
{'email': new_email},
content_type='application/json'
)
You can user iteritems on dictionaries to loop
for index, p in prospects_json.iteritems():
prospect={
'email': p['email'],
}
or alternatively
for index in prospect_json:
prospect={
'email': prospect_json[ index ]['email']
}
Adding to Guillaume Vincent's answer, from Django 2.1 we no longer need to use json.dumps for passing the data.
Changed in Django 2.1:
The JSON serialization described above was added. In older versions, you can call json.dumps() on data before passing it to post() to achieve the same thing.
Related
I'm having trouble testing one of my endpoints:
#require_http_methods(["POST"])
def store(request):
try:
body_unicode = request.body.decode('utf-8')
body = ast.literal_eval(body_unicode)
new_short_url = body['short_url']
original_url = body['original_url']
check_parameters(new_short_url, original_url)
Url.objects.create(short_url=new_short_url, original_url=original_url)
return HttpResponse('Created', status=201)
except KeyError as error:
return HttpResponse('Missing {}'.format(error.args), status=400)
except (AttributeError, IntegrityError, ValidationError) as error:
return HttpResponse(error.args, status=400)
As you can see, this endpoint only accepts POST requests and when trying to pass data from my tests, it arrives in the request.body, so I implemented my logic to get the data from there:
def test_create_url_ok(self):
creation_data = {
"short_url": "ab",
"original_url": "https://stackoverflow.com/"
}
response = self.client.post(reverse('store'), data=creation_data, content_type="application/json")
self.assertEqual(response.status_code, 201)
This works, but the problem is that when sending requests from my templates, data is not in the request.body, but in the request.POST. How to send data in the request.POST from my tests?
So, the thing is that vanilla django accepts data from request.POST when the content_type='multipart/form-data'. Since I was using 'json' it wasn't working (though it works when using DjangoRestFramework views):
When using default django tests, this content_type is the default, so you don't need to specify any:
I have a post method under View Set. I need to write a unit test case for the method. when I pass param its give None. How should I pass both param and data(payload).
views.py
#action(detail=True, methods=['post'])
def complete_task(self, request, *args, **kwargs):
"""
Method for complete the task
input post request : task_id : str, variable_return:boolean, request data: dict
output Response : gives whether task is completed or not
"""
try:
get_task_id = self.request.query_params.get("task_id")
get_process_variables = request.data
print(get_task_id)
print(get_process_variables)
complete_task = CamundaWriteMixins.complete_task(url=CAMUNDA_URL, task_id=get_task_id,
process_variable_data=get_process_variables)
print("compl", complete_task)
return Response({"task_status": str(complete_task)})
except Exception as error:
return Response(error)
test.py
def test_completed_task(self):
self.client = Client()
url = reverse('complete-task')
data = {"variables": {
"dept_status": {"value": "approved", "type": "String"}}
}
response = self.client.post(url, data=data, params={"task_id": "000c29840512"},
headers={'Content-Type': 'application/json'})
print(response.data)
self.assertTrue(response.data)
I have tried above test case method which is getting request data but I got param None.
Thanks in Advance,.
if you just modify your request a bit and add query param as part of your url then i guess you are good to go.
Example:
response = self.client.post(f'{url}?task_id=000c29840512', data=data,
headers={'Content-Type': 'application/json'})
you can refer the official documentation for the example: https://docs.djangoproject.com/en/4.0/topics/testing/tools/
I have views like:
class StudentAPIPerformanceReport(
generics.RetrieveAPIView,
):
def get(self, request, *args, **kwargs):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="report.csv"'
writer = csv.writer(response)
for student in Student.objects.filter(pk=self.kwargs['pk']):
assigned_courses = CourseParticipant.objects.filter(student=student)
completed_courses = assigned_courses.filter(completed=True)
headings = (
"student full Name",
"number of assigned courses to student",
"number of completed courses by student"
)
rows = (
student.full_name,
assigned_courses.count(),
completed_courses.count()
)
writer.writerow(headings)
writer.writerow(rows)
return response
Urls:
path(
'student/report/<int:pk>/',
StudentAPIPerformanceReport.as_view(),
name='student_performance'
)
And test for it view:
class StudentAPIPerformanceReportTestCase(APITestCase):
def setUp(self):
self.student_obj = Student.objects.create(
first_name='test',
last_name='student',
email='test_student#gmail.com',
)
self.course_obj = Course.objects.create(
name='test',
)
student_obj = CourseParticipant.objects.create(
course_id=self.course_obj.pk,
student_id=self.student_obj.pk,
)
def test_student_unassigned_from_course(self):
data_id = self.student_obj.pk
rud_url = api_reverse('student:student_performance', kwargs={'pk': data_id})
get_response = self.client.get(rud_url, data_id)
self.assertEqual(get_response.status_code, status.HTTP_200_OK)
But i have Traceback:
Error
Traceback (most recent call last):
File "/home/project/test_task/student/tests.py", line 120, in test_student_unassigned_from_course
get_response = self.client.get(rud_url, data_id)
File "/home/project/test_task/venv/lib/python3.7/site-packages/rest_framework/test.py", line 292, in get
response = super(APIClient, self).get(path, data=data, **extra)
File "/home/project/test_task/venv/lib/python3.7/site-packages/rest_framework/test.py", line 199, in get
'QUERY_STRING': urlencode(data or {}, doseq=True),
File "/home/project/test_task/venv/lib/python3.7/site-packages/django/utils/http.py", line 93, in urlencode
for key, value in query:
TypeError: 'int' object is not iterable
Api which i wont to test just make some csv file in format:
test student,1,0
How i can test it? I will be grateful for the help
The error you are receiving is from the way you call self.client.get.
TypeError: 'int' object is not iterable
is about the second parameter that you are passing in this particular line:
get_response = self.client.get(rud_url, data_id)
# ______________________________________^
data_id itself is the student object's primary key, which is a simple integer id.
If you look closer at the Django docs here, the second parameter of get() is data, which is a dictionary, not an integer. That's why it tries to iterate this parameter, but it is not iterable.
Since you've already attached your primary key in the url (in the reverse()), there is no need to attach it as a query parameter of the GET request performed by self.client.get().
Once you remove data_id from the get() function call
get_response = self.client.get(rud_url)
it should work and you will successfully get the CSV file in the response, which you can test further.
How can i print the id from the response in the below code.The user does exist in the DB.Also i come across this error.
from django.test import Client
c = Client(enforce_csrf_checks=False)
response = c.post('/reg/_user/', {'firstname': 'test', 'lastname' : '_test'})
views get_user
def _user(request):
try:
response_dict = {}
qd = request.POST
firstname = qd.__getitem__('firstname')
lastname = qd.__getitem__('lastname')
up = UserProfile.objects.get(first_name=firstname,last_name=lastname)
print up.id
return up.id
except:
pass
Error:
response = c.post('/reg/_user/', {'firstname': 'test', 'lastname' : '_test'})
File "/usr/local/lib/python2.7/dist-packages/django/test/client.py", line 483, in post
response = super(Client, self).post(path, data=data, content_type=content_type, **extra)
File "/usr/local/lib/python2.7/dist-packages/django/test/client.py", line 302, in post
return self.request(**r)
File "/usr/local/lib/python2.7/dist-packages/django/test/client.py", line 444, in request
six.reraise(*exc_info)
File "/usr/local/lib/python2.7/dist-packages/django/core/handlers/base.py", line 201, in get_response
response = middleware_method(request, response)
File "/usr/local/lib/python2.7/dist-packages/django/middleware/clickjacking.py", line 30, in process_response
if response.get('X-Frame-Options', None) is not None:
AttributeError: 'UserProfile' object has no attribute 'get'
The problem is not with your tests, but with the view itself. In Django a view always has to return a HttpResponse object. Sometimes this is achieved using a shortcut function like render(), but it in turn also returns an HttpResponse object.
If for some reason you just want to return an otherwise empty page with this single value you could change
return up.id
to
return HttpResponse(up.id)
Also, I wonder: Did you create the view just to test UserProfile and don't use it as a view on the actual site? If so, this code doesn't belong in a view, it should be put into the unittest itself. You should only use the test client to test actual, real views.
On an mostly unrelated, but quite important note. This:
try:
# your view code
except:
pass
is a strong antipattern. Why would you want to silence all the potential problems? You should really stop doing that.
I have this middleware
import logging
request_logger = logging.getLogger('api.request.logger')
class LoggingMiddleware(object):
def process_response(self, request, response):
request_logger.log(logging.DEBUG,
"GET: {}. POST: {} response code: {}. response "
"content: {}".format(request.GET, request.DATA,
response.status_code,
response.content))
return response
The problem is that request in process_response method has no .POST nor .DATA nor .body. I am using django-rest-framework and my requests has Content-Type: application/json
Note, that if I put logging to process_request method - it has .body and everything I need. However, I need both request and response in a single log entry.
Here is complete solution I made
"""
Api middleware module
"""
import logging
request_logger = logging.getLogger('api.request.logger')
class LoggingMiddleware(object):
"""
Provides full logging of requests and responses
"""
_initial_http_body = None
def process_request(self, request):
self._initial_http_body = request.body # this requires because for some reasons there is no way to access request.body in the 'process_response' method.
def process_response(self, request, response):
"""
Adding request and response logging
"""
if request.path.startswith('/api/') and \
(request.method == "POST" and
request.META.get('CONTENT_TYPE') == 'application/json'
or request.method == "GET"):
request_logger.log(logging.DEBUG,
"GET: {}. body: {} response code: {}. "
"response "
"content: {}"
.format(request.GET, self._initial_http_body,
response.status_code,
response.content), extra={
'tags': {
'url': request.build_absolute_uri()
}
})
return response
Note, this
'tags': {
'url': request.build_absolute_uri()
}
will allow you to filter by url in sentry.
Andrey's solution will break on concurrent requests. You'd need to store the body somewhere in the request scope and fetch it in the process_response().
class RequestLoggerMiddleware(object):
def process_request(self, request):
request._body_to_log = request.body
def process_response(self, request, response):
if not hasattr(request, '_body_to_log'):
return response
msg = "method=%s path=%s status=%s request.body=%s response.body=%s"
args = (request.method,
request.path,
response.status_code,
request._body_to_log,
response.content)
request_logger.info(msg, *args)
return response
All answers above have one potential problem -- big request.body passed to the server. In Django request.body is a property. (from framework)
#property
def body(self):
if not hasattr(self, '_body'):
if self._read_started:
raise RawPostDataException("You cannot access body after reading from request's data stream")
try:
self._body = self.read()
except IOError as e:
six.reraise(UnreadablePostError, UnreadablePostError(*e.args), sys.exc_info()[2])
self._stream = BytesIO(self._body)
return self._body
Django framework access body directly only in one case. (from framework)
elif self.META.get('CONTENT_TYPE', '').startswith('application/x-www-form-urlencoded'):
As you can see, property body read the entire request into memory. As a result, your server can simply crash. Moreover, it becomes vulnerable to DoS attack.
In this case I would suggest using another method of HttpRequest class. (from framework)
def readlines(self):
return list(iter(self))
So, you no longer need to do this
def process_request(self, request):
request._body_to_log = request.body
you can simply do:
def process_response(self, request, response):
msg = "method=%s path=%s status=%s request.body=%s response.body=%s"
args = (request.method,
request.path,
response.status_code,
request.readlines(),
response.content)
request_logger.info(msg, *args)
return response
EDIT: this approach with request.readlines() has problems. Sometimes it does not log anything.
It's frustrating and surprising that there is no easy-to-use request logging package in Django.
So I created one myself. Check it out: https://github.com/rhumbixsf/django-request-logging.git
Uses the logging system so it is easy to configure. This is what you get with DEBUG level:
GET/POST request url
POST BODY if any
GET/POST request url - response code
Response body
It is like accessing the form data to create a new form.
You must use request.POST for this (perhaps request.FILES is something you'd log as well).
class LoggingMiddleware(object):
def process_response(self, request, response):
request_logger.log(logging.DEBUG,
"GET: {}. POST: {} response code: {}. response "
"content: {}".format(request.GET, request.POST,
response.status_code,
response.content))
return response
See Here for request properties.
You can use like below:
"""
Middleware to log requests and responses.
"""
import socket
import time
import json
import logging
request_logger = logging.getLogger(__name__)
class RequestLogMiddleware:
"""Request Logging Middleware."""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
log_data = {}
# add request payload to log_data
req_body = json.loads(request.body.decode("utf-8")) if request.body else {}
log_data["request_body"] = req_body
# request passes on to controller
response = self.get_response(request)
# add response payload to our log_data
if response and response["content-type"] == "application/json":
response_body = json.loads(response.content.decode("utf-8"))
log_data["response_body"] = response_body
request_logger.info(msg=log_data)
return response
# Log unhandled exceptions as well
def process_exception(self, request, exception):
try:
raise exception
except Exception as e:
request_logger.exception("Unhandled Exception: " + str(e))
return exception
You can also check this out - log requests via middleware explains this
Also note, that response.content returns bytestring and not unicode string so if you need to print unicode, you need to call response.content.decode("utf-8").
You cannot access request.POST (or equivalently request.body) in the process_response part of the middleware. Here is a ticket raising the issue. Though you can have it in the process_request part. The previous answers give a class-based middleware. Django 2.0+ and 3.0+ allow function based middlewares.
from .models import RequestData # Model that stores all the request data
def requestMiddleware(get_response):
# One-time configuration and initialization.
def middleware(request):
# Code to be executed for each request before
# the view (and later middleware) are called.
try : metadata = request.META ;
except : metadata = 'no data'
try : data = request.body ;
except : data = 'no data'
try : u = str(request.user)
except : u = 'nouser'
response = get_response(request)
w = RequestData.objects.create(userdata=u, metadata=metadata,data=data )
w.save()
return response
return middleware
Model RequestData looks as follows -
class RequestData(models.Model):
time = models.DateTimeField(auto_now_add=True)
userdata = models.CharField(max_length=10000, default=' ')
data = models.CharField(max_length=20000, default=' ')
metadata = models.CharField(max_length=20000, default=' ')