I am using Django REST framework to upload a large csv file and extract the data from the file and save the data in the data. By large file I mean file of like 10 to 50mb but when I upload a file its taking way longer then expected like 10 to 15mins but the endpoint keeps on processing and does not returns the response here is how my views.py looks like:
from asyncore import read
from django.shortcuts import render
from rest_framework.views import APIView
from rest_framework.decorators import api_view
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.response import Response
import pandas as pd
from .models import ExtractedData
from .urls import urlpatterns
from django.urls import path
# Create your views here.
class FileUploadView(APIView):
parser_classes = ( MultiPartParser, FormParser)
def put(self, request, format=None):
file_obj = request.FILES['file']
df = pd.read_csv(file_obj)
dict_data = df.to_dict(orient='records')
for dict in dict_data:
ExtractedData.objects.get_or_create(data=dict)
return Response({'details':"File Saved Succesfully"}, status=204)
Related
I have a view that create two csv and my goal is to zip them and add to a model.FileField
zip = zipfile.ZipFile('myzip.zip','w')
zip.writestr('file1.csv', file1.getvalue())
zip.writestr('file2.csv', file2.getvalue())
I have tried this, the zip is upload but when I download it I have the error 'the archive is damaged or unknown format'
Mymodel.objects.create(zip = File(open('myzip.zip','rb'))
This example just worked for me,
from django.core.files import File
from django.http import HttpResponse
from .models import ZipFile
import zipfile
from django.views import View
class ZipWriteView(View):
def get(self, request, *args, **kwargs):
with zipfile.ZipFile("myzip.zip", "w") as zip_obj:
zip_obj.write("polls/views.py") # add file 1
zip_obj.write("polls/admin.py") # add file 2
with open(zip_obj.filename, "rb") as f:
ZipFile.objects.create(file=File(f))
return HttpResponse("Done")
I get 500(Internal Server Error) for the below code, cant able to identify the problem. Please help me
urls.py
from django.conf.urls import url
from .audfile import daywisedetail,hourwisedetail
urlpatterns = [
url(r'^pyapi/api/daywise/(?P<start_date>[\w-]+)/(?P<end_date>[\w-]+)/$',daywisedetail.as_view()),
url(r'^pyapi/api/hourwise/(?P<start_date_val>[\w-]+)/(?P<end_date_val>[\w-]+)/$',hourwisedetail.as_view()),
]
audfile.py
from rest_framework.views import APIView
from django.http import JsonResponse
from django.http import HttpResponse
from django.shortcuts import render
from django.views.generic import View
from django.template.loader import get_template
from rest_framework.response import Response
from rest_framework import status
import MySQLdb
sdatabase = MySQLdb.connect(host="localhost",user="root",passwd="password",db="main")
class daywisedetail(APIView):
def get(self,request,start_date=None,end_date=None,format=None):
db = sdatabase
cursor = db.cursor()
daydetail = """select id, values, name, email from day_users where date(reg_date) between '%s' and '%s' """ % (start_date,end_date)
cursor.execute(daydetail)
getdaydetailresult = cursor.fetchall()
mainlist=[]
for v0 in getdaydetailresult:
for v1 in v0:
mainlist.append(v1)
data = {
"hours": mainlist
}
return Response(data)
class hourwisedetail(APIView):
def get(self,request,start_date_val=None,end_date_val=None,format=None):
db = sdatabase
cursor = db.cursor()
hourwisedata = """select id, values, name, email from hour_users where date(reg_date) between '%s' and '%s' """ % (start_date_val,end_date_val)
cursor.execute(hourwisedata)
hourwisedataresult = cursor.fetchall()
mainlist1=[]
for v2 in hourwisedataresult:
for v3 in v2:
mainlist1.append(v3)
data1 = {
"days": mainlist1
}
return Response(data1)
If i have single class(daywisedetail), api is working properly, if i am adding second class(hourwisedetail), both the apis are showning 500 responce.
cant able to find where is the bug, please help me.
I have developed an API (Python 3.5, Django 1.10, DRF 3.4.2) that uploads a video file to my media path when I request it from my UI. That part is working fine. I try to write a test for this feature but cannot get it to run successfully.
#views.py
import os
from rest_framework import views, parsers, response
from django.conf import settings
class FileUploadView(views.APIView):
parser_classes = (parsers.FileUploadParser,)
def put(self, request, filename):
file = request.data['file']
handle_uploaded_file(file, filename)
return response.Response(status=204)
def handle_uploaded_file(file, filename):
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
if not os.path.exists(dir_name):
os.makedirs(dir_name)
file_path = os.path.join(dir_name, new_filename)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
and
#test.py
import tempfile
import os
from django.test import TestCase
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from rest_framework.test import APIRequestFactory
from myapp.views import FileUploadView
class UploadVideoTestCase(TestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp(suffix=None, prefix=None, dir=None)
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(), 'video')
factory = APIRequestFactory()
request = factory.put('file_upload/'+filename,
{'file': uploaded_file}, format='multipart')
view = FileUploadView.as_view()
response = view(request, filename)
print(response)
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
In this test, I need to use an existing video file ('media/testfiles/vid.mp4') and upload it since I need to test some processings on the video data after: that's why I reset the MEDIA_ROOT using mkdtemp.
The test fails since the file is not uploaded. In the def put of my views.py, when I print request I get <rest_framework.request.Request object at 0x10f25f048> and when I print request.data I get nothing. But if I remove the FileUploadParser in my view and use request = factory.put('file_upload/' + filename, {'filename': filename}, format="multipart") in my test, I get <QueryDict: {'filename': ['vid']}> when I print request.data.
So my conclusion is that the request I generate with APIRequestFactory is incorrect. The FileUploadParseris not able to retrieve the raw file from it.
Hence my question: How to generate a file upload (test) request with Django REST Framework's APIRequestFactory?
Several people have asked questions close to this one on SO but I had no success with the proposed answers.
Any help on that matter will be much appreciated!
It's alright now! Switching from APIRequestFactory to APIClient, I managed to have my test running.
My new test.py:
import os
import tempfile
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from django.urls import reverse
from rest_framework.test import APITestCase, APIClient
from django.contrib.auth.models import User
class UploadVideoTestCase(APITestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp()
User.objects.create_user('michel')
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(),
content_type='multipart/form-data')
client = APIClient()
user = User.objects.get(username='michel')
client.force_authenticate(user=user)
url = reverse('file_upload:upload_view', kwargs={'filename': filename})
client.put(url, {'file': uploaded_file}, format='multipart')
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
Below, testing file upload using APIRequestFactory as requested (and ModelViewSet).
from rest_framework.test import APIRequestFactory, APITestCase
from my_project.api.views import MyViewSet
from io import BytesIO
class MyTestCase(APITestCase):
def setUp(self):
fd = BytesIO(b'Test File content') # in-memory file to upload
fd.seek(0) # not needed here, but to remember after writing to fd
reqfactory = APIRequestFactory() # initialize in setUp if used by more tests
view = MyViewSet({'post': 'create'}) # for ViewSet {action:method} needed, for View, not.
request = factory.post('/api/new_file/',
{
"title": 'test file',
"fits_file": self.fd,
},
format='multipart') # multipart is default, but for clarification that not json
response = view(request)
response.render()
self.assertEqual(response.status_code, 201)
Note that there is no authorization for clarity, as with: 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.AllowAny'].
I am trying to implement url callbacks. And trying to test it. But seems like it is not working. I have been following this article for callbacks implementation.
I have defined two urls in urls.py
from django.conf.urls import patterns, include, url
from django.contrib import admin
urlpatterns = patterns('',
url(r'^admin/', include(admin.site.urls)),
url(r'^agent205', 'agent205.views.test'),
url(r'^agent206', 'agent205.views.test2'),
)
and their views in views.py
__author__ = 'rai'
from django.shortcuts import HttpResponse, render_to_response, render
from django.http.request import HttpRequest
import urllib, urllib2, json
from django.contrib.auth.decorators import login_required
import json
from rest_framework.views import APIView
from django.views.decorators.csrf import csrf_exempt
#csrf_exempt
def test(request):
data = {'foo': 'bar', 'hello': 'world'}
print request.body
return HttpResponse(json.dumps(data), content_type='application/json')
#csrf_exempt
def test2(request):
return HttpResponse(json.dumps(request.body), content_type='application/json')
Then I test from postman like
I am getting HTTP 200 OK response instead of getting 202 Accepted. What should I do for callback to work? Or am I missing something
If your issue is to return a 202 HTTP status code instead of the default 200, you could try to use the status parameter as follows:
#csrf_exempt
def test(request):
data = {'foo': 'bar', 'hello': 'world'}
print request.body
return HttpResponse(json.dumps(data), content_type='application/json', status=202)
I am trying to create custom templates for a simple contact form as per the django docs but I am getting a NameError. Looks like a simple issue but I can't figure it out. Any help will be greatly appreciated. The error message is:
"NameError at /contact/
name 'wizardcustomtemplate' is not defined"
where 'wizardcustomtemplate' is the app. Here is my code:
urls.py
from django.conf.urls import patterns, include, url
from django.contrib import admin
admin.autodiscover()
from wizardcustomtemplate.forms import SubjectForm, SenderForm, MessageForm
from wizardcustomtemplate.views import ContactWizard
urlpatterns = patterns('',
url(r'^admin/', include(admin.site.urls)),
url(r'^contact/$', ContactWizard.as_view(FORMS)),
)
views.py
import os
from django.shortcuts import render
from django.shortcuts import render_to_response
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.core.mail import send_mail
from django.core.context_processors import csrf
from django.contrib.formtools.wizard.views import SessionWizardView
from django.contrib.formtools.wizard.views import WizardView
from django.core.files.storage import FileSystemStorage
from django.core.files import File
FORMS = [("0", wizardcustomtemplate.forms.SubjectForm),
("1", wizardcustomtemplate.forms.SenderForm),
("2", wizardcustomtemplate.forms.MessageForm)
]
TEMPLATES = {"0": "wizardcustomtemplate/subject.html",
"1": "wizardcustomtemplate/sender.html",
"2": "wizardcustomtemplate/message.html"
}
class ContactWizard(SessionWizardView):
def get_template_names(self):
return [TEMPLATES[self.steps.current]]
def done(self, form_list, **kwargs):
form_data = process_form_data(form_list)
return render_to_response('wizardcustomtemplate/thanks.html', {'form_data': form_data})
def process_form_data(form_list):
form_data = [form.cleaned_data for form in form_list]
return form_data
forms.py
from django import forms
class SubjectForm(forms.Form):
subject = forms.CharField(max_length = 100,initial='Wizard')
class SenderForm(forms.Form):
sender = forms.EmailField(initial='abcd#efgh.org')
class MessageForm(forms.Form):
message = forms.CharField(initial='How r u?')
The form wizard works fine if I don't use the custom templates (FORMS, TEMPLATES etc.) Please let me know if you need additional information.
Solved it by adding import wizardcustomtemplate in views.py as suggested by #Rohan.