I am using Django REST framework to upload a large csv file and extract the data from the file and save the data in the data. By large file I mean file of like 10 to 50mb but when I upload a file its taking way longer then expected like 10 to 15mins but the endpoint keeps on processing and does not returns the response here is how my views.py looks like:
from asyncore import read
from django.shortcuts import render
from rest_framework.views import APIView
from rest_framework.decorators import api_view
from rest_framework.parsers import MultiPartParser, FormParser
from rest_framework.response import Response
import pandas as pd
from .models import ExtractedData
from .urls import urlpatterns
from django.urls import path
# Create your views here.
class FileUploadView(APIView):
parser_classes = ( MultiPartParser, FormParser)
def put(self, request, format=None):
file_obj = request.FILES['file']
df = pd.read_csv(file_obj)
dict_data = df.to_dict(orient='records')
for dict in dict_data:
ExtractedData.objects.get_or_create(data=dict)
return Response({'details':"File Saved Succesfully"}, status=204)
I am trying to read csv file in django but I don't know how to achieve it. I am fresher recently joined organization and working singly, please any one help me
from django.shortcuts import render
from django.http import HttpResponse
import csv
from csv import reader
def fun(request):
with open(r"C:\Users\Sagar\Downloads\ULB_Sp_ThreePhaseUpdate.csv") as file:
reader = csv.reader(file)
ip = request.GET["id"]
flag = False
for rec in reader:
if rec[1]=="id":
return HttpResponse("MID: ",rec[2])
return render(request, "index.html")
I am having trouble uploading excel file to my django application. It is a very simple application that should allow a user to upload an excel file with 3 columns. The application will read the contents of this file and process it into bunch of calculations
here is my forms.py:
class InputForm(forms.Form):
FileLocation = forms.FileField(label='Import Data',required=True,widget=forms.FileInput(attrs={'accept': ".xlsx"}))
settings.py:
FILE_UPLOAD_HANDLERS = ["django_excel.ExcelMemoryFileUploadHandler",
"django_excel.TemporaryExcelFileUploadHandler"]
views.py:
import xlrd
from django.shortcuts import render_to_response, render
from django.conf.urls.static import static
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.template.context_processors import csrf
from io import TextIOWrapper
from WebApp.forms import *
from django.core.mail import send_mail
from django.utils.safestring import mark_safe
from django.db import connection
import os
import csv
def analyze(request):
if request.method == 'POST':
form = InputForm(request.POST,request.FILES['FileLocation'])
if form.is_valid():
book = xlrd.open_workbook(request.FILES('FileLocation'))
for sheet in book.sheets():
number_of_rows = sheet.nrows
number_of_columns = sheet.ncols
print(number_of_rows)
I upload the file in the form and it gives me an error:
AttributeError at /app/analyze/
'ExcelInMemoryUploadedFile' object has no attribute 'get'
Request Method: POST
Request URL: http://127.0.0.1:8000/data/analyze/
Django Version: 1.11
Exception Type: AttributeError
Exception Value:
Exception Location: C:\Python36\lib\site-packages\django\forms\widgets.py in value_from_datadict, line 367
Python Executable: C:\Python36\python.exe
Python Version: 3.6.4
I am also able to upload a .csv file successfully using the following views.py code:
def analyze(request):
c={}
context = RequestContext(request)
c.update(csrf(request))
abc=['a','b','c']
if request.method == 'POST':
form = InputForm(request.POST,request.FILES)
dataType = request.POST.get("DataType")
print(dataType)
if form.is_valid():
cd = form.cleaned_data #print (cd)
a = TextIOWrapper(request.FILES['FileLocation'].file,encoding='ascii',errors='replace')
#print (request.FILES.keys())
data = csv.reader(a)
row1csv = next(data)
region = row1csv[0]
metric = row1csv[2]
I have tried django-excel with same error.
You're correctly initialising your form for the .CSV case but not in your Excel case:
form = InputForm(request.POST, request.FILES)
Don't initialise using request.FILES['FileLocation'] as that's passing the wrong type to the form. It's expecting a MultiValueDict of uploaded files, not a single uploaded file. That's why it fails when calling get on it.
Next, you can't pass an ExcelInMemoryUploadedFile to xlrd.get_workbook(). You need to save the file to disk first, then pass it's path to the get_workbook() method. The documentation of django-excel gives some easier methods:
book = request.FILES['FileLocation'].get_book() # note the square brackets!
or to directly access a sheet:
sheet = request.FILES['FileLocation'].get_sheet('sheet1')
I have developed an API (Python 3.5, Django 1.10, DRF 3.4.2) that uploads a video file to my media path when I request it from my UI. That part is working fine. I try to write a test for this feature but cannot get it to run successfully.
#views.py
import os
from rest_framework import views, parsers, response
from django.conf import settings
class FileUploadView(views.APIView):
parser_classes = (parsers.FileUploadParser,)
def put(self, request, filename):
file = request.data['file']
handle_uploaded_file(file, filename)
return response.Response(status=204)
def handle_uploaded_file(file, filename):
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
if not os.path.exists(dir_name):
os.makedirs(dir_name)
file_path = os.path.join(dir_name, new_filename)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
and
#test.py
import tempfile
import os
from django.test import TestCase
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from rest_framework.test import APIRequestFactory
from myapp.views import FileUploadView
class UploadVideoTestCase(TestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp(suffix=None, prefix=None, dir=None)
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(), 'video')
factory = APIRequestFactory()
request = factory.put('file_upload/'+filename,
{'file': uploaded_file}, format='multipart')
view = FileUploadView.as_view()
response = view(request, filename)
print(response)
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
In this test, I need to use an existing video file ('media/testfiles/vid.mp4') and upload it since I need to test some processings on the video data after: that's why I reset the MEDIA_ROOT using mkdtemp.
The test fails since the file is not uploaded. In the def put of my views.py, when I print request I get <rest_framework.request.Request object at 0x10f25f048> and when I print request.data I get nothing. But if I remove the FileUploadParser in my view and use request = factory.put('file_upload/' + filename, {'filename': filename}, format="multipart") in my test, I get <QueryDict: {'filename': ['vid']}> when I print request.data.
So my conclusion is that the request I generate with APIRequestFactory is incorrect. The FileUploadParseris not able to retrieve the raw file from it.
Hence my question: How to generate a file upload (test) request with Django REST Framework's APIRequestFactory?
Several people have asked questions close to this one on SO but I had no success with the proposed answers.
Any help on that matter will be much appreciated!
It's alright now! Switching from APIRequestFactory to APIClient, I managed to have my test running.
My new test.py:
import os
import tempfile
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from django.urls import reverse
from rest_framework.test import APITestCase, APIClient
from django.contrib.auth.models import User
class UploadVideoTestCase(APITestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp()
User.objects.create_user('michel')
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(),
content_type='multipart/form-data')
client = APIClient()
user = User.objects.get(username='michel')
client.force_authenticate(user=user)
url = reverse('file_upload:upload_view', kwargs={'filename': filename})
client.put(url, {'file': uploaded_file}, format='multipart')
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
Below, testing file upload using APIRequestFactory as requested (and ModelViewSet).
from rest_framework.test import APIRequestFactory, APITestCase
from my_project.api.views import MyViewSet
from io import BytesIO
class MyTestCase(APITestCase):
def setUp(self):
fd = BytesIO(b'Test File content') # in-memory file to upload
fd.seek(0) # not needed here, but to remember after writing to fd
reqfactory = APIRequestFactory() # initialize in setUp if used by more tests
view = MyViewSet({'post': 'create'}) # for ViewSet {action:method} needed, for View, not.
request = factory.post('/api/new_file/',
{
"title": 'test file',
"fits_file": self.fd,
},
format='multipart') # multipart is default, but for clarification that not json
response = view(request)
response.render()
self.assertEqual(response.status_code, 201)
Note that there is no authorization for clarity, as with: 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.AllowAny'].
I have pisa producing .pdfs in django in the browser fine, but what if I want to automatically write the file to disk? What I want to do is to be able to generate a .pdf version file at specified points in time and save it in a uploads directory, so there is no browser interaction. Is this possible?
Yes it is possible. for example, using code from Greg Newman as a starter:
from django.template.loader import get_template
from django.template import Context
import ho.pisa as pisa
import cStringIO as StringIO
import cgi
def write_pdf(template_src, context_dict, filename):
template = get_template(template_src)
context = Context(context_dict)
html = template.render(context)
result = open(filename, 'wb') # Changed from file to filename
pdf = pisa.pisaDocument(StringIO.StringIO(
html.encode("UTF-8")), result)
result.close()
You just need to call write_pdf with a template, data in a dict and a file name.