Read uploaded fasta file in django using Bio library - django

in index.html I used
<input type="file" name="upload_file">
in views.py
from Bio import SeqIO
def index(request):
if request.method == "POST":
try:
text_file = request.FILES['upload_file']
list_1, list_2 = sequence_extract_fasta(text_file)
context = {'files': text_file}
return render(request, 'new.html', context)
except:
text_file = ''
context = {'files': text_file}
return render(request, 'index.html')
def sequence_extract_fasta(fasta_files):
# Defining empty list for the Fasta id and fasta sequence variables
fasta_id = []
fasta_seq = []
# opening a given fasta file using the file path
with open(fasta_files, 'r') as fasta_file:
print("pass")
# extracting multiple data in single fasta file using biopython
for record in SeqIO.parse(fasta_file, 'fasta'): # (file handle, file format)
print(record.seq)
# appending extracted fasta data to empty lists variables
fasta_seq.append(record.seq)
fasta_id.append(record.id)
# returning fasta_id and fasta sequence to both call_compare_fasta and call_reference_fasta
return fasta_id, fasta_seq
The method sequence_extract_fasta(fasta_files) work with python. But not on the Django framework. If I can find the temporary location of the uploaded file then using the path, I may be able to call the method. Is there any efficient way to solve this? your help is highly appreciated. Thank you for your time.

I found a one way of doing this.
def sequence_extract_fasta(fasta_file):
# Defining empty list for the Fasta id and fasta sequence variables
fasta_id = []
fasta_seq = []
# fasta_file = fasta_file.chunks()
print(fasta_file)
# opening given fasta file using the file path
# crating a backup file with original uploaded file data
with open('data/temp/name.bak', 'wb+') as destination:
for chunk in fasta_file.chunks():
destination.write(chunk)
# opening created backup file and reading
with open('data/temp/name.bak', 'r') as fasta_file:
# extracting multiple data in single fasta file using biopython
for record in SeqIO.parse(fasta_file, 'fasta'): # (file handle, file format)
fasta_seq.append(record.seq)
fasta_id.append(record.id)
# returning fasta_id and fasta sequence to both call_compare_fasta and call_reference_fasta
return fasta_id, fasta_seq

Related

How to zip multiple uploaded file in Django before saving it to database?

I am trying to compress a folder before saving it to database/file storage system using Django. For this task I am using ZipFile library. Here is the code of view.py:
class BasicUploadView(View):
def get(self, request):
file_list = file_information.objects.all()
return render(self.request, 'fileupload_app/basic_upload/index.html',{'files':file_list})
def post(self, request):
zipfile = ZipFile('test.zip','w')
if request.method == "POST":
for upload_file in request.FILES.getlist('file'): ## index.html name
zipfile.write(io.BytesIO(upload_file))
fs = FileSystemStorage()
content = fs.save(upload_file.name,upload_file)
data = {'name':fs.get_available_name(content), 'url':fs.url(content)}
zipfile.close()
return JsonResponse(data)
But I am getting the following error:
TypeError: a bytes-like object is required, not 'InMemoryUploadedFile'
Is there any solution for this problem? Since I may have to upload folder with large files, do I have to write a custom TemporaryFileUploadHandler for this purpose? I have recently started working with Django and it is quite new to me. Please help me with some advice.
InMemoryUploadedFile is an object that contains more than just file you should open file and read it content ( InMemoryUploadedFile.file is the file)
InMemoryUploadedFile.open()
You should open file with open() and then read() it's content, also you should check if you have uploaded files correctly also you could use with syntax for both zip and file
https://www.pythonforbeginners.com/files/with-statement-in-python

how to load csv file data into pandas using request.FILES(django 1.11) without saving file on server

i just want to upload .csv file via form, directly in to pandas dataframe in django without saving physically file on to server.
def post(self, request, format=None):
try:
from io import StringIO, BytesIO
import io
print("data===",request.FILES['file'].read().decode("utf-8"))
# print("file upload FILES data=====",pd.read_csv(request.FILES['file'].read(), sep=','))
#print(request.FILES)
print("file upload data df=====11")
mm = pd.read_csv( BytesIO(request.FILES['file'].read().decode("utf-8")))
print("dataframe data=====",mm)
# import io, csv
# urlData = request.FILES['file']
# data = [row for row in (csv.reader(urlData))]
# print("file upload data df=====222",data)
# mm = pd.read_csv()
#excel_file = request.FILES['file']
# movies = pd.read_excel(request.FILES['file'])
except Exception as e:
print(e)
log.debug("Error in CheckThreadStatus api key required "+str(e))
return Response(responsejson('api key required', status=404))
the ans is straight forward: that is
pd.read_csv(request.FILES['file'])
works perfectly fine, the mistake i was doing is that.. my csv file was not in correct format.
Check With
pd.read_csv('data.csv') # doctest: +SKIP
If using post method you can try
getFile = request.FILE['file_name']
pd.read_csv(getFile) # doctest: +SKIP
You can use StringIO for reading and decoding your csv :
import csv
from io import StringIO
csv_file = request.FILES["csv_file"]
content = StringIO(csv_file.read().decode('utf-8'))
reader = csv.reader(content)
After reading you can populate your database like this :
csv_rows = [row for row in reader]
field_names = csv_rows[0] # Get the header row
del csv_rows[0] # Deleting header after storing it's values in field_names
for index, row in enumerate(csv_rows):
data_dict = dict(zip(field_names, row))
Model.objects.update_or_create(id=row[0],
defaults=data_dict
)
Make sure to validate data before inserting, if the data is critical.
HINT: use django forms to validate for you.
from django import forms

Serving Zip file Django

I'm following this solution (Serving dynamically generated ZIP archives in Django) to serve some zip files from django.
The idea is to select the files from a database using some check boxes, but I'm trying to make the example work with just 2 images.
import os
import zipfile
import StringIO
from django.http import HttpResponse
def getfiles(request):
# Files (local path) to put in the .zip
# FIXME: Change this (get paths from DB etc)
filenames = ["/home/../image1.png", "/home/../image2.png"]
# Folder name in ZIP archive which contains the above files
# E.g [thearchive.zip]/somefiles/file2.txt
# FIXME: Set this to something better
zip_subdir = "somefiles"
zip_filename = "%s.zip" % zip_subdir
# Open StringIO to grab in-memory ZIP contents
s = StringIO.StringIO()
# The zip compressor
zf = zipfile.ZipFile(s, "w")
for fpath in filenames:
# Calculate path for file in zip
fdir, fname = os.path.split(fpath)
zip_path = os.path.join(zip_subdir, fname)
# Add file, at correct path
zf.write(fpath, zip_path)
# Must close zip for all contents to be written
zf.close()
# Grab ZIP file from in-memory, make response with correct MIME-type
resp = HttpResponse(s.getvalue(), mimetype = "application/x-zip-compressed")
# ..and correct content-disposition
resp['Content-Disposition'] = 'attachment; filename=%s' % zip_filename
return resp
I wrote the getfile(request) on my views.py and i make a call from the index view
def index(request):
if request.method == 'POST': # If the form has been submitted...
resp = getfiles(request)
form = FilterForm(request.POST) # A form bound to the POST data
# do some validation and get latest_events from database
context = {'latest_events_list': latest_events_list, 'form': form}
return render(request, 'db_interface/index.html', context)
I know the getfile() method is called, because if I put names of unexistents files I got an error, but I dont get any download neither an error if the filenames are correct (I put the full path /home/myuser/xxx/yyy/Project/app/static/app/image1.png).
I tried with the django server and with the apache2/nginx server I have for production
I also tried using content_type = 'application/force-download'
Thanks

django RequestFactory file upload

I try to create a request, using RequestFactory and post with file, but I don't get request.FILES.
from django.test.client import RequestFactory
from django.core.files import temp as tempfile
tdir = tempfile.gettempdir()
file = tempfile.NamedTemporaryFile(suffix=".file", dir=tdir)
file.write(b'a' * (2 ** 24))
file.seek(0)
post_data = {'file': file}
request = self.factory.post('/', post_data)
print request.FILES # get an empty request.FILES : <MultiValueDict: {}>
How can I get request.FILES with my file ?
If you open the file first and then assign request.FILES to the open file object you can access your file.
request = self.factory.post('/')
with open(file, 'r') as f:
request.FILES['file'] = f
request.FILES['file'].read()
Now you can access request.FILES like you normally would. Remember that when you leave the open block request.FILES will be a closed file object.
I made a few tweaks to #Einstein 's answer to get it to work for a test that saves the uploaded file in S3:
request = request_factory.post('/')
with open('my_absolute_file_path', 'rb') as f:
request.FILES['my_file_upload_form_field'] = f
request.FILES['my_file_upload_form_field'].read()
f.seek(0)
...
Without opening the file as 'rb' I was getting some unusual encoding errors with the file data
Without f.seek(0) the file that I uploaded to S3 was zero bytes
You need to provide proper content type, proper file object before updating your FILES.
from django.core.files.uploadedfile import File
# Let django know we are uploading files by stating content type
content_type = "multipart/form-data; boundary=------------------------1493314174182091246926147632"
request = self.factory.post('/', content_type=content_type)
# Create file object that contain both `size` and `name` attributes
my_file = File(open("/path/to/file", "rb"))
# Update FILES dictionary to include our new file
request.FILES.update({"field_name": my_file})
the boundary=------------------------1493314174182091246926147632 is part of the multipart form type. I copied it from a POST request done by my webbrowser.
All the previous answers didn't work for me. This seems to be an alternative solution:
from django.core.files.uploadedfile import SimpleUploadedFile
with open(file, "rb") as f:
file_upload = SimpleUploadedFile("file", f.read(), content_type="text/html")
data = {
"file" : file_upload
}
request = request_factory.post("/api/whatever", data=data, format='multipart')
Be sure that 'file' is really the name of your file input field in your form.
I got that error when it was not (use name, not id_name)

Django (audio) File Validation

I'm experimenting with a site that will allow users to upload audio files. I've read every doc that I can get my hands on but can't find much about validating files.
Total newb here (never done any file validation of any kind before) and trying to figure this out. Can someone hold my hand and tell me what I need to know?
As always, thank you in advance.
You want to validate the file before it gets written to disk. When you upload a file, the form gets validated then the uploaded file gets passed to a handler/method that deals with the actual writing to the disk on your server. So in between these two operations, you want to perform some custom validation to make sure it's a valid audio file
You could:
check if the the file is less then a certain size (good practice)
then check if the submitted file has a certain content type (i.e. an audio file)
this is pretty useless as someone could easily spoof it
then check that the file ends in a certain extension (or extensions)
this is also pretty useless
try read the file and see if it's actually audio
(I haven't tested this code)
models.py
class UserSong(models.Model):
title = models.CharField(max_length=100)
audio_file = models.FileField()
forms.py
class UserSongForm(forms.ModelForm):
# Add some custom validation to our file field
def clean_audio_file(self):
file = self.cleaned_data.get('audio_file',False):
if file:
if file._size > 4*1024*1024:
raise ValidationError("Audio file too large ( > 4mb )")
if not file.content-type in ["audio/mpeg","audio/..."]:
raise ValidationError("Content-Type is not mpeg")
if not os.path.splitext(file.name)[1] in [".mp3",".wav" ...]:
raise ValidationError("Doesn't have proper extension")
# Here we need to now to read the file and see if it's actually
# a valid audio file. I don't know what the best library is to
# to do this
if not some_lib.is_audio(file.content):
raise ValidationError("Not a valid audio file")
return file
else:
raise ValidationError("Couldn't read uploaded file")
views.py
from utils import handle_uploaded_file
def upload_file(request):
if request.method == 'POST':
form = UserSongForm(request.POST, request.FILES)
if form.is_valid():
# If we are here, the above file validation has completed
# so we can now write the file to disk
handle_uploaded_file(request.FILES['file'])
return HttpResponseRedirect('/success/url/')
else:
form = UploadFileForm()
return render_to_response('upload.html', {'form': form})
utils.py
# from django's docs
def handle_uploaded_file(f):
ext = os.path.splitext(f.name)[1]
destination = open('some/file/name%s'%(ext), 'wb+')
for chunk in f.chunks():
destination.write(chunk)
destination.close()
https://docs.djangoproject.com/en/dev/topics/http/file-uploads/#file-uploads
https://docs.djangoproject.com/en/dev/ref/forms/fields/#filefield
https://docs.djangoproject.com/en/dev/ref/files/file/#django.core.files.File