According to the official django documentation about uploads, small files are saved to memory and big files are saved to disk.
I would like to know how to save uploaded files to disk? Is it possible?
This is my code so far. But it only works on memory. When I try to write the file to disk, or the file is big in size the app crashes.
views.py
# ...
def spreadsheet_form(request, id = None):
if is_admin_user(request):
instance = get_object_or_404(Spreadsheet, id=id) if id is not None else None
form = SpreadsheetForm(request.POST or None, request.FILES or None, instance=instance)
if form.is_valid():
spreadsheet = form.save(commit=False)
spreadsheet.name = request.POST['name']
spreadsheet.spreadsheet_file = request.FILES['spreadsheet_file'].name
spreadsheet.size = request.FILES['spreadsheet_file'].size
spreadsheet.save()
handle_uploaded_file(request.FILES['spreadsheet_file'])
return redirect('/spreadsheets/')
return render_to_response("pages/spreadsheet_form.html", {"form": form,"id":id},context_instance=RequestContext(request))
else:
return redirect('/', False)
# ...
def handle_uploaded_file(f):
with open(f.name, 'wb+') as destination:
for chunk in f.chunks():
destination.write(chunk)
models.py
# ...
class Spreadsheet(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=256)
spreadsheet_file = models.FileField(upload_to='spreadsheets/')
size = models.CharField(max_length=32)
created_at = models.DateTimeField(auto_now=True)
# ...
def __unicode__(self):
return u'%s' % (self.name )
settings.py
FILE_UPLOAD_HANDLERS = (
"django.core.files.uploadhandler.MemoryFileUploadHandler",
"django.core.files.uploadhandler.TemporaryFileUploadHandler",
)
FILE_UPLOAD_MAX_MEMORY_SIZE = 7000000
FILE_UPLOAD_TEMP_DIR = '/tmp'
Thanks in advance
There's no such thing as 'save to disk' in the App Engine world. The closest was Blobstore, and now it's GCS.
django-nonrel includes a django storage class to upload to Blobstore. Follow this:
http://www.allbuttonspressed.com/blog/django/2010/06/Uploads-to-Blobstore-and-GridFS-with-Django
You don't need to fiddle with the FILE_UPLOAD_HANDLERS, the defaults are fine.
Related
Currently working on a django social media application where next to posting every post to the feed, the information of each upload should create a pdf document, containing caption, image, created_at and image_id.
I´ve put the canvas into the upload functions, so that both will be created on the same click. So far, i get a pdf (can't get the attributes from the post into the pdf tho) and the post is uploaded just fine.
How do I get the posted data into the pdf?
And how do save that pdf to a folder within the project instead of starting an automatic download? The user should not be able to notice the pdf converting. It is just for back office - but very necessary due to the social media website being a part of a big installation. So how do I get these pdfs?
Here is the code:
views.py
def upload(request):
if request.method == 'POST':
#user = request.user.username
image = request.FILES.get('image_upload')
caption = request.POST['caption']
new_post = Post.objects.create( image=image, caption=caption)
new_post.save()
#create pdf
buffer = io.BytesIO()
p = canvas.Canvas(buffer)
p.drawString(100, 100, "Hello world.")
p = request.FILES.get('post_pdf')
p.drawText('caption')
p.drawImage('image')
p.showPage()
p.save()
buffer.seek(0)
return FileResponse(buffer, as_attachment=True, filename='hello.pdf')
return redirect('/')
else:
return redirect('/')
models.py
class Post(models.Model):
id = models.UUIDField(primary_key=True, default=uuid.uuid4)
image = models.ImageField(upload_to='post_images')
caption = models.TextField(max_length=300)
created_at = models.DateTimeField(auto_now_add=True)
number_of_likes = models.IntegerField(default=0)
number_of_dislikes = models.IntegerField(default=0)
#interaction_count = models.IntegerField(default=0)
#engagement_count = models.IntegerField(null=True)#number_of_dislikes + number_of_likes
def __str__(self):
return self.caption
To add the value of Post.caption into the pdf, use the value of new_post.caption, change this:
p.drawText('caption')
for this:
p.drawText(new_post.caption)
Same for other fields.
This is not as hard as it seems,
so Let's see if you are successful in creating a pdf and now you have to store it in background instead of downloading.
file_name = request.FILES["file_name"]. #suppose file_name is file name
file_name = default_storage.save(rf"{basePath}/media/whatsapp/file_name.pdf", file_name) #{basePath}/media/whatsapp/ is the path name where we want it to be stored
This code return a TypeError as expected str, bytes or os.PathLike object, not InMemoryUploadedFile
I don't know how to pass user data in the form of file and image to my code.py file for making changes to the original.
views.py
def home(request):
new_image = None
file = None
form = ScanForm()
if request.method == 'POST':
form = ScanForm(request.POST, request.FILES)
if form.is_valid():
image = request.FILES['image']
xml_file = request.FILES['xml_file']
new_image = code.create(image, code.search(
xml_file)[0], code.search(xml_file)[1])
form.save()
return render(request, 'app/home.html', {'form': form, 'new_image': new_image})
else:
form = ScanForm()
return render(request, 'app/home.html', {'form': form, 'new_image': new_image})
printing image and xml_file successfully prints out their names
forms.py
class ScanForm(forms.ModelForm):
class Meta:
model = Scan
fields = '__all__'
models.py
class Scan(models.Model):
image = models.ImageField(upload_to='images')
xml_file = models.FileField(upload_to='files')
processed_at = models.DateTimeField(auto_now_add=True)
description = models.CharField(max_length=500, null=True)
class Meta:
ordering = ['-processed_at']
def __str__(self):
return self.description
Here is the code for manipulation of image according to the data in the xml
code.py
def search(path):
new = []
object_names = []
object_values = []
txt = Path(path).read_text()
txt.strip()
names = et.fromstring(txt).findall('object')
for i in names:
object_names.append(i[0].text)
values = et.fromstring(txt).findall('object/bndbox')
for i in values:
for j in i:
object_values.append(int(j.text))
return object_names, object_values
def create(image, object_names, object_values):
img = cv.imread(image)
on = len(object_names)
ov = len(object_values)
for i in list(range(0, ov, on)):
cv.rectangle(img, (object_values[i], object_values[i+1]),
(object_values[i+2], object_values[i+3]), (0, 0, 255), thickness=5)
return img
This code.py works fine if tested by passing data manually using local folder.
Here is the Traceback:
Traceback image
pathlib.Path() handles file paths, not memory objects. request.FILES are the data attached to the POST request. During your handling of a POST request, you can validate this data and decide to save it to the server disk.
If you would like your image processing to read the file from the server disk, you have to save the new model instance first. You can then access the file's path on disk through the name attribute of the model's ImageField, see Using files in models.
If you want to handle the uploaded data before saving it to disk, you can read it as follows:
txt = request.FILES["xml_file"].read()
See UploadedFile.read()
models.py
class FileUpload(models.Model):
File_Name = models.CharField(max_length=255, blank=True)
File_path = models.FileField(upload_to='')
Description = models.CharField(max_length=255, blank=True)
Upload_Date = models.DateTimeField(auto_now_add=True)
forms.py
class FileUploadForm(forms.Form):
class Meta:
model = FileUpload
File_Name = forms.CharField(label="File Name",max_length=255)
Description = forms.CharField(label="Description", max_length=255)
I'm new in Django.I need help. How to upload images in the database and view those images? Thanks in advance!
here paths are stored in database and images are stored in a folder. But I don't need that. I want to save images and path to the database and I need to view that image. Please help!
views.py:
def uploadfile(request):
print('inside upload logic')
if request.method == 'POST':
form = FileUploadForm(request.POST, request.FILES)
if form.is_valid():
# ImageUpload(request.FILES['File_Name'])
myfile = request.FILES['File_Name']
fs = FileSystemStorage()
filename = fs.save(myfile.name, myfile)
uploaded_file_url = fs.url(filename)
newdoc = FileUpload(File_Name=myfile.name, File_path=uploaded_file_url, Description=request.POST['Description'])
newdoc.save()
#return HttpResponse("File uploaded successfuly")
return render(request, 'Login/fileupload.html')
else:
form = FileUploadForm()
return render(request, 'Login/fileupload.html', {
'form': form
})
You normally shouldn't store the image data in your database. If you need to upload and store images, use the ImageField or FileField and follow the instructions from the docs to save the image. The only thing you need to do is:
form = FileUploadForm(request.POST, request.FILES)
if form.is_valid():
uploaded_file = FileUpload(
File_path=request.FILES['File_path'], # or whatever you've called the file input
File_name=form.cleaned_data['File_Name'],
Description=form.cleaned_data['Description'])
uploaded_file.save()
It would be easier to use a ModelForm in your case, then you only need to save the form:
if form.is_valid():
form.save()
This will automatically save the file in the correct way. No need to do the saving manually like you are doing.
To view the image is as simple as this:
file_upload = FileUpload.objects.get(id=34)
file_url = file_upload.File_path.url # url relative to `MEDIA_ROOT`
# or in a template
{% load static %}
{% get_media_prefix %}{{ file_upload.File_path.url }}
If you really need to store the images as binary blobs to your database (but beware that it almost never makes sense to do so), use Django's BinaryField, as described here.
This also means you will have to handle transforming the image back and forth from a binary blob to a proper image file. Consider also storing the content type (image/jpg or image/png or image/webp) since you will need that to properly re-create the file.
I'm using PIL to resize an uploaded photo before saving. Note that I'm using formsets to upload the pictures. I'm using BytesIO to open the file. At the last step, I get the error - '_io.BytesIO' object has no attribute 'name'. Why is this?
def fsbo_create_listing(request):
PhotoFormSet = formset_factory(OwnerListingPhotoForm, extra=15)
if request.method == 'POST':
form = OwnerListingForm(request.POST)
photo_formset = PhotoFormSet(request.POST, request.FILES)
if form.is_valid() and photo_formset.is_valid():
form.instance.user = request.user
form.save()
for i in photo_formset:
if i.instance.pk and i.instance.photo == '':
i.instance.delete()
elif i.cleaned_data:
temp = i.save(commit=False)
temp.listing = form.instance
temp.save() # Where the error happens
def clean_photo(self):
picture = self.cleaned_data.get('photo')
# I had to import ImageFieldFile. If picture is already uploaded, picture would still be retrieved as ImageFieldFile. The following line checks the variable type of `picture` to determine whether the cleaning should proceed.
if type(picture) != ImageFieldFile:
image_field = self.cleaned_data.get('photo')
image_file = BytesIO(image_field.read())
image = Image.open(image_file)
image = ImageOps.fit(image, (512,512,), Image.ANTIALIAS)
image_file = BytesIO()
image.save(image_file, 'JPEG', quality=90)
image_field.file = image_file
#if picture._size > 2*1024*1024:
#raise ValidationError("Image file too large. Max size is 2MB.")
return picture
class OwnerListingPhoto(models.Model):
listing = models.ForeignKey(OwnerListing, on_delete=models.CASCADE, related_name='owner_listing_photo')
photo = models.ImageField(upload_to=owner_listing_folder_name)
The issue is that new versions of Django default to using MemoryFileUploadHandler, which doesn't create a temporary file, and therefore there is no file "name." See related Django ticket.
You'll probably have to modify your code a bit to make this work, but you can at least start getting the name property by setting:
FILE_UPLOAD_HANDLERS = [
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
In your settings.py file.
You may find the code I've used to solve almost the exact same issue as helpful.
def clean_logo_file(self):
logo_file_field = self.cleaned_data.get('logo_file')
if logo_file_field:
try:
logo_file = logo_file_field.file
with Image.open(logo_file_field.file.name) as image:
image.thumbnail((512, 512), Image.ANTIALIAS)
image.save(logo_file, format=image.format)
logo_file_field.file = logo_file
return logo_file_field
except IOError:
logger.exception("Error during image resize.")
Additional information on upload handlers.
If file is bigger than 2.5mb (2621440 bytes) - Django will
use TemporaryFileUploadHandler.
Otherwise Django will use MemoryFileUploadHandler.
You can change FILE_UPLOAD_MAX_MEMORY_SIZE (doc) in settings.py
Or change FILE_UPLOAD_HANDLERS (doc) as Nostalg.io mentioned above.
My example with Django Rest Framework serializers:
Broken code:
# models.py
class ImageModel(Model):
image = models.ImageField(upload_to='images/', null=False, blank=False)
# serializers.py
class ImageSerializer(serializers.ModelSerializer):
class Meta:
model = ImageModel
fields = ["id", "image"]
read_only_fields = ["id"]
def validate_image(self, user_img):
img = Image.open(user_img)
... # process image here
img_io = io.BytesIO()
img.save(img_io, format='JPEG', quality=100)
filename = "%s.jpg" % user_img.name.split('.')[0]
user_img.name = "%s.jpg" % user_img.name.split('.')[0]
user_img.file = img_io # BAD IDEA!!!
# This overrides django's tempfile._TemporaryFileWrapper() with _io.BytesIO() !!!
...
return user_img # if picture bigger than 2.5mb -> gives an error!
Fixed code:
#settings.py
FILE_UPLOAD_HANDLERS = [
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
# serializers.py
class ImageSerializer(serializers.ModelSerializer):
class Meta:
model = ImageModel
fields = ["id", "image"]
read_only_fields = ["id"]
def validate_image(self, user_img):
img = Image.open(user_img)
... # process image here
# override old TemporaryFile image with edited image
path_to_tmp = user_img.file.name
new_filename = "%s.jpeg" % user_img.name.split('.')[0]
# set new image name
img.save(path_to_tmp, format='JPEG', quality=100)
user_img.name = new_filename
...
return user_img # no errors more :)
It might be more rational to process image by rewriting save() method in models.py, but I convert images in serializers.py because of handly ValidationError() :)
I have an app that lets people upload files, represented as UploadedFiles. However, I want to make sure that users only upload xml files. I know I can do this using magic, but I don't know where to put this check - I can't put it in the clean function since the file is not yet uploaded when clean runs, as far as I can tell.
Here's the UploadedFile model:
class UploadedFile(models.Model):
"""This represents a file that has been uploaded to the server."""
STATE_UPLOADED = 0
STATE_ANNOTATED = 1
STATE_PROCESSING = 2
STATE_PROCESSED = 4
STATES = (
(STATE_UPLOADED, "Uploaded"),
(STATE_ANNOTATED, "Annotated"),
(STATE_PROCESSING, "Processing"),
(STATE_PROCESSED, "Processed"),
)
status = models.SmallIntegerField(choices=STATES,
default=0, blank=True, null=True)
file = models.FileField(upload_to=settings.XML_ROOT)
project = models.ForeignKey(Project)
def __unicode__(self):
return self.file.name
def name(self):
return os.path.basename(self.file.name)
def save(self, *args, **kwargs):
if not self.status:
self.status = self.STATE_UPLOADED
super(UploadedFile, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
os.remove(self.file.path)
self.file.delete(False)
super(UploadedFile, self).delete(*args, **kwargs)
def get_absolute_url(self):
return u'/upload/projects/%d' % self.id
def clean(self):
if not "XML" in magic.from_file(self.file.url):
raise ValidationError(u'Not an xml file.')
class UploadedFileForm(forms.ModelForm):
class Meta:
model = UploadedFile
exclude = ('project',)
Validating files is a common challenge, so I would like to use a validator:
import magic
from django.utils.deconstruct import deconstructible
from django.template.defaultfilters import filesizeformat
#deconstructible
class FileValidator(object):
error_messages = {
'max_size': ("Ensure this file size is not greater than %(max_size)s."
" Your file size is %(size)s."),
'min_size': ("Ensure this file size is not less than %(min_size)s. "
"Your file size is %(size)s."),
'content_type': "Files of type %(content_type)s are not supported.",
}
def __init__(self, max_size=None, min_size=None, content_types=()):
self.max_size = max_size
self.min_size = min_size
self.content_types = content_types
def __call__(self, data):
if self.max_size is not None and data.size > self.max_size:
params = {
'max_size': filesizeformat(self.max_size),
'size': filesizeformat(data.size),
}
raise ValidationError(self.error_messages['max_size'],
'max_size', params)
if self.min_size is not None and data.size < self.min_size:
params = {
'min_size': filesizeformat(self.min_size),
'size': filesizeformat(data.size)
}
raise ValidationError(self.error_messages['min_size'],
'min_size', params)
if self.content_types:
content_type = magic.from_buffer(data.read(), mime=True)
data.seek(0)
if content_type not in self.content_types:
params = { 'content_type': content_type }
raise ValidationError(self.error_messages['content_type'],
'content_type', params)
def __eq__(self, other):
return (
isinstance(other, FileValidator) and
self.max_size == other.max_size and
self.min_size == other.min_size and
self.content_types == other.content_types
)
Then you can use FileValidator in your models.FileField or forms.FileField as follows:
validate_file = FileValidator(max_size=1024 * 100,
content_types=('application/xml',))
file = models.FileField(upload_to=settings.XML_ROOT,
validators=[validate_file])
From django 1.11, you can also use FileExtensionValidator.
from django.core.validators import FileExtensionValidator
class UploadedFile(models.Model):
file = models.FileField(upload_to=settings.XML_ROOT,
validators=[FileExtensionValidator(allowed_extensions=['xml'])])
Note this must be used on a FileField and won't work on a CharField (for example), since the validator validates on value.name.
ref: https://docs.djangoproject.com/en/dev/ref/validators/#fileextensionvalidator
For posterity: the solution is to use the read method and pass that to magic.from_buffer.
class UploadedFileForm(ModelForm):
def clean_file(self):
file = self.cleaned_data.get("file", False)
filetype = magic.from_buffer(file.read())
if not "XML" in filetype:
raise ValidationError("File is not XML.")
return file
class Meta:
model = models.UploadedFile
exclude = ('project',)
I think what you want to do is to clean the uploaded file in Django's Form.clean_your_field_name_here() methods - the data is available on your system by then if it was submitted as normal HTTP POST request.
Also if you consider this inefficient explore the options of different Django file upload backends and how to do streaming processing.
If you need to consider the security of the system when dealing with uploads
Make sure uploaded file has correct extension
Make sure the mimetype matches the file extension
In the case you are worried about user's uploading exploit files (for attacking against your site)
Rewrite all the file contents on save to get rid of possible extra (exploit) payload (so you cannot embed HTML in XML which the browser would interpret as a site-origin HTML file when downloading)
Make sure you use content-disposition header on download
Some more info here: http://opensourcehacker.com/2013/07/31/secure-user-uploads-and-exploiting-served-user-content/
Below is my example how I sanitize the uploaded images:
class Example(models.Model):
image = models.ImageField(upload_to=filename_gen("participant-images/"), blank=True, null=True)
class Example(forms.ModelForm):
def clean_image(self):
""" Clean the uploaded image attachemnt.
"""
image = self.cleaned_data.get('image', False)
utils.ensure_safe_user_image(image)
return image
def ensure_safe_user_image(image):
""" Perform various checks to sanitize user uploaded image data.
Checks that image was valid header, then
:param: InMemoryUploadedFile instance (Django form field value)
:raise: ValidationError in the case the image content has issues
"""
if not image:
return
assert isinstance(image, InMemoryUploadedFile), "Image rewrite has been only tested on in-memory upload backend"
# Make sure the image is not too big, so that PIL trashes the server
if image:
if image._size > 4*1024*1024:
raise ValidationError("Image file too large - the limit is 4 megabytes")
# Then do header peak what the image claims
image.file.seek(0)
mime = magic.from_buffer(image.file.getvalue(), mime=True)
if mime not in ("image/png", "image/jpeg"):
raise ValidationError("Image is not valid. Please upload a JPEG or PNG image.")
doc_type = mime.split("/")[-1].upper()
# Read data from cStringIO instance
image.file.seek(0)
pil_image = Image.open(image.file)
# Rewrite the image contents in the memory
# (bails out with exception on bad data)
buf = StringIO()
pil_image.thumbnail((2048, 2048), Image.ANTIALIAS)
pil_image.save(buf, doc_type)
image.file = buf
# Make sure the image has valid extension (can't upload .htm image)
extension = unicode(doc_type.lower())
if not image.name.endswith(u".%s" % extension):
image.name = image.name + u"." + extension
I found an interesting package who can do upload file validation recently. You can see the package here. the package approach is similar with sultan answer, thus we can just implement it right away.
from upload_validator import FileTypeValidator
validator = FileTypeValidator(
allowed_types=['application/msword'],
allowed_extensions=['.doc', '.docx']
)
file_resource = open('sample.doc')
# ValidationError will be raised in case of invalid type or extension
validator(file_resource)