I've been trying to get the tests running on upload forms. But, whenever I run the tests, it says that file is of wrong type.
Upload form saves the file with a randomly generated file name:
class Video(models.Model):
def get_generated_path(self):
# Generates random path for video file upload
original_file = models.CharField(null=True)
uploaded_file = models.FileField(storage=FileSystemStorage(location=
settings.MEDIA_ROOT), upload_to=get_generated_path)
video_name = models.TextField()
And form looks like:
class VideoForm(forms.Form):
video_file = forms.FileField()
video_name = forms.CharField()
def clean_video_file(forms.Form):
content = self.cleaned_data['video_file']
content_type = content.content_type.split('/')[0]
if content_type in settings.CONTENT_TYPES:
if content._size > settings.MAX_UPLOAD_SIZE:
raise forms.ValidationError(_('Please keep filesize under %s.
Current filesize %s') % (filesizeformat(
settings.MAX_UPLOAD_SIZE), filesizeformat(content._size)))
else:
raise forms.ValidationError(_('File type is not supported,
content type is: %s' % content_type))
return content
Most of the remaining logic is in views:
def upload_video(request):
try:
# Check if user is authenticated
if form.is_valid():
video_file = request.FILES['video_file']
video_name = form.cleaned_data['video_name']
save_video = Video.objects.create(
original_file = 'uploaded_videos' + user.username,
video_name = video_name)
return HTTPResponseRedirect('next-page')
except Exception, e:
...
The tests are written as:
def test_video_form(TestCase):
user = #Create a dummy user
test_video = SimpleUploadedFile('test_video.flv', open(
'path/to/test/video', 'rb'))
form = VideoForm(user, {'video_name': test_video}, )
self.assertTrue(form.is_valid())
The above test always fails since it says that the file type of 'test_video.flv' is plain/text. I've checked the 'test_video.flv' and its of correct type.
How to pass these files to the upload form and test it.
SimpleUploadedFile has the content_type text/plain by default. It is why your testing code goes fail in clean_video_file and this line:
raise forms.ValidationError(_('File type is not supported,
content type is: %s' % content_type))
is printing:
File type is not supported,
content type is: text/plain
Pass the content_type = 'video'in the SimpleUploadedFile line as shown below.
def test_video_form(TestCase):
user = #Create a dummy user
test_video = SimpleUploadedFile('test_video.flv', open(
'path/to/test/video', 'rb'), content_type='video') # Notice the change here.
form = VideoForm(user, {'video_name': test_video}, )
self.assertTrue(form.is_valid())
Related
This is my Django Model where I'm trying to store the content of an uploaded file in field attachments:
class CsvFile(models.Model):
processed = models.BooleanField(default=False)
uid = models.UUIDField(unique=True, default=str(uuid4()))
date = models.DateField(null=False, default=datetime.datetime.now().date())
time = models.TimeField(null=False, default=datetime.datetime.now().time())
original_filename = models.CharField(max_length=600, blank=True)
attachment = models.TextField(blank=True)
file = models.FileField(upload_to=f"csv/", blank=True)
def save_file_content_to_attachment(self, file):
try:
with file.open('r') as f:
self.attachment = f.read()
except (FileNotFoundError, ValueError):
self.attachment = ''
def save(self, *args, **kwargs):
# Save the uploaded file to the csv_path field
self.original_filename = self.file.name
# Print a message to help troubleshoot the issue
print(f"Saving file content to attachment for file {self.file.name}")
# Save the file content to the attachment field
self.save_file_content_to_attachment(self.file)
super(CsvFile, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
# Delete the file from storage
try:
default_storage.delete(self.file.name)
except FileNotFoundError:
pass # File does not exist, so we can ignore the exception
super(CsvFile, self).delete(*args, **kwargs)
Unfotunately an upload of a file fails with I/O error
Here is the full backtrace
https://hastebin.skyra.pw/mizawicane.css
Can somebody shed light into why this is not working?
In case others run into the same problem.
This is how I solved it:
def save_file_content_to_attachment(self, file):
# Make sure the file is open in read mode
if not file.closed:
file.open('r')
# Make sure the file is a file-like object that supports reading
if hasattr(file, 'read'):
try:
# Read the contents of the file and save them to the attachment field
self.attachment = file.read()
except FileNotFoundError:
# If the file is not found, set the attachment field to an empty string
self.attachment = ''
else:
# If the file is not a file-like object that supports reading, set the attachment field to an empty string
self.attachment = ''
I'm using PIL to resize an uploaded photo before saving. Note that I'm using formsets to upload the pictures. I'm using BytesIO to open the file. At the last step, I get the error - '_io.BytesIO' object has no attribute 'name'. Why is this?
def fsbo_create_listing(request):
PhotoFormSet = formset_factory(OwnerListingPhotoForm, extra=15)
if request.method == 'POST':
form = OwnerListingForm(request.POST)
photo_formset = PhotoFormSet(request.POST, request.FILES)
if form.is_valid() and photo_formset.is_valid():
form.instance.user = request.user
form.save()
for i in photo_formset:
if i.instance.pk and i.instance.photo == '':
i.instance.delete()
elif i.cleaned_data:
temp = i.save(commit=False)
temp.listing = form.instance
temp.save() # Where the error happens
def clean_photo(self):
picture = self.cleaned_data.get('photo')
# I had to import ImageFieldFile. If picture is already uploaded, picture would still be retrieved as ImageFieldFile. The following line checks the variable type of `picture` to determine whether the cleaning should proceed.
if type(picture) != ImageFieldFile:
image_field = self.cleaned_data.get('photo')
image_file = BytesIO(image_field.read())
image = Image.open(image_file)
image = ImageOps.fit(image, (512,512,), Image.ANTIALIAS)
image_file = BytesIO()
image.save(image_file, 'JPEG', quality=90)
image_field.file = image_file
#if picture._size > 2*1024*1024:
#raise ValidationError("Image file too large. Max size is 2MB.")
return picture
class OwnerListingPhoto(models.Model):
listing = models.ForeignKey(OwnerListing, on_delete=models.CASCADE, related_name='owner_listing_photo')
photo = models.ImageField(upload_to=owner_listing_folder_name)
The issue is that new versions of Django default to using MemoryFileUploadHandler, which doesn't create a temporary file, and therefore there is no file "name." See related Django ticket.
You'll probably have to modify your code a bit to make this work, but you can at least start getting the name property by setting:
FILE_UPLOAD_HANDLERS = [
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
In your settings.py file.
You may find the code I've used to solve almost the exact same issue as helpful.
def clean_logo_file(self):
logo_file_field = self.cleaned_data.get('logo_file')
if logo_file_field:
try:
logo_file = logo_file_field.file
with Image.open(logo_file_field.file.name) as image:
image.thumbnail((512, 512), Image.ANTIALIAS)
image.save(logo_file, format=image.format)
logo_file_field.file = logo_file
return logo_file_field
except IOError:
logger.exception("Error during image resize.")
Additional information on upload handlers.
If file is bigger than 2.5mb (2621440 bytes) - Django will
use TemporaryFileUploadHandler.
Otherwise Django will use MemoryFileUploadHandler.
You can change FILE_UPLOAD_MAX_MEMORY_SIZE (doc) in settings.py
Or change FILE_UPLOAD_HANDLERS (doc) as Nostalg.io mentioned above.
My example with Django Rest Framework serializers:
Broken code:
# models.py
class ImageModel(Model):
image = models.ImageField(upload_to='images/', null=False, blank=False)
# serializers.py
class ImageSerializer(serializers.ModelSerializer):
class Meta:
model = ImageModel
fields = ["id", "image"]
read_only_fields = ["id"]
def validate_image(self, user_img):
img = Image.open(user_img)
... # process image here
img_io = io.BytesIO()
img.save(img_io, format='JPEG', quality=100)
filename = "%s.jpg" % user_img.name.split('.')[0]
user_img.name = "%s.jpg" % user_img.name.split('.')[0]
user_img.file = img_io # BAD IDEA!!!
# This overrides django's tempfile._TemporaryFileWrapper() with _io.BytesIO() !!!
...
return user_img # if picture bigger than 2.5mb -> gives an error!
Fixed code:
#settings.py
FILE_UPLOAD_HANDLERS = [
'django.core.files.uploadhandler.TemporaryFileUploadHandler',
]
# serializers.py
class ImageSerializer(serializers.ModelSerializer):
class Meta:
model = ImageModel
fields = ["id", "image"]
read_only_fields = ["id"]
def validate_image(self, user_img):
img = Image.open(user_img)
... # process image here
# override old TemporaryFile image with edited image
path_to_tmp = user_img.file.name
new_filename = "%s.jpeg" % user_img.name.split('.')[0]
# set new image name
img.save(path_to_tmp, format='JPEG', quality=100)
user_img.name = new_filename
...
return user_img # no errors more :)
It might be more rational to process image by rewriting save() method in models.py, but I convert images in serializers.py because of handly ValidationError() :)
I am having the following issue with filedownload from product views in Django product site:
The issue: Filesize of downloaded file is practically 1kb whilst it should be a normal image filesize (20kb in my example).
So the to-download file is present in static folder of the product instance.id (static_cdn/protected/instance.id/image.jpg -- context: product site where user can upload a file to the corresponding product view).
However, whenever I try to download it from the product view, it downloads the file with the right filename (including the added instance.id number before the filename), but the filesize is almost null. I think it has to do the with the class ProductDownloadView.
Please find the relevant codes below:
views.py:
class ProductDownloadView(MultiSlugMixin, DetailView):
model = Product
def get(self, request, *args, **kwargs):
obj = self.get_object()
filepath = os.path.join(settings.PROTECTED_ROOT, obj.media.path)
response = HttpResponse(file(filepath), content_type="application/force-download")
response["Content-Disposition"] = "attachment;filename=%s" % (obj.media.name)
response["X-SendFile"] = str(obj.media.name)
return response
models.py
class Product(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True)
managers = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name="managers_product")
media = models.FileField(blank=True,
null=True,
upload_to=download_media_location,
storage=FileSystemStorage(location=settings.PROTECTED_ROOT))
def __unicode__(self):
return self.title
def get_absolute_url(self):
view_name = "products:detail_slug"
return reverse(view_name, kwargs={"slug": self.slug})
def get_download(self):
view_name = "products:download_slug"
url = reverse(view_name, kwargs={"slug": self.slug})
return url
Please find below the printed obj, filepath and response variables:
print obj:
pr8
print filepath:
C:\Users\xx\xx\xx\market_place\static_cdn\protected\8\Beach.jpg
print response:
Content-Type: application/force-download
Content-Disposition: attachment;filename=8/Beach.jpg
X-SendFile: 8/Beach.jpg
���� JFIF �� C
[21/Jun/2017 02:17:05] "GET /products/pr8/download/ HTTP/1.1" 200 52
I think I've found the answer. I got the file-download to work by using the open method instead of the file method. Because of this solution I am deviating from a tutorial, but at least I got the job done.
So I got it working by changing the following rule:
response = HttpResponse(file(filepath), content_type="application/force-download")
into:
response = HttpResponse(open(filepath, "rb"), content_type="application/force-download")
So basically adding a mode to the function. Even the file method works after adding the mode "rb".
try this:
response = HttpResponse(content_type="image/jpeg")
response['X-Sendfile'] = obj.media.path
response['Content-Disposition'] = 'attachment; filename=%s' % smart_str(photo.image.name)
I have an app that lets people upload files, represented as UploadedFiles. However, I want to make sure that users only upload xml files. I know I can do this using magic, but I don't know where to put this check - I can't put it in the clean function since the file is not yet uploaded when clean runs, as far as I can tell.
Here's the UploadedFile model:
class UploadedFile(models.Model):
"""This represents a file that has been uploaded to the server."""
STATE_UPLOADED = 0
STATE_ANNOTATED = 1
STATE_PROCESSING = 2
STATE_PROCESSED = 4
STATES = (
(STATE_UPLOADED, "Uploaded"),
(STATE_ANNOTATED, "Annotated"),
(STATE_PROCESSING, "Processing"),
(STATE_PROCESSED, "Processed"),
)
status = models.SmallIntegerField(choices=STATES,
default=0, blank=True, null=True)
file = models.FileField(upload_to=settings.XML_ROOT)
project = models.ForeignKey(Project)
def __unicode__(self):
return self.file.name
def name(self):
return os.path.basename(self.file.name)
def save(self, *args, **kwargs):
if not self.status:
self.status = self.STATE_UPLOADED
super(UploadedFile, self).save(*args, **kwargs)
def delete(self, *args, **kwargs):
os.remove(self.file.path)
self.file.delete(False)
super(UploadedFile, self).delete(*args, **kwargs)
def get_absolute_url(self):
return u'/upload/projects/%d' % self.id
def clean(self):
if not "XML" in magic.from_file(self.file.url):
raise ValidationError(u'Not an xml file.')
class UploadedFileForm(forms.ModelForm):
class Meta:
model = UploadedFile
exclude = ('project',)
Validating files is a common challenge, so I would like to use a validator:
import magic
from django.utils.deconstruct import deconstructible
from django.template.defaultfilters import filesizeformat
#deconstructible
class FileValidator(object):
error_messages = {
'max_size': ("Ensure this file size is not greater than %(max_size)s."
" Your file size is %(size)s."),
'min_size': ("Ensure this file size is not less than %(min_size)s. "
"Your file size is %(size)s."),
'content_type': "Files of type %(content_type)s are not supported.",
}
def __init__(self, max_size=None, min_size=None, content_types=()):
self.max_size = max_size
self.min_size = min_size
self.content_types = content_types
def __call__(self, data):
if self.max_size is not None and data.size > self.max_size:
params = {
'max_size': filesizeformat(self.max_size),
'size': filesizeformat(data.size),
}
raise ValidationError(self.error_messages['max_size'],
'max_size', params)
if self.min_size is not None and data.size < self.min_size:
params = {
'min_size': filesizeformat(self.min_size),
'size': filesizeformat(data.size)
}
raise ValidationError(self.error_messages['min_size'],
'min_size', params)
if self.content_types:
content_type = magic.from_buffer(data.read(), mime=True)
data.seek(0)
if content_type not in self.content_types:
params = { 'content_type': content_type }
raise ValidationError(self.error_messages['content_type'],
'content_type', params)
def __eq__(self, other):
return (
isinstance(other, FileValidator) and
self.max_size == other.max_size and
self.min_size == other.min_size and
self.content_types == other.content_types
)
Then you can use FileValidator in your models.FileField or forms.FileField as follows:
validate_file = FileValidator(max_size=1024 * 100,
content_types=('application/xml',))
file = models.FileField(upload_to=settings.XML_ROOT,
validators=[validate_file])
From django 1.11, you can also use FileExtensionValidator.
from django.core.validators import FileExtensionValidator
class UploadedFile(models.Model):
file = models.FileField(upload_to=settings.XML_ROOT,
validators=[FileExtensionValidator(allowed_extensions=['xml'])])
Note this must be used on a FileField and won't work on a CharField (for example), since the validator validates on value.name.
ref: https://docs.djangoproject.com/en/dev/ref/validators/#fileextensionvalidator
For posterity: the solution is to use the read method and pass that to magic.from_buffer.
class UploadedFileForm(ModelForm):
def clean_file(self):
file = self.cleaned_data.get("file", False)
filetype = magic.from_buffer(file.read())
if not "XML" in filetype:
raise ValidationError("File is not XML.")
return file
class Meta:
model = models.UploadedFile
exclude = ('project',)
I think what you want to do is to clean the uploaded file in Django's Form.clean_your_field_name_here() methods - the data is available on your system by then if it was submitted as normal HTTP POST request.
Also if you consider this inefficient explore the options of different Django file upload backends and how to do streaming processing.
If you need to consider the security of the system when dealing with uploads
Make sure uploaded file has correct extension
Make sure the mimetype matches the file extension
In the case you are worried about user's uploading exploit files (for attacking against your site)
Rewrite all the file contents on save to get rid of possible extra (exploit) payload (so you cannot embed HTML in XML which the browser would interpret as a site-origin HTML file when downloading)
Make sure you use content-disposition header on download
Some more info here: http://opensourcehacker.com/2013/07/31/secure-user-uploads-and-exploiting-served-user-content/
Below is my example how I sanitize the uploaded images:
class Example(models.Model):
image = models.ImageField(upload_to=filename_gen("participant-images/"), blank=True, null=True)
class Example(forms.ModelForm):
def clean_image(self):
""" Clean the uploaded image attachemnt.
"""
image = self.cleaned_data.get('image', False)
utils.ensure_safe_user_image(image)
return image
def ensure_safe_user_image(image):
""" Perform various checks to sanitize user uploaded image data.
Checks that image was valid header, then
:param: InMemoryUploadedFile instance (Django form field value)
:raise: ValidationError in the case the image content has issues
"""
if not image:
return
assert isinstance(image, InMemoryUploadedFile), "Image rewrite has been only tested on in-memory upload backend"
# Make sure the image is not too big, so that PIL trashes the server
if image:
if image._size > 4*1024*1024:
raise ValidationError("Image file too large - the limit is 4 megabytes")
# Then do header peak what the image claims
image.file.seek(0)
mime = magic.from_buffer(image.file.getvalue(), mime=True)
if mime not in ("image/png", "image/jpeg"):
raise ValidationError("Image is not valid. Please upload a JPEG or PNG image.")
doc_type = mime.split("/")[-1].upper()
# Read data from cStringIO instance
image.file.seek(0)
pil_image = Image.open(image.file)
# Rewrite the image contents in the memory
# (bails out with exception on bad data)
buf = StringIO()
pil_image.thumbnail((2048, 2048), Image.ANTIALIAS)
pil_image.save(buf, doc_type)
image.file = buf
# Make sure the image has valid extension (can't upload .htm image)
extension = unicode(doc_type.lower())
if not image.name.endswith(u".%s" % extension):
image.name = image.name + u"." + extension
I found an interesting package who can do upload file validation recently. You can see the package here. the package approach is similar with sultan answer, thus we can just implement it right away.
from upload_validator import FileTypeValidator
validator = FileTypeValidator(
allowed_types=['application/msword'],
allowed_extensions=['.doc', '.docx']
)
file_resource = open('sample.doc')
# ValidationError will be raised in case of invalid type or extension
validator(file_resource)
I've tried to import a csv file into a database by tweaking the modelform inside the admin doing this:
models.py:
class Data(models.Model):
place = models.ForeignKey(Places)
time = models.DateTimeField()
data_1 = models.DecimalField(max_digits=3, decimal_places=1)
data_2 = models.DecimalField(max_digits=3, decimal_places=1)
data_3 = models.DecimalField(max_digits=4, decimal_places=1)
Forms.py:
import csv
class DataImport(ModelForm):
file_to_import = forms.FileField()
class Meta:
model = Data
fields = ("file_to_import", "place")
def save(self, commit=False, *args, **kwargs):
form_input = DataImport()
self.place = self.cleaned_data['place']
file_csv = request.FILES['file_to_import']
datafile = open(file_csv, 'rb')
records = csv.reader(datafile)
for line in records:
self.time = line[1]
self.data_1 = line[2]
self.data_2 = line[3]
self.data_3 = line[4]
form_input.save()
datafile.close()
Admin.py:
class DataAdmin(admin.ModelAdmin):
list_display = ("place", "time")
form = DataImport
admin.site.register(Data, DataAdmin)
But i'm stuck trying to import the file i put in "file_to_import" field. Getting AttributeError in forms.py : 'function' object has no attribute 'FILES'.
What i'm doing wrong?
After a long search i found an answer: Create a view inside the admin using a standard form
Form:
class DataInput(forms.Form):
file = forms.FileField()
place = forms.ModelChoiceField(queryset=Place.objects.all())
def save(self):
records = csv.reader(self.cleaned_data["file"])
for line in records:
input_data = Data()
input_data.place = self.cleaned_data["place"]
input_data.time = datetime.strptime(line[1], "%m/%d/%y %H:%M:%S")
input_data.data_1 = line[2]
input_data.data_2 = line[3]
input_data.data_3 = line[4]
input_data.save()
The view:
#staff_member_required
def import(request):
if request.method == "POST":
form = DataInput(request.POST, request.FILES)
if form.is_valid():
form.save()
success = True
context = {"form": form, "success": success}
return render_to_response("imported.html", context,
context_instance=RequestContext(request))
else:
form = DataInput()
context = {"form": form}
return render_to_response("imported.html", context,
context_instance=RequestContext(request))
The rest is part of this post:
http://web.archive.org/web/20100605043304/http://www.beardygeek.com/2010/03/adding-views-to-the-django-admin/
Take a look at django-admin-import, it does more or less exactly what you want -- you can upload a XLS (not a CSV, but that should not matter) and lets you assign columns to model fields. Default values are also supported.
https://pypi.org/project/django-admin-import/
Additionally, it does not take away the possibility to modify individual records by hand because you don't have to replace the default model form used in the administration.
In the save() method, you don't have any access to the request object - you can see that it's not passed in. Normally you would expect to have a NameError there, but I suspect that you've got a function elsewhere in the file called request().
At the point of saving, all the relevant data should be in cleaned_data: so you should be able to do
file_csv = self.cleaned_data['file_to_import']
At that point you'll have another problem, which is when you get to open - you can't do that, as file_to_import is not a file on the server filesystem, it's an in-memory file that has been streamed from the client. You should be able to pass file_csv directly to csv.reader.