saving image/doc or media files dynamicallly created folder from input form in django - django

I want to make a Notes Platform where teachers can upload Notes and others can access it . The problem is whenever I am uploading the image it is going to the same folder and I am not able to categorise it according to year,branch,subject, and unit-wise.
Since all the media files are uploaded to the same folder I am unable to get the logic how will i able to fetch the media/image/doc file when a user query for the notes.
my model form is :-
class Note(models.Model):
year_choices = (
( 1 , 'First' ),
( 2 , 'Second'),
( 3 , 'Third' ),
( 4 , 'Fourth')
)
branch_choices = (
( 'IT','IT' ),
( 'EE','EE' ),
( 'CSE','CSE'),
( 'EC','EC' ),
( 'ME','ME' ),
( 'CE','CE' ),
)
unit_choices = ((1,'1'),(2,'2'),(3,'3'),(4,'4'),(5,'5'),
(6,'6'),(7,'7'),(8,'8'),(9,'9'),(10,'10'))
branch = models.CharField(max_length=55,choices=branch_choices)
year = models.IntegerField(choices = year_choices)
subject_name = models.CharField(max_length = 15)
unit = models.IntegerField(choices=unit_choices)
location = 'images'
picture = models.ImageField(upload_to = location)
My notes uploading form and searchform(for searching) is field is :-
class notesform(forms.ModelForm):
class Meta:
model = Note
fields = [ 'year','branch','subject_name','unit','picture' ]
class searchform(forms.ModelForm):
class Meta:
model = Note
fields = [ 'year','branch','subject_name','unit' ]
My notes adding function logic is :-
def addNotes(request):
if request.user.is_authenticated():
if request.method == "POST":
form = notesform(request.POST,request.FILES)
if form.is_valid():
profile = Note()
profile.year = form.cleaned_data["year"]
profile.branch = form.cleaned_data["branch"]
profile.subject_name = form.cleaned_data["subject_name"]
profile.picture = form.cleaned_data["picture"]
post = form.save(commit=False)
post.save()
return redirect('Notes', pk=post.pk)
else:
form = notesform()
return render(request, 'newNotes.html', {'form': form})
else:
return redirect('/accounts/login/')
I want to make upload in such a way that when the teacher fill the form and send upload the media the files upload according to the fields data he will be filling in. For ex:- Teacher fill the form as "First year" then "CSE branch" then "Data Structure" ad "Unit 1" , then it goes to the dynamic media folder "media/images/1/CSE/DATA_STRUCTURE/UNIT_1/".So that i can easily implement the search query .
If it can not applied in this way please suggest some other ways.

The problem is whenever I am uploading the image it is going to the same folder and I am not able to categorise it according to year,branch,subject, and unit-wise.
Instead of creating a new directory for very unit (which is ultimately going to result in a huge directory tree), you can provide a well structured name to uploaded picture and save it in the images directory itself.
Define a function to generate new name for picture:
def generate_picture_name(instance, filename):
url = "images/{0}_{1}_{2}_{3}.jpg".format(
instance.branch, instance.year, instance.subject_name, instance.unit)
return url
And update picture field to use generate_picture_name for saving image file.
picture = models.ImageField(upload_to = generate_picture_name)
Now the image file will be saved as:
media/images/CSE_1_DATA_STRUCTURE_UNIT_1.jpg

Related

Form fields are empty when created using instance = instance

I have two models prodcut_prices and WrongPrice.
In WrongPrice the user can correct report wrong prices - when the price is reported it should also be updated in product_price.
My problem is, even though I instantiate product_price at the very beginning as instance_productprice, all of its required fields returns the "this field has to be filled out" error.
How come those field are not set when im using the instance instance_productprice = product_prices.objects.filter(id=pk)[0] ? Note, that all fields in product_prices are always non-empty since they are being pulled from the product_price model, which is handled in another view, thus that is not the issue.
def wrong_price(request,pk):
#Get the current price object
instance_productprice = product_prices.objects.filter(id=pk)[0]
#Get different values
wrong_link = instance_productprice.link
img_url = instance_productprice.image_url
wrong_price = instance_productprice.last_price
domain = instance_productprice.domain
# Create instances
instance_wrongprice = WrongPrice(
link=wrong_link,
correct_price=wrong_price,
domain = domain)
if request.method == "POST":
form_wrong_price = wrong_price_form(request.POST,instance=instance_wrongprice)
# Update values in product_prices
form_product_price = product_prices_form(request.POST,instance=instance_productprice)
form_product_price.instance.start_price = form_wrong_price.instance.correct_price
form_product_price.instance.last_price = form_wrong_price.instance.correct_price
if form_wrong_price.is_valid() & form_product_price.is_valid():
form_wrong_price.save()
form_product_price.save()
messages.success(request, "Thanks")
return redirect("my_page")
else:
messages.error(request, form_product_price.errors) # Throws empty-field errors,
messages.error(request, form_wrong_price.errors)
return redirect("wrong-price", pk=pk)
else:
form_wrong_price = wrong_price_form(instance=instance_wrongprice)
return render(request, "my_app/wrong_price.html",context={"form":form_wrong_price,"info":{"image_url":img_url}})
I am bit confused about how you implemented it. You have passed a instance of WrongPrice price through the form, which is unnecessary, you could have used initial:
wrong_values = dict(
link=wrong_link,
correct_price=wrong_price,
domain = domain
)
form_wrong_price = wrong_price_form(initial= wrong_values)
Then you are adding values to product_prices_form from instance of form_wrong_price. I don't see why you need a form again here. You can simple use:
form_wrong_price = wrong_price_form(request.POST, initial= wrong_values)
if form_wrong_price.is_valid():
instance = form_wrong_price.save()
instance_productprice.start_price = instance.correct_price
instance_productprice.last_price = instance.correct_price
instance_productprice.save()
Finally, please use PascalCase when defining class names. And you can get the product prices by product_prices.objects.get(id=pk)(instead of filter()[0]).

How to classify an image from Azure Storage in Django using a Tensorflow model

I am developing a django application where the user chooses a machine learning model from a drop down list and uploads an image for classification. This image was initially saved in the project directory (bad, I know) so that I can use it in the classification.
Now I save these images in Azure Storage, but at the moment I can't find a way to access them without having to save them locally to classify them, so I think I'll have to temporarily save them in the project directory and once I use them in the ml model, then I remove the images.
I would like to deploy this application to Azure web service, so I consider it is a bad idea to save and delete images in the project directory.
You can see app's form here
models.py
image is saved in Azure Storage, the other fields in Azure Database for PostgreSQL.
class UploadedImage(models.Model):
image = models.ImageField(upload_to='%Y/%m/%d/')
uploaded = models.DateTimeField(auto_now_add=False, auto_now=True)
title = models.CharField(max_length=50)
prediction = models.FloatField(null=True, blank=True)
def __str__(self):
return self.title
forms.py
class UploadImageForm(forms.ModelForm):
EXTRA_CHOICES = [
('MOB', 'MobileNetV2'),
('VGG', 'VGG-19'),
('CNN', 'CNN 3BI'),
]
predicted_with = forms.ChoiceField(label="Modelo Predictivo",
choices=EXTRA_CHOICES, required=True,
widget=forms.Select(attrs={'class': 'form-control'})
)
class Meta:
model = UploadedImage
fields = [
'image',
]
widgets = {
'image': forms.FileInput(attrs={'class':'custom-file-input'}),
}
views.py
def make_prediction(image_to_predict, model='MOB'):
tf.keras.backend.reset_uids()
folders = {'VGG': 'vgg', 'MOB': 'mobilenet', 'CNN': 'cnn3', 'MSG': 'mobile_sin_gpu'}
model_as_json = 'upload_images/model/%s/modelo.json' % (folders[model])
weights = 'upload_images/model/%s/modelo.h5' % (folders[model])
json_file = open(model_as_json, 'r')
loaded_json_model = json_file.read()
json_file.close()
model = tf.keras.models.model_from_json(loaded_json_model)
model.load_weights(weights)
image = [image_to_predict]
data = img_preprocessing.create_data_batches(image)
return model.predict(data)
def upload_image_view(request):
if request.method == 'POST':
form = forms.UploadImageForm(request.POST, request.FILES)
if form.is_valid():
m = form.save(commit=False)
try:
pred = make_prediction(m.image.path, form.cleaned_data['predicted_with'])[0][0]
if pred > 0.5:
# Code continue...
if status == 200:
m.prediction = pred
m.title = m.image.path
m.save()
# Code continue...
The above snippet worked when I initially saved the images in the project directory but when I started saving the images in Azure Storage I started getting this error:
This backend doesn't support absolute paths.
So I changed the following line: pred = make_prediction(m.image.name, form.cleaned_data['predicted_with'])[0][0]
However now I have this error: NewRandomAccessFile failed to Create/Open: image-100.png : The system cannot find the file specified. ; No such file or directory [[{{node ReadFile}}]] [[IteratorGetNext]] [Op:__inference_predict_function_845] Function call stack: predict_function
For this reason I think my solution would be to temporarily save the image in the project directory, use it in the model and then delete it, however, I do not think it is ideal.
What approach is appropriate to follow in this case?

Limit Amount Of Files A User Can Upload

I have a multi-file upload and want to limit users to 3 uploads each. My problem is that I need to know how many files a user has already created in the DB and how many they are currently uploading (they can upload multiple files at once, and can upload multiple times).
I have attempted many things, including:
Creating a validator (the validator was passed the actual file being added, not a model, so I couldn't access the model to get it's id to call if StudentUploadedFile.objects.filter(student_lesson_data=data.id).count() >= 4:).
Doing the validation in clean(self): (clean was only passed one instance at a time and the DB isn't updated till all files are cleaned, so I could count the files already in the DB but couldn't count how many were currently being uploaded).
Using a pre-save method (If the DB was updated between each file being passed to my pre-save method it would work, but the DB is only updated after all the files being uploaded have passed through my pre-save method).
My post-save attempt:
#receiver(pre_save, sender=StudentUploadedFile)
def upload_file_pre_save(sender, instance, **kwargs):
if StudentUploadedFile.objects.filter(student_lesson_data=instance.data.id).count() >= 4:
raise ValidationError('Sorry, you cannot upload more than three files')
edit:
models.py
class StudentUploadedFile(models.Model):
student_lesson_data = models.ForeignKey(StudentLessonData, related_name='student_uploaded_file', on_delete=models.CASCADE)
student_file = models.FileField(upload_to='module_student_files/', default=None)
views.py
class StudentUploadView(View):
def get(self, request):
files_list = StudentUploadedFile.objects.all()
return render(self.request, 'users/modules.html', {'student_files': files_list})
def post(self, request, *args, **kwargs):
form = StudentUploadedFileForm(self.request.POST, self.request.FILES)
form.instance.student_lesson_data_id = self.request.POST['student_lesson_data_id']
if form.is_valid():
uploaded_file = form.save()
# pass uploaded_file data and username so new file can be added to students file list using ajax
# lesson_id is used to output newly added file to corresponding newly_added_files div
data = {'is_valid': True, 'username': request.user.username, 'file_id': uploaded_file.id, 'file_name': uploaded_file.filename(),
'lesson_id': uploaded_file.student_lesson_data_id, 'file_path': str(uploaded_file.student_file)}
else:
data = {'is_valid': False}
return JsonResponse(data)
template.py
<form id='student_uploaded_file{{ item.instance.id }}'>
{% csrf_token %}
<a href="{% url 'download_student_uploaded_file' username=request.user.username file_path=item.instance.student_file %}" target='_blank'>{{ item.instance.filename }}</a>
<a href="{% url 'delete_student_uploaded_file' username=request.user.username file_id=item.instance.id %}" class='delete' id='{{ item.instance.id }}'>Delete</a>
</form>
js
$(function () {
// open file explorer window
$(".js-upload-photos").on('click', function(){
// concatenates the id from the button pressed onto the end of fileupload class to call correct input element
$("#fileupload" + this.id).click();
});
$('.fileupload_input').each(function() {
$(this).fileupload({
dataType: 'json',
done: function(e, data) { // process response from server
// add newly added files to students uploaded files list
if (data.result.is_valid) {
$("#newly_added_files" + data.result.lesson_id).prepend("<form id='student_uploaded_file" + data.result.file_id +
"'><a href='/student_hub/" + data.result.username + "/download_student_uploaded_file/" +
data.result.file_path + "' target='_blank'>" + data.result.file_name + "</a><a href='/student_hub/" + data.result.username +
"/delete_student_uploaded_file/" + data.result.file_id + "/' class='delete' id=" + data.result.file_id + ">Delete</a></form>")
}
}
});
});
UPDATE:
forms.py
class StudentUploadedFileForm(forms.ModelForm):
student_file = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True}))
view.py
class StudentUploadView(View):
model = StudentUploadedFile
max_files_per_lesson = 3
def post(self, request, *args, **kwargs):
lesson_data_id = request.POST['student_lesson_data_id']
current_files_count = self.model.objects.filter(
student_lesson_data_id=lesson_data_id
).count()
avail = self.max_files_per_lesson - current_files_count
file_list = request.FILES.getlist('student_file')
print(len(file_list))
if avail - len(file_list) < 0:
return JsonResponse(data={
'is_valid': False,
'reason': f'Too many files: you can only upload {avail}.'
})
else:
for f in file_list:
print(f)
data = {'test': True}
return JsonResponse(data)
Thank you.
I've tried using a PyPi package and it works flawlessly. I'm gonna go out on a limb here and assume that you are open to editing the package code to fix any errors you encounter due to compatibility issues since most of the packages that haven't been updated in quite a while might face them.
To solve the problem of limiting the number of files a user can upload, django-multiuploader package would be of enormous help and would honestly do more than you ask for. And yes, it uses JQuery form for uploading multiple files.
How to use it?
Installation and pre-usage steps
Installation
pip install django-multiuploader
python3 manage.py syncdb
python3 manage.py migrate multiuploader
In your settings.py file :
MULTIUPLOADER_FILES_FOLDER = ‘multiuploader’ # - media location where to store files
MULTIUPLOADER_FILE_EXPIRATION_TIME = 3600 # - time, when the file is expired (and it can be cleaned with clean_files command).
MULTIUPLOADER_FORMS_SETTINGS =
{
'default': {
'FILE_TYPES' : ["txt","zip","jpg","jpeg","flv","png"],
'CONTENT_TYPES' : [
'image/jpeg',
'image/png',
'application/msword',
'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/vnd.ms-excel',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/vnd.ms-powerpoint',
'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/vnd.oasis.opendocument.text',
'application/vnd.oasis.opendocument.spreadsheet',
'application/vnd.oasis.opendocument.presentation',
'text/plain',
'text/rtf',
],
'MAX_FILE_SIZE': 10485760,
'MAX_FILE_NUMBER':5,
'AUTO_UPLOAD': True,
},
'images':{
'FILE_TYPES' : ['jpg', 'jpeg', 'png', 'gif', 'svg', 'bmp', 'tiff', 'ico' ],
'CONTENT_TYPES' : [
'image/gif',
'image/jpeg',
'image/pjpeg',
'image/png',
'image/svg+xml',
'image/tiff',
'image/vnd.microsoft.icon',
'image/vnd.wap.wbmp',
],
'MAX_FILE_SIZE': 10485760,
'MAX_FILE_NUMBER':5,
'AUTO_UPLOAD': True,
},
'video':{
'FILE_TYPES' : ['flv', 'mpg', 'mpeg', 'mp4' ,'avi', 'mkv', 'ogg', 'wmv', 'mov', 'webm' ],
'CONTENT_TYPES' : [
'video/mpeg',
'video/mp4',
'video/ogg',
'video/quicktime',
'video/webm',
'video/x-ms-wmv',
'video/x-flv',
],
'MAX_FILE_SIZE': 10485760,
'MAX_FILE_NUMBER':5,
'AUTO_UPLOAD': True,
},
'audio':{
'FILE_TYPES' : ['mp3', 'mp4', 'ogg', 'wma', 'wax', 'wav', 'webm' ],
'CONTENT_TYPES' : [
'audio/basic',
'audio/L24',
'audio/mp4',
'audio/mpeg',
'audio/ogg',
'audio/vorbis',
'audio/x-ms-wma',
'audio/x-ms-wax',
'audio/vnd.rn-realaudio',
'audio/vnd.wave',
'audio/webm'
],
'MAX_FILE_SIZE': 10485760,
'MAX_FILE_NUMBER':5,
'AUTO_UPLOAD': True,
}}
Take note of that MAX_FILE_NUMBER, right within their lies the answer to your question. Have a look at the source once you install this and try implementing it on your own if you want. It might be fun.
Refer for further instructions :
django-multiuploader package on pypi
I guess that you can use multi file upload in Django hasn't trickled to the community yet. Excerpt:
If you want to upload multiple files using one form field, set the multiple HTML attribute of field’s widget:
# forms.py
from django import forms
class FileFieldForm(forms.Form):
file_field = forms.FileField(widget=forms.ClearableFileInput(attrs={'multiple': True}))
Your form and view structure is also very contrived, excluding fields from a form and then setting values injected via HTML form on the model instance. However, with the code shown, the model instance would never exist as the form has no pk field.
Anyway - to focus on the problem that needs fixing...
In the form, request.FILES is now an array:
class StudentUploadView(View):
model = StudentUploadedFile
max_files_per_lesson = 3
def post(request, *args, **kwargs):
lesson_data_id = request.POST['student_lesson_data_id']
current_files_count = self.model.objects.filter(
student_lesson_data_id=lesson_data_id
).count()
avail = self.max_files_per_lesson - current_files_count
file_list = request.FILES.get_list('student_file')
if avail - len(file_list) < 0:
return JsonResponse(data={
'is_valid': False,
'reason': f'Too many files: you can only upload {avail}.'
})
else:
# create one new instance of self.model for each file
...
Addressing comments:
From an aesthetic perspective, you can do a lot with styling...
However, uploading async (separate POST requests), complicates validation and user experience a lot:
The first file can finish after the 2nd, so which are you going to deny if the count is >3.
Frontend validation is hackable, so you can't rely on it, but backend validation is partitioned into several requests, which from the user's point of view is one action.
But with files arriving out of order, some succeeding and some failing, how are you going to provide feedback to the user?
If 1, 3 and 4 arrive, but user cares more about 1, 2, 3 - user has to
take several actions to correct the situation.
One post request:
There are no out of order arrivals
You can use a "everything fails or everything succeeds" approach, which is transparent to the end user and easy to correct.
It's likely that file array order is user preferred order, so even if you allow partial success, you're likely to do the right thing.
So the gist of it is that you are using jQuery .each to upload images via AJAX. Each POST request to your Django view is a single file upload, but there might be multiple requests at the same time.
Try this:
forms.py:
class StudentUploadedFileForm(forms.ModelForm):
class Meta:
model = StudentUploadedFile
fields = ('student_file', )
def __init__(self, *args, **kwargs):
"""Accept a 'student_lesson_data' parameter."""
self._student_lesson_data = kwargs.pop('student_lesson_data', None)
super(StudentUploadedFileForm, self).__init__(*args, **kwargs)
def clean(self):
"""
Ensure that the total number of student_uploaded_file instances that
are linked to the student_lesson_data parameter are within limits."""
cleaned_data = super().clean()
filecount = self._student_lesson_data.student_uploaded_file.count()
if filecount >= 3:
raise forms.ValidationError("Sorry, you cannot upload more than three files")
return cleaned_data
views.py:
class StudentUploadView(View):
def get(self, request):
# stuff ...
def post(self, request, *args, **kwargs):
sld_id = request.POST.get('student_lesson_data_id', None)
student_lesson_data = StudentLessonData.objects.get(id=sld_id)
form = StudentUploadedFileForm(
request.POST,
request.FILES,
student_lesson_data=student_lesson_data
)
if form.is_valid():
uploaded_file = form.save()
# other stuff ...

Django: Math on Models

I need to do some math against a value and I'm very confused about F(). I've read through the docs and searched for examples but I'm missing some fundamentals. Could you help with a solution and give some pointers about how to make sense of this? My attempts below are commented out. I'm just trying to convert Mb to GB. If I could get a 2 decimal value that would be be really wonderful.
class DatastoreInfo(models.Model):
[ ... ]
total_capacity = models.IntegerField(db_column='Total_Capacity', blank=True, null=True)
[ ... ]
class Meta:
managed = False
db_table = 'Datastore_Info'
def ecsdatastores(request):
form = myForm()
results = {}
if request.method == 'POST':
if 'listxyz' in request.POST:
form = myForm(request.POST)
taf = form['taf'].value()
results = DatastoreInfo.objects.filter(f_hostname=c_name)
# results = DatastoreInfo.objects.filter(f_hostname=c_name, total_capacity=F('total_capacity') / 1000)
# results = DatastoreInfo.objects.update(total_capacity=F('total_capacity') / 1000).filter(f_hostname=c_name)
return render(request, 'dpre/datastoreinfo.html', {'form': form , 'results': results})
You almost are there. You need to organize your query so first you filter down to the applicable rows, and then you apply the update with the F function as the value.
# First take your filtered rows
results = DatastoreInfo.objects.filter(f_hostname=c_name)
# Then, on the filtered queryset, you can apply the update.
results.update(total_capacity=F('total_capacity') / 1000)
You can see this is just like the reporter example from the Django F Expression Documentation.
reporter = Reporters.objects.filter(name='Tintin')
reporter.update(stories_filed=F('stories_filed') + 1)
Edit: As a side note you may want to divide by 1024, not 1000, if converting from GB to MB.

flask-admin incorrect relative url

I am trying to create a simple admin for editing a mongo collection. I have posted the code below. It all works perfectly locally or as a docker container. However when I deploy this in our micro-service architecture the app lives at: SERVER_NAME/TEAM_NAME/APP_NAME/.
Flask routes set with #app.route work correctly. However the urls in the admin templates are not correct and always start directly at SERVER_NAME ignoring team-name and app-name. The actual pages and resources are located at the correct urls but the urls for the static resources are not found. How do I make sure the urls generated within flask-admin also take into account the relative url?
The code:
import os
import flask_admin
from wtforms import form, fields
from flask_admin.contrib.pymongo import ModelView, filters
# User admin
class WordPairsForm(form.Form):
text = fields.StringField("Text")
language = fields.SelectField("Language", choices=[("de", "german"), ("en", "english"), ("pl", "polish")])
label = fields.SelectField("Label", choices=[("badword", "bad word"), ("no_stay", "no overnight stay")])
active = fields.BooleanField("Active", default="checked")
class WordPairsView(ModelView):
column_list = ("text", "language", "label", "active")
column_sortable_list = ("text", "language", "label", "active")
column_searchable_list = ("text",)
column_filters = (
filters.FilterLike("text", "Text"),
filters.FilterNotLike("text", "Text"),
filters.FilterEqual("language", "Language", options=[("de", "german"), ("en", "english"), ("pl", "polish")]),
filters.FilterEqual("label", "Label", options=[("badword", "bad word"), ("no_stay", "no overnight stay")]),
filters.BooleanEqualFilter("active", "Active")
)
form = WordPairsForm
def create_form(self):
_form = super(WordPairsView, self).create_form()
return _form
def edit_form(self, obj):
_form = super(WordPairsView, self).edit_form(obj)
return _form
def get_list(self, *args, **kwargs):
count, data = super(WordPairsView, self).get_list(*args, **kwargs)
return count, data
def get_url
def add_admin(app):
admin = flask_admin.Admin(
app,
name="CQAS Admin",
url=os.getenv(
"F_ADMIN_URL",
"/admin"
),
static_url_path=os.getenv("F_ADMIN_STATIC_URL", None),
subdomain=os.getenv("F_ADMIN_SUBDOMAIN", None),
endpoint=os.getenv("F_ADMIN_ENDPOINT", None)
)
admin.add_view(WordPairsView(app.data.data, "WordPairs"))