DRF test multiple image upload - django

I'm trying to test multiple image upload to my server. Here is the serializer:
class ImageSerializer(serializers.ModelSerializer):
image = serializers.ListField(
child=serializers.ImageField(allow_empty_file=True)
)
ImageFactory:
def get_image():
image = Image.new("RGB", (2000, 2000))
file = tempfile.NamedTemporaryFile(suffix=".jpg")
image.save(file)
return file
Test:
def test_upload_multiple_images(self):
self.image = get_image()
with open(self.image.name, "rb") as file:
payload = {
"image": [file, file]
}
response = self.client.post(
reverse("gallery-list", args=[self.item.pk]),
data=payload,
format="multipart"
)
When testing via Postman, images from the array are saved correctly. However when using the test case, I get the following message from the response:
{'image': [{'message': 'The submitted file is empty.', 'code': 'empty'}]}
Before adding allow_empty_file=True, there were two of those messages being returned.
Has anyone got any idea why that would happen?

The problem here is that you're sending the same Image object to the database to be saved twice. Once the first Image object has been decoded to a file to be saved in the database, the Image object will become unreadable, and since its the same object as the second, an error will be thrown.
So if you send the same Image object as two different items, only the first one will be readable. To avoid this you'll have to send two different image objects.
Your ImageFactory can be refactored to:
def get_image(count=1):
images = []
for i in list(range(count)):
file = io.BytesIO()
image = Image.new('RGBA', size=(100, 100), color=(155, 0, 0))
image.save(file, 'png')
file.name = 'test.png'
file.seek(0)
images.append(file)
return images[0] if count == 1 else images
and your Test:
def test_upload_multiple_images(self):
self.images = get_image(2)
payload = {
"image": self.images
}
response = self.client.post(
reverse("gallery-list", args=[self.item.pk]),
data=payload,
format="multipart"
)

Related

Vertex AI batch prediction results in ('ValueError', 1)

I have a predictor that works successfully when run on a local container, but when that container is pushed to the Artifact Registry, and imported to a Vertex AI model, and run in Batch Prediction mode, it returns an empty prediction file and an error file that simply says ('ValueError', 1)
The request is made using Flask.
code (main.py) is
if 'AIP_HEALTH_ROUTE' in os.environ:
AIP_HEALTH_ROUTE = os.environ['AIP_HEALTH_ROUTE']
else:
AIP_HEALTH_ROUTE = '/health'
if 'AIP_PREDICT_ROUTE' in os.environ:
AIP_PREDICT_ROUTE = os.environ['AIP_PREDICT_ROUTE']
else:
AIP_PREDICT_ROUTE = '/predict'
#app.route(AIP_HEALTH_ROUTE, methods=['GET'])
def health():
response = {"message": "OK", "code": "SUCCESS"}
return make_response(jsonify(response), 200)
#app.route(AIP_PREDICT_ROUTE, methods=['POST'])
def predict():
try:
instances = pd.DataFrame(request.json.get("instances"))
# creates a model using model and encoders stored at given GCS location
model = model_active.CustomPredictor('datascience-modelartifacts/my-model/actives')
processed_data = model.process_score_data(instances)
predictions = model.predict(processed_data)
response = {"predictions": predictions}
status = 200
except Exception as e:
response = {"error": str(e)}
status = 500
return make_response(jsonify(response), status)
For local testing, the data looks like:
dat = {"instances": [
{"F1": 1000854828492, "F2": 3000076437878, "F3": 19.99, ...},
{"F1": 1000222326963, "F2": 3000127917915, "F3": 19.99, ...},
... ]}
I call this with
import requests
url = 'http://localhost:5000/predict'
headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
r = requests.post(url, json=dat, headers=headers)
and in this case everything works as expected (I get a list of predicted classes returned).
I can successfully deploy this model as an endpoint and provide this data as test and get predictions returned.
However, for Vertex batch prediction (which is what I ultimately want to enable since we only need this model once a month), I have tried providing this data as both JSONL and CSV and I keep receiving output with no successful predictions and an error file that simply says ('ValueError', 1)
Does anyone have any suggestions what this error means and what is causing it?

<HttpResponse status_code=200, "image/png"> is not JSON serializable

I have build a CNN model and able to classify well. But i want trying to use Django to classify the class of an image when an image is passed in the url. Here are few thing i tried. the prediction function in my apps.py
def prediction(image_loc):
image_path = 'D:/'
image = cv2.imread(os.path.join(image_path,image_loc))
print("image in matrix is ", image)
output = image.copy()
# Pre-Process the image for classification
image = cv2.resize(image, (96, 96))
image = image.astype("float") / 255.0
image = img_to_array(image)
image = np.expand_dims(image, axis=0)
# Load the trained convolutional neural network and the label binarizer
print("[INFO] Loading Model...")
model_RP = load_model(os.path.join(os.getcwd(), "model\cnn_model_RP.hdf5"))
lb_path = os.path.join(os.getcwd(),"model\labelbin_RP")
lb_RP = pickle.loads(open(lb_path, "rb").read())
print("[INFO] Model Loading Complete.")
# Classify the input image
print("[INFO] classifying image...")
proba = model_RP.predict(image)[0]
idx = np.argmax(proba)
label = lb_RP.classes_[idx]
#we will mark our prediction as "correct" of the input image filename contains the predicted label text
#(obviously this makes the assumption that you have named your testing image files this way)
filename = image_loc[image_loc.rfind(os.path.sep) + 1:]
correct = "correct" if filename.rfind(label) != -1 else "incorrect"
# Build the label and draw the label on the image
label = "{}: {:.2f}% ({})".format(label, proba[idx] * 100, correct)
output = imutils.resize(output, width=400)
cv2.putText(output, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2)
# show the output image
print("[INFO] {}".format(label))
#plt.imshow(output)
#plt.show()
#return(plt.show())
return(label) #- WORKING
#return HttpResponse(output, content_type = 'image/png')
#resp = HttpResponse("", content_type = 'image/png')
#resp.write('output')
#return resp
If i return just the label it works, below is Code snippet from my apps.py
label = "{}: {:.2f}% ({})".format(label, proba[idx] * 100, correct)
return (label)
2.a Here is the problem. I am trying to return the image as well with label on it. which i am not successful.
label = "{}: {:.2f}% ({})".format(label, proba[idx] * 100, correct)
output = imutils.resize(output, width=400)
cv2.putText(output, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2)
return HttpResponse(output, content_type = 'image/png')
2.b Here is the 2nd approach to return the image, unfortunately failed.
label = "{}: {:.2f}% ({})".format(label, proba[idx] * 100, correct)
output = imutils.resize(output, width=400)
cv2.putText(output, label, (10, 25), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2)
resp = HttpResponse("", content_type = 'image/png')
resp.write('output')
return resp
Here is my views.py
#api_view(['GET']) # Decorator
def call_model(request):
#if request.method == 'GET':
# sentence is the query we want to get the prediction for
params = request.GET.get('image_loc')
# predict method used to get the prediction
resp = prediction(image_loc = params)
# returning JSON response
return Response(resp)
Stack trace of error, when i run the below on the browser.
http://127.0.0.1:8000/model/img?format=json&image_loc=bulbasaur_plush.png
TypeError at /model/img
<HttpResponse status_code=200, "image/png"> is not JSON serializable
Request Method: GET
Request URL: http://127.0.0.1:8000/model/img?format=json&image_loc=bulbasaur_plush.png
Django Version: 2.2.12
Exception Type: TypeError
Exception Value:
<HttpResponse status_code=200, "image/png"> is not JSON serializable
Exception Location: C:\Anaconda3\envs\tf35\lib\json\encoder.py in default, line 179
Python Executable: C:\Anaconda3\envs\tf35\python.exe
Python Version: 3.5.5
Python Path:
['D:\\Discern\\Term 3\\deploy',
'C:\\Anaconda3\\envs\\tf35\\python35.zip',
'C:\\Anaconda3\\envs\\tf35\\DLLs',
'C:\\Anaconda3\\envs\\tf35\\lib',
'C:\\Anaconda3\\envs\\tf35',
'C:\\Users\\prade\\AppData\\Roaming\\Python\\Python35\\site-packages',
'C:\\Anaconda3\\envs\\tf35\\lib\\site-packages',
'C:\\Anaconda3\\envs\\tf35\\lib\\site-packages\\win32',
'C:\\Anaconda3\\envs\\tf35\\lib\\site-packages\\win32\\lib',
'C:\\Anaconda3\\envs\\tf35\\lib\\site-packages\\Pythonwin']
Server time: Fri, 29 May 2020 04:35:18 +0000
Tried to looks at similar posts but they are related to text, so could not get my issue solved.
Could someone help please.
It looks like resp is a HttpResponse object (can't tell more, you did not gave code for prediction).
You have to make something JSON serializable within the Response() constructor (usually a dict, or a list, with only string / int / boolean).
(BTW, the #api_view decorator of Django Rest Framework is outdated, #action is prefered with last version)

Generate multiple PDFs and zip them for download, all in a single view

I am using xhtml2pdf to generate PDFs in my Django View. The idea is to loop over all the instances that are there in the query, then for each instance create a PDF, then add all the generated PDFs to one zip File for download. The xtml2pdf logic is working okay but the looping logic is what gives me headache.
So this is my function so far:
def bulk_cover_letter(request, ward_id, school_cat_id, cheque_number):
school_type = SchoolType.objects.get(id=school_cat_id)
schools_in_school_type = Applicant.objects.filter(
school_type=school_type, ward_id=ward_id, award_status='awarded'
).order_by().values_list('school_name', flat=True).distinct()
for school in schools_in_school_type:
beneficiaries = Applicant.objects.filter(school_type=school_type, ward_id=ward_id, award_status='awarded', school_name=school)
total_amount_to_beneficiaries = Applicant.objects.filter(school_type=school_type, ward_id=ward_id, award_status='awarded', school_name=school).aggregate(total=Sum('school_type__amount_allocated'))
context = {
'school_name' : school,
'beneficiaries' : beneficiaries,
'total_amount_to_beneficiaries' : total_amount_to_beneficiaries,
'title' : school + ' Disbursement Details',
'cheque_number': cheque_number
}
response = HttpResponse('<title>Cover Letter</title>', content_type='application/pdf')
filename = "%s.pdf" %(cheque_number)
content = "inline; filename=%s" %(filename)
response['Content-Disposition'] = content
template = get_template('cover_letter.html')
html = template.render(context)
result = io.BytesIO()
pdf = pisa.CreatePDF(
html, dest=response, link_callback=link_callback)
if not pdf.error:
# At this point I can generate a single PDF.
# But no idea on what to do next.
# The zipping logic should follow here after looping all the instances - (schools)
From that Point I have no idea on what to do next. Any help will be highly appreciated.
Try this:
Utils.py
def render_to_pdf(template_src, context_dict={}):
template = get_template(template_src)
html = template.render(context_dict)
buffer = BytesIO()
p = pisa.pisaDocument(BytesIO(html.encode("ISO-8859-1")), buffer)
pdf = buffer.getvalue()
buffer.close()
if not p.err:
return pdf#HttpResponse(result.getvalue(), content_type='application/pdf')
return None
def generate_zip(files):
mem_zip = BytesIO()
with zipfile.ZipFile(mem_zip, mode="w",compression=zipfile.ZIP_DEFLATED) as zf:
for f in files:
zf.writestr(f[0], f[1])
return mem_zip.getvalue()
Views.py
def generate_attendance_pdf(modeladmin, request, queryset):
template_path = 'student/pdf_template.html'
files = []
for q in queryset:
context = {
'firstname': q.firstname,
'lastname': q.lastname,
'p_firstname': q.bceID.firstname
}
pdf = render_to_pdf(template_path, context)
files.append((q.firstname + ".pdf", pdf))
full_zip_in_memory = generate_zip(files)
response = HttpResponse(full_zip_in_memory, content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="{}"'.format('attendnace.zip')
return response
Obviously, you have to modify the context/names to what you need.
Credit to -> Neil Grogan https://www.neilgrogan.com/py-bin-zip/
If you need to generate several PDF files and send them as a response in a zip file then you can store the reports in memory and set it as dest when you call pisa.CreatePDF. Then have a list of reports in memory, zip them, and send as a Django response specifying another content type.
For example:
reports = tempfile.TemporaryDirectory()
report_files = {}
for school in schools_in_school_type:
# ... same code that renerates `html`
mem_fp = BytesIO()
pisa.CreatePDF(html, dest=mem_fp)
report_files[filename] = mem_fp
mem_zip = BytesIO()
with zipfile.ZipFile(mem_zip, mode="w") as zf:
for filename, content in report_files.items():
zf.write(filename, content)
response = HttpResponse(mem_zip, content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="{}"'.format('cover_letters.zip')
This still generates an error of [Errno 2] No such file or directory: 'cheque_number.pdf'.

How to resize images before uploading them in Flask?

I have four images i want to upload them, but their sizes are very big it takes long time to upload them into the site.
I want to resize each one of them, i wrote a small route that handle the request:
#team_route.route('/team/dashboard/add/product', methods=['GET', 'POST'])
#requires_auth
#master_login_required(role='master')
def team_add_product():
form = AddProduct()
imagesList = []
size = 1024, 764
if request.method == 'POST' and form.is_submitted():
product = Goods()
file = request.files.getlist("image[]")
if file:
for zipfile in file:
fi = Image.open(BytesIO(zipfile.stream.read()))
im.thumbnail(size)
img2 = im.rotate(-90, expand=True)
img2.seek(0)
img2.save(UPLOAD_FOLDER + '/crops/' + zipfile)
When i hit upload it gives me this error:
AttributeError: 'JpegImageFile' object has no attribute 'read'
Please any help solving this ?
You have to remove .stream
zipfile.read() its the only you need.

ionic 2 upload image to django rest

I am trying to upload an image from Ionic 2 app to Django-powered website through Django Rest API.
The API is working and tested through Postman but I always get HTTP 400 BAD Request error in Ionic.
Here is my code in Ionic:
openCamera(){
var options = {
sourceType: Camera.PictureSourceType.CAMERA,
destinationType: Camera.DestinationType.DATA_URL
};
Camera.getPicture(options).then((imageData) => {
this.imageName = imageData;
this.imageURL = 'data:image/jpeg;base64,' + imageData;
}, (err) => {
this.showAlert(err);
});
}
Upload file (I am serving my Django project on my local PC with IP address 192.168.22.4):
transferData(auth){
let headers = new Headers();
headers.append('Authorization', auth);
let formData = new FormData();
formData.append('image', this.imageURL, this.imageName);
this.http.post("http://192.168.22.4/api-imageUpload", formData, {headers: headers}).subscribe(res => {
let status = res['status'];
if(status == 200){
this.showAlert( "The image was successfully uploaded!");
}else{
this.showAlert("upload error");
}
}, (err) => {
var message = "Error in uploading file " + err
this.showAlert(message);
});
}
On Django, here is my serializer:
class ImageDetailsSerializer(serializers.ModelSerializer):
image = serializers.ImageField(max_length=None, use_url=True)
class Meta:
model = ImageDetails
fields= ('image','status','category', 'user') ####status, category has default value
and views.py:
class ImageDetailsViewSet(generics.ListCreateAPIView):
queryset = ImageDetails.objects.all()
serializer_class = ImageDetailsSerializer
I am not sure if my code in uploading file is correct. I am trying to pass the data through Form data since the form works well in my API. Is this method correct? Are there any other methods to get this work?
Note: I have tried to use Transfer Cordova plugin but it is not working.
I finally solved the problem. The HTTP 400 indicates that there is a syntax error somewhere in the code and that is the encoding used in the uploaded photo. Mobile data uses base64 encoding. When sending requests, the file will then be converted to a Unicode string.
On the other hand, Django-Rest uses normal encoding for images, thus by default, it cannot support base64 image. But luckily, this plugin is already available at GitHub.
You just need to install the plugin and import it on your serializers.py:
from drf_extra_fields.fields import Base64ImageField
class ImageDetailsSerializer(serializers.ModelSerializer):
image = Base64ImageField()
class Meta:
model = ImageDetails
fields= ('image','status','category', 'user')
On Ionic side, you have to submit the actual image not the imageURL. In my case I just have to tweak my code to:
transferData(auth){
let headers = new Headers();
headers.append('Authorization', auth);
let formData = new FormData();
formData.append('category', 1);
formData.append('status', 'Y')
formData.append('image', this.imageName);
this.http.post("http://192.168.22.4/api-imageUpload", formData, {headers: headers}).subscribe(res => {
let status = res['status'];
if(status == 201){
var message = "The image was successfully uploaded!";
this.showAlert(message);
}else{
var message = "upload error";
this.showAlert(message);
}
}, (err) => {
var message = "Error in uploading file " + err;
this.showAlert(message);
});
In order to inspect what's ongoing with the request:
from rest_framework.exceptions import ValidationError
class ImageDetailsViewSet(generics.ListCreateAPIView):
queryset = ImageDetails.objects.all()
serializer_class = ImageDetailsSerializer
def create(self, request, *args, **kwargs):
serializer = self.get_serializer(data=request.data)
if not serializer.is_valid():
print(serializer.errors) # or better use logging if it's configured
raise ValidationError(serialize.errors)
self.perform_create(serializer)
headers = self.get_success_headers(serializer.data)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
Even without Base64 it is possible using ionic native components for file-transfer and image-picker see here: https://gist.github.com/AndreasDickow/9d5fcd2c608b4726d16dda37cc880a7b