I am working on a project on django 1.9 and using django-rest-framework.
Views.py:
#parser_classes((FileUploadParser,))
class upload_image(APIView):
def post(self, request, format=None):
file_obj = request.data['images']
upload_s3 = FileUpload()
folder_name = 'images'
file_url = upload_s3.put(file_obj, folder_name, file_obj.name)
class File_Upload:
def __init__(self):
self.env = {
"aws_access_key_id": "*************",
"aws_secret_access_key": "***********",
"region_name": "Asia Pacific (Singapore)"
}
self.bucketname = "my_bucket"
self.session = Session(**self.env)
self.s3_client = self.session.client('s3')
def put(self, bytes, folder, file_name):
self.s3_client.put_object(Body=bytes, Bucket=self.bucketname, Key=folder + "/" + file_name)
return "https://s3.%s.amazonaws.com/%s/%s/%s" % (self.env['region_name'], self.bucketname, folder, file_name)
On python shell I am able to upload as well as download images to and from amazon s3. I am using boto for this.
Below is the code that I am using :
>>> import boto
>>> import sys
>>> from boto.s3.connection import S3Connection
>>> from boto.s3.key import Key
>>> conn = boto.connect_s3("ACESS_KEY_ID","SECRET_ACCESS_KEY",host="s3-ap-southeast-1.amazonaws.com",is_secure = False)
>>> k = bucket.get_key("KEY_NAME")
>>> k.get_contents_to_filename("test.file")
>>> k.set_canned_acl('public-read')
>>> url = k.generate_url(0,query_auth=False,force_http=True)
I get a working url. But the problem comes when I want to do the same in the views.py(above code), I get this error:
S3ResponseError: 400 Bad Request
Request Method: POST Request
URL: http://127.0.0.1:8000/admin/imagedata/image/add/
Django Version: 1.9
Exception Type: S3ResponseError
Exception Value: S3ResponseError: 400
Bad Request
Exception Location:/Library/Python/2.7/site-packages/boto/s3/bucket.py in_get_key_internal, line 235
Python Executable: /usr/bin/python Python Version: 2.7.10
kindly tell me where the problem is?
Related
I am using AWS CodeStar (Lambda + API Gateway) to build my serverless API. My lambda function works well in the Lambda console but strangely throws this error when I run the code on AWS CodeStar:
"message": "Internal server error"
Kindly help me with this issue.
import json
import os
import bz2
import pprint
import hashlib
import sqlite3
import re
from collections import namedtuple
from gzip import GzipFile
from io import BytesIO
from botocore.vendored import requests
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
def handler(event, context):
logger.info('## ENVIRONMENT VARIABLES')
logger.info(os.environ)
logger.info('## EVENT')
logger.info(event)
n = get_package_list()
n1 = str(n)
dat = {"total_pack":n1}
return {'statusCode': 200,
'headers': {'Content-Type': 'application/json'},
'body': json.dumps(dat)
}
def get_package_list():
url = "http://amazonlinux.us-east-2.amazonaws.com/2/core/2.0/x86_64/c60ceaf6dfa3bc10e730c9e803b51543250c8a12bb009af00e527a598394cd5e/repodata/primary.sqlite.gz"
db_filename = "dbfile"
resp = requests.get(url, stream=True)
remote_data = resp.raw.read()
cached_fh = BytesIO(remote_data)
compressed_fh = GzipFile(fileobj=cached_fh)
with open(os.path.join('/tmp',db_filename), "wb") as local_fh:
local_fh.write(compressed_fh.read())
package_obj_list = []
db = sqlite3.connect(os.path.join('/tmp',db_filename))
c = db.cursor()
c.execute('SELECT name FROM packages')
for package in c.fetchall():
package_obj_list.append(package)
no_of_packages = len(package_obj_list)
return no_of_packages
Expected Result: should return an Integer (no_of_packages).
I have written code in python that uploads a file to Google Drive, but after it uploads I cannot delete it from local drive, because I get error "Access Denied", but if I exit out of function then I can delete the file. So my question is how can I delete the file from inside the function?
GDriveUpload.py
import os
import httplib2
import ntpath
import oauth2client
from googleapiclient.discovery import build
from googleapiclient.http import MediaFileUpload
# Copy your credentials here
_CLIENT_ID = 'YOUR_CLIENT_ID'
_CLIENT_SECRET = 'YOUR_CLIENT_SECRET'
_REFRESH_TOKEN = 'YOUR_REFRESH_TOKEN'
_PARENT_FOLDER_ID = 'YOUR_PARENT_FOLDER_ID'
_DATA_FILE = 'datafile.dat'
# ====================================================================================
# Upload file to Google Drive
def UploadFile(client_id, client_secret, refresh_token, parent_folder_id, local_file, DeleteOnExit=False):
cred = oauth2client.client.GoogleCredentials(None,client_id,client_secret,refresh_token,None,'https://accounts.google.com/o/oauth2/token',None)
http = cred.authorize(httplib2.Http())
drive_service = build('drive', 'v2', http=http)
media_body = MediaFileUpload(local_file, mimetype='application/octet-stream', chunksize=5242880, resumable=True)
body = {
'title': (ntpath.basename(local_file)),
'parents': [{'id': parent_folder_id}],
'mimeType': 'application/octet-stream'
}
request = drive_service.files().insert(body=body, media_body=media_body)
response = None
while response is None:
status, response = request.next_chunk()
if status:
print "Uploaded %.2f%%" % (status.progress() * 100)
if DeleteOnExit == True:
os.remove(local_file)
# ====================================================================================
if __name__ == '__main__':
UploadFile(_CLIENT_ID, _CLIENT_SECRET, _REFRESH_TOKEN, _PARENT_FOLDER_ID, _DATA_FILE, DeleteOnExit=True)
I could get HTTP Basic Authentication to work using requests:
import requests
request = requests.post(url, auth=(user, pass), data={'a':'whatever'})
And also using urllib2 and urllib:
import urllib2, urllib
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, url, user, pass)
auth_handler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
content = urllib2.urlopen(url, urllib.urlencode({'a': 'whatever'}))
The problem is I get an unauthorized error when I try the same thing with mechanize:
import mechanize, urllib
from base64 import b64encode
browser = mechanize.Browser()
b64login = b64encode('%s:%s' % (user, pass))
browser.addheaders.append(('Authorization', 'Basic %s' % b64login ))
request = mechanize.Request(url)
response = mechanize.urlopen(request, data=urllib.urlencode({'a':'whatever}))
error:
HTTPError: HTTP Error 401: UNAUTHORIZED
The code I tried with mechanize could be trying to authenticate in a different way than the other two code snippets. So the question is how could the same authentication process be achieved in mechanize.
I am using python 2.7.12
The header should have been added to the request instead of the browser. In fact the browser variable isn't even needed.
import mechanize, urllib
from base64 import b64encode
b64login = b64encode('%s:%s' % (user, pass))
request = mechanize.Request(url)
request.add_header('Authorization', 'Basic %s' % b64login )
response = mechanize.urlopen(request, data=urllib.urlencode({'a':'whatever'}))
I have developed an API (Python 3.5, Django 1.10, DRF 3.4.2) that uploads a video file to my media path when I request it from my UI. That part is working fine. I try to write a test for this feature but cannot get it to run successfully.
#views.py
import os
from rest_framework import views, parsers, response
from django.conf import settings
class FileUploadView(views.APIView):
parser_classes = (parsers.FileUploadParser,)
def put(self, request, filename):
file = request.data['file']
handle_uploaded_file(file, filename)
return response.Response(status=204)
def handle_uploaded_file(file, filename):
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
if not os.path.exists(dir_name):
os.makedirs(dir_name)
file_path = os.path.join(dir_name, new_filename)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
and
#test.py
import tempfile
import os
from django.test import TestCase
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from rest_framework.test import APIRequestFactory
from myapp.views import FileUploadView
class UploadVideoTestCase(TestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp(suffix=None, prefix=None, dir=None)
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(), 'video')
factory = APIRequestFactory()
request = factory.put('file_upload/'+filename,
{'file': uploaded_file}, format='multipart')
view = FileUploadView.as_view()
response = view(request, filename)
print(response)
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
In this test, I need to use an existing video file ('media/testfiles/vid.mp4') and upload it since I need to test some processings on the video data after: that's why I reset the MEDIA_ROOT using mkdtemp.
The test fails since the file is not uploaded. In the def put of my views.py, when I print request I get <rest_framework.request.Request object at 0x10f25f048> and when I print request.data I get nothing. But if I remove the FileUploadParser in my view and use request = factory.put('file_upload/' + filename, {'filename': filename}, format="multipart") in my test, I get <QueryDict: {'filename': ['vid']}> when I print request.data.
So my conclusion is that the request I generate with APIRequestFactory is incorrect. The FileUploadParseris not able to retrieve the raw file from it.
Hence my question: How to generate a file upload (test) request with Django REST Framework's APIRequestFactory?
Several people have asked questions close to this one on SO but I had no success with the proposed answers.
Any help on that matter will be much appreciated!
It's alright now! Switching from APIRequestFactory to APIClient, I managed to have my test running.
My new test.py:
import os
import tempfile
from django.conf import settings
from django.core.files import File
from django.core.files.uploadedfile import SimpleUploadedFile
from django.urls import reverse
from rest_framework.test import APITestCase, APIClient
from django.contrib.auth.models import User
class UploadVideoTestCase(APITestCase):
def setUp(self):
settings.MEDIA_ROOT = tempfile.mkdtemp()
User.objects.create_user('michel')
def test_video_uploaded(self):
"""Video uploaded"""
filename = 'vid'
file = File(open('media/testfiles/vid.mp4', 'rb'))
uploaded_file = SimpleUploadedFile(filename, file.read(),
content_type='multipart/form-data')
client = APIClient()
user = User.objects.get(username='michel')
client.force_authenticate(user=user)
url = reverse('file_upload:upload_view', kwargs={'filename': filename})
client.put(url, {'file': uploaded_file}, format='multipart')
dir_name = settings.MEDIA_ROOT + '/scene/' + filename + '/cam1'
new_filename = 'orig.mp4'
file_path = os.path.join(dir_name, new_filename)
self.assertTrue(os.path.exists(file_path))
Below, testing file upload using APIRequestFactory as requested (and ModelViewSet).
from rest_framework.test import APIRequestFactory, APITestCase
from my_project.api.views import MyViewSet
from io import BytesIO
class MyTestCase(APITestCase):
def setUp(self):
fd = BytesIO(b'Test File content') # in-memory file to upload
fd.seek(0) # not needed here, but to remember after writing to fd
reqfactory = APIRequestFactory() # initialize in setUp if used by more tests
view = MyViewSet({'post': 'create'}) # for ViewSet {action:method} needed, for View, not.
request = factory.post('/api/new_file/',
{
"title": 'test file',
"fits_file": self.fd,
},
format='multipart') # multipart is default, but for clarification that not json
response = view(request)
response.render()
self.assertEqual(response.status_code, 201)
Note that there is no authorization for clarity, as with: 'DEFAULT_PERMISSION_CLASSES': ['rest_framework.permissions.AllowAny'].
I am trying to print log on the local environment of Google App engine. It seems the way it should be but still i am not able to print the log. Need some helping hand here?
I need this output on the standard console.
import webapp2
from google.appengine.api import urlfetch
from Webx import WebxClass
import json
import logging
class SearchHandler(webapp2.RequestHandler):
def __init__(self,*args, **kwargs):
super(SearchHandler,self).__init__(*args, **kwargs)
self.result=[]
self.searchPortals = [WebxClass()]
self.asa = []
def handleCallBack(self,rpc,portalObject):
try:
rr = rpc.get_result()
if rr.status_code == 200:
if isinstance(portalObject, WebxClass):
resultList=portalObject.getResultList(rr.content)
self.result.extend(resultList)
except urlfetch.DownloadError:
self.result = 'Error while fetching from portal - ' + portalObject.getName()
def getSearchResult(self):
rpcs=[]
searchKeyword=self.request.get('searchString')
logging.error("------------------------------")
for portal in self.searchPortals:
rpc = urlfetch.create_rpc(deadline=5)
rpc.callback = lambda: self.handleCallBack(rpc, portal)
urlfetch.make_fetch_call(rpc, portal.getSearchURL(searchKeyword))
rpcs.append(rpc)
for rpc in rpcs:
rpc.wait()
self.response.status_int = 200
self.response.headers['Content-Type'] = 'application/json'
self.response.headers.add_header("Access-Control-Allow-Origin", "*")
self.response.write(json.dumps(self.result))
app = webapp2.WSGIApplication([
webapp2.Route(r'/search', methods=['GET'], handler='Torrent.SearchHandler:getSearchResult')
], debug=True)
def main():
logging.getLogger().setLevel(logging.DEBUG)
logging.debug("------------------------------")
app.run()
if __name__ == '__main__':
main()