I'm using Python2.7, django==1.7 and uwsgi for streaming video/mp4 file to iPhone player.
My code is as below:
def stream(request):
with open('/path/video.mp4', 'r') as video_file:
response = HttpResponse(video_file.read(), content_type='video/mp4')
response['Content-Disposition'] = 'inline; filename=%s' % 'video.mp4'
return response
video_file.close
When i use some small video (less than 1MB), it streams in browser, but in iPhone palyer i have this error:
[uwsgi-http key: 127.0.0.1:8008 client_addr: 192.168.0.172
client_port: 14563] hr_write(): Broken pipe [plugins/http/http.c line
564]
And when the video size is more that 5MB, it doesn't stream in both (means browser and iPhone player) with same error.
I tried to do that by chunk chunk returning using StreamHttpRespose as below:
def read(chunksize=8192):
with open('/path/video.mp4', 'rb') as video_file:
byte = video_file.read(chunksize)
while byte:
yield byte
return StreamingHttpResponse(read(), content_type='video/mp4')
But there is the same error: Broken pipe.
fyi I can stream pdf and image files. This problem is only with mp4 files. And also i changed the content_type to 'video-mpeg', the browser downloaded that, while i want to prevent file downloading.
What's your idea? Any solution!!?
I had the same problem and did a lot of digging before finding a workable solution!
Apparently the Accept Ranges header is needed for HTML5 video controls to work (https://stackoverflow.com/a/24977085/4264463). So, we need to both parse the requested range from HTTP_RANGE and return Content-Range with the response. The generator that is passed to StreamingHttpResponse also needs to return content based on this range as well (by offset and length). I've found the follow snippet that works great (from http://codegist.net/snippet/python/range_streamingpy_dcwatson_python):
import os
import re
import mimetypes
from wsgiref.util import FileWrapper
from django.http.response import StreamingHttpResponse
range_re = re.compile(r'bytes\s*=\s*(\d+)\s*-\s*(\d*)', re.I)
class RangeFileWrapper(object):
def __init__(self, filelike, blksize=8192, offset=0, length=None):
self.filelike = filelike
self.filelike.seek(offset, os.SEEK_SET)
self.remaining = length
self.blksize = blksize
def close(self):
if hasattr(self.filelike, 'close'):
self.filelike.close()
def __iter__(self):
return self
def __next__(self):
if self.remaining is None:
# If remaining is None, we're reading the entire file.
data = self.filelike.read(self.blksize)
if data:
return data
raise StopIteration()
else:
if self.remaining <= 0:
raise StopIteration()
data = self.filelike.read(min(self.remaining, self.blksize))
if not data:
raise StopIteration()
self.remaining -= len(data)
return data
def stream_video(request, path):
range_header = request.META.get('HTTP_RANGE', '').strip()
range_match = range_re.match(range_header)
size = os.path.getsize(path)
content_type, encoding = mimetypes.guess_type(path)
content_type = content_type or 'application/octet-stream'
if range_match:
first_byte, last_byte = range_match.groups()
first_byte = int(first_byte) if first_byte else 0
last_byte = int(last_byte) if last_byte else size - 1
if last_byte >= size:
last_byte = size - 1
length = last_byte - first_byte + 1
resp = StreamingHttpResponse(RangeFileWrapper(open(path, 'rb'), offset=first_byte, length=length), status=206, content_type=content_type)
resp['Content-Length'] = str(length)
resp['Content-Range'] = 'bytes %s-%s/%s' % (first_byte, last_byte, size)
else:
resp = StreamingHttpResponse(FileWrapper(open(path, 'rb')), content_type=content_type)
resp['Content-Length'] = str(size)
resp['Accept-Ranges'] = 'bytes'
return resp
After a lot of search, i didn't find my solution.
So, i tried to create a stream-server easily using nodejs from html5-video-streamer.js reference as below:
var http = require('http'),
fs = require('fs'),
url = require('url'),
basePath = '/var/www/my_project/media/',
baseUrl = 'Your Domain or IP',
basePort = 8081;
http.createServer(function (req, res) {
// Get params from request.
var params = url.parse(req.url, true).query,
filePath = basePath + params.type + '/' + params.name,
stat = fs.statSync(filePath),
total = stat.size;
if (req.headers['range']) {
var range = req.headers.range,
parts = range.replace(/bytes=/, "").split("-"),
partialstart = parts[0],
partialend = parts[1],
start = parseInt(partialstart, 10),
end = partialend ? parseInt(partialend, 10) : total-1,
chunksize = (end-start)+1;
var file = fs.createReadStream(filePath, {start: start, end: end});
res.writeHead(206, { 'Content-Range' : 'bytes ' + start + '-' + end + '/' + total,
'Accept-Ranges' : 'bytes',
'Content-Length' : chunksize,
'Content-Type' : 'video/mp4' });
file.pipe(res);
// Close file at end of stream.
file.on('end', function(){
file.close();
});
}
else {
res.writeHead(206, { 'Content-Length' : total,
'Content-Type' : 'video/mp4' });
var file = fs.createReadStream(filePath);
file.pipe(res);
// Close file at end of stream.
file.on('end', function(){
file.close();
});
}
}).listen(basePort, baseUrl);
Now i have separate stream-server with nodejs that streams mp4 files beside python project that provides my APIs.
I'm aware It's not my solution, but it works for me ;)
Related
I now have a function in views.py that determines the user input options and uses multi-threaded new a class
views.py
try:
poc_instance = None
upload_file = request.FILES['file']
ext = upload_file.name.split('.')[-1]
poc = request.POST.get('poc_list')
input_payload = request.POST.get('input_payload')
if upload_file.size != 0 and poc != "" and input_payload != "":
if ext in ['txt']:
uuid_str = uuid.uuid4().hex
upload_file.name = uuid_str + '.txt'
filename = os.path.join('upload/',upload_file.name)
saveFile(upload_file,filename)
with open(filename,'r') as f:
line = f.read().split()
if poc == "test":
poc_instance = test(line,input_payload)
poc_t = Thread(target=run_test,args=(poc_instance,))
poc_t.start()
result = poc_instance.get_data()
Suppose I have a test class that returns a response after url requests and sends it back to the front-end template, but since the thread is running in the background, I can't use get_data to return the result of the response, what can I do to solve this problem?
class test:
def __init__(self,ip,payload):
self.ip = ip
self.payload = payload
self.result = []
def entry(self):
headers={
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3",
}
url = "https://xxxxx.com"
res = requests.get(url=url,headers=headers,verify=False,proxies=proxies)
def get_data(self):
return self.result
I tried not to use thread when calling the class, but this causes the current user to use it while other users have to wait, so I have to use multi-thread to call the class, but this in turn causes the result that my front-end cannot receive the response
I am trying to upload video through twitter API from my website. I scraped their github library code's async upload file for large files. I am uploading the data in chunks. This is the code:
(Note I am using static file size and chunks for the testing purpose would definitely appreciate a dynamic method suggestion)
MEDIA_ENDPOINT_URL = 'https://upload.twitter.com/1.1/media/upload.json'
POST_TWEET_URL = 'https://api.twitter.com/1.1/statuses/update.json'
CONSUMER_KEY = 'xxx'
CONSUMER_SECRET = 'xxx'
ACCESS_TOKEN = 'xxx-xxx'
ACCESS_TOKEN_SECRET = 'xxx'
VIDEO_FILENAME = request.FILES['video']
VIDEO_SIZE = 59467
oauth = OAuth1(CONSUMER_KEY,
client_secret=CONSUMER_SECRET,
resource_owner_key=ACCESS_TOKEN,
resource_owner_secret=ACCESS_TOKEN_SECRET)
request_data = {
'command': 'INIT',
'media_type': 'video/mp4',
'total_bytes': VIDEO_SIZE,
'media_category': 'tweet_video'
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
print req
media_id = req.json()['media_id']
print('Media ID: %s' % str(media_id))
segment_id = 0
bytes_sent = 0
vid_file = VIDEO_FILENAME
while bytes_sent < VIDEO_SIZE:
chunk = vid_file.read(59467)
print('APPEND')
request_data = {
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment_id
}
files = {
'media': chunk
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, files=files, auth=oauth)
if req.status_code < 200 or req.status_code > 299:
print(req.status_code)
print(req.text)
segment_id = segment_id + 1
bytes_sent = vid_file.tell()
print('%s of %s bytes uploaded' % (str(bytes_sent), str(VIDEO_SIZE)))
print('Upload chunks complete.')
request_data = {
'command': 'FINALIZE',
'media_id': media_id,
'media_category': 'tweet_video'
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
print(req.json())
processing_info = req.json().get('processing_info', None)
print(req.status_code)
time.sleep(5)
request_data = {
'status': 'I just uploaded a video with the #TwitterAPI.',
'media_ids': req.json()['media_id_string'],
'media_category': 'tweet_video'
}
req = requests.post(url=POST_TWEET_URL, data=request_data, auth=oauth)
print(req.json())
context = {
'r': req
}
return render_to_response('dashboard/manage_content/display.html', context)
I am hit by the following error:
{"errors":[{"code":324,"message":"Not valid video"}]}
I am uploading a mp4 file of 1.6 mb size. Please let me know if you need any more info.
The solution was to use the actual size of the file instead of just a part of it and make sure you use a function to dynamically get the size. Static size does not work even if all the chunks get uploaded.
I want to get an JSON response from the API and give a response with the selected field. I have fetched the response but I am not able to create a response with the new value. I am very new to python world and in learning stage.
def search(request):
if request.method == 'POST':
searchQry = request.POST.get("searchQry","")
nodeIP = settings.NODEIP
params = {'search':searchQry}
apiResponse = requests.get(url = nodeIP, params = params)
data = apiResponse.json()
newArray = {}
nodeName = 'RPID'
if nodeName == 'RPID':
for x in data:
newArray['cphNumber'] = x["data"]["cphNumber"]
newArray['farmName'] = x['data']['farmName']
newArray['addressLine1'] = x['data']['addressLine1']
return HttpResponse(json.dumps(newArray))
else:
return HttpResponse('Unauthrozed Access')
My response array looks likes this :
[{"data": {"cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13", "postalCode": "13"},"id": "4c1b935664e6f684e89ee363f473ce3567599d4b9da0f5889565d5b6f0b84440"},{"data": {"cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13","postalCode": "13"},"id": "7cbe7be9797896545410ed6c4dcc18064525037bc19fbe9272f9baabbb3216ec"},{"data": { "cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13","postalCode": "13"},"id": "7df10c0b7b84434d5ace6811a1b2752a5e5bca13b691399ccac2a6ee79d17797"}]
In response I am getting only one array. I know I have to do something like newArray[0]['cphNumber'] But I am getting an error. Can you please help me to solve this .
I have made a few Django projects after having read the tutorial but I am by no means an expert in Django.
I am trying to take a screenshot of the current page and store it (if one does not exist).
To achieve this, we require a few things:
function to get screen shot of current page
function to async post this image to a view which should store it
view that stores the posted image
However, the screen shot function results in a Blob and I am having trouble getting a Django view to properly handle this.
A demo project is available here: https://gitlab.com/SumNeuron/so_save_blob
Function for screenshot
const screenshot = (function() {
function urlsToAbsolute(nodeList) {
if (!nodeList.length) {
return [];
}
var attrName = 'href';
if (nodeList[0].__proto__ === HTMLImageElement.prototype
|| nodeList[0].__proto__ === HTMLScriptElement.prototype) {
attrName = 'src';
}
nodeList = [].map.call(nodeList, function (el, i) {
var attr = el.getAttribute(attrName);
if (!attr) {
return;
}
var absURL = /^(https?|data):/i.test(attr);
if (absURL) {
return el;
} else {
return el;
}
});
return nodeList;
}
function addOnPageLoad_() {
window.addEventListener('DOMContentLoaded', function (e) {
var scrollX = document.documentElement.dataset.scrollX || 0;
var scrollY = document.documentElement.dataset.scrollY || 0;
window.scrollTo(scrollX, scrollY);
});
}
function capturePage(){
urlsToAbsolute(document.images);
urlsToAbsolute(document.querySelectorAll("link[rel='stylesheet']"));
var screenshot = document.documentElement.cloneNode(true);
var b = document.createElement('base');
b.href = document.location.protocol + '//' + location.host;
var head = screenshot.querySelector('head');
head.insertBefore(b, head.firstChild);
screenshot.style.pointerEvents = 'none';
screenshot.style.overflow = 'hidden';
screenshot.style.webkitUserSelect = 'none';
screenshot.style.mozUserSelect = 'none';
screenshot.style.msUserSelect = 'none';
screenshot.style.oUserSelect = 'none';
screenshot.style.userSelect = 'none';
screenshot.dataset.scrollX = window.scrollX;
screenshot.dataset.scrollY = window.scrollY;
var script = document.createElement('script');
script.textContent = '(' + addOnPageLoad_.toString() + ')();';
screenshot.querySelector('body').appendChild(script);
var blob = new Blob([screenshot.outerHTML], {
type: 'text/html'
});
return blob;
}
return capturePage
})()
Function to async post Blob
function setupAjaxWithCSRFToken() {
// using jQuery
var csrftoken = jQuery("[name=csrfmiddlewaretoken]").val();
function csrfSafeMethod(method) {
// these HTTP methods do not require CSRF protection
return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
}
// set csrf header
$.ajaxSetup({
beforeSend: function(xhr, settings) {
if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
xhr.setRequestHeader("X-CSRFToken", csrftoken);
}
}
});
}
function asyncSubmitBlob( url, blob ) {
var fd = new FormData();
fd.append('image', blob);
$.ajax({
url: url,
type: "POST",
data: fd,
contentType: false,
processData: false,
success: function(response){ console.log(response) },
error: function(data){ console.log(data) }
})
}
So to submit a screenshot of the current page:
setupAjaxWithCSRFToken()
const page = window.location.pathname;
const blob_url = "{% url 'my-app:post_blob' 'REPLACE' %}".replace(/REPLACE/,page == '/' ? '' : page)
asyncSubmitBlob( blob_url, screenshot() )
View to store the posted blob image
urls.py
...
from django.urls import include, path
...
app_name='my-app'
url_patterns=[
...
path('post_blob/', views.post_blob, {'page':'/'},name='post_blob'),
path('post_blob/<page>', views.post_blob,name='post_blob'),
...
]
views.py
from .models import PageBlob
...
def post_blob(request, page):
if request.FILES: # save screenshot of specified page
try:
pb = PageBlob.objects.all().filter(page=page))
if not pb.count():
pb = PageBlob()
pb.page = page
pb.blob = request.FILES['image']
pb.save()
return HttpResponse('Blob Submitted')
except:
return HttpResponse('[App::my-app]\tError when requesting page_image({page})'.format(page=page))
else: # return screenshot of requested page
try:
# get objects storing screenshot for requested page
pb = PageBlob.objects.all().filter(page=page)
# if one exists
if pb.count():
pb = pb[0]
## this just returns the string literal "blob"
return HttpResponse(str(pb.blob))
return HttpResponse('[App::my-app]\tNo blob for {page}'.format(page=page))
except:
return HttpResponse('[App::my-app]\tError when trying to retrieve blob for {page}'.format(page=page))
return HttpResponse('Another response')
models.py
class PageBlob(models.Model):
page = models.CharField(max_length=500)
blob = models.TextField(db_column='data', blank=True)
But I can not seem to faithfully capture and retrieve the blob.
Many S.O. questions of storing blobs use the model approach with import base64 to encode and decode the blob. One even recommends using the BinaryField. However, Django's documentation states firmly that BinaryField is not a replacement for the handling of static files.
So how could I achieve this?
S.O. posts I have found helpful to get this far
Upload an image blob from Ajax to Django
How to screenshot website in JavaScript client-side / how Google did it? (no need to access HDD)
How to download data in a blob field from database in django?
passing blob parameter to django
Returning binary data with django HttpResponse
https://djangosnippets.org/snippets/1597/
Django Binary or BLOB model field
Django CSRF check failing with an Ajax POST request
How can javascript upload a blob?
I developed an Angular application where the user can handle brands.
When creating/updating a brand, the user can also upload a logo. All data are sent to the DB via a REST API built using the Django REST Framework.
Using the Django REST Framework API website I'm able to upload files, but using Angular when I send data thu the API I get an error.
I also tried to encode the File object to base64 using FileReader, but I get the same error from Django.
Can you help me understanding the issue?
Models:
export class Brand {
id: number;
name: string;
description: string;
is_active: boolean = true;
is_customer_brand: boolean = false;
logo_img: Image;
}
export class Image {
id: number;
img: string; // URL path to the image (full size)
img_md: string; // medium size
img_sm: string; // small
img_xs: string; // extra-small/thumbnail
}
Service:
import { Injectable } from '#angular/core';
import { Http, Response } from '#angular/http';
import { Headers, RequestOptions } from '#angular/http';
import { Observable } from 'rxjs/Observable';
import 'rxjs/add/operator/map';
import 'rxjs/add/operator/catch';
import { Brand } from './brand';
const endpoint = 'http://127.0.0.1:8000/api/brands/'
#Injectable()
export class BrandService {
private brands: Array<Brand>;
constructor(private http: Http) { }
list(): Observable<Array<Brand>> {
return this.http.get(endpoint)
.map(response => {
this.brands = response.json() as Brand[];
return response.json();
})
.catch(this.handleError);
}
create(brand: Brand): Observable<Brand> {
console.log(brand);
return this.http.post(endpoint+'create/', brand)
.map(response => response.json())
.catch(this.handleError);
}
get(id): Observable<Brand> {
return this.http.get(endpoint+id)
.map(response => response.json())
.catch(this.handleError);
}
private handleError(error:any, caught:any): any {
console.log(error, caught);
}
}
Error from the browser console:
"{"logo_img":{"img":["The submitted data was not a file. Check the
encoding type on the form."]}}"
Django Serializer:
class BrandSerializer(ModelSerializer):
is_active = BooleanField(required=False)
logo_img = ImageSerializer(required=False, allow_null=True)
class Meta:
model = Brand
fields = [
'id',
'name',
'description',
'is_active',
'is_customer_brand',
'logo_img',
]
def update(self, instance, validated_data):
image = validated_data.get('logo_img',None)
old_image = None
if image:
image = image.get('img',None)
brand_str = validated_data['name'].lower().replace(' ','-')
ext = validated_data['logo_img']['img'].name.split('.')[-1].lower()
filename = '{0}.{1}'.format(brand_str,ext)
user = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
image_serializer_class = create_image_serializer(path='logos', filename=filename, created_by=user, img_config = {'max_w':3000.0,'max_h':3000.0,'max_file_size':1.5,'to_jpeg':False})
image_serializer = image_serializer_class(data=validated_data['logo_img'])
image_serializer.is_valid()
validated_data['logo_img'] = image_serializer.save()
old_image = instance.logo_img
super(BrandSerializer, self).update(instance,validated_data)
if old_image: # Removing old logo
old_image.img.delete()
old_image.img_md.delete()
old_image.img_sm.delete()
old_image.img_xs.delete()
old_image.delete()
return instance
def create(self, validated_data):
image = validated_data.get('logo_img',None)
print(image)
if image:
print(image)
image = image.get('img',None)
print(image)
brand_str = validated_data['name'].lower().replace(' ','-')
ext = validated_data['logo_img']['img'].name.split('.')[-1].lower()
filename = '{0}.{1}'.format(brand_str,ext)
user = None
request = self.context.get('request')
if request and hasattr(request, 'user'):
user = request.user
image_serializer_class = create_image_serializer(path='logos', filename=filename, created_by=user, img_config = {'max_w':3000.0,'max_h':3000.0,'max_file_size':1.5,'to_jpeg':False})
image_serializer = image_serializer_class(data=validated_data['logo_img'])
image_serializer.is_valid()
validated_data['logo_img'] = image_serializer.save()
return super(BrandSerializer, self).create(validated_data)
When posting a new brand to the server with files, I have three main choices:
Base64 encode the file, at the expense of increasing the data size by around 33%.
Send the file first in a multipart/form-data POST, and return an ID to the client. The client then sends the metadata with the ID, and the server re-associates the file and the metadata.
Send the metadata first, and return an ID to the client. The client then sends the file with the ID, and the server re-associates the file and the metadata.
The Base64 encoding will involve unacceptable payload.
So I choose to use multipart/form-data.
Here's how I implemented it in Angular's service:
create(brand: Brand): Observable<Brand> {
let headers = new Headers();
let formData = new FormData(); // Note: FormData values can only be string or File/Blob objects
Object.entries(brand).forEach(([key, value]) => {
if (key === 'logo_img') {
formData.append('logo_img_file', value.img);
} else {
formData.append(key, value);
});
return this.http.post(endpoint+'create/', formData)
.map(response => response.json())
.catch(this.handleError);
}
IMPORTANT NOTE: Since there's no way to have nested fields using FormData, I cannot append formData.append('logo_img', {'img' : FILE_OBJ }). I had change the API in order to receive the file in one field called logo_img_file.
Hope that my issue helped someone.