I am trying to upload video through twitter API from my website. I scraped their github library code's async upload file for large files. I am uploading the data in chunks. This is the code:
(Note I am using static file size and chunks for the testing purpose would definitely appreciate a dynamic method suggestion)
MEDIA_ENDPOINT_URL = 'https://upload.twitter.com/1.1/media/upload.json'
POST_TWEET_URL = 'https://api.twitter.com/1.1/statuses/update.json'
CONSUMER_KEY = 'xxx'
CONSUMER_SECRET = 'xxx'
ACCESS_TOKEN = 'xxx-xxx'
ACCESS_TOKEN_SECRET = 'xxx'
VIDEO_FILENAME = request.FILES['video']
VIDEO_SIZE = 59467
oauth = OAuth1(CONSUMER_KEY,
client_secret=CONSUMER_SECRET,
resource_owner_key=ACCESS_TOKEN,
resource_owner_secret=ACCESS_TOKEN_SECRET)
request_data = {
'command': 'INIT',
'media_type': 'video/mp4',
'total_bytes': VIDEO_SIZE,
'media_category': 'tweet_video'
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
print req
media_id = req.json()['media_id']
print('Media ID: %s' % str(media_id))
segment_id = 0
bytes_sent = 0
vid_file = VIDEO_FILENAME
while bytes_sent < VIDEO_SIZE:
chunk = vid_file.read(59467)
print('APPEND')
request_data = {
'command': 'APPEND',
'media_id': media_id,
'segment_index': segment_id
}
files = {
'media': chunk
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, files=files, auth=oauth)
if req.status_code < 200 or req.status_code > 299:
print(req.status_code)
print(req.text)
segment_id = segment_id + 1
bytes_sent = vid_file.tell()
print('%s of %s bytes uploaded' % (str(bytes_sent), str(VIDEO_SIZE)))
print('Upload chunks complete.')
request_data = {
'command': 'FINALIZE',
'media_id': media_id,
'media_category': 'tweet_video'
}
req = requests.post(url=MEDIA_ENDPOINT_URL, data=request_data, auth=oauth)
print(req.json())
processing_info = req.json().get('processing_info', None)
print(req.status_code)
time.sleep(5)
request_data = {
'status': 'I just uploaded a video with the #TwitterAPI.',
'media_ids': req.json()['media_id_string'],
'media_category': 'tweet_video'
}
req = requests.post(url=POST_TWEET_URL, data=request_data, auth=oauth)
print(req.json())
context = {
'r': req
}
return render_to_response('dashboard/manage_content/display.html', context)
I am hit by the following error:
{"errors":[{"code":324,"message":"Not valid video"}]}
I am uploading a mp4 file of 1.6 mb size. Please let me know if you need any more info.
The solution was to use the actual size of the file instead of just a part of it and make sure you use a function to dynamically get the size. Static size does not work even if all the chunks get uploaded.
Related
Hi everyone New to cosing and I am getting Hashed Signature wrong, Can anyone help me with Dumbed down version of AWS V4 API signature,
Thanks a Million
//Script start
def execute() {
def accountPropertyService = services.getAccountPropertyService()
// List<String> entityUrls = context.getStringListVariable("entityUrls")
def sessionAccessKey = context.getStringVariable("sessionAccessKey")
def sessionSecretKey = context.getStringVariable("sessionSecretKey")
def sessionExpiry = context.getStringVariable("sessionExpiry")
def sessionToken = context.getStringVariable("sessionToken")
def apiClientKey = accountPropertyService.getAccountProperty("rl-api-clientKey").getValue()
//Generate signature & Timestamp
def date = new Date()
Calendar c = Calendar.getInstance()
c.add(Calendar.MINUTE, 5)
SimpleDateFormat formatDate = new SimpleDateFormat("yyyyMMdd")
SimpleDateFormat formatAmzDate = new SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'")
def dateString = formatDate.format(date)
def amzDateTime = formatAmzDate.format(c.getTime())
def signature = getSignatureKey(sessionAccessKey, dateString, "us-east-1", "execute-api")
def signatureString = convertbyte(signature)
context.logInfo("Signature : $signatureString")
context.logInfo("AMZ Date : $amzDateTime")
Unirest.setTimeouts(0, 0)
HttpResponse<String> response = Unirest.get("https://api-staging.rightsline.com/v4/XXXX-item/42262XX")
.header("x-api-key", apiClientKey)
.header("X-Amz-Security-Token", sessionToken)
.header("X-Amz-Date", amzDateTime)
.header("Authorization", "AWS4-HMAC-SHA256 Credential=$sessionAccessKey/20230118/us-east-1/execute-api/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token;x-api-key, Signature=$signatureString")
.asString()
context.logInfo("AWS4-HMAC-SHA256 Credential=$sessionAccessKey/20230118/us-east-1/execute-api/aws4_request, SignedHeaders=host;x-amz-date;x-amz-security-token;x-api-key, Signature=$signatureString\"")
//Get & Parse response
def responseCode = response.getStatus()
def responseBody = response.getBody()
context.logInfo("Response Code : $responseCode")
context.logInfo("Response Body : $responseBody")
if (responseCode > 299) {
throw new Exception("Unexpected response code : $responseCode")
} else {
def responseJson = new JsonSlurper().parseText(responseBody)
}
//return "process"
// }
} else {
context.logInfo("No entity urls to process")
//return "done"
}
static byte[] HmacSHA256(String data, byte[] key) throws Exception {
String algorithm = "HmacSHA256"
Mac mac = Mac.getInstance(algorithm)
mac.init(new SecretKeySpec(key, algorithm))
return mac.doFinal(data.getBytes("UTF-8"))
}
static byte[] getSignatureKey(String key, String dateStamp, String regionName, String serviceName) throws Exception {
byte[] kSecret = ("AWS4" + key).getBytes("UTF-8")
byte[] kDate = HmacSHA256(dateStamp, kSecret)
byte[] kRegion = HmacSHA256(regionName, kDate)
byte[] kService = HmacSHA256(serviceName, kRegion)
byte[] kSigning = HmacSHA256("aws4_request", kService)
return kSigning
}
public static String convertbyte(byte[] bytes) {
StringBuffer hexString = new StringBuffer();
for (int j = 0; j < bytes.length; j++) {
String hex = Integer.toHexString(0xff & bytes[j]);
if (hex.length() == 1) hexString.append('0');
hexString.append(hex);
}
return hexString.toString();
}
When I execute this def execute() is the first to execute, I have comprated the code with Postman only signature field is not accurate .Please help out as I am stuck in this for mostly two days
I want to get an JSON response from the API and give a response with the selected field. I have fetched the response but I am not able to create a response with the new value. I am very new to python world and in learning stage.
def search(request):
if request.method == 'POST':
searchQry = request.POST.get("searchQry","")
nodeIP = settings.NODEIP
params = {'search':searchQry}
apiResponse = requests.get(url = nodeIP, params = params)
data = apiResponse.json()
newArray = {}
nodeName = 'RPID'
if nodeName == 'RPID':
for x in data:
newArray['cphNumber'] = x["data"]["cphNumber"]
newArray['farmName'] = x['data']['farmName']
newArray['addressLine1'] = x['data']['addressLine1']
return HttpResponse(json.dumps(newArray))
else:
return HttpResponse('Unauthrozed Access')
My response array looks likes this :
[{"data": {"cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13", "postalCode": "13"},"id": "4c1b935664e6f684e89ee363f473ce3567599d4b9da0f5889565d5b6f0b84440"},{"data": {"cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13","postalCode": "13"},"id": "7cbe7be9797896545410ed6c4dcc18064525037bc19fbe9272f9baabbb3216ec"},{"data": { "cphNumber": "321","farmName": "313","addressLine1": "13","addressLine2": "13","region": "13","postalCode": "13"},"id": "7df10c0b7b84434d5ace6811a1b2752a5e5bca13b691399ccac2a6ee79d17797"}]
In response I am getting only one array. I know I have to do something like newArray[0]['cphNumber'] But I am getting an error. Can you please help me to solve this .
I am trying to upload large images to AWS S3 using the Multipart Upload API. From UI, i am sending the chunks(blob) of an image and when the last part arrives, completing the upload and getting the upload file url. It is working very nicely.
Sample Code:
public UploadPartResponse UploadChunk(Stream stream, string fileName, string uploadId, List<PartETag> eTags, int partNumber, bool lastPart)
{
stream.Position = 0;
//Step 1: build and send a multi upload request
if (partNumber == 1)
{
var initiateRequest = new InitiateMultipartUploadRequest
{
BucketName = _settings.Bucket,
Key = fileName
};
var initResponse = _s3Client.InitiateMultipartUpload(initiateRequest);
uploadId = initResponse.UploadId;
}
//Step 2: upload each chunk (this is run for every chunk unlike the other steps which are run once)
var uploadRequest = new UploadPartRequest
{
BucketName = _settings.Bucket,
Key = fileName,
UploadId = uploadId,
PartNumber = partNumber,
InputStream = stream,
IsLastPart = lastPart,
PartSize = stream.Length
};
var response = _s3Client.UploadPart(uploadRequest);
//Step 3: build and send the multipart complete request
if (lastPart)
{
eTags.Add(new PartETag
{
PartNumber = partNumber,
ETag = response.ETag
});
var completeRequest = new CompleteMultipartUploadRequest
{
BucketName = _settings.Bucket,
Key = fileName,
UploadId = uploadId,
PartETags = eTags
};
try
{
var res = _s3Client.CompleteMultipartUpload(completeRequest);
return res.Location;
}
catch
{
//do some logging and return null response
return null;
}
}
response.ResponseMetadata.Metadata["uploadid"] = uploadRequest.UploadId;
return response;
}
Now, i need to get the thumbnail of the uploaded image and upload that image too in a Thumbnails directory.
So basically, when the last part(chunk) arrives for the original image, i am completing the upload and retrieving the file url. At that time, i need to upload the thumbnail also and get back the thumbnail url.
I saw that people are referring of lambda function but don't know how to incorporate into my multipart api code setup.
Can anyone give me some direction here? Thanks in advance.
I'm following the api's guide about resumable uploads.
I managed to get a response after step 1 ("create the video"),
with a uri and a upload_link.
About the second part, things are not as clear.
It only says which headers should I sent, but there are two things I don't get,
first - where do I need to put the "upload_link"?
Should the call be like this:
/me/{upload_link}? (of course im also adding the access token, etc)
second, what about the actual file? I guess i should send it in the same method, but how? No word about it.
This is the code for the PATCH request:
public string UploadPatch(
string uploadlink,
string method)
{
var headers = new WebHeaderCollection()
{
{ "Tus-Resumable", "1.0.0" },
{ "Upload-Offest", "0" }
};
method = method.ToUpper();
string body = "";
string contentType = "application/offset+octet-stream";
return Helpers.HTTPUpload(uploadlink, method, headers, body, contentType);
}
And HTTPUpload():
public static string HTTPPatch(string url, string method,
WebHeaderCollection headers, string payload,
string contentType)
{
HttpWebRequest request = (HttpWebRequest)WebRequest.CreateHttp(url);
if (Proxy != null) request.Proxy = Proxy;
request.Headers = headers;
request.Method = method;
request.Accept = "application/vnd.vimeo.*+json; version=3.1";
request.ContentType = contentType;
request.KeepAlive = false;
if (!String.IsNullOrWhiteSpace(payload))
{
var streamBytes = Helpers.ToByteArray(payload);
request.ContentLength = streamBytes.Length;
Stream reqStream = request.GetRequestStream();
reqStream.Write(streamBytes, 0, streamBytes.Length);
reqStream.Close();
}
HttpWebResponse response = (HttpWebResponse)(request.GetResponse());
Debug.WriteLine(((HttpWebResponse)response).StatusDescription);
var dataStream = response.GetResponseStream();
StreamReader reader = new StreamReader(dataStream);
string responseFromServer = reader.ReadToEnd();
reader.Close();
dataStream.Close();
response.Close();
Debug.WriteLine(String.Format("Response from URL {0}:", url), "HTTPFetch");
Debug.WriteLine(responseFromServer, "HTTPFetch");
return responseFromServer;
}
Thanks
The upload_link is the URL where you upload the video to. In other words, make your call to the https://[...].cloud.vimeo.com/upload?[...] URL instead of the https://api.vimeo.com host that is used for other API requests.
Additionally, when you make a request to that cloud.vimeo.com upload_link, only provide the required headers as specified in the documentation.
https://developer.vimeo.com/api/upload/videos#resumable-approach
The code is VB.Net, but you can change to C#
'Imports / use these classes
'Imports System.Net
'Imports Newtonsoft.Json
'Imports System.IO
Protected Sub Button1_Click(sender As Object, e As EventArgs) Handles Button1.Click
Try
'Receive the video from File Upload
If Not IsNothing(fuVideo.PostedFile) Then
'You will need this for SSL
System.Net.ServicePointManager.SecurityProtocol = (SecurityProtocolType.Tls Or (SecurityProtocolType.Tls11 Or SecurityProtocolType.Tls12))
'Path to save the video Save
Dim vFilePath As String = Server.MapPath("App_data/Videos")
Dim vFileNameAndPath As String = vFilePath & "/" & fuVideo.PostedFile.FileName
'Save Video
fuVideo.PostedFile.SaveAs(vFileNameAndPath)
'Get the size
Dim vSize As String = New FileInfo(vFileNameAndPath).Length()
'Vimeo URL
Dim vVimeURL As String = "https://api.vimeo.com/me/videos"
Dim wc As WebClient = New WebClient()
wc.Headers.Clear()
wc.Headers.Add("Authorization", "bearer XXXXXXXXXXXXXXXXX") 'Use your App Code
wc.Headers.Add("Content-Type", "application/json")
wc.Headers.Add("Accept", "application/vnd.vimeo.*+json;version=3.4")
wc.Encoding = System.Text.Encoding.UTF8
'txtName is a text box, so you can give a Title to the Video
Dim vData As String = "{ ""upload"": {""approach"": ""tus"",""size"": """ & vSize & """ }, ""name"" : """ & txtName.Text & """ }"
Dim vimeoTicket = JsonConvert.DeserializeObject(wc.UploadString(vVimeURL, "POST", vData))
wc.Headers.Clear()
wc.Headers.Add("Content-Type", "application/offset+octet-stream")
wc.Headers.Add("Accept", "application/vnd.vimeo.*+json;version=3.4")
wc.Headers.Add("Tus-Resumable", "1.0.0")
wc.Headers.Add("Upload-Offset", "0")
Dim vupload_link As String = vimeoTicket("upload")("upload_link").Value 'Json from Vimeo has the upload_link
Dim vResponse As Byte() = wc.UploadFile(vupload_link, "PATCH", vFileNameAndPath)
Response.Write(System.Text.Encoding.Unicode.GetString(vResponse)) ' If everething is ok, vResponse is Nothing
End If
Catch ex As Exception
ltrErro.Text = "Error"
End Try
End Sub
for this may look at the sample code below:-
I am using a nuget library vimeo-dot-net also at https://github.com/mfilippov/vimeo-dot-net, this has a wrapper built around upload, delete etc.
public ActionResult UploadChapterVideoVimeo(HttpPostedFileBase file, string productID = "")
{
if (file != null){
var authCheck = Task.Run(async () => await vimeoClient.GetAccountInformationAsync()).Result;
if (authCheck.Name != null)
{
BinaryContent binaryContent = new BinaryContent(file.InputStream, file.ContentType);
int chunkSize = 0;
int contenetLength = file.ContentLength;
int temp1 = contenetLength / 1024;
if (temp1 > 1)
{
chunkSize = temp1 / 1024;
chunkSize = chunkSize * 1048576;
}
else
{ chunkSize = chunkSize * 1048576; }
binaryContent.OriginalFileName = file.FileName;
var d = Task.Run(async () => await vimeoClient.UploadEntireFileAsync(binaryContent, chunkSize, null)).Result;
vmodel.chapter_vimeo_url = "VIMEO-" + d.ClipUri;
}
return RedirectToAction("ProductBuilder", "Products", new { productId = EncryptedProductID, message = "Successfully Uploaded video", type = 1 });
}
}
catch (Exception exc)
{
return RedirectToAction("ProductBuilder", "Products", new { productId = EncryptedProductID, message = "Failed to Uploaded video " + exc.Message, type = 0 });
}
}
return null; }
I'm using Python2.7, django==1.7 and uwsgi for streaming video/mp4 file to iPhone player.
My code is as below:
def stream(request):
with open('/path/video.mp4', 'r') as video_file:
response = HttpResponse(video_file.read(), content_type='video/mp4')
response['Content-Disposition'] = 'inline; filename=%s' % 'video.mp4'
return response
video_file.close
When i use some small video (less than 1MB), it streams in browser, but in iPhone palyer i have this error:
[uwsgi-http key: 127.0.0.1:8008 client_addr: 192.168.0.172
client_port: 14563] hr_write(): Broken pipe [plugins/http/http.c line
564]
And when the video size is more that 5MB, it doesn't stream in both (means browser and iPhone player) with same error.
I tried to do that by chunk chunk returning using StreamHttpRespose as below:
def read(chunksize=8192):
with open('/path/video.mp4', 'rb') as video_file:
byte = video_file.read(chunksize)
while byte:
yield byte
return StreamingHttpResponse(read(), content_type='video/mp4')
But there is the same error: Broken pipe.
fyi I can stream pdf and image files. This problem is only with mp4 files. And also i changed the content_type to 'video-mpeg', the browser downloaded that, while i want to prevent file downloading.
What's your idea? Any solution!!?
I had the same problem and did a lot of digging before finding a workable solution!
Apparently the Accept Ranges header is needed for HTML5 video controls to work (https://stackoverflow.com/a/24977085/4264463). So, we need to both parse the requested range from HTTP_RANGE and return Content-Range with the response. The generator that is passed to StreamingHttpResponse also needs to return content based on this range as well (by offset and length). I've found the follow snippet that works great (from http://codegist.net/snippet/python/range_streamingpy_dcwatson_python):
import os
import re
import mimetypes
from wsgiref.util import FileWrapper
from django.http.response import StreamingHttpResponse
range_re = re.compile(r'bytes\s*=\s*(\d+)\s*-\s*(\d*)', re.I)
class RangeFileWrapper(object):
def __init__(self, filelike, blksize=8192, offset=0, length=None):
self.filelike = filelike
self.filelike.seek(offset, os.SEEK_SET)
self.remaining = length
self.blksize = blksize
def close(self):
if hasattr(self.filelike, 'close'):
self.filelike.close()
def __iter__(self):
return self
def __next__(self):
if self.remaining is None:
# If remaining is None, we're reading the entire file.
data = self.filelike.read(self.blksize)
if data:
return data
raise StopIteration()
else:
if self.remaining <= 0:
raise StopIteration()
data = self.filelike.read(min(self.remaining, self.blksize))
if not data:
raise StopIteration()
self.remaining -= len(data)
return data
def stream_video(request, path):
range_header = request.META.get('HTTP_RANGE', '').strip()
range_match = range_re.match(range_header)
size = os.path.getsize(path)
content_type, encoding = mimetypes.guess_type(path)
content_type = content_type or 'application/octet-stream'
if range_match:
first_byte, last_byte = range_match.groups()
first_byte = int(first_byte) if first_byte else 0
last_byte = int(last_byte) if last_byte else size - 1
if last_byte >= size:
last_byte = size - 1
length = last_byte - first_byte + 1
resp = StreamingHttpResponse(RangeFileWrapper(open(path, 'rb'), offset=first_byte, length=length), status=206, content_type=content_type)
resp['Content-Length'] = str(length)
resp['Content-Range'] = 'bytes %s-%s/%s' % (first_byte, last_byte, size)
else:
resp = StreamingHttpResponse(FileWrapper(open(path, 'rb')), content_type=content_type)
resp['Content-Length'] = str(size)
resp['Accept-Ranges'] = 'bytes'
return resp
After a lot of search, i didn't find my solution.
So, i tried to create a stream-server easily using nodejs from html5-video-streamer.js reference as below:
var http = require('http'),
fs = require('fs'),
url = require('url'),
basePath = '/var/www/my_project/media/',
baseUrl = 'Your Domain or IP',
basePort = 8081;
http.createServer(function (req, res) {
// Get params from request.
var params = url.parse(req.url, true).query,
filePath = basePath + params.type + '/' + params.name,
stat = fs.statSync(filePath),
total = stat.size;
if (req.headers['range']) {
var range = req.headers.range,
parts = range.replace(/bytes=/, "").split("-"),
partialstart = parts[0],
partialend = parts[1],
start = parseInt(partialstart, 10),
end = partialend ? parseInt(partialend, 10) : total-1,
chunksize = (end-start)+1;
var file = fs.createReadStream(filePath, {start: start, end: end});
res.writeHead(206, { 'Content-Range' : 'bytes ' + start + '-' + end + '/' + total,
'Accept-Ranges' : 'bytes',
'Content-Length' : chunksize,
'Content-Type' : 'video/mp4' });
file.pipe(res);
// Close file at end of stream.
file.on('end', function(){
file.close();
});
}
else {
res.writeHead(206, { 'Content-Length' : total,
'Content-Type' : 'video/mp4' });
var file = fs.createReadStream(filePath);
file.pipe(res);
// Close file at end of stream.
file.on('end', function(){
file.close();
});
}
}).listen(basePort, baseUrl);
Now i have separate stream-server with nodejs that streams mp4 files beside python project that provides my APIs.
I'm aware It's not my solution, but it works for me ;)