How to get result of AWS lambda function running with step function - amazon-web-services

I am using AWS step function to invoke lambda function like this.
return stepfunctions.startExecution(params).promise().then((result) => {
console.log(result);
console.log(result.output);
return result;
})
And result is
{ executionArn: 'arn:aws:states:eu-west-2:695510026694:...........:7c197be6-9dca-4bef-966a-ae9ad327bf23',
startDate: 2018-07-09T07:35:14.930Z }
But i want the result as output of final lambda function
I am going through https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#sendTaskSuccess-property
There are multible function there i am confused which one could be used to get back result of final lambda function.
Same question is there on stackoverflow Api gateway get output results from step function? i dont want to call any function periodically and keep checking status.Even if i use DescribeExecution function periodically i will only get the status of execution but not the result i wanted. Is there any way or any function which returns promise and is resolved once all the lambda has executed and give back the result

You can't get back a result from a step function execution in a synchronous way.
Instead of polling the result of the step function on completion send a result to an SNS topic or SQS queue for further processing in the final lambda function or model the whole process in the step function state machine.

After doing some study and looking at various tutorial i realized that this stackoverflow answer Api gateway get output results from step function? gives a easier approach to solve the problem and get final result from step function, yes i am not sure about another approach and how to implement any new answer is always appreciated
This is my code to implement the same approach this might help someone.
// in function first start step function execution using startExecution()
var params = {
stateMachineArn: 'some correct ARN',
input: JSON.stringify(body)
};
return stepfunctions.startExecution(params).promise().then((result) => {
var paramsStatus = {
executionArn: result.executionArn
};
var finalResponse = new Promise(function(resolve,reject){
var checkStatusOfStepFunction = setInterval(function(){
//on regular interval keep checking status of step function
stepfunctions.describeExecution(paramsStatus, function(err, data) {
console.log('called describeExecution:', data.status);
if (err){
clearInterval(checkStatusOfStepFunction);
reject(err);
}
else {
if(data.status !== 'RUNNING'){
// once we get status is not running means step function execution is now finished and we get result as data.output
clearInterval(checkStatusOfStepFunction);
resolve(data.output);
}
}
});
},200);
});
return finalResponse
})

To be able to get the result of step function (example: combined gateway & step function). You need to:
1. startExecution,
2. wait for your state machine to finish the execution (to be sure make wait equivalent to timeout of your state machine => wait = TimeoutSeconds of your state machine)
3. call describeExecution with the receive executionArn from startExecution.
Note that startExecution is an async function and it's not waiting for the result.
In my case, I'm using Lambda named init to execute the 3 discussed steps:
Code lambda Init:
const AWS = require('aws-sdk')
exports.handler = async (event) => {
const stepFunctions = new AWS.StepFunctions();
const reqBody = event.body || {};
const params = {
stateMachineArn: process.en.stateMachineArn,
input: JSON.stringify(reqBody)
}
return stepFunctions.startExecution(params).promise()
.then(async data => {
console.log('==> data: ', data)
await new Promise(r => setTimeout(r, 6000));
return stepFunctions.describeExecution({ executionArn: data.executionArn }).promise();
})
.then(result => {
return {
statusCode: 200,
message: JSON.stringify(result)
}
})
.catch(err => {
console.error('err: ', err)
return {
statusCode: 500,
message: JSON.stringify({ message: 'facing error' })
}
})
}
Code stateMachine
Make sure that in your statemachine your returning "ResultPath".
{
"Comment": "Annoucement validation",
"StartAt": "contact-validation",
"Version": "1.0",
"TimeoutSeconds": 5,
"States": {
"contact-validation": {
"Type": "Task",
"Resource": "arn:aws:xxxxxxx:function:scam-detection-dev-contact",
"ResultPath": "$.res",
"Next": "WaitSeconds"
},
"WaitSeconds": {
"Type": "Wait",
"Seconds": 1,
"Next": "Result"
},
"Result": {
"Type": "Pass",
"ResultPath": "$.res",
"End": true
}
}
}

Related

Dropzone.js + AWS S3 stalling queue

I'm trying to impliment a dropzone.js uploader to amazon S3 using the aws-sdk.js for the browser. But when I exceed the 'parallelUploads' maximum in the settings, the queue never completes. I'm using the approach in the following link:
amazon upload
relevant parts of my code:
var dz = new Dropzone("#DZContainer", {
acceptedFiles: "image/*,.jpg,.jpeg,.png,.gif",
autoQueue: true,
autoProcessQueue: true,
parallelUploads: 10,
clickable: [".uploadButton"],
accept: function(file, done){
let params = {
"Bucket": "upload-bucket",
"Key": getFullKey(file.name),
Body: file,
Region: "us-east-1,
ContentType: file.type
}
file.s3upload = AWS.S3.ManagedUpload(params);
if (typeof(done) === 'function') done();
},
canceled: function(file) {
if (file.s3upload) file.s3upload.abort();
},
init: function () {
this.on('removedfile', function (file) {
if (file.s3upload) file.s3upload.abort();
});
}
)
dz.uploadFiles = function (files) {
for (var j = 0; j < files.length; j++) {
var file = files[j];
dz.SendFile(file);
}
};
dz.SendFile = function(file) {
file.s3upload.send(function (err, data) {
if (err) {
console.err(err)
dz.emit("error", file, err.message);
} else {
dz.emit("complete", file);
}
});
if I drag in (or use the clickable) more than 10 files, the first 10 complete but it never processes the rest of the queue. What am I missing? All help is appreciated
EDIT: With a little more digging into Dropzone, it looks as though the file status is never getting set to complete. I see a function called _finished() in the dropzone code, but I'm having a hard time figuring out what specifically is supposed to trigger that function. I have tried dz.emit("complete", file) listed below as well as adding dz.emit("success",file) but my breakpoint at the first line of the _finished() function never triggers. Thus the file.status never gets set to completed.
Does anyone know when/what/how _finished() is supposed to be run?
As mentioned in the edit, I was able to track down where the .status was not properly getting set. This seemed to be in a private Dropzone function called _finished()
With further examination, I noticed that _finished() seemed to also be calling emit("complete", file) after setting file.status to Dropzone.SUCCESS and also emitting "success". It then checks if autoProcessQueue is set and if it is, returns the result of a processQueue() call.
I had a hard time figuring out what triggered this function as it was on an onload event that eventually realized was tied to an XHTTPRequest object used by the internal uploader (which is being overridden by the S3 uploader)
So I modified the function to emulate what the Dropzone._finished() was doing and it's behaving as expected:
dz.SendFile = function(file) {
file.s3upload.send(function (err, data) {
if (err) {
console.err(err)
dz.emit("error", file, err.message);
} else {
file.status = Dropzone.SUCCESS;
dz.emit("success", file, data, err);
dz.emit("complete", file);
if(dz.options.autoProcessQueue)
dz.processQueue()
}
});

Self invoking lambda invocation timing out

We're trying to develop a self-invoking lambda to process S3 files in chunks. The lambda role has the policies needed for the invocation attached.
Here's the code for the self-invoking lambda:
export const processFileHandler: Handler = async (
event: S3CreateEvent,
context: Context,
callback: Callback,
) => {
let bucket = loGet(event, 'Records[0].s3.bucket.name');
let key = loGet(event, 'Records[0].s3.object.key');
let totalFileSize = loGet(event, 'Records[0].s3.object.size');
const lastPosition = loGet(event, 'position', 0);
const nextRange = getNextSizeRange(lastPosition, totalFileSize);
context.callbackWaitsForEmptyEventLoop = false;
let data = await loadDataFromS3ByRange(bucket, key, nextRange);
await database.connect();
log.debug(`Successfully connected to the database`);
const docs = await getParsedDocs(data, lastPosition);
log.debug(`upserting ${docs.length} records to database`);
if (docs.length) {
try {
// upserting logic
log.debug(`total documents added: ${await docs.length}`);
} catch (err) {
await recurse(nextRange.end, event, context);
log.debug(`error inserting docs: ${JSON.stringify(err)}`);
}
}
if (nextRange.end < totalFileSize) {
log.debug(`Last ${context.getRemainingTimeInMillis()} milliseconds left`);
if (context.getRemainingTimeInMillis() < 10 * 10 * 10 * 6) {
log.debug(`Less than 6000 milliseconds left`);
log.debug(`Invoking next iteration`);
await recurse(nextRange.end, event, context);
callback(null, {
message: `Lambda timed out processing file, please continue from LAST_POSITION: ${nextRange.start}`,
});
}
} else {
callback(null, { message: `Successfully completed the chunk processing task` });
}
};
Where recurse is an invocation call to the same lambda. Rest of the things work as expected it just times out whenever the call stack comes on this invocation request:
const recurse = async (position: number, event: S3CreateEvent, context: Context) => {
let newEvent = Object.assign(event, { position });
let request = {
FunctionName: context.invokedFunctionArn,
InvocationType: 'Event',
Payload: JSON.stringify(newEvent),
};
let resp = await lambda.invoke(request).promise();
console.log('Invocation complete', resp);
return resp;
};
This is the stack trace logged to CloudWatch:
{
"errorMessage": "connect ETIMEDOUT 63.32.72.196:443",
"errorType": "NetworkingError",
"stackTrace": [
"Object._errnoException (util.js:1022:11)",
"_exceptionWithHostPort (util.js:1044:20)",
"TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)"
]
}
Not a good idea to create a self-invoking lambda function. In case of an error (could also be a bad handler call on AWS side) a lambda function might re-run several times. Very hard to monitor and debug.
I would suggest using Step Functions. I believe this tutorial can help Iterating a Loop Using Lambda
From the top of my head, if you prefer not dealing with Step Functions, you could create a Lambda trigger for an SQS queue. Then you pass a message to the queue if you want to run the lambda function another time.

Step Function Triggered In a loop

I am starting a step function from Lambda and the Lambda function is tied to an API Gateway. For some reason, when I try to test the Lambda function, I see hundreds of executions failed and running in loop. I just triggered the step function once. I am missing something here. Can you please advise.
const AWS = require("aws-sdk");
const uuidv4 = require("uuid/v4");
/*----------------------------------------------------------------------- */
/* Implementation */
/*----------------------------------------------------------------------- */
exports.handler = async event => {
var _dt = await ExecuteStepFunction()
return _dt;
}
function ExecuteStepFunction() {
const stepFunctions = new AWS.StepFunctions();
return new Promise((res, rej) => {
var params = {
stateMachineArn: 'arn:aws:states:us-east-1:xxxxxxxxxxxxx:stateMachine:xxTestSateMachine',
input: JSON.stringify(''),
name: uuidv4()
};
stepFunctions.startExecution(params, function (err, data) {
if (err) {
rej(err);
}
else {
res(data);
}
});
});
}
I tried thIS approach provided in the this link (https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-api-gateway.html) where the API gateway directly triggers the step function but I am receiving the following error. After trying to fix this, I move to the above option of starting the function using the API.
{
"__type": "com.amazon.coral.service#UnrecognizedClientException",
"message": "The security token included in the request is invalid"
}

Alexa function works in AWS lamda, but not from the Service Simulator

I am relatively new to AWS and Alexa skills. I am building a simple custom skill that gives you a dressing advice depending on the weather.
I have 2 custom intents : dressingTodayIntent & dressingTomorrowIntent. In the Service Simulator of the developer portal, my two intents don't work, I do get a lambda response though, but with an undefined outputSpeech, like this:
{
"version": "1.0",
"response": {
"outputSpeech": {
"type": "SSML",
"ssml": "<speak> undefined </speak>"
},
"card": null,
"reprompt": null,
"speechletResponse": {
"outputSpeech": {
"id": null,
"ssml": "<speak> undefined </speak>"
},
"card": null,
"directives": null,
"reprompt": null,
"shouldEndSession": true
}
},
"sessionAttributes": {}
}
Could it be a scope issue in my intent code?
'DressingTodayIntent': function() {
var dressingAdvice;
var speechOutput = getJSON('https://api.darksky.net/forecast/9e0495a835ed823a705a9a567eee982a/48.861317,2.348764?units=si&exclude=currently,minutely,hourly,alerts,flags',
function(err, forecast) {
if (err) {
console.log('Error occurred while trying to retrieve weather data', err);
} else {
dressingAdvice = getDressingAdvice(forecast, true);
console.log("one " + dressingAdvice);
}
console.log("two " + dressingAdvice);
return dressingAdvice;
});
console.log("three " + speechOutput);
this.response.cardRenderer("Your dressing advice for today:", speechOutput);
this.response.speak(speechOutput);
this.emit(':responseReady');
},
In AWS Lambda, I see a correct output for the first 2 logs, and an error for the 3rd one:
first log: "one " + dressingAdvice, as expected
second log: "two " + dressingAdvice, as expected
third log: "three " + undefined
Thank you for you help!
When you say "tested from AWS Lambda", I assume that you mean using the AWS console to send a JSON test message to the Lambda, then looking at the response JSON to determine if it is correct?
If so, make sure that it matches the JSON sent to/from the Alexa test page in the dev portal. Sounds like they might be different.
Also, make sure that you are linked to the correct ARN in the Alexa skill.
The undefined is likely a variable scope issue in the code.
I noticed in your response that you don't have any sessionAttributes. Is your code setting or pulling the value for the response from a session value? If so, the values need to be sent back with the sessionAttributes.
I figured out what was wrong, I needed to move the response code into the callback function, like this:
'DressingTodayIntent': function() {
var speechOutput;
var self = this;
var dressingAdvice = getJSON('https://api.darksky.net/forecast/9e0495a835ed823a705a9a567eee982a/48.861317,2.348764?units=si!ude=currently,minutely,hourly,alerts,flags',
function(err, forecast) {
if (err) {
console.log('Error occurred while trying to retrieve weather data', err);
} else {
speechOutput = getDressingAdvice(forecast, true);
}
self.response.cardRenderer("Your dressing advice for today:", speechOutput);
self.response.speak(speechOutput);
self.emit(':responseReady');
});
},

AWS javascript SDK request.js send request function execution time gradually increases

I am using aws-sdk to push data to Kinesis stream.
I am using PutRecord to achieve realtime data push.
I am observing same delay in putRecords as well in case of batch write.
I have tried out this with 4 records where I am not crossing any shard limit.
Below is my node js http agent configurations. Default maxSocket value is set to infinity.
Agent {
domain: null,
_events: { free: [Function] },
_eventsCount: 1,
_maxListeners: undefined,
defaultPort: 80,
protocol: 'http:',
options: { path: null },
requests: {},
sockets: {},
freeSockets: {},
keepAliveMsecs: 1000,
keepAlive: false,
maxSockets: Infinity,
maxFreeSockets: 256 }
Below is my code.
I am using following code to trigger putRecord call
event.Records.forEach(function(record) {
var payload = new Buffer(record.kinesis.data, 'base64').toString('ascii');
// put record request
evt = transformEvent(payload );
promises.push(writeRecordToKinesis(kinesis, streamName, evt ));
}
Event structure is
evt = {
Data: new Buffer(JSON.stringify(payload)),
PartitionKey: payload.PartitionKey,
StreamName: streamName,
SequenceNumberForOrdering: dateInMillis.toString()
};
This event is used in put request.
function writeRecordToKinesis(kinesis, streamName, evt ) {
console.time('WRITE_TO_KINESIS_EXECUTION_TIME');
var deferred = Q.defer();
try {
kinesis.putRecord(evt , function(err, data) {
if (err) {
console.warn('Kinesis putRecord %j', err);
deferred.reject(err);
} else {
console.log(data);
deferred.resolve(data);
}
console.timeEnd('WRITE_TO_KINESIS_EXECUTION_TIME');
});
} catch (e) {
console.error('Error occured while writing data to Kinesis' + e);
deferred.reject(e);
}
return deferred.promise;
}
Below is output for 3 messages.
WRITE_TO_KINESIS_EXECUTION_TIME: 2026ms
WRITE_TO_KINESIS_EXECUTION_TIME: 2971ms
WRITE_TO_KINESIS_EXECUTION_TIME: 3458ms
Here we can see gradual increase in response time and function execution time.
I have added counters in aws-sdk request.js class. I can see same pattern in there as well.
Below is code snippet for aws-sdk request.js class which executes put request.
send: function send(callback) {
console.time('SEND_REQUEST_TO_KINESIS_EXECUTION_TIME');
if (callback) {
this.on('complete', function (resp) {
console.timeEnd('SEND_REQUEST_TO_KINESIS_EXECUTION_TIME');
callback.call(resp, resp.error, resp.data);
});
}
this.runTo();
return this.response;
},
Output for send request:
SEND_REQUEST_TO_KINESIS_EXECUTION_TIME: 1751ms
SEND_REQUEST_TO_KINESIS_EXECUTION_TIME: 1816ms
SEND_REQUEST_TO_KINESIS_EXECUTION_TIME: 2761ms
SEND_REQUEST_TO_KINESIS_EXECUTION_TIME: 3248ms
Here you can see it is increasing gradually.
Can anyone please suggest how can I reduce this delay?
3 seconds to push single record to Kinesis is not at all acceptable.