Is there a way to trigger an AWS Lambda on a dynamic timer? Currently, I am utilizing scheduled-events to trigger the lambda, but this is a set timer. Is there a way to dynamically set a time for the Lambda to be triggered from within the Lambda?
The idea here is that this Lambda does specific checks and executes code to know when it should run next (because I only want this lambda to run when it needs to). I want to 1) determine the next time it needs to run and 2) set the time from within the Lambda code.
I see there are a lot of resources that are used for triggering Lambda functions (SNS, Kinesis, etc.), but I cant seem to find a good way to dynamically kick one off.
This can be accomplished by setting a CloudWatch event rule to trigger your Lambda function. On each invocation of your Lambda function, the function will need to determine its next run time and modify the event rule appropriately.
var AWS = require("aws-sdk");
exports.handler = function(event, context) {
var cloudwatchevents = new AWS.CloudWatchEvents();
var intervals = Array(3, 5, 7);
var nextInterval = intervals[Math.floor(Math.random()*intervals.length)];
var currentTime = new Date().getTime(); // UTC Time
var nextTime = dateAdd(currentTime, "minute", nextInterval);
var nextMinutes = nextTime.getMinutes();
var nextHours = nextTime.getHours();
// =================================
// DO YOUR WORK HERE
// =================================
var scheduleExpression = "cron(" + nextMinutes + " " + nextHours + " * * ? *)";
var params = {
Name: "YOUR CLOUDWATCH EVENT RULE NAME",
ScheduleExpression: scheduleExpression
};
cloudwatchevents.putRule(params, function(err, data) {
if (err) {
console.log(err, err.stack);
}
else {
console.log(data);
}
})
};
var dateAdd = function(date, interval, units) {
var ret = new Date(date); // don't change original date
switch(interval.toLowerCase()) {
case 'year' : ret.setFullYear(ret.getFullYear() + units); break;
case 'quarter': ret.setMonth(ret.getMonth() + 3*units); break;
case 'month' : ret.setMonth(ret.getMonth() + units); break;
case 'week' : ret.setDate(ret.getDate() + 7*units); break;
case 'day' : ret.setDate(ret.getDate() + units); break;
case 'hour' : ret.setTime(ret.getTime() + units*3600000); break;
case 'minute' : ret.setTime(ret.getTime() + units*60000); break;
case 'second' : ret.setTime(ret.getTime() + units*1000); break;
default : ret = undefined; break;
}
return ret;
}
You should be able to swap my random determination with your own scheduling logic and insert whatever work you need in place of my comment.
You will need to substitute your event rule's name for "YOUR CLOUDWATCH EVENT RULE NAME" in my snippet.
Great question for a blog: AWS Lambda Functions That Dynamically Schedule Their Next Runtime
This can now be accomplished without polling using a step function. You can find more information on AWS, but basically you would define a state machine for your step function that uses the Wait state and the TimestampPath field. Your state machine might end up looking something like
{
"SartAt": "WaitState",
"States": {
"WaitState": {
"Type": "Wait",
"TimestampPath": "$.timestamp",
"Next": "ExecuteLambda"
},
"ExecuteLambda": {
"Type": "Task",
"Resource": "lambda-arn",
"End": true,
"Retry": [
{ ... },
...
]
}
}
Assuming you're using Node, you could then invoke the step function with the following code:
const AWS = require('aws-sdk');
const stepFunctions = new AWS.StepFunctions();
await stepFunctions.startExecution({
stateMachineArn: process.env.STATE_MACHINE_ARN,
name: "unique-name",
input: JSON.stringify({
timestamp: (new Date(/*Date you want to start execution*/)).toISOString(),
// Any extra context you want to pass to the step function.
}),
}).promise();
You could create a CloudWatch rule to run at a particular time. CloudWatch rules can be run periodically or with a cron like syntax that lets you specify a single run time. You'll likely need to clean these up later though.
You can keep executing the lambda on a schedule but persist a value in a data store such as DynamoDB or S3 that tells the lambda when it should run next.
The lambda will keep executing periodically but you can control when it actually does what its intended for. There are billing considerations here (the lambda will continue consuming a minimal amount of resources in the background) but hopefully things shouldnt get too out of hand. Probably simpler than trying to manage the triggers from within the same lambda.
Related
I am trying to delete items in my Dynamodb table using Cloud Watch event triggered Lambda. This lambda scans the dynamo table and deletes all expired items. My code seems to be working when I test it using the test event in the console (i.e it deletes all the expired items). But when lambda gets triggered automatically using the Cloud Watch event it does not delete, event though I see that the lambda is being triggered.
exports.handler = async function () {
var params = {
TableName: TABLE_NAME
}
try {
const data = await docClient.scan(params).promise();
const items = data.Items;
if (items.length != 0) {
Promise.all(items.map(async (item) => {
const expirationDT = new Date(item.ExpiresAt);
const now = new Date();
if (now > expirationDT) {
console.log("Deleting item with otc: " + item.Otc + " and name: " + item.SecretName);
const deleteParams = {
TableName: TABLE_NAME,
Key: {
"Otc": item.Otc,
"SecretName": item.SecretName,
},
};
try {
await docClient.delete(deleteParams).promise();
} catch (err) {
console.log("The Secret was not deleted due to: ", err.message);
}
}
}))
}
} catch (err) {
console.log("The items were not able to be scanned due to : ", err.message)
}}
I know using DynamoDB TTL is an option, but I need these deletions to be somewhat precise, and TTL can sometimes take up to 48 hours, and I am aware I can use a filter when retrieving records to counter-act that. Just wondering what's wrong with my code here.
You need to await Promise.all or your lambda will end execution before it resolves
await Promise.all(items.map(async (item) => {
const expirationDT = new Date(item.ExpiresAt);
const now = new Date();
// ...
track the time elapsed in lambda function stop it at 9-10 min .
save the point at which it stopped and continue untill task is completed
compulsory use lambda function
I support John Rotenstein's response, stating that if you're using lambda this way, you probably shouldn't be using lambda. Out of my own curiosity though, I think the code you'd be looking for is something along the following lines (written in Node.JS)
let AWS = require('aws-sdk');
let functionName = 'YOUR_LAMBDA_FUNCTION_NAME'
let timeThreshholdInMillis = 5000 // 5 seconds
exports.handler = async (event, context, callback) => {
let input = JSON.parse(event['foo']); // input from previous lambda, omitted try-catch for brevity
let done // set this variable to true when your code is done
let interval = setInterval(() => {
if (done) callback(null, true)
let remainingTime = context.get_remaining_time_in_millis();
if (remainingTime < timeThreshholdInMillis) {
// Going to restart
clearInterval(interval);
// save your progress (local or remote) to `input` variable
let lambda = new AWS.Lambda();
return lambda.invoke({
FunctionName: functionName,
Payload: JSON.stringify({ 'foo': input }) // Pass your progress here
}, (err, data) => {
// kill container
if (err) callback(err)
else callback(null, data)
});
}
}, 1000);
}
Edit: an example
This is to clarify how 'passing progress' would work in recursive lambdas.
Let's say you want to increment a variable (+1) every second and you want to do this in Lambda.
Givens
We will increment the variable by 1 once every 1000 ms.
Lambda will run until remaining time is < 5000 ms.
Lambda execution timeout is 60,000 ms (1 minute).
Lambda function pseudo code:
function async (event, context) {
let counter = event.counter || 0
setInterval(() => {
if (context.get_remaining_time_in_millis() < 5000) {
// start new lambda and pass `counter` variable
// so it continues counting and we don't lose progress
lambda.invoke({ payload: { counter } })
} else {
// Add 1 to the counter variable
counter++
}
}, 1000)
}
Not sure what you are trying out, but have a look at AWS Step Functions to better orchestrate your serverless recursion fun.
Also be aware of costs.
Getting Started:
https://aws.amazon.com/getting-started/hands-on/create-a-serverless-workflow-step-functions-lambda/
Example:
https://docs.aws.amazon.com/step-functions/latest/dg/sample-project-transfer-data-sqs.html
I'm using lambda triggers to detect an insertion into a DynamoDB table (Tweets). Once triggered, I want to take the message in the event, and get the sentiment for it using Comprehend. I then want to update a second DynamoDB table (SentimentAnalysis) where I ADD + 1 to a value depending on the sentiment.
This works fine if I manually insert a single item, but I want to be able to use the Twitter API to insert bulk data into my DynamoDB table and have every tweet analysed for its sentiment. The lambda function works fine if the count specified in the Twitter params is <= 5, but anything above causes an issue with the update in the SentimentAnalysis table, and instead the trigger keeps repeating itself with no sign of progress or stopping.
This is my lambda code:
let AWS = require("aws-sdk");
let comprehend = new AWS.Comprehend();
let documentClient = new AWS.DynamoDB.DocumentClient();
exports.handler = (event, context) => {
event.Records.forEach(record => {
if (record.eventName == "INSERT") {
//console.log(JSON.stringify(record.dynamodb.NewImage.tweet.S));
let params = {
LanguageCode: "en",
Text: JSON.stringify(record.dynamodb.NewImage.tweet.S)
};
comprehend.detectSentiment(params, (err, data) => {
if (err) {
console.log("\nError with call to Comprehend:\n " + JSON.stringify(err));
} else {
console.log("\nSuccessful call to Comprehend:\n " + data.Sentiment);
//when comprehend is successful, update the sentiment analysis data
//we can use the ADD expression to increment the value of a number
let sentimentParams = {
TableName: "SentimentAnalysis",
Key: {
city: record.dynamodb.NewImage.city.S,
},
UpdateExpression: "ADD " + data.Sentiment.toLowerCase() + " :pr",
ExpressionAttributeValues: {
":pr": 1
}
};
documentClient.update(sentimentParams, (err, data) => {
if (err) {
console.error("Unable to read item " + JSON.stringify(sentimentParams.TableName));
} else {
console.log("Successful Update: " + JSON.stringify(data));
}
});
}
});
}
});
};
This is the image of a successful call, it works with the first few tweets
This is the unsuccessful call right after the first image. The request is always timed out
The timeout is why it’s happening repeatedly. If the lambda times out or otherwise errs it will cause the batch to be reprocessed. You need to handle this because the delivery is “at least once”. You also need to figure out the cause of the timeout. It might be as simple as smaller batches, or a more complex solution using step functions. You might just be able to increase the timeout on the lambda.
I am using AWS step function to invoke lambda function like this.
return stepfunctions.startExecution(params).promise().then((result) => {
console.log(result);
console.log(result.output);
return result;
})
And result is
{ executionArn: 'arn:aws:states:eu-west-2:695510026694:...........:7c197be6-9dca-4bef-966a-ae9ad327bf23',
startDate: 2018-07-09T07:35:14.930Z }
But i want the result as output of final lambda function
I am going through https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#sendTaskSuccess-property
There are multible function there i am confused which one could be used to get back result of final lambda function.
Same question is there on stackoverflow Api gateway get output results from step function? i dont want to call any function periodically and keep checking status.Even if i use DescribeExecution function periodically i will only get the status of execution but not the result i wanted. Is there any way or any function which returns promise and is resolved once all the lambda has executed and give back the result
You can't get back a result from a step function execution in a synchronous way.
Instead of polling the result of the step function on completion send a result to an SNS topic or SQS queue for further processing in the final lambda function or model the whole process in the step function state machine.
After doing some study and looking at various tutorial i realized that this stackoverflow answer Api gateway get output results from step function? gives a easier approach to solve the problem and get final result from step function, yes i am not sure about another approach and how to implement any new answer is always appreciated
This is my code to implement the same approach this might help someone.
// in function first start step function execution using startExecution()
var params = {
stateMachineArn: 'some correct ARN',
input: JSON.stringify(body)
};
return stepfunctions.startExecution(params).promise().then((result) => {
var paramsStatus = {
executionArn: result.executionArn
};
var finalResponse = new Promise(function(resolve,reject){
var checkStatusOfStepFunction = setInterval(function(){
//on regular interval keep checking status of step function
stepfunctions.describeExecution(paramsStatus, function(err, data) {
console.log('called describeExecution:', data.status);
if (err){
clearInterval(checkStatusOfStepFunction);
reject(err);
}
else {
if(data.status !== 'RUNNING'){
// once we get status is not running means step function execution is now finished and we get result as data.output
clearInterval(checkStatusOfStepFunction);
resolve(data.output);
}
}
});
},200);
});
return finalResponse
})
To be able to get the result of step function (example: combined gateway & step function). You need to:
1. startExecution,
2. wait for your state machine to finish the execution (to be sure make wait equivalent to timeout of your state machine => wait = TimeoutSeconds of your state machine)
3. call describeExecution with the receive executionArn from startExecution.
Note that startExecution is an async function and it's not waiting for the result.
In my case, I'm using Lambda named init to execute the 3 discussed steps:
Code lambda Init:
const AWS = require('aws-sdk')
exports.handler = async (event) => {
const stepFunctions = new AWS.StepFunctions();
const reqBody = event.body || {};
const params = {
stateMachineArn: process.en.stateMachineArn,
input: JSON.stringify(reqBody)
}
return stepFunctions.startExecution(params).promise()
.then(async data => {
console.log('==> data: ', data)
await new Promise(r => setTimeout(r, 6000));
return stepFunctions.describeExecution({ executionArn: data.executionArn }).promise();
})
.then(result => {
return {
statusCode: 200,
message: JSON.stringify(result)
}
})
.catch(err => {
console.error('err: ', err)
return {
statusCode: 500,
message: JSON.stringify({ message: 'facing error' })
}
})
}
Code stateMachine
Make sure that in your statemachine your returning "ResultPath".
{
"Comment": "Annoucement validation",
"StartAt": "contact-validation",
"Version": "1.0",
"TimeoutSeconds": 5,
"States": {
"contact-validation": {
"Type": "Task",
"Resource": "arn:aws:xxxxxxx:function:scam-detection-dev-contact",
"ResultPath": "$.res",
"Next": "WaitSeconds"
},
"WaitSeconds": {
"Type": "Wait",
"Seconds": 1,
"Next": "Result"
},
"Result": {
"Type": "Pass",
"ResultPath": "$.res",
"End": true
}
}
}
Is it possible to run alexa skill locally with ngrok instead AWS? I built a skill in AWS Lambda but I would rather use my own server. What can I do to run Alexa locally?
I tried https://github.com/alexa-js/alexa-app-server but it makes any sense because I would need to rewrite my whole code :( The better solution is http://docs.bespoken.tools/en/latest/tutorials/tutorial_lambda_nodejs/ but it isn't the best. It just works only for a wellcome intent and freezes after that :(
Terminal Logs from bespken command:
BST: v0.9.35 Node: v7.8.0
Your URL for Alexa Skill configuration:
https://proxy.bespoken.tools?node-id=33efccba-2246-477f-bbb8-2e1e510cce9d
INFO 2017-04-25T20:27:20.628Z Connected - proxy.bespoken.tools:5000
INFO 2017-04-25T20:27:26.812Z RequestReceived: POST /?node-id=33efccba-2246-477f-bbb8-2e1e510cce9d ID: 1493152039146
INFO 2017-04-25T20:27:26.815Z Forwarding localhost:10000
Current hour: 24
Warning: Application ID is not set
INFO 2017-04-25T20:27:27.939Z ResponseReceived ID: 1493152039146
INFO 2017-04-25T20:28:10.755Z RequestReceived: POST /?node-id=33efccba-2246-477f-bbb8-2e1e510cce9d ID: 1493152078963
INFO 2017-04-25T20:28:10.756Z Forwarding localhost:10000
Warning: Application ID is not set
INFO 2017-04-25T20:28:11.157Z ResponseReceived ID: 1493152078963
INFO 2017-04-25T20:28:51.073Z RequestReceived: POST /?node-id=33efccba-2246-477f-bbb8-2e1e510cce9d ID: 1493152113739
INFO 2017-04-25T20:28:51.073Z Forwarding localhost:10000
Warning: Application ID is not set
INFO 2017-04-25T20:28:51.995Z ResponseReceived ID: 1493152113739
Yes, there are several solutions for running your node lambda locally. I've been using node-lambda, for example. Like most solutions it is oriented to users who want to test locally and then easily deploy to AWS Lambda.
If you want to run them yourself, I would note that MS and IBM have made their implementations of lambda open-source (here's MS's and IBM's). I haven't actually tried it myself, and I would note that with AWS, GCP, and Azure all providing Lambda services for node the market for these is healthy and the lock-in is minimal so I feel less need to be able to run it myself than for something like Dynamo.
But I also recommend that you continue to pursue BST. I'm using some of my own pieces for testing my skills because I got started before I heard of their stuff, but what I have tried of their's (BSTAlexa) is very useful and I see that they provide some of the other pieces you need for easy and effective testing of your skill.
Here's some sample code that you can use to easily run a Lambda locally, call this file AlexaLambda.js:
const log = require('console');
var AWS = require('aws-sdk');
AWS.config.region = "us-east-1";
AWS.config.update({
accessKeyId: "----",
secretAccessKey: "----",
});
/**
* Wraps the actual underlying Alexa lambda initialization in a
* Promise. Injects test mocks where appropriate.
*/
var initializerPromise = new Promise(function(fulfill, reject) {
// Mock out certain imports here if you want but not necessary
/*
var Module = require('module');
var originalRequire = Module.prototype.require;
Module.prototype.require = function() {
if ((arguments[0] == 'S3FeedService') ||
(arguments[0] == './lib/S3FeedService')) {
return MockS3Service;
} else if ((arguments[0] == 'WebsocketService') ||
(arguments[0] == './lib/WebsocketService')) {
return WSMockService;
} else if ((arguments[0] == 'SQSService') ||
(arguments[0] == './lib/SQSService')) {
return SQSMockService;
} else {
return originalRequire.apply(this, arguments);
}
};*/
// Import your actual lambda here.
var lambda = require('../src/index.js');
fulfill(lambda);
});
/**
* The Alexa Lambda context object which is called upon completion
* of lambda execution. Also wraps the callback which contains the
* test assertion code of the caller.
* #param callback - must be of the form function(error, result) {};
* #returns
*/
function Context(callback) {
this.clientContext = {"env": {}};
this.callback = callback;
}
Context.prototype.done = function(error, result) {
if (typeof error != "undefined" && error) {
this.callback(error, null);
} else {
this.callback(null, result);
}
}
Context.prototype.succeed = function(result) {
this.callback(null, result);
}
Context.prototype.fail = function(error) {
this.callback(error, null);
}
/**
* The AlexaLambda object that's exposed for test cases.
* #returns
*/
function AlexaLambda() {
}
/**
* Executes the lambda function, provided an inputEvent and a
* callback.
* #param inputEvent - the input event that includes the intent.
* #param callback - called upon completion of lambda execution.
*/
AlexaLambda.prototype.execute = function(inputEvent, callback) {
initializerPromise.then(function(lambda) {
var context = new Context(callback);
lambda.handler(inputEvent, context);
});
}
/**
* Export the lambda class, importers instantiate via new AlexaLambda();
*/
module.exports = AlexaLambda;
Then you can use this 'AlexaLambda' in your tests like so (in my case, I'm using Mocha):
var AlexaLambda = require('./AlexaLambda');
var Event = require('./Event'); // My 'fake' Event class
describe("Guest User Test", function() {
var alexaLambda = new AlexaLambda();
it("Alexa, open/launch 60db", function(done) {
var event = Event.createLaunchEvent();
alexaLambda.execute(event, function(error, result) {
validateYourResultHere();
done();
})
});
Then it's just a matter of running your test via whatever framework you're using.
You can test your alexa skill locally by following the following tutorial:
How to test Alexa locally