AWS - Sending 1000's of emails from Lambda / Node.js - amazon-web-services

I have a "main" Lambda function that gets triggered by SNS. It pulls a list of recipients from the database and it needs to send each of them a message based on a template, replacing things like first name and such.
The way I have it setup is I created another Lambda function called "email-send" which is subscribed to "email-send" topic. The "main" Lambda then loops through the recipients list and publishes messages to "email-send" with a proper payload (from, to, subject, message). This might eventually need to process 1000's of emails in a single batch.
Is this a good approach to my requirements? Perhaps Lambda/SNS is not a way to go? If so, what would you recommend.
With this setup I am running into issues when my "main" function finishes running and somehow "sns.publish" does not get triggered in my loop. I assume because I am not letting it finish. But I am not sure how to fix it, being a loop.
Here is the snippet from my Lambda function:
exports.handler = (event, context, callback) => {
// code is here to pull data into "data" array
// process records
for (var i = 0; i < data.length; i++) {
var sns = new aws.SNS();
sns.publish({
Message: JSON.stringify({ from: data[i].from, to: data[i].to, subject: subject, body: body }),
TopicArn: 'arn:aws:sns:us-west-2:XXXXXXXX:email-send'
}, function(err, data) {
if (err) {
console.log(err.stack);
} else {
console.log('SNS pushed!');
}
});
}
context.succeed("success");
};
Thanks for any assistance.

Your code is doing this...
Begin calling sns.publish() 1000 times
Return (through context.succeed())
You didn't wait for those 1000 calls to finish!
What your code should do is...
Begin calling sns.publish() 1000 times
When all calls to sns.publish() has returned, then return. (context.succeed is old so we should use callback() instead).
Something like this...
// Instantiate the client only once instead of data.length times
const sns = new aws.SNS();
exports.handler = (event, context, callback) => {
const snsCalls = []
for (var i = 0; i < data.length; i++) {
snsCalls.push(sns.publish({
Message: JSON.stringify({
from: data[i].from,
to: data[i].to,
subject: subject,
body: body
}),
TopicArn: 'arn:aws:sns:us-west-2:XXXXXXXX:email-send'
}).promise();
}
return Promise.all(snsCalls)
.then(() => callback(null, 'Success'))
.catch(err => callback(err));
};

I think that a better approach is using AWS Lambda API.
That way, you don't need SNS.
For example:
var lambda = new AWS.Lambda({region: AWS_REGION});
function invokeWorkerLambda(task, callback) {
var params = {
FunctionName: WORKER_LAMBDA_NAME,
InvocationType: 'Event',
Payload: JSON.stringify({.....})
};
lambda.invoke(params, function(err, data) {
if (err) {
console.error(err, err.stack);
callback(err);
} else {
callback(null, data);
}
});
}
As you can see, you don't need SNS for lambda function's invocation.
Important: Another suggestion is to create an Array of invocations (functions) and later execute them as follow:
async.parallel(invocations, function(err) {
if (err) {
console.error(err, err.stack);
callback(err);
}
});
Take a look at this link where I got a lot of knowledge about Lambda invocation: https://cloudonaut.io/integrate-sqs-and-lambda-serverless-architecture-for-asynchronous-workloads/

Related

Publishing SNS message from lambda (node js) result in timeout error

I have used a very simple code slightly modified from AWS provided example:
exports.handler = async (event) => {
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2'});
// Create publish parameters
var params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS().publish(params).promise();
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
// Handle promise's fulfilled/rejected states
publishTextPromise.then(
function(data) {
console.log("Message ${params.Message} send sent to the topic ${params.TopicArn}");
console.log("MessageID is " + data.MessageId);
response.result = 'Success';
}).catch(
function(err) {
console.error(err, err.stack);
response.result = 'Error';
});
return response;
};
And I am getting a timeout error when testing this service. 3 seconds is the limit.
Since this is a very simply process, I suppose it shouldn't take more than 3 seconds to execute.
I have checked my IAM setting and grant my profile (admin profile) to have full access to SNS service. But the error still persists. I am wondering what is being wrong here and how should I fix this?
I am not sure why you're getting the timeout, but your code isn't supposed to work the way you might expect.
See that you're returning your response outside your .then() code, meaning that your code is going to return before your .then() code is even run (Promises are asynchronous).
Since you're already using Node 8, you're better off using async/await instead of using the old .then().catch() approach.
I have refactored your code a little bit and it works just fine. I have kept the original parameters for your convenience. See how the code is much easier to read and debug.
'use strict';
// Load the AWS SDK for Node.js
const AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2'});
const sns = new AWS.SNS()
module.exports.handler = async (event) => {
const params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
try {
const data = await sns.publish(params).promise();
response.messageId = data.MessageId,
response.result = 'Success'
} catch (e) {
console.log(e.stack)
response.result = 'Error'
}
return response
};
If for whatever reason you don't want to use async/await, you then need to move the return of your function inside your .then() code, as well as return as soon as the promise is invoked, like this:
'use strict';
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2''});
// Create publish parameters
var params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
module.exports.handler = async (event) => {
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS().publish(params).promise();
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
// Handle promise's fulfilled/rejected states
return publishTextPromise.then(
function(data) {
console.log("Message ${params.Message} send sent to the topic ${params.TopicArn}");
console.log("MessageID is " + data.MessageId);
response.result = 'Success';
return response;
}).catch(
function(err) {
console.error(err, err.stack);
response.result = 'Error';
return response
});
};
I highly recommend you go with approach #1 though.

Add information into DynamoDB using Alexa Lambda function

I've been stuck on this code for about a day now.
I am just trying to add information into DynamoDB through a launch request using Alexa.
I get the following error code:
"errorMessage": "RequestId: f96ae2cb-1dbf-11e7-a267-b7cf2f2c95a0 Process exited before completing request"
The information actually gets inserted into DynamoDB, but I can't add more functions to the program because of the error.
From what I understand, it may be a problem with the callback.
I have tried many different ways to "callback" or return something, but I haven't figured out how to avoid the error.
If I uncomment this.emit(':tell', "Hello, inserting Apples into DynamoDB"); the error goes away, but no information gets inserted.
What am I doing wrong and how can I fix it?
Below is my code;
'use strict';
var Alexa = require('alexa-sdk');
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();
exports.handler = function(event, context, callback) {
var alexa = Alexa.handler(event, context);
alexa.registerHandlers(handlers);
alexa.execute();
};
var handlers = {
'LaunchRequest': function(event, context, callback) {
// this.emit(':tell', "Hello, inserting Apples into DynamoDB");
var params = {
Item: {
date: Date.now(),
message: "Apples"
},
TableName: '_yourTableName'
};
dynamo.putItem(params, function(err, data) {
if (err) {
callback(err, null);
} else {
callback(null, data);
}
});
context.done();
}
};
This is because you are adding values to Dynamodb, which is a callback but context.done(); is written outside the callback. Before dynamoDb completes the operation it will call context.done(); hence it will exit the process

Sending SNS push through lambda - no message?

I have a problem with my lambda function. It takes the data of the devices and sends the push notification (information that someone added you to friends). However, it often happens that the push notificaiton is not sent. Sometimes, I have to activate the function couple of times (through simultaneous clicking on the button) to make it send push notifications. When testing the below function in Lambda it does not send any push notifications. What might be the issue?
Thank you in advance,
John
console.log("Loading friend-request function");
var AWS = require("aws-sdk");
exports.handler = function(event, context) {
var senderID = event.senderID;
var receiverID = event.receiverID;
var message = event.message;
var eventText = JSON.stringify(event);
console.log("Received event:", eventText);
var sns = new AWS.SNS();
var params = {
Message: message,
Subject: "Test SNS From Lambda",
TargetArn: receiverID
};
context.succeed(message);
sns.publish(params, context.done);
};
Remember that the sns.publish() function is asynchronous, i.e. you must wait for it to complete before you can call the context.succeed(). Otherwise, the Lambda function may terminate before the message has been published.
exports.handler = function(event, context) {
// same as before...
var params = {...}
// do not call context.succeed() yet
sns.publish(params, function(err, data) {
if (err) {
console.log('Failed to publish SNS message');
context.fail(err);
}
else {
console.log('SNS message published successfully');
context.succeed(data);
}
});
};

Amazon Lambda to SNS notifications not working

I have a use case of pushing SNS notifications from Amazon lambda. I have written the following code, with the IAM role having the permission to invoke SNS. Even with the Kenesis trigger, this Lambda function is unable to send any notification to SNS. I confirmed this by, subscribing my email id to the SNS.
[EDIT]: Just a follow up question. I now need to query DynamoDB and based on the output, need to call different end point of SNS. Now, when I query DynamoDB, the execution stops after DynamoDB query and not even progressing for SNS checks.
Following is my lambda function code.
console.log('Loading function');
var AWS = require('aws-sdk');
exports.handler = function(event, context) {
event.Records.forEach(function(record) {
//var payload = new Buffer(record.kinesis.data, 'base64').toString('ascii');
var payload = record.kinesis.data;
console.log('Decoded payload:', payload);
var dynamodb = new AWS.DynamoDB();
var dynamodb_params = {
Key: {
dataSource: {
S: payload
}
},
TableName: 'TableName',
AttributesToGet: [
'attribute' // ...
],
ConsistentRead: false,
};
var sns_endpoint = null;
dynamodb.getItem(dynamodb_params, function(err, data) {
if (err) {
console.log(err.stack);
context.done(err, 'Errors while querying dynamodb!');
} else {
console.log(data);
sns_endpoint = data.Item.sns.S;
console.log("Result= " + data);
console.log("Item= " + data.Item);
console.log("sns= " + data.Item.sns);
console.log("value= " + data.Item.sns.S);
console.log("sns_endpoint= " + sns_endpoint);
context.done(null, 'Querying dynamodb succeeded!');
}
});
if( sns_endpoint != null ) {
console.log("sns_endpoint= " + sns_endpoint);
var sns_params = {
Message: payload,
Subject: 'Event Notification From Lambda',
TopicArn: sns_endpoint
};
var sns = new AWS.SNS();
sns.publish(sns_params, function(err, data) {
if (err) {
console.log(err.stack);
context.done(err, 'Errors while putting to SNS!');
} else {
console.log(data);
context.done(null, 'Putting to SNS succeeded!');
}
});
}
});
};
You are calling an asynchronous function, sns.publish(), within a forEach loop. Then you are immediately calling context.succeed(). As soon as context.succeed, context.fail or context.done are called, your Lambda function will exit. You need to modify your code to only call one of those after all asynchronous function calls have completed.

AWS Lambda subscribed to SNS topic doesn't process all messages to the topic

I am building an alerting system powered by Amazon services.
I drop a file onto S3 daily which spawns a lambda function (let's call it the Generator function) to process the file.
Generator builds alerts based on this file and posts multiple messages to an SNS topic (let's call it Outbox) - one message for each recipient calculated by Generator.
I have a second lambda function (let's call it Courier) subscribed to Outbox which should take each message and do something with it.
The Generator code:
// 'Generator' function
exports.handler = function (event, context) {
console.log('Reading options from event:\n', util.inspect(event, {depth: 5}));
var users = {};
var userSubscriptions = {};
var alerts = {};
var artists = {};
var tracks = {};
async.waterfall([
function downloadSubscribersFile (next) {
// Do stuff
},
function downloadAndFormatActionsFile (next) {
// Download data file and analyse
},
function publishAlerts (next) {
// Now we have alerts built, we need to mail them out
var recipients = Object.keys(alerts);
async.each(recipients, function (recipient, callback) {
var recipientAlert = alerts[recipient];
console.log(util.inspect(recipientAlert, { depth: 10 }));
if (alerts[recipient].actions.artists.length < 1) {
return callback();
}
var params = {
TopicArn: SNS_TOPIC_ARN,
Subject: recipient,
Message: JSON.stringify(recipientAlert)
};
sns.publish(params, function (err, data) {
if (err) {
console.log(err);
return callback(err);
} else {
console.log('PUBLISHED MESSAGE: \n', util.inspect(data, { depth: 10 }));
console.log('MESSAGE WAS: \n', util.inspect(params, { depth: 10 }));
}
return callback();
})
}, function (err) {
if (err) return next(err);
next();
})
}
], function (err) {
if (err) {
console.log('Error: ', err);
} else {
console.log('Process successful');
}
context.done();
})
}
And the other function:
// 'Courier' function
console.log('Loading function');
exports.handler = function(event, context) {
console.log(JSON.stringify(event, null, 2));
console.log('From SNS:', event.Records[0].Sns.Message);
context.succeed();
};
When my Generator function is invoked, I can see 12 messages should be posted to the SNS topic. There are no errors recorded when publishing these messages, and yet the Courier function only fires once.
I'm wondering if anyone's had any similar issues with this and whether there's something I'm missing here. It could be that there's something I haven't configured correctly in AWS but I'm pretty confident everything is set up as it should be.
UPDATE:
After looking at the messages I'm attempting to send, it seems the message that WAS picked up by SNS seemed to be the one with the smallest payload. I'm wondering if SNS is able to cope with lots of small, frequent messages to a topic...?
Yes, SNS can cope with high throughput on a single topic. However, there is a maximum message size 256KB, so if your messages are larger than that it might be the cause.
I see your Generator function is logging the messages, are you seeing 12 logged messages with message ids? I see you have a variable alerts which you are hoping to get multiple recipients from, but I don't see where you are setting it.
My advice: Add more logging before you sent the message to verify that what you think should be happening is actually happening.