In our scalable AWS serverless application (DynamoDB, Lambda, API Gateway WebSockets) we want to broadcast messages from a Lambda to very many connected browser sessions simultaneously.
We have many topics, and to limit the load on the system we want to limit each browser session to subscribe to max. two topics.
The AWS example app linked in this official tutorial does this by looping through the connectionIds and doing an await postToConnection for each of them, which doesn't scale well as each postToConnection takes 40 ms (average):
const AWS = require('aws-sdk');
const ddb = new AWS.DynamoDB.DocumentClient();
exports.handler = async function (event, context) {
let connections;
try {
connections = await ddb.scan({ TableName: process.env.table }).promise();
} catch (err) {
return {
statusCode: 500,
};
}
const callbackAPI = new AWS.ApiGatewayManagementApi({
apiVersion: '2018-11-29',
endpoint:
event.requestContext.domainName + '/' + event.requestContext.stage,
});
const message = JSON.parse(event.body).message;
const sendMessages = connections.Items.map(async ({ connectionId }) => {
if (connectionId !== event.requestContext.connectionId) {
try {
await callbackAPI
.postToConnection({ ConnectionId: connectionId, Data: message })
.promise();
} catch (e) {
console.log(e);
}
}
});
try {
await Promise.all(sendMessages);
} catch (e) {
console.log(e);
return {
statusCode: 500,
};
}
return { statusCode: 200 };
};
How can we make this broadcast scalable?
In some of my routes the lambda function times out after 6 seconds when it is using an existing mongodb connection.
I am sure the function itself is not the issue as I have added lots of logs and it takes 0.1 seconds to finish but the lambda does not return anything for 6+ seconds. In most of the tutorials online the db connection is in the same folder as the lambda function but I have moved it out to its own package as I have 5+ endpoints.
This is the most basic endpoint that times out every now and then
module.exports.getAll = async (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false;
try {
await databaseModels.db();
const x = await databaseModels.items.model.find({});
callback(null, {
statusCode: 200,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(x)
})
} catch (error) {
callback(error, {
statusCode: 500,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(error)
})
}
};
This is my connect to database function which I call at the start of every lambda function
const url = ``
let isConnected;
module.exports = connectToDatabase = () => {
if (isConnected) {
console.log('=> using existing database connection');
return Promise.resolve();
}
console.log('=> using new database connection');
return mongoose.connect(url).then(db => {
isConnected = db.connections[0].readyState;
});
};
In one of the turorials I found the author opened a connection for each request and closed it when the computation was done. Is this a good practice?
Debugging lambda is frustrating.
I've got a very simple lambda function:
const AWS = require('aws-sdk')
const dynamodb = new AWS.DynamoDB.DocumentClient({region: 'ap-southeast-2'});
exports.handler = async (event, context, callback) => {
const params = {
TableName: 'people-dev',
Item: {
id: '1',
name: 'person',
email: 'person#example.com'
}
};
dynamodb.put(params, (err, data) => {
if(err) {
callback(err, null)
} else{
callback(null, data)
}
});
};
The test response is:
Response:
null
Request ID:
"3d7e9329-3843-4760-917d-4b4d4781dbd7"
Function Logs:
START RequestId: 3d7e9329-3843-4760-917d-4b4d4781dbd7 Version: $LATEST
END RequestId: 3d7e9329-3843-4760-917d-4b4d4781dbd7
REPORT RequestId: 3d7e9329-3843-4760-917d-4b4d4781dbd7 Duration: 243.13 ms Billed Duration: 300 ms Memory Size: 128 MB Max Memory Used: 29 MB
Nothing is being written to Dynamo.
Nothing useful is logged in cloudwatch.
Yes, the function has full permissions to DynamoDB.
Put async back to its place as using callbacks is outdated and much more error prone. Use the built-in promise() methods available on the node.js aws-sdk and just await on these promises. If you want to deal with errors, just surround your code with a try/catch block.
const AWS = require('aws-sdk')
const dynamodb = new AWS.DynamoDB.DocumentClient({region: 'ap-southeast-2'});
exports.handler = async (event) => {
const params = {
TableName: 'people-dev',
Item: {
id: '1',
name: 'person',
email: 'person#example.com'
}
};
await dynamodb.put(params).promise()
return {
statusCode: 200,
body: JSON.stringify({message: 'Success'})
}
};
More on async/await
The issue was that the handler was executing asynchronously.
exports.handler = async (event, context, ....
Changing it to the following, fixed the problem:
exports.handler = function (event, context, ....
I have used a very simple code slightly modified from AWS provided example:
exports.handler = async (event) => {
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2'});
// Create publish parameters
var params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS().publish(params).promise();
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
// Handle promise's fulfilled/rejected states
publishTextPromise.then(
function(data) {
console.log("Message ${params.Message} send sent to the topic ${params.TopicArn}");
console.log("MessageID is " + data.MessageId);
response.result = 'Success';
}).catch(
function(err) {
console.error(err, err.stack);
response.result = 'Error';
});
return response;
};
And I am getting a timeout error when testing this service. 3 seconds is the limit.
Since this is a very simply process, I suppose it shouldn't take more than 3 seconds to execute.
I have checked my IAM setting and grant my profile (admin profile) to have full access to SNS service. But the error still persists. I am wondering what is being wrong here and how should I fix this?
I am not sure why you're getting the timeout, but your code isn't supposed to work the way you might expect.
See that you're returning your response outside your .then() code, meaning that your code is going to return before your .then() code is even run (Promises are asynchronous).
Since you're already using Node 8, you're better off using async/await instead of using the old .then().catch() approach.
I have refactored your code a little bit and it works just fine. I have kept the original parameters for your convenience. See how the code is much easier to read and debug.
'use strict';
// Load the AWS SDK for Node.js
const AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2'});
const sns = new AWS.SNS()
module.exports.handler = async (event) => {
const params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
try {
const data = await sns.publish(params).promise();
response.messageId = data.MessageId,
response.result = 'Success'
} catch (e) {
console.log(e.stack)
response.result = 'Error'
}
return response
};
If for whatever reason you don't want to use async/await, you then need to move the return of your function inside your .then() code, as well as return as soon as the promise is invoked, like this:
'use strict';
// Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
// Set region
AWS.config.update({region: 'ap-southeast-2''});
// Create publish parameters
var params = {
Message: 'This is a sample message',
Subject: 'Test SNS From Lambda',
TopicArn: 'arn:aws:sns:ap-southeast-2:577913011449:TestTopic'
};
module.exports.handler = async (event) => {
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS().publish(params).promise();
let response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
// Handle promise's fulfilled/rejected states
return publishTextPromise.then(
function(data) {
console.log("Message ${params.Message} send sent to the topic ${params.TopicArn}");
console.log("MessageID is " + data.MessageId);
response.result = 'Success';
return response;
}).catch(
function(err) {
console.error(err, err.stack);
response.result = 'Error';
return response
});
};
I highly recommend you go with approach #1 though.
Now day we can use the AWS StepFunctions if we want to make an lambda function to call another one.
But for now I need do support the code in production that was written before the StepFunctions time.
For that reason I need to understand how it works. I was trying to create a very simple lambda calling another lambda function trough AWS-SDk.
I have the follow serverless.yml
service: lambdaCallLambda
provider:
name: aws
runtime: nodejs6.10
functions:
hello:
handler: handler.hello
funcOne:
handler: handler.funcOne
funcTwo:
handler: handler.funcTwo
#Must install aws-sdk. #npm install --save aws-sdk
And this is the handler.js:
'use strict';
//https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html
var Lambda = require('aws-sdk/clients/lambda');
module.exports.hello = (event, context, callback) => {
const response = {
statusCode: 200,
body: JSON.stringify({
message: 'hello',
input: event,
}),
};
callback(null, response);
};
module.exports.funcOne = (event, context, callback) => {
var text='';
var i = 0;
for (i = 0; i < 5; i++) {
text += "The number is " + i + "\n";
}
console.log(text);
//https://docs.aws.amazon.com/general/latest/gr/rande.html
const lambda = new Lambda({
region: 'us-east-1'
});
console.log('control 3');
/*
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#constructor-property
To invoke a Lambda function
This operation invokes a Lambda function
https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html
Payload - JSON that you want to provide to your Lambda function as input.
*/
var params = {
ClientContext: "lambdaCallLambda",
FunctionName: "lambdaCallLambda-dev-funcOne",
InvocationType: "Event",
LogType: "Tail",
Payload: '{"jsonKey2":123}',
Qualifier: "1"
};
lambda.invoke(params, function(err, data) {
if (err){
console.log('control error\n');
console.log(err, err.stack); // an error occurred
}
else{
console.log('control OK\n');
console.log(data); // successful response
}
/*
data = {
FunctionError: "",
LogResult: "",
Payload: <Binary String>,
StatusCode: 123
}
*/
});
};
module.exports.funcTwo = async (event, context) => {
return 2;
//return '{"funcTwo":20000}';
//console.log("funcTwo = " + event);
};
After deploy sls deploy and call funcOne I get this 2 outputs:
LOCAL:
sls invoke local --function funcOne
Serverless: INVOKING INVOKE
The number is 0
The number is 1
The number is 2
The number is 3
The number is 4
control 3
control OK
{ StatusCode: 202, Payload: '' }
Invoking remotely in AWS:
sls invoke --function funcOne
{
"errorMessage": "Unexpected token (",
"errorType": "SyntaxError",
"stackTrace": [
" ^",
"SyntaxError: Unexpected token (",
"createScript (vm.js:56:10)",
"Object.runInThisContext (vm.js:97:10)",
"Module._compile (module.js:542:28)",
"Object.Module._extensions..js (module.js:579:10)",
"Module.load (module.js:487:32)",
"tryModuleLoad (module.js:446:12)",
"Function.Module._load (module.js:438:3)",
"Module.require (module.js:497:17)",
"require (internal/module.js:20:19)"
]
}
Error --------------------------------------------------
Invoked function failed
For debugging logs, run again after setting the "SLS_DEBUG=*" environment variable.
Get Support --------------------------------------------
Docs: docs.serverless.com
Bugs: github.com/serverless/serverless/issues
Issues: forum.serverless.com
Your Environment Information -----------------------------
OS: linux
Node Version: 8.11.3
Serverless Version: 1.29.2
Does someone knows hat is happening here? Specially for the first scenario where I dont have any error.
This is what I get from the documentation
Parameters:
err (Error) — the error object returned from the request. Set to null if the request is successful.
data (Object) — the de-serialized data returned from the request. Set to null if a request error occurs. The data object has the following properties:
Status — (Integer)
It will be 202 upon success.
Update
After Eduardo Díaz suggestion -
I have changed lambda.invoke to:
lambda.invoke({
FunctionName: 'lambdaCallLambda-dev-funcOne',
Payload: JSON.stringify(event, null, 2)
}, function(error, data) {
if (error) {
console.log('control ErrorFoncOne\n');
context.done('error', error);
}
if(data.Payload){
console.log('control SuccessFoncOne\n');
context.succeed(data)
}
});
And this what I get for Local and Remote:
{
"errorMessage": "Unexpected token (",
"errorType": "SyntaxError",
"stackTrace": [
"Module.load (module.js:487:32)",
"tryModuleLoad (module.js:446:12)",
"Function.Module._load (module.js:438:3)",
"Module.require (module.js:497:17)",
"require (internal/module.js:20:19)"
]
}
It is a SyntaxError. There is a "(" somewhere.
I have found another developer with the same error here.
Note:
No error logs in CloudWatch
Try to send the event in the payload:
lambda.invoke({
FunctionName: 'name_lambda_function',
Payload: JSON.stringify(event, null, 2)
}, function(error, data) {
if (error) {
context.done('error', error);
}
if(data.Payload){
context.succeed(data.Payload)
}
});
I strongly suspect this problem is rooted in your handler signature for funcTwo:
module.exports.funcTwo = async (event, context) => {
NodeJS 6.10 does not support async/await. Node always complains about the token after the async token, for whatever reason. If you don't use a fat arrow function:
module.exports.funcTwo = async function(event, context) {
Node will complain: Unexpected token function.
Options
Deploy the function to NodeJS 8.10 instead.
Get rid of the async keyword in the handler signature.
Use a build tool (like serverless-webpack) to transpile the function down to ES6 (or lower).
Note: If you stick with the 6.10 runtime, I think you'll want to do something like context.succeed(2); or callback(null, 2);, rather than return 2;. Simply using a return statement does seem to work on 8.10.
I have found the issue. We need JSON.stringfy in the playload in lamdda.invoke method. No extra parameters are needed. Only the JSON as per the documentation.
lambda.invoke({
FunctionName: 'lambdaCallLambda-dev-funcTwo',
Payload: JSON.stringify({"jsonKey2":i})
...
From the AWS documentation for playload we have:
Payload
JSON that you want to provide to your Lambda function as input.
Note:
the async in front of the functions
lambda.invoke({
FunctionName: 'lambdaCallLambda-dev-funcTwo',
Payload: JSON.stringify({"jsonKey2":i})
}, async function(error, data) { ...
and
module.exports.funcTwo = async(event, context, callback) => { ...
Gives me this output:
Loop nb=1
Loop nb=2
Loop nb=3
Loop nb=4
Loop nb=5
{"message":"hello from funcTwo","event":{"jsonKey2":3}}
{"message":"hello from funcTwo","event":{"jsonKey2":2}}
{"message":"hello from funcTwo","event":{"jsonKey2":1}}
{"message":"hello from funcTwo","event":{"jsonKey2":5}}
{"message":"hello from funcTwo","event":{"jsonKey2":4}}
While the absence of async gives me:
Loop nb=1
Loop nb=2
Loop nb=3
Loop nb=4
Loop nb=5
{"message":"hello from funcTwo","event":{"jsonKey2":1}}
{"message":"hello from funcTwo","event":{"jsonKey2":2}}
{"message":"hello from funcTwo","event":{"jsonKey2":3}}
{"message":"hello from funcTwo","event":{"jsonKey2":4}}
{"message":"hello from funcTwo","event":{"jsonKey2":5}}
I'm going to share the handle.js code just in case someone else needs it:
'use strict';
//https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html
var Lambda = require('aws-sdk/clients/lambda');
module.exports.hello = (event, context, callback) => {
const response = {
statusCode: 200,
body: JSON.stringify({
message: 'hello',
input: event,
}),
};
callback(null, response);
};
/*
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#constructor-property
To invoke a Lambda function
This operation invokes a Lambda function
https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html
Payload - JSON that you want to provide to your Lambda function as input.
Serverless Framework: Lambdas Invoking Lambdas
https://lorenstewart.me/2017/10/02/serverless-framework-lambdas-invoking-lambdas/
How to escape async/await hell
https://medium.freecodecamp.org/avoiding-the-async-await-hell-c77a0fb71c4c
Iterating a Loop Using Lambda
https://docs.aws.amazon.com/step-functions/latest/dg/tutorial-create-iterate-pattern-section.html
AWS Lambda “Process exited before completing request”
https://stackoverflow.com/questions/31627950/aws-lambda-process-exited-before-completing-request
Class: AWS.Lambda
https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html#constructor-property
Invoke
https://docs.aws.amazon.com/lambda/latest/dg/API_Invoke.html#API_Invoke_RequestSyntax
Programming Model(Node.js)
https://docs.aws.amazon.com/lambda/latest/dg/programming-model.html
AWS Lambda Examples
https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/lambda-examples.html
*/
//The event is a file inserted in S3. This function funcOne reads the file and loop trough all quantities of products.
//Invoking a lamda function funcTwo for each process in the loop.
module.exports.funcOne = (event, context, callback) => {
//https://docs.aws.amazon.com/general/latest/gr/rande.html
const lambda = new Lambda({
region: 'us-east-1'
});
//Loop
//nbProducts = loop trough the products JSON list in S3
var nbProducts=5;
for (let i = 1; i <= nbProducts; i++) {
console.log('Loop nb='+i+'\n');
lambda.invoke({
FunctionName: 'lambdaCallLambda-dev-funcTwo',
Payload: JSON.stringify({"jsonKey2":i})
}, async function(error, data) {
if (error) {
//console.log('control ErrorFoncOne\n');
context.done('error', error);
}
if(data.Payload){
//console.log('control SuccessFoncOne\n');
console.log(data.Payload);
//context.succeed(data)
}
});
}
};
module.exports.funcTwo = async(event, context, callback) => {
callback(null, { message: 'hello from funcTwo', event });
};
just a change of one line would get you the respose
do change in params
'use strict';
//https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Lambda.html
var Lambda = require('aws-sdk/clients/lambda');
module.exports.hello = (event, context, callback) => {
const response = {
statusCode: 200,
body: JSON.stringify({
message: 'hello',
input: event,
}),
};
callback(null, response);
};
module.exports.funcOne = (event, context, callback) => {
var text='';
var i = 0;
for (i = 0; i < 5; i++) {
text += "The number is " + i + "\n";
}
console.log(text);
//https://docs.aws.amazon.com/general/latest/gr/rande.html
const lambda = new Lambda({
region: 'us-east-1'
});
console.log('control 3');
var params = {
ClientContext: "lambdaCallLambda",
FunctionName: "lambdaCallLambda-dev-funcOne",
InvocationType: "RequestResponse", /* changed this line from
"Event" to "RequestResponse"*/
LogType: "Tail",
Payload: '{"jsonKey2":123}',
Qualifier: "1"
};
lambda.invoke(params, function(err, data) {
if (err){
console.log('control error\n');
console.log(err, err.stack); // an error occurred
}
else{
console.log('control OK\n');
console.log(data); // successful response
}
/*
data = {
FunctionError: "",
LogResult: "",
Payload: <Binary String>,
StatusCode: 123
}
*/
});
};
module.exports.funcTwo = async (event, context) => {
return 2;
//return '{"funcTwo":20000}';
//console.log("funcTwo = " + event);
};