Lambda in Private VPC taking long time to use S3 API - amazon-web-services

I created a lambda function to move an object from one S3 bucket to another. The lambda function exists in a VPC.
const AWS = require("aws-sdk");
const s3 = new AWS.S3({apiVersion: '2006-03-01'});
const sourceBucket = "source-bucket";
const destinationBucket = "destination-bucket";
const copyObjectAsync = (params) => new Promise((resolve, reject) => {
s3.copyObject(params, (err, data) => {
if (err) reject(err);
else resolve(data);
});
});
const deleteObjectAsync = (params) => new Promise((resolve, reject) => {
s3.deleteObject(params, (err, data) => {
if (err) reject(err);
else resolve(data);
});
});
exports.handler = async (event) => {
const objectKey = event.Records[0].s3.object.key;
try {
console.log(`Start moving: ${objectKey}`);
const copySource = "/" + sourceBucket + "/" + objectKey;
await copyObjectAsync({
Bucket: destinationBucket,
CopySource: copySource,
Key: objectKey,
ContentType: 'application/x-gzip',
});
console.log(`Copied successfully - ${objectKey}`);
await deleteObjectAsync({
Bucket: sourceBucket,
Key: objectKey,
});
console.log(`Deleted successfully - ${objectKey}`);
} catch (err) {
if (err.code === 'NoSuchKey') {
console.log(`${objectKey} already moved`);
return;
}
console.log(err.stack.substr(0, 200));
// Throw to retry?
throw err;
}
}
I know the S3 endpoints, S3 policies, IAM role, etc have been set up correctly because this function does work but I noticed there's a massive delay when calling s3.copyObject or s3.deleteObject. For example, in the execution below, there's a ~4 minute wait time to move a test file with "hello world" in it.
CloudWatch Logs
09:39:52 START RequestId: <GUID> Version: $LATEST
09:39:52 2019-11-10T09:39:52.676Z <GUID> INFO Start moving: source-bucket/input/tenbatsu_copy_files_2.txt
09:44:16 2019-11-10T09:44:16.017Z <GUID> INFO Copied successfully - source-bucket/input/tenbatsu_copy_files_2.txt
09:44:16 2019-11-10T09:44:16.220Z <GUID> INFO Deleted successfully - source-bucket/input/tenbatsu_copy_files_2.txt
09:44:16 END RequestId: <GUID>
09:44:16 REPORT RequestId: <GUID> Duration: 263579.63 ms Billed Duration: 263600 ms Memory Size: 128 MB Max Memory Used: 99 MB Init Duration: 380.26 ms
2+ minutes to copy and then times out before deleting
09:56:35 START RequestId: <GUID> Version: $LATEST
09:56:35 2019-11-10T09:56:35.363Z <GUID> INFO Start moving: source-bucket/input/tenbatsu_copy_files_3.txt
09:58:46 2019-11-10T09:58:46.380Z <GUID> INFO Copied successfully - source-bucket/input/tenbatsu_copy_files_3.txt
10:01:35 END RequestId: <GUID>
10:01:35 REPORT RequestId: <GUID> Duration: 300100.16 ms Billed Duration: 300000 ms Memory Size: 128 MB Max Memory Used: 99 MB Init Duration: 369.49 ms
10:01:35 2019-11-10T10:01:35.460Z <GUID> Task timed out after 300.10 seconds
What could the issue be?

Related

Why does AWS Lambda function log in CloudWatch but doesn't show console.log output?

I am using CloudWatch from the AWS console and my Lambda function has a log group and log stream and I can see the recent invocations, however, the only thing that shows up in the logs is Start, End, and Report Request ID as shown below:
START RequestId: fe8ca243-288e-48f1-b585-c5d84bfb1bd7 Version: $LATEST
END RequestId: fe8ca243-288e-48f1-b585-c5d84bfb1bd7
REPORT RequestId: fe8ca243-288e-48f1-b585-c5d84bfb1bd7 Duration: 2.71 ms Billed Duration: 3 ms Memory Size: 128 MB Max Memory Used: 56 MB Init Duration: 147.61 ms
Lambda Function:
var aws = require('aws-sdk');
var ddb = new aws.DynamoDB({apiVersion: '2012-10-08'});
exports.handler = async (event, context) => {
console.log("event: ", event);
const tableName = process.env.TABLE_NAME;
const region = process.env.REGION;
console.log("table=" + tableName + " -- region=" + region);
aws.config.update({region: region});
// If the required parameters are present, proceed
if (event.request.userAttributes.sub) {
// -- Write data to DDB
let ddbParams = {
Item: {
'id': {S: event.request.userAttributes.sub},
'username': {S: event.userName},
},
TableName: tableName
};
// Call DynamoDB
try {
await ddb.putItem(ddbParams).promise()
console.log("Success");
} catch (err) {
console.log("Error", err);
}
console.log("Success: Everything executed correctly");
context.done(null, event);
} else {
// Nothing to do, the user's email ID is unknown
console.log("Error: Nothing was written to DDB or SQS");
context.done(null, event);
}
};
Why can't I see the console.log output?

cannot get aws lambda to send sms via aws sns

I have copied the example in the various guides on the internet, including the fairly recent topic on SO:
How to send SMS using Amazon SNS from a AWS lambda function
I've implemented successfully the code from https://docs.aws.amazon.com/sdk-for-javascript/v2/developer-guide/sns-examples-sending-sms.html on my local nodejs server, however when i do the same on Lambda nothing happens, not even console.log???
This is the code i am using in Lambda:
const AWS = require('aws-sdk');
exports.handler = async (event) => {
AWS.config.update({region: 'eu-west-2'});
var params = {
Message: 'TEXT_MESSAGE', /* required */
PhoneNumber: '+1346879',
};
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS({apiVersion: '2010-03-31'}).publish(params).promise();
// Handle promise's fulfilled/rejected states
publishTextPromise.then(
function(data) {
console.log("MessageID is " + data.MessageId);
const response = {
statusCode: 200,
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data)
};
console.log('Server response function');
return response;
}).catch(
function(error) {
//console.error(err, err.stack);
const response = {
statusCode: 500,
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({error: 'Internal server error', message: error})
};
return response;
});
}
Needless to say i tried various flavours of what seems to be an elementary piece of code...
Logs say exactly this:
2020-10-29T00:29:21.820+00:00 START RequestId: 8dbaedac-1f98-4319-9ac5-acba1d8860c5 Version: $LATEST
2020-10-29T00:29:22.242+00:00 Lambda Insights extension initializing.
2020-10-29T00:29:22.242+00:00 EXTENSION Name: cloudwatch_lambda_agent State: Ready Events: [INVOKE,SHUTDOWN]
2020-10-29T00:29:22.838+00:00 END RequestId: 8dbaedac-1f98-4319-9ac5-acba1d8860c5
2020-10-29T00:29:22.838+00:00
Copy
REPORT RequestId: 8dbaedac-1f98-4319-9ac5-acba1d8860c5 Duration: 594.62 ms Billed Duration: 600 ms Memory Size: 128 MB Max Memory Used: 99 MB Init Duration: 448.90 ms
REPORT RequestId: 8dbaedac-1f98-4319-9ac5-acba1d8860c5 Duration: 594.62 ms Billed Duration: 600 ms Memory Size: 128 MB Max Memory Used: 99 MB Init Duration: 448.90 ms
I have set permissions in the same exact way both on my user and on lambda:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": "sns:*",
"Resource": "*"
}
]
}
Any thought?
I think that you have to return a promise to ensure all the tasks will be executed till the end.
If your code performs an asynchronous task, return a promise to make sure that it finishes running. When you resolve or reject the promise, Lambda sends the response or error to the invoker.
const AWS = require('aws-sdk');
exports.handler = async (event) => {
AWS.config.update({region: 'eu-west-2'});
var params = {
Message: 'TEXT_MESSAGE', /* required */
PhoneNumber: '+1346879',
};
// Create promise and SNS service object
var publishTextPromise = new AWS.SNS({apiVersion: '2010-03-31'}).publish(params).promise();
return new Promise((resolve, reject) => {
// Handle promise's fulfilled/rejected states
publishTextPromise.then((data) => {
console.log("MessageID is " + data.MessageId);
const response = {
statusCode: 200,
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify(data)
};
console.log('Server response function');
resolve(response);
}).catch((error) => { reject(Error(error)); });
});
}
Check the lambda execution time, try to increase it to 15 mins.
This SNS is having two types of messages 1.Promotional 2.Transactional
By default it choose promotional. If end users is having DND enabled to his number. Then your message will not delivered.
If it is related to transactional use the below object for sending transactional messages.
{
Message: 'Message',
PhoneNumber: '+XXX',
MessageAttributes: {
'AWS.SNS.SMS.SMSType': {
DataType: 'String',
StringValue: 'Transactional'
}
}

AWS Cognito User Pool AdminCreateUser in lambda returns no error nor data

I created an AWS Lambda function to test new Cognito User Pool account creation but no account was created in user pool and no error shown in console log. I checked AWS Cloud Watch but no error was reported in cloud watch also.
START RequestId: ..... Version: $LATEST
END RequestId: .....
REPORT RequestId: ..... Duration: 80.12 ms Billed Duration: 100 ms Memory Size: 128 MB Max Memory Used: 84 MB
I changed my Lambda function to something simpler; describeUserPool. Still no error was thrown and no user pool info was printed in console. I added a console.log print within the describeUserPool callback function but that did not print.
The lambda function was created using AWS Console Lambda inline editor. Lambda function has AmazonESCognitoAccess policy attached (this policy has List/Read/Write access level to Cognito User Pools).
Can anyone shed some lights on what I've gotten wrong?
Thank you very much in advance.
var aws = require('aws-sdk');
aws.config.update({
accessKeyId: 'access_key_id',
secretAccessKey: 'secret_access_key',
region: 'us-east-1',
apiVersion: '2016-04-18'
});
var cognito = new aws.CognitoIdentityServiceProvider();
var params = {
UserPoolId: 'us-region-user-pool-id'
};
exports.handler = async (event) => {
cognito.describeUserPool(params, function(err, data) {
console.log('hello from inside function');
if (err) {
console.log(err);
} else {
console.log(data);
}
});
};
If I included an incorrect params in AdminCreateUser it reported error
var aws = require('aws-sdk');
aws.config.update({
accessKeyId: 'access_key_id',
secretAccessKey: 'secret_access_key',
region: 'us-east-1',
apiVersion: '2016-04-18'
});
var cognito = new aws.CognitoIdentityServiceProvider();
var params = {
UserPoolId: 'us-region-user-pool-id'
Username: 'someone',
TemporaryPassword: '11223344',
DesiredDeliveryMediums: 'EMAIL',
MessageAction: 'SUPPRESS',
UserAttributes: [
{
Name: 'Email',
Value: 'someone#example.com'
},
{
Name: 'Family_Name',
Value: 'One'
},
{
Name: 'Given_Name',
Value: 'Some'
},
{
Name: 'Phone_Number_verified',
Value: 'True'
},
{
Name: 'Email_verified',
Value: 'True'
}
],
};
exports.handler = async (event) => {
cognito.adminCreateUser(params, function(err, data) {
console.log('hello from inside function');
if (err) {
console.log(err);
} else {
console.log(data);
}
});
};
I received the following error message in Lambda inline editor console:
Response:
null
Request ID:
"6e50fdb4-e437-4b84-be7b-3caaeb5b0a98"
Function Logs:
ncCredentials (/var/runtime/node_modules/aws-sdk/lib/config.js:391:24)
at Config.getCredentials (/var/runtime/node_modules/aws-sdk/lib/config.js:411:9) {
code: 'MultipleValidationErrors',
errors: [
InvalidParameterType: Expected params.DesiredDeliveryMediums to be an Array
at ParamValidator.fail (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:50:37)
at ParamValidator.validateType (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:232:10)
at ParamValidator.validateList (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:99:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:90:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:88:21)
at ParamValidator.validate (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:34:10)
at Request.VALIDATE_PARAMETERS (/var/runtime/node_modules/aws-sdk/lib/event_listeners.js:126:42)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:106:20)
at callNextListener (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:96:12) {
code: 'InvalidParameterType',
time: 2020-05-31T21:59:35.116Z
},
InvalidParameterType: Expected params.UserAttributes[3].Value to be a string
at ParamValidator.fail (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:50:37)
at ParamValidator.validateType (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:232:10)
at ParamValidator.validateString (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:154:32)
at ParamValidator.validateScalar (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:130:21)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:94:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:88:21)
at ParamValidator.validateList (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:103:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:90:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14) {
code: 'InvalidParameterType',
time: 2020-05-31T21:59:35.116Z
},
InvalidParameterType: Expected params.UserAttributes[4].Value to be a string
at ParamValidator.fail (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:50:37)
at ParamValidator.validateType (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:232:10)
at ParamValidator.validateString (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:154:32)
at ParamValidator.validateScalar (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:130:21)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:94:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:88:21)
at ParamValidator.validateList (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:103:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:90:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14) {
code: 'InvalidParameterType',
time: 2020-05-31T21:59:35.116Z
}
],
time: 2020-05-31T21:59:35.173Z
}
END RequestId: 6e50fdb4-e437-4b84-be7b-3caaeb5b0a98
REPORT RequestId: 6e50fdb4-e437-4b84-be7b-3caaeb5b0a98 Duration: 122.82 ms Billed Duration: 200 ms Memory Size: 128 MB Max Memory Used: 80 MB Init Duration: 356.12 ms
Found this post cognito admin not giving error. Suggestion by #thopaw solved the issue that I was having.
By changing it to promise() base it works as expected.
Thank you guys for your time.
In case anyone came across this, below is the updated code.
exports.handler = async (event, context) => {
console.log('starts');
var data;
try {
const data = await cognito.adminCreateUser(params).promise();
} catch (error) {
console.log(error);
}
};

Sending an email from an AWS Lambda function

I've created a basic lambda function that should send an email from the SES service, but it is not working.
I've followed this tutorial: here. I've created the proper IAM policy and attached it to the function, but I don't think that's the problem.
I've created a test config for the function with and the control flow is not reaching inside of sendEmail, and I'm not sure why.
const aws = require('aws-sdk')
const ses = new aws.SES({ region: 'us-east-1' });
exports.handler = async (event, context, callback) => {
const printOut = JSON.stringify(event);
console.log(`Received event: ${printOut}`);
const params = {
Destination: {
ToAddresses: ["xxxxx.xxxx#gmail.com"]
},
Message: {
Body: {
Text: {
Data: event['desc']
}
},
Subject: {
Data: "Email from xxxxxxx Contact Us Form"
}
},
Source: "xxxxxxxxxxxxxxxxx#gmail.com"
};
console.log(`sending: ${JSON.stringify(params)}`);
ses.sendEmail(params, function (err, data) {
callback(null, { err: err, data: data });
if (err) {
console.error(err);
context.fail(err);
} else {
console.log(data);
context.succeed(event);
}
console.log(`Done!`);
});
};
Here is my current output:
Response:
null
Request ID:
"xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx"
Function Logs:
START RequestId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx Version: $LATEST
2019-11-12T01:28:48.352Z xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx INFO Received event: {"name":"Redacted","email":"xxxxx.xxxx#gmail.com","desc":"Test message"}
2019-11-12T01:28:48.372Z xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx INFO sending: {"Destination":{"ToAddresses":["xxxxx.xxxx#gmail.com"]},"Message":{"Body":{"Text":{"Data":"Test message"}},"Subject":{"Data":"Email from xxxxxx Contact Us Form"}},"Source":"xxxxxxxxxxxxxxx#gmail.com"}
END RequestId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx
REPORT RequestId: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxx Duration: 590.10 ms Billed Duration: 600 ms Memory Size: 128 MB Max Memory Used: 94 MB Init Duration: 359.55 ms
As you can see, the console outs/errors inside the sendEmail function are never being called. Not really sure why.
I did make sure to verify both of the email addresses I'm testing with, but no dice.
You have to either make it non-async or return a promise back to the caller (you can return that promise directly to the runtime). Like this:
const aws = require('aws-sdk')
const ses = new aws.SES({region: 'us-east-1'});
exports.handler = async (event, context, callback) => {
const printOut = JSON.stringify(event);
console.log(`Received event: ${printOut}`);
const params = {
Destination: {
ToAddresses: ["xxxxx.xxxx#gmail.com"]
},
Message: {
Body: {
Text: {
Data: event['desc']
}
},
Subject: {
Data: "Email from xxxxxxx Contact Us Form"
}
},
Source: "xxxxxxxxxxxxxxxxx#gmail.com"
};
console.log(`sending: ${JSON.stringify(params)}`);
return ses.sendEmail(params, null).promise();
};

Amazon ElasticTranscoder, How to wait for job completion?

I have a node.js lambda , triggered on S3 event. An Elastic Transcoder job is initiated like so:
let AWS = require('aws-sdk');
let s3 = new AWS.S3({apiVersion: '2012–09–25'});
let eltr = new AWS.ElasticTranscoder({apiVersion: '2012–09–25', region: 'us-west-2'});
exports.handler = (event, context, callback) => {
let pipelineId = 'keystone';
let bucket = event.Records[0].s3.bucket.name;
let key = event.Records[0].s3.object.key;
let etParams = {
PipelineId: pipelineId,
Input: {
Key: key,
FrameRate: 'auto',
Resolution: 'auto',
AspectRatio: 'auto',
Interlaced: 'auto',
Container: 'auto'
},
Outputs: [{
Key: key,
PresetId: '1351620000001-000010'
}]
};
eltr.createJob(etParams, function(err, data) {
if (err) {
console.log("ET error", err, err.stack);
} else {
console.log("Calling waitFor for Job Id:", data.Job.Id);
eltr.waitFor("jobComplete", {Id: data.Job.Id}, function(err, data) {
if (err) {
console.log("ET waitFor Error", err, err.stack);
} else {
console.log("ET Job finished", data, data.Job.Output.Key);
}
});
}
});
};
The transcoding process times out:
START RequestId: 82c0a1ce-5cf3-11e7-81aa-a3362402de83 Version: $LATEST
2017-06-29T17:51:03.509Z 82c0a1ce-5cf3-11e7-81aa-a3362402de83 Creating Job { PipelineId: 'keystone',
Input:
{ Key: 'f04d62af47.mp4',
FrameRate: 'auto',
Resolution: 'auto',
AspectRatio: 'auto',
Interlaced: 'auto',
Container: 'auto' },
Outputs:
[ { Key: 'f04d62af47.mp4',
PresetId: '1351620000001-000010' } ] }
2017-06-29T17:51:04.829Z 82c0a1ce-5cf3-11e7-81aa-a3362402de83 Calling waitFor for Job Id: 1498758664450-jxhdlx
END RequestId: 82c0a1ce-5cf3-11e7-81aa-a3362402de83
REPORT RequestId: 82c0a1ce-5cf3-11e7-81aa-a3362402de83 Duration: 3001.65 ms Billed Duration: 3000 ms Memory Size: 128 MB Max Memory Used: 37 MB
2017-06-29T17:51:06.260Z 82c0a1ce-5cf3-11e7-81aa-a3362402de83 Task timed out after 3.00 seconds
The above log output is repeated 3 times (lambda's three tries?)
I'm sure I'm missing something, can anybody please point the mistake?
All calls made to AWS Lambda must complete execution within 300 seconds. The default timeout is 3 seconds, but you can set the timeout to any value between 1 and 300 seconds.
And on your two retries conjecture, you are correct. If AWS Lambda is unable to fully process an asynchronous event then it will automatically retry the invocation twice, with delays between retries.