Aws lambda function addPermission error: PolicyLengthExceededException - amazon-web-services

I am creating a cloudwatch event which at a specific time in future is supposed to call a aws lambda function . The i am using aws nodejs sdk as described here: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/index.html
the code block to create the cloudwatch event looks like this:
module.exports.createReservationReminder = function (reservationModel, user, restaurant) {
return new Promise(function (resolve, reject) {
const ruleName = "rsv_" + reservationModel.reservationId;
const description = "Reservation reminder of `" + user.name + "` # `" + restaurant.title + "` on `" + reservationModel.time + "`";
let reservationTime = reservationModel.time;
let lambdaFunctionName = module.exports.buildLamdaFunctionArn("restaurant")
let alertTime = moment(reservationTime).tz(AppConfig.defaultTimezone).subtract( // Create alert 45 minute before a reservation
45,
'minutes'
);
let lambda = new AWS.Lambda({
accessKeyId: AppConfig.accessKeyId,
secretAccessKey: AppConfig.secretAccessKey,
region: AppConfig.region
});
let scheduleExpression1 = "cron(" + alertTime.utc().format('m H D MMM ? YYYY') + ')';
let ruleParams = {
Name: ruleName,
Description: description,
ScheduleExpression: scheduleExpression1,
State: 'ENABLED',
};
cloudwatchevents.deleteRule({Name: ruleName}, function (err, deleteRuleData) { //remove if a previous rule was created halfway
cloudwatchevents.putRule(ruleParams, function (err, ruleData) { //create the rule
if (err) {
reject(err)
}
else {
let lambdaPermission = {
FunctionName: lambdaFunctionName,
StatementId: ruleName,
Action: 'lambda:InvokeFunction',
Principal: 'events.amazonaws.com',
SourceArn: ruleData.RuleArn
};
let removePermission = {
FunctionName: lambdaFunctionName,
StatementId: ruleName,
}
//now to create the rule's target, need to add permission to lambda
lambda.removePermission(removePermission, function (err, removeLambdaData) { //remove if rule of same name was added as permission to this lambda before, ignore if rule not found error is thrown
lambda.addPermission(lambdaPermission, function (err, lamdaData) { //now add the permission
if (err) {
reject(err) // FAIL : throws error PolicyLengthExceededException after ~50 cloudwatch events are registered to this lambda function
}
else {
let targetParams = {
Rule: ruleName,
Targets: [
{
Arn: module.exports.buildLamdaFunctionArn("restaurant"),
Id: ruleName,
Input: JSON.stringify({
func: "notifyUserOfUpcomingReservation",
data: {
reservationId: reservationModel.reservationId
}
}),
},
]
};
cloudwatchevents.putTargets(targetParams, function (err, targetData) {
if (err) {
reject(err)
}
else {
resolve(targetData)
}
})
}
})
})
}
});
})
})
}
Above function works fine for the first ~50 times ( so I can easily make reminder for the 50 reservations. ) However , it will always fail eventually with:
PolicyLengthExceededException
Lambda function access policy is limited to 20 KB.
HTTP Status Code: 400
Which makes sense, as policy document can not be too big.
So what is the correct way to approach this problem : make unlimited cloudwatch event reminder with a lambda function target .

create a role and add that policy or permission for that role and then your lambda can assume role and run.
you can use aws STS module for that.
Rather than create and removing permission each time. STS will assume role temporarily then execute the code.

Related

Lambda not deleting DynamoDB records when triggered with Cloud Watch events

I am trying to delete items in my Dynamodb table using Cloud Watch event triggered Lambda. This lambda scans the dynamo table and deletes all expired items. My code seems to be working when I test it using the test event in the console (i.e it deletes all the expired items). But when lambda gets triggered automatically using the Cloud Watch event it does not delete, event though I see that the lambda is being triggered.
exports.handler = async function () {
var params = {
TableName: TABLE_NAME
}
try {
const data = await docClient.scan(params).promise();
const items = data.Items;
if (items.length != 0) {
Promise.all(items.map(async (item) => {
const expirationDT = new Date(item.ExpiresAt);
const now = new Date();
if (now > expirationDT) {
console.log("Deleting item with otc: " + item.Otc + " and name: " + item.SecretName);
const deleteParams = {
TableName: TABLE_NAME,
Key: {
"Otc": item.Otc,
"SecretName": item.SecretName,
},
};
try {
await docClient.delete(deleteParams).promise();
} catch (err) {
console.log("The Secret was not deleted due to: ", err.message);
}
}
}))
}
} catch (err) {
console.log("The items were not able to be scanned due to : ", err.message)
}}
I know using DynamoDB TTL is an option, but I need these deletions to be somewhat precise, and TTL can sometimes take up to 48 hours, and I am aware I can use a filter when retrieving records to counter-act that. Just wondering what's wrong with my code here.
You need to await Promise.all or your lambda will end execution before it resolves
await Promise.all(items.map(async (item) => {
const expirationDT = new Date(item.ExpiresAt);
const now = new Date();
// ...

Lambda not able to execute the aws sdk API

I am trying to create a lambda function which will check if a particular repository exist in codecommit.
Lamda service role is having admin priviledge. Below is the code.
The lambda is unable to call getRepository method. It is niether giving any exception nor passing. Any help on this?
console.log("Before calling cc") This is last printed statement. After that I am not getting any success or error log.
const CloudFormation = require('aws-sdk/clients/cloudformation');
const Codecommit = require('aws-sdk/clients/codecommit');
exports.handler = async (event) => {
try{
console.log("event",event);
console.log("event",JSON.stringify(event));
var repositoryName = event.detail.repositoryName;
var cfn = new CloudFormation({
region: "ap-northeast-1"
});
var cc = new Codecommit({
region: "ap-northeast-1"
});
const stackName = repositoryName+"-infra-stack";
var cloneUrl;
console.log("RepositoryName"+repositoryName);
console.log("StackName"+stackName);
var codeCommitParam = {
repositoryName: repositoryName
};
try{
console.log("Before calling cc")
cc.getRepository(codeCommitParam, function(err, data) {
if (err){
console.log(err, err.stack);
}else {
console.log(data.repositoryMetadata.cloneUrlHttp);
cloneUrl=data.repositoryMetadata.cloneUrlHttp;
console.log("Clone url "+cloneUrl);
checkStackDescription();
}
});
}catch(error){
console.log(error);
}
}
I believe this is coming down to the JavaScript in the Lambda being invoked asynchronously so the Lambda is finishing invoking before the callback processes the response.
Try updating to use this synchronously by updating to the below syntax.
console.log("Before calling cc")
let result = await cc.getRepository(codeCommitParam).promise();
console.log(result);
Be aware that result could either be an error or valid response.

UpdateFunctionCode in lambda does not update the code

I have followed this blog to update the code of a lambda function using a jar file stored in a S3 bucket. the execution was succeded, but it is not updating the code of target lambda function
Code snippet
console.log('Loading function');
var AWS = require('aws-sdk');
var lambda = new AWS.Lambda();
exports.handler = function(event, context) {
var functionName = "runJarFile";
var bucket = "jarfiletest2";
var key = "lambda-java-example-0.0.1-SNAPSHOT.jar.zip";
console.log("uploaded to lambda function: " + functionName);
var params = {
FunctionName: functionName,
S3Key: key,
S3Bucket: bucket,
Publish: true
};
lambda.updateFunctionCode(params, function(err, data) {
if (err) {
console.log(err, err.stack);
context.fail(err);
} else {
console.log(data);
context.succeed(data);
}
});
};
Thanks in advance
It's difficult to comment on this without knowing the details about the destination function. What's the output of the GetFunction API call of that Lambda, before and after calling the UpdateFunctionConfig call?
I am interested to see the SHA-256 hash of the code, and the last modified timestamp off that API call before and after calling UpdateFunctionConfig:
{
...
"CodeSha256": "5tT2qgzYUHoqwR616pZ2dpkn/0J1FrzJmlKidWaaCgk=",
"LastModified": "2019-09-24T18:20:35.054+0000"
...
}
If the values are exactly the same, can you add this check as per the blog post to see if the bucket and the object exists?
if (bucket == "YOUR_BUCKET_NAME" && key == "YOUR_CODE.zip" && version) {
// your code
} else {
context.succeed("skipping zip " + key + " in bucket " + bucket + " with version " + version);
}
Pls try to remove 'Publish: true' to call the latest version not the specified version

Self invoking lambda invocation timing out

We're trying to develop a self-invoking lambda to process S3 files in chunks. The lambda role has the policies needed for the invocation attached.
Here's the code for the self-invoking lambda:
export const processFileHandler: Handler = async (
event: S3CreateEvent,
context: Context,
callback: Callback,
) => {
let bucket = loGet(event, 'Records[0].s3.bucket.name');
let key = loGet(event, 'Records[0].s3.object.key');
let totalFileSize = loGet(event, 'Records[0].s3.object.size');
const lastPosition = loGet(event, 'position', 0);
const nextRange = getNextSizeRange(lastPosition, totalFileSize);
context.callbackWaitsForEmptyEventLoop = false;
let data = await loadDataFromS3ByRange(bucket, key, nextRange);
await database.connect();
log.debug(`Successfully connected to the database`);
const docs = await getParsedDocs(data, lastPosition);
log.debug(`upserting ${docs.length} records to database`);
if (docs.length) {
try {
// upserting logic
log.debug(`total documents added: ${await docs.length}`);
} catch (err) {
await recurse(nextRange.end, event, context);
log.debug(`error inserting docs: ${JSON.stringify(err)}`);
}
}
if (nextRange.end < totalFileSize) {
log.debug(`Last ${context.getRemainingTimeInMillis()} milliseconds left`);
if (context.getRemainingTimeInMillis() < 10 * 10 * 10 * 6) {
log.debug(`Less than 6000 milliseconds left`);
log.debug(`Invoking next iteration`);
await recurse(nextRange.end, event, context);
callback(null, {
message: `Lambda timed out processing file, please continue from LAST_POSITION: ${nextRange.start}`,
});
}
} else {
callback(null, { message: `Successfully completed the chunk processing task` });
}
};
Where recurse is an invocation call to the same lambda. Rest of the things work as expected it just times out whenever the call stack comes on this invocation request:
const recurse = async (position: number, event: S3CreateEvent, context: Context) => {
let newEvent = Object.assign(event, { position });
let request = {
FunctionName: context.invokedFunctionArn,
InvocationType: 'Event',
Payload: JSON.stringify(newEvent),
};
let resp = await lambda.invoke(request).promise();
console.log('Invocation complete', resp);
return resp;
};
This is the stack trace logged to CloudWatch:
{
"errorMessage": "connect ETIMEDOUT 63.32.72.196:443",
"errorType": "NetworkingError",
"stackTrace": [
"Object._errnoException (util.js:1022:11)",
"_exceptionWithHostPort (util.js:1044:20)",
"TCPConnectWrap.afterConnect [as oncomplete] (net.js:1198:14)"
]
}
Not a good idea to create a self-invoking lambda function. In case of an error (could also be a bad handler call on AWS side) a lambda function might re-run several times. Very hard to monitor and debug.
I would suggest using Step Functions. I believe this tutorial can help Iterating a Loop Using Lambda
From the top of my head, if you prefer not dealing with Step Functions, you could create a Lambda trigger for an SQS queue. Then you pass a message to the queue if you want to run the lambda function another time.

Why AWS Lambda function invoked multiple times for a single event?

I am trying to create AWS Lambda function that does following process.
Receive S3 "Put" event
Get fileA from S3
Get fileB from S3 that invoked lambda
Launch just one EC2 instance
Create tags for the new EC2 instance
Problem: Multiple(5) instances are launched unexpectedly.
An instance is successfully created, but 4 other instances are also launched. 5 instances in total are launched.
Logs
In the Log Streams for this function, I found 4 Streams for this invocation. Each Stream doesn't show any errors or exceptions, but it seems that the function is executed repeatedly.
Trial
I guessed that the function has been timed out and then re-run.
Then, I changed Timeout from 5s to 60s and put a file on S3.
It somehow effected. Only 2 Log Streams appeared, first one shows that the function has been executed just once, second shows the function has been executed twice. Number of launched instances is 3.
However, I have no idea why multiple(3) instances are launched.
Any comments are welcome!
Thank you in advance :-)
My Lambda function
My Lambda function is following. (It's simplified to hide credential informations but it doesn't lose its basic structure)
var AWS = require('aws-sdk');
function composeParams(data, config){
var block_device_name = "/dev/xvdb";
var security_groups = [
"MyGroupName"
];
var key_name = 'mykey';
var security_group_ids = [
"sg-xxxxxxx"
];
var subnet_id = "subnet-xxxxxxx";
// Configurations for a new EC2 instance
var params = {
ImageId: 'ami-22d27b22', /* required */
MaxCount: 1, /* required */
MinCount: 1, /* required */
KeyName: key_name,
SecurityGroupIds: security_group_ids,
InstanceType: data.instance_type,
BlockDeviceMappings: [
{
DeviceName: block_device_name,
Ebs: {
DeleteOnTermination: true,
Encrypted: true,
VolumeSize: data.volume_size,
VolumeType: 'gp2'
}
}
],
Monitoring: {
Enabled: false /* required */
},
SubnetId: subnet_id,
UserData: new Buffer(config).toString('base64'),
DisableApiTermination: false,
InstanceInitiatedShutdownBehavior: 'stop',
DryRun: data.dry_run,
EbsOptimized: false
};
return params;
}
exports.handler = function(event, context) {
// Get the object from the event
var s3 = new AWS.S3({ apiVersion: '2006-03-01' });
var bucket = event.Records[0].s3.bucket.name;
var key = event.Records[0].s3.object.key;
// Get fileA
var paramsA = {
Bucket: bucket,
Key: key
};
s3.getObject(paramsA, function(err, data) {
if (err) {
console.log(err);
} else {
var dataA = JSON.parse(String(data.Body));
// Get fileB
var paramsB = {
Bucket: bucket,
Key: 'config/config.yml'
};
s3.getObject(paramsB, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
var config = data.Body;
/* Some process */
// Launch EC2 Instance
var ec2 = new AWS.EC2({ region: REGION, apiVersion: '2015-04-15' });
var params = composeParams(dataA, config);
ec2.runInstances(params, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log(data);
// Create tags for instance
for (var i=0; i<data.Instances.length; i++){
var instance = data.Instances[i];
var params = {
Resources: [ /* required */
instance.InstanceId
],
Tags: [ /* required */
{
Key: 'Name',
Value: instance_id
},
{
Key: 'userID',
Value: dataA.user_id
}
],
DryRun: dataA.dry_run
};
ec2.createTags(params, function(err, data) {
if (err) {
console.log(err, err.stack);
} else {
console.log("Tags created.");
console.log(data);
}
});
}
}
});
}
});
}
});
};
Solved.
Adding context.succeed(message); to the last part of the nested callback prevents the repeated execution of the function.
ec2.createTags(params, function(err, data) {
if (err) {
console.log(err, err.stack);
context.fail('Failed');
} else {
console.log("Tags created.");
console.log(data);
context.succeed('Completed');
}
});
Check in cloudwatch event that context.aws_request_id value for each invokation. If it is
same than it is retry because aws function got some error raised.
make your lambda idempotent
different than it is because of connection timeout from your aws
lambda client. check aws client configuration request timeout and
connect timeout values.
I was having the same problem with the newer runtime (Node.JS v4.3). Call
context.callbackWaitsForEmptyEventLoop = false;
before calling
callback(...)
Maximum Event Age
When a function returns an error before execution, Lambda returns the event to the queue and attempts to run the function again for up to 6 hours by default. With Maximum Event Age, you can configure the lifetime of an event in the queue from 60 seconds to 6 hours. This allows you to remove any unwanted events based on the event age.
Maximum Retry Attempts
When a function returns an error after execution, Lambda attempts to run it two more times by default. With Maximum Retry Attempts, you can customize the maximum number of retries from 0 to 2. This gives you the option to continue processing new events with fewer or no retries.
under the Configuration > Asynchronous invocation > Retry Attempts
you can set it to 0-2
Source:
https://aws.amazon.com/about-aws/whats-new/2019/11/aws-lambda-supports-max-retry-attempts-event-age-asynchronous-invocations/