Is there a way to trigger Lambda Function declared in the very same CFN template that was used to create a stack when the given stack is being deleted?
Preferably I'd like to implement somewhat opposite to THIS snippet (that is: a relatively simple solution that omits e.g. the need of creating SNS topics etc.).
In advance thx for any advices. Best regards! //M
You can achieve the desired behaviour by creating a CustomResource and hooking it up with your lambda function.
From the documentation page:
With AWS Lambda functions and custom resources, you can run custom code in response to stack events (create, update, and delete)
The lambda function needs to react to a Delete event, so it has to be written to allow for that.
Example lambda function source code
Here's an example for the AWS lambda function configuration, which needs to react to a delete event:
import cfnresponse
def lambda_handler(event, context):
response = {}
if event['RequestType'] == 'Delete':
... # do stuff
response['output'] = ' Delete event.'
cfnresponse.send(event, context, cfnresponse.SUCCESS, response)
Example lambda function
Here's a CloudFormation snippet for the lambda function:
"MyDeletionLambda": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Code": {
"ZipFile": {
"Fn::Join": [
"\n",
[
"import cfnresponse",
"",
"def lambda_handler(event, context):",
" response = {}",
" if event['RequestType'] == 'Delete':",
" response['output'] = 'Delete event.'",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, response)"
]
]
}
},
"Handler": "index.lambda_handler",
"Runtime": "python3.8"
}
}
}
Example custom resource
Here's a CloudFormation snippet for a custom resource:
"OnStackDeletion": {
"Type": "Custom::LambdaDependency",
"Properties": {
"ServiceToken": {
"Fn::GetAtt": [
"MyDeletionLambda",
"Arn"
]
}
}
}
Related
I'm currently working on the implementation of SNS-notifications being the intermediary between S3-bucket-uploads and an upload-handler Lambda Function.
The information flow should look like this:
Upload of a file to an S3-bucket
This should trigger an SNS-topic "upload-notification"
Lambda-function ("upload-handler") are subscribed to this SNS-topic:
depending on which bucket received a file upload, a certain
lambda-function should be triggered by the SNS-notification
--> How can I obtain information like the "S3 bucket name" etc. from the event that triggered the SNS-notification?
I hope for a possibility like with lambda functions where you can extract information e.g. from a JSON-object produced by SNS.
If that doesn't exist, I'd be delighted to learn about other approaches, but somehow I need to extract this information programmatically/automatically from SNS to hand it over to the upload-handler lambda function in Step 3.
Details on the terraform definition blocks:
1. aws_sns_topic_subscription:
resource "aws_sns_topic_subscription" "start_from_upload_topic" {
topic_arn = var.upload_notification_topic_arn
protocol = "lambda"
endpoint = module.start_from_upload_handler.arn
}
2. aws_s3_bucket_notification:
resource "aws_s3_bucket_notification" "start_from_upload_handler" {
for_each = local.input_bucket_id_map
bucket = each.value
topic {
topic_arn = module.upload_notification.topic_setup.topic_arn
events = ["s3:ObjectCreated:*"]
}
}
3. SNS-module "upload_notification"
module "upload_notification" {
source = "../../modules/sns_topic"
name = "${var.platform_settings.prefix}-upload-notification"
key_arn = var.platform_settings.logging_settings.logging_key_arn
allowed_producers = [
"s3.amazonaws.com",
"lambda.amazonaws.com",
"edgelambda.amazonaws.com",
"events.amazonaws.com",
"states.amazonaws.com",
]
allowed_consumers = ["lambda.amazonaws.com",
"edgelambda.amazonaws.com",
"events.amazonaws.com",
"states.amazonaws.com",
]
tags = local.tags
}
As per documentation, the S3 bucket name (amongst other data like region, event time, bucket ARN, source IP etc.) will be inside the event message that is passed through to the Lambda from S3 via SNS in your case.
Records[0].s3.bucket.name
{
"Records":[
{
"s3":{
"bucket":{
"name":"bucket-name",
...
},
...
},
...
}
]
}
SNS needs to be set-up in conjunction with the Lambda-function and S3-uploads like so (in terraform, excluding KMS for this example):
resource "aws_lambda_permission" "start_from_upload_sns_topic" {
statement_id = "AllowExecutionFromSNStopic"
action = "lambda:InvokeFunction"
function_name = module.start_from_upload_handler.arn
principal = "sns.amazonaws.com"
source_arn = var.upload_notification_topic_arn
}
resource "aws_s3_bucket_notification" "start_from_upload_handler" {
for_each = var.input_bucket_name_map
bucket = each.value
topic {
topic_arn = var.upload_notification_topic_arn
events = ["s3:ObjectCreated:*"]
}
}
resource "aws_sns_topic_subscription" "start_from_upload_sns_topic" {
topic_arn = var.upload_notification_topic_arn
protocol = "lambda"
endpoint = module.start_from_upload_handler.arn
}
The JSON-object the lambda-function receives from the SNS-topic looks like so:
{
"Records": [
{
"EventSource": "aws:sns",
"EventVersion": "1.0",
"EventSubscriptionArn": "arn:aws:sns:eu-central-1:...",
"Sns": {
"Type": "Notification",
"MessageId": "...",
"TopicArn": "arn:aws:sns:eu-central-1:....",
"Subject": "Amazon S3 Notification",
"Message": "{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"eu-central-1\",\"eventTime\":\"2021-10-27T15:29:38.959Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:...\"},\"requestParameters\":{\"sourceIPAddress\":\"....\"},\"responseElements\":{\"x-amz-request-id\":\"..\",\"x-amz-id-2\":\"...\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"tf-s3-topic-...\",\"bucket\":{\"name\":\"test-bucket-name\",\"ownerIdentity\":{\"principalId\":\"....\"},\"arn\":\"arn:aws:s3:::test-bucket-name\"},\"object\":{\"key\":\"test_file.json\",\"size\":189,\"eTag\":\"....\",\"versionId\":\"...\",\"sequencer\":\"...\"}}}]}",
"Timestamp": "2021-10-27T15:29:40.086Z",
"SignatureVersion": "1",
"Signature": "...",
"SigningCertUrl": "https://sns.eu-central-1.amazonaws.com/SimpleNotificationService...",
"UnsubscribeUrl": "https://sns.eu-central-1.amazonaws.com....",
"MessageAttributes": {}
}
}
]
}
We're interested in the "Message" - body of the incoming JSON-object, and this finally looks indeed like #Ermiya Eskandary mentioned in his answer pointing to the S3-notification-JSON-event-structure:
{
'Records': [
{
's3': {
'bucket': {
'arn': 'arn:aws:s3:...',
'name': 'bucket-name',
},
'object': {
'key': 'upload_file_name.json',
},
},
}
]
}
The take-away here is that one needs to bear in mind that the incoming JSON emitted by SNS has several top-layer dictionary keywords, which need to be "stripped-off" or "dug-through" in order to get to the actual S3-upload-event in the SNS-Message-body, which comes as a string-JSON-format which needs to be loaded into a proper dictionary-object.
Moreover, it is paramount to subscribe the lambda-function to the SNS-topic and allow the SNS-topic in turn to invoke said lambda-function.
I was able to setup AutoScaling events as rules in EventBridge to trigger SSM Commands, but I've noticed that with my chosen Target Value the event is passed to all my active EC2 Instances. My Target key is a tag shared by those instances, so my mistake makes sense now.
I'm pretty new to EventBridge, so I was wondering if there's a way to actually target the instance that triggered the AutoScaling event (as in extracting the "InstanceId" that's present in the event data and use that as my new Target Value). I saw the Input Transformer, but I think that just transforms the event data to pass to the target.
Thanks!
EDIT - help with js code for Lambda + SSM RunCommand
I realize I can achieve this by setting EventBridge to invoke a Lambda function instead of the SSM RunCommand directly. Can anyone help with the javaScript code to call a shell command on the ec2 instance specified in the event data (event.detail.EC2InstanceId)? I can't seem to find a relevant and up-to-date base template online, and I'm not familiar enough with js or Lambda. Any help is greatly appreciated! Thanks
Sample of Event data, as per aws docs
{
"version": "0",
"id": "12345678-1234-1234-1234-123456789012",
"detail-type": "EC2 Instance Launch Successful",
"source": "aws.autoscaling",
"account": "123456789012",
"time": "yyyy-mm-ddThh:mm:ssZ",
"region": "us-west-2",
"resources": [
"auto-scaling-group-arn",
"instance-arn"
],
"detail": {
"StatusCode": "InProgress",
"Description": "Launching a new EC2 instance: i-12345678",
"AutoScalingGroupName": "my-auto-scaling-group",
"ActivityId": "87654321-4321-4321-4321-210987654321",
"Details": {
"Availability Zone": "us-west-2b",
"Subnet ID": "subnet-12345678"
},
"RequestId": "12345678-1234-1234-1234-123456789012",
"StatusMessage": "",
"EndTime": "yyyy-mm-ddThh:mm:ssZ",
"EC2InstanceId": "i-1234567890abcdef0",
"StartTime": "yyyy-mm-ddThh:mm:ssZ",
"Cause": "description-text"
}
}
Edit 2 - my Lambda code so far
'use strict'
const ssm = new (require('aws-sdk/clients/ssm'))()
exports.handler = async (event) => {
const instanceId = event.detail.EC2InstanceId
var params = {
DocumentName: "AWS-RunShellScript",
InstanceIds: [ instanceId ],
TimeoutSeconds: 30,
Parameters: {
commands: ["/path/to/my/ec2/script.sh"],
workingDirectory: [],
executionTimeout: ["15"]
}
};
const data = await ssm.sendCommand(params).promise()
const response = {
statusCode: 200,
body: "Run Command success",
};
return response;
}
Yes, but through Lambda
EventBridge -> Lambda (using SSM api) -> EC2
Thank you #Sándor Bakos for helping me out!! My JavaScript ended up not working for some reason, so I ended up just using part of the python code linked in the comments.
1. add ssm:SendCommand permission:
After I let Lambda create a basic role during function creation, I added an inline policy to allow Systems Manager's SendCommand. This needs access to your documents/*, instances/* and managed-instances/*
2. code - python 3.9
import boto3
import botocore
import time
def lambda_handler(event=None, context=None):
try:
client = boto3.client('ssm')
instance_id = event['detail']['EC2InstanceId']
command = '/path/to/my/script.sh'
client.send_command(
InstanceIds = [ instance_id ],
DocumentName = 'AWS-RunShellScript',
Parameters = {
'commands': [ command ],
'executionTimeout': [ '60' ]
}
)
You can do this without using lambda, as I just did, by using eventbridge's input transformers.
I specified a new automation document that called the document I was trying to use (AWS-ApplyAnsiblePlaybooks).
My document called out the InstanceId as a parameter and is passed this by the input transformer from EventBridge. I had to pass the event into lambda just to see how to parse the JSON event object to get the desired instance ID - this ended up being
$.detail.EC2InstanceID
(it was coming from an autoscaling group).
I then passed it into a template that was used for the runbook
{"InstanceId":[<instance>]}
This template was read in my runbook as a parameter.
This was the SSM playbook inputs I used to run the AWS-ApplyAnsiblePlaybook Document, I just mapped each parameter to the specified parameters in the nested playbook:
"inputs": {
"InstanceIds": ["{{ InstanceId }}"],
"DocumentName": "AWS-ApplyAnsiblePlaybooks",
"Parameters": {
"SourceType": "S3",
"SourceInfo": {"path": "https://testansiblebucketab.s3.amazonaws.com/"},
"InstallDependencies": "True",
"PlaybookFile": "ansible-test.yml",
"ExtraVariables": "SSM=True",
"Check": "False",
"Verbose": "-v",
"TimeoutSeconds": "3600"
}
See the document below for reference. They used a document that was already set up to receive the variable
https://docs.aws.amazon.com/systems-manager/latest/userguide/automation-tutorial-eventbridge-input-transformers.html
This is the full automation playbook I used, most of the parameters are defaults from the nested playbook:
{
"description": "Runs Ansible Playbook on Launch Success Instances",
"schemaVersion": "0.3",
"assumeRole": "<Place your automation role ARN here>",
"parameters": {
"InstanceId": {
"type": "String",
"description": "(Required) The ID of the Amazon EC2 instance."
}
},
"mainSteps": [
{
"name": "RunAnsiblePlaybook",
"action": "aws:runCommand",
"inputs": {
"InstanceIds": ["{{ InstanceId }}"],
"DocumentName": "AWS-ApplyAnsiblePlaybooks",
"Parameters": {
"SourceType": "S3",
"SourceInfo": {"path": "https://testansiblebucketab.s3.amazonaws.com/"},
"InstallDependencies": "True",
"PlaybookFile": "ansible-test.yml",
"ExtraVariables": "SSM=True",
"Check": "False",
"Verbose": "-v",
"TimeoutSeconds": "3600"
}
}
}
]
}
I have a lambda function that's supposed to process my SQS messages. However, it doesn't seem to be triggered automatically even though I have my SQS as its event trigger. Below is my code.
import json
import boto3
def lambda_handler(event, context):
max=2 # Number of messages to process # a time
if "max" in event:
max=int(event["max"])
else:
max=1
sqs = boto3.resource('sqs') # Get access to resource
queue = sqs.get_queue_by_name(QueueName='mysqs.fifo') # Get queue by name
count=0
# Process messages
for message in queue.receive_messages(MaxNumberOfMessages=max):
body = json.loads(message.body) # Attribute list
print("test")
payload = message[body]
print(str(payload))
count+=1
message.delete()
return {
'statusCode': 200,
'body': str(count)
}
When an AWS Lambda function is configured with Amazon SQS as a trigger, the messages are passed in via the event variable. The code inside the function should not call Amazon SQS directly.
Here is an example of an event being passed to the function:
{
"Records": [
{
"messageId": "11d6ee51-4cc7-4302-9e22-7cd8afdaadf5",
"receiptHandle": "AQEBBX8nesZEXmkhsmZeyIE8iQAMig7qw...",
"body": "Test message.",
"attributes": {
"ApproximateReceiveCount": "1",
"SentTimestamp": "1573251510774",
"SequenceNumber": "18849496460467696128",
"MessageGroupId": "1",
"SenderId": "AIDAIO23YVJENQZJOL4VO",
"MessageDeduplicationId": "1",
"ApproximateFirstReceiveTimestamp": "1573251510774"
},
"messageAttributes": {},
"md5OfBody": "e4e68fb7bd0e697a0ae8f1bb342846b3",
"eventSource": "aws:sqs",
"eventSourceARN": "arn:aws:sqs:us-east-2:123456789012:fifo.fifo",
"awsRegion": "us-east-2"
}
]
}
Your code could then access the messages with:
for record in event['Records']:
payload = record['body']
print(payload)
count += 1
There is no need to delete the message -- it will be automatically deleted once the Lambda function successfully executes.
See: Using AWS Lambda with Amazon SQS - AWS Lambda
I would like to use the Serverless Application Model(SAM) and CloudFormation to create a simple lambda function which gets triggered when an object is created in a S3 bucket(e.g. thescore-cloudfront-trial). How do I enable the trigger from the S3 bucket to the Lambda Function? Below is my python3 boto3 code.
region = 'us-east-1'
import boto3
test_lambda_template = {
'AWSTemplateFormatVersion': '2010-09-09',
'Transform': 'AWS::Serverless-2016-10-31',
'Resources': {
'CopyS3RajivCloudF': {
'Type': 'AWS::Serverless::Function',
'Properties': {
"CodeUri": 's3://my-tmp/CopyS3Lambda',
"Handler": 'lambda.handler',
"Runtime": 'python3.6',
"Timeout": 300,
"Role": 'my_existing_role_arn'
},
'Events': {
'Type': 'S3',
'Properties': {
"Bucket": "thescore-cloudfront-trial",
"Events": 's3:ObjectCreated:*'
}
}
},
'SrcBucket': {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": 'thescore-cloudfront-trial',
}
}
}
}
conf = config.get_aws_config('development')
client = aws.client(conf, 'cloudformation', region_name=region)
response = client.create_change_set(
StackName='RajivTestStack',
TemplateBody=json.dumps(test_lambda_template),
Capabilities=['CAPABILITY_IAM'],
ChangeSetName='a',
Description='Rajiv ChangeSet Description',
ChangeSetType='CREATE'
)
response = client.execute_change_set(
ChangeSetName='a',
StackName='RajivTestStack',
)
I figured it out with caveats
Caveat 1: The trigger notification will show up in S3 console but not in the Lambda console. My existing python deploy scripts using boto3 s3 and lambda clients(which I want to replace) show the notification in both consoles.
Caveat 2: For monitoring, I see my lambda trigger only when I switch to see the lambda alias view. But I haven't specified an alias for my lambda. So I don't know why I don't see it in the non alias view(just seeing the LATEST version)
I had to modify the Events key like this:
'Events': {
'RajivCopyEvent': {
'Type': 'S3',
'Properties': {
"Bucket": {"Ref": "SrcBucket"},
"Events": "s3:ObjectCreated:*"
}
}
}
I'm trying to create a stack with CloudFormation. The stack needs to take some data files from a central S3 bucket and copy them to it's own "local" bucket.
I've written a lambda function to do this, and it works when I run it in the Lambda console with a test event (the test event uses the real central repository and successfully copies the file to a specified repo).
My current CloudFormation script does the following things:
Creates the "local" S3 bucket
Creates a role that the Lambda function can use to access the buckets
Defines the Lambda function to move the specified file to the "local" bucket
Defines some Custom resources to invoke the Lambda function.
It's at step 4 where it starts to go wrong - the Cloudformation execution seems to freeze here (CREATE_IN_PROGESS). Also, when I try to delete the stack, it seems to just get stuck on DELETE_IN_PROGRESS instead.
Here's how I'm invoking the Lambda function in the CloudFormation script:
"DataSync": {
"Type": "Custom::S3DataSync",
"Properties": {
"ServiceToken": { "Fn::GetAtt" : [ "S3DataSync", "Arn" ] },
"InputFile": "data/provided-as-ip-v6.json",
"OutputFile": "data/data.json"
}
},
"KeySync1": {
"Type": "Custom::S3DataSync",
"Properties": {
"ServiceToken": { "Fn::GetAtt" : [ "S3DataSync", "Arn" ] },
"InputFile": "keys/1/public_key.pem"
}
},
"KeySync2": {
"Type": "Custom::S3DataSync",
"Properties": {
"ServiceToken": { "Fn::GetAtt" : [ "S3DataSync", "Arn" ] },
"InputFile": "keys/2/public_key.pem"
}
}
And the Lambda function itself:
exports.handler = function(event, context) {
var buckets = {};
buckets.in = {
"Bucket":"central-data-repository",
"Key":"sandbox" + "/" + event.ResourceProperties.InputFile
};
buckets.out = {
"Bucket":"sandbox-data",
"Key":event.ResourceProperties.OutputFile || event.ResourceProperties.InputFile
};
var AWS = require('aws-sdk');
var S3 = new AWS.S3();
S3.getObject(buckets.in, function(err, data) {
if (err) {
console.log("Couldn't get file " + buckets.in.Key);
context.fail("Error getting file: " + err)
}
else {
buckets.out.Body = data.Body;
S3.putObject(buckets.out, function(err, data) {
if (err) {
console.log("Couln't write to S3 bucket " + buckets.out.Bucket);
context.fail("Error writing file: " + err);
}
else {
console.log("Successfully copied " + buckets.in.Key + " to " + buckets.out.Bucket + " at " + buckets.out.Key);
context.succeed();
}
});
}
});
}
Your Custom Resource function needs to send signals back to CloudFormation to indicate completion, status, and any returned values. You will see CREATE_IN_PROGRESS as the status in CloudFormation until you notify it that your function is complete.
The generic way of signaling CloudFormation is to post a response to a pre-signed S3 URL. But there is a cfn-response module to make this easier in Lambda functions. Interestingly, the two examples provided for Lambda-backed Custom Resources use different methods:
Walkthrough: Refer to Resources in Another Stack - uses the cfn-response module
Walkthrough: Looking Up Amazon Machine Image IDs - uses pre-signed URLs.
Yup, i did the same thing. We need to upload(PUT request) the status of our request.(Need to send the status as SUCCESS)