Creating an ALB Target Group in CloudFormation - amazon-web-services

I'm trying to create an Application Load Balancer in CloudFormation, with a target group that forwards traffic to EC2 instances. Here is the relevant snippet, where ELBSubnets, ECSCluster, taskdefinition, and VpcId are passed in as parameters:
"EcsElasticLoadBalancer" : {
"Type" : "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties" : {
"Subnets" : { "Ref" : "ELBSubnets" },
"SecurityGroups": [
{ "Ref": "ELBAccessSecurityGroup" }
]
}
},
"LoadBalancerListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": { "Ref": "TargetGroup" }
}],
"LoadBalancerArn": { "Ref": "EcsElasticLoadBalancer" },
"Port": 80,
"Protocol": "HTTP"
}
},
"TargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "TargetGroup" ] ] },
"Port": 80,
"Protocol": "HTTP",
"VpcId": { "Ref": "VpcId" }
},
"DependsOn": [ "EcsElasticLoadBalancer" ]
},
"service": {
"Type": "AWS::ECS::Service",
"Properties" : {
"Cluster": { "Ref": "ECSCluster" },
"DesiredCount": "1",
"LoadBalancers": [
{
"ContainerName": "main-app",
"ContainerPort": 3000,
"TargetGroupArn": { "Ref": "TargetGroup" }
}
],
"Role" : {"Ref":"ECSServiceRole"},
"TaskDefinition" : {"Ref":"taskdefinition"}
}
},
"ECSServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ecs.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "ecs-service",
"PolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:Describe*",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"ec2:Describe*",
"ec2:AuthorizeSecurityGroupIngress"
],
"Resource": "*"
}
]
}
}
]
}
}
I get the following error when creating the service:
The target group with targetGroupArn arn:aws:elasticloadbalancing:us-east-1:xxxxxxxx:targetgroup/AlbServiceStack-TargetGroup/6ba9c037c26cdb36 does not have an associated load balancer.
What am I missing? In the documentation there doesn't seem to be a way to specify a load balancer for the target group.

Got it working - the problem was twofold:
The following lines were missing from the Role PolicyDocument:
"elasticloadbalancing:DeregisterTargets"
"elasticloadbalancing:RegisterTargets"
The service needed "DependsOn": [ "LoadBalancerListener" ] as an additional attribute.
Updated template looks like this:
"EcsElasticLoadBalancer" : {
"Type" : "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties" : {
"Subnets" : { "Ref" : "ELBSubnets" },
"SecurityGroups": [
{ "Ref": "ELBAccessSecurityGroup" }
]
}
},
"LoadBalancerListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": { "Ref": "TargetGroup" }
}],
"LoadBalancerArn": { "Ref": "EcsElasticLoadBalancer" },
"Port": 80,
"Protocol": "HTTP"
}
},
"TargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "TargetGroup" ] ] },
"Port": 80,
"Protocol": "HTTP",
"VpcId": { "Ref": "VpcId" }
},
"DependsOn": [ "EcsElasticLoadBalancer" ]
},
"service": {
"Type": "AWS::ECS::Service",
"DependsOn": [ "LoadBalancerListener" ],
"Properties" : {
"Cluster": { "Ref": "ECSCluster" },
"DesiredCount": "1",
"LoadBalancers": [
{
"ContainerName": "main-app",
"ContainerPort": 3000,
"TargetGroupArn": { "Ref": "TargetGroup" }
}
],
"Role" : {"Ref":"ECSServiceRole"},
"TaskDefinition" : {"Ref":"taskdefinition"}
}
},
"ECSServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ecs.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "ecs-service",
"PolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:Describe*",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"ec2:Describe*",
"ec2:AuthorizeSecurityGroupIngress",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:RegisterTargets"
],
"Resource": "*"
}
]
}
}
]
}
}

Related

AWS Firehose DynamicPartitioningConfiguration/Prefix simply won't work

I've been slamming my head against the wall for the past 2 days trying to get dynamic partitioning to work in the delivery stream I'm creating using a serverless application model template. I've tried multiple configurations, but it seems as soon as I fill in a prefix, enable dynamic partitioning and add a metadata extraction processor to get myself a field I can use as the prefix my stack stops working, I can generate alerts in aws and see the logs from the lambda processor I also added, but no files reach the bucket, I also can't seem to configure cloudwatch logs for the stream as well, since I've tried creating them through the script and manually and neither at least have logs pertaining to any error....
I'll leave my script here, hopefully I'm doing something wrong:
{
"AWSTemplateFormatVersion": "2010-09-09",
"Transform": "AWS::Serverless-2016-10-31",
"Description": "An AWS Serverless Application.",
"Parameters": {
"BucketArn": {
"Type": "String"
}
},
"Resources": {
"DeliveryPolicy": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "K-POLICY",
"Roles": [
{
"Ref": "DeliveryRole"
}
],
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
{
"Ref": "BucketArn"
}
]
},
{
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction",
"lambda:GetFunctionConfiguration"
],
"Resource": [
{
"Fn::GetAtt": [
"Processor",
"Arn"
]
}
]
}
]
}
}
},
"DeliveryRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"firehose.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
}
}
},
"DeliveryStream": {
"Type": "AWS::KinesisFirehose::DeliveryStream",
"Properties": {
"DeliveryStreamName": "K-STREAM",
"DeliveryStreamType": "DirectPut",
"ExtendedS3DestinationConfiguration": {
"BucketARN": {
"Ref": "BucketArn"
},
"Prefix": "!{partitionKeyFromQuery:symb}",
"ErrorOutputPrefix": "errors/!{firehose:random-string}/!{firehose:error-output-type}/!{timestamp:yyyy/MM/dd}",
"RoleARN": {
"Fn::GetAtt": [
"DeliveryRole",
"Arn"
]
},
"DynamicPartitioningConfiguration": {
"Enabled": true
},
"ProcessingConfiguration": {
"Enabled": true,
"Processors": [
{
"Type": "MetadataExtraction",
"Parameters": [
{
"ParameterName": "MetadataExtractionQuery",
"ParameterValue": "{symb: .TICKER_SYMBOL}"
},
{
"ParameterName": "JsonParsingEngine",
"ParameterValue": "JQ-1.6"
}
]
},
{
"Type": "Lambda",
"Parameters": [
{
"ParameterName": "LambdaArn",
"ParameterValue": {
"Fn::GetAtt": [
"Processor",
"Arn"
]
}
}
]
}
]
}
}
}
},
"Processor": {
"Type": "AWS::Serverless::Function",
"Properties": {
"Handler": "K-PROCESSOR::KRATOS_PROCESSOR.Functions::FunctionHandler",
"FunctionName": "K-PROCESSOR",
"Runtime": "dotnetcore3.1",
"CodeUri": "./staging/app/3cs-int-k",
"MemorySize": 256,
"Timeout": 60,
"Role": null,
"Policies": [
"AWSLambdaBasicExecutionRole"
]
}
}
}
}

Need help on CloudFormation template and AWS lambda for pulling events from SQS to S3 via lambda

I am new to AWS CloudFormation, and I am trying to capture events from SQS Queue and place them in S3 bucket via AWS lambda. Flow of events is
SNS --> SQS <-- Lambda ---> S3 bucket.
I am trying to achieve above flow using cloudFormation template.I am getting below error message after deploying my CloudFormation template. Any help you can provide would be greatly appreciated. Thank you
11:51:56 2022-01-13 17:51:47,930 - INFO - ...
11:52:53 2022-01-13 17:52:48,511 - ERROR - Stack myDemoApp shows a rollback status ROLLBACK_IN_PROGRESS.
11:52:53 2022-01-13 17:52:48,674 - INFO - The following root cause failure event was found in the myDemoApp stack for resource EventStreamLambda:
11:52:53 2022-01-13 17:52:48,674 - INFO - Resource handler returned message: "Error occurred while GetObject. S3 Error Code: NoSuchKey.
S3 Error Message: The specified key does not exist. (Service: Lambda, Status Code: 400, Request
ID: 5f2f9882-a863-4a58-90bd-7e0d0dfdf4d5, Extended Request ID: null)" (RequestToken: 0a95acb4-a677-0a2d-d0bc-8b7487a858ad, HandlerErrorCode: InvalidRequest)
11:52:53 2022-01-13 17:52:48,674 - INFO - ..
MY lambda function is :
import json
import logging
import boto3
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logging.basicConfig(level=logging.INFO,
format='%(asctime)s: %(levelname)s: %(message)s')
def lambda_handler(event, context):
logger.info(f"lambda_handler -- event: {json.dumps(event)}")
s3_bucket = boto3.resource("3")
event_message = json.loads(event["Records"][0]["body"])
s3_bucket.put_object(Bucket="S3DeployBucket", key="data.json", Body=json.dumps(event_message))
My complete CloudFormation template is :
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "myDemoApp Resource Stack",
"Mappings": {
},
"Parameters": {
"S3DeployBucket": {
"Default": "myDemoApp-deploy-bucket",
"Description": "Bucket for deployment configs and artifacts for myDemoApp",
"Type": "String"
},
"EnvName": {
"Description": "Platform environment name for myDemoApp",
"Type": "String"
},
"AuditRecordKeyArn": {
"Description": "ARN for audit record key encryption for myDemoApp",
"Type": "String"
},
"ParentVPCStack": {
"Description": "The name of the stack containing the parent VPC for myDemoApp",
"Type": "String"
},
"StackVersion": {
"Description": "The version of this stack of myDemoApp",
"Type": "String"
},
"EventLogFolderName": {
"Type": "String",
"Description": "folder name for the logs for the event stream of myDemoApp",
"Default": "event_log_stream"
},
"EventLogPartitionKeys": {
"Type": "String",
"Description": "The partition keys that audit logs will write to S3. Use Hive-style naming conventions for automatic Athena/Glue comprehension.",
"Default": "year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/hour=!{timestamp:HH}"
},
"AppEventSNSTopicArn": {
"Description": "Events SNS Topic of myDemoApp",
"Type": "String"
},
"ReportingEventsRetentionDays": {
"Default": "2192",
"Description": "The number of days to retain a record used for reporting.",
"Type": "String"
}
},
"Resources": {
"AppEventSQSQueue": {
"Type": "AWS::SQS::Queue"
},
"AppEventSnsSubscription": {
"Type": "AWS::SNS::Subscription",
"Properties": {
"TopicArn": {
"Ref": "AppEventSNSTopicArn"
},
"Endpoint": {
"Fn::GetAtt": [
"AppEventSQSQueue",
"Arn"
]
},
"Protocol": "sqs"
}
},
"S3DeployBucket": {
"Type": "AWS::S3::Bucket",
"DeletionPolicy": "Retain",
"UpdateReplacePolicy": "Retain",
"Properties": {
"BucketEncryption": {
"ServerSideEncryptionConfiguration": [
{
"ServerSideEncryptionByDefault": {
"KMSMasterKeyID": {
"Ref": "AuditRecordKeyArn"
},
"SSEAlgorithm": "aws:kms"
}
}
]
},
"VersioningConfiguration": {
"Status": "Enabled"
},
"LifecycleConfiguration": {
"Rules": [
{
"ExpirationInDays": {
"Ref": "ReportingEventsRetentionDays"
},
"Status": "Enabled"
}
]
}
}
},
"EventStreamLogGroup": {
"Type": "AWS::Logs::LogGroup"
},
"EventLogStream": {
"Type": "AWS::Logs::LogStream",
"Properties": {
"LogGroupName": {
"Ref": "EventStreamLogGroup"
}
}
},
"EventStreamSubscriptionRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "sns.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
},
"Policies": [
{
"PolicyName": "SNSSQSAccessPolicy",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": {
"Action": [
"sqs:*"
],
"Effect": "Allow",
"Resource": "*"
}
}
}
]
}
},
"EventDeliveryRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": "sqs.amazonaws.com"
},
"Action": "sts:AssumeRole",
"Condition": {
"StringEquals": {
"sts:ExternalId": {
"Ref": "AWS::AccountId"
}
}
}
}
]
}
}
},
"EventSqsQueuePolicy": {
"Type": "AWS::SQS::QueuePolicy",
"Properties": {
"PolicyDocument": {
"Version": "2012-10-17",
"Id": "SqsQueuePolicy",
"Statement": [
{
"Sid": "Allow-SNS-SendMessage",
"Effect": "Allow",
"Principal": "*",
"Action": [
"sqs:SendMessage",
"sqs:ReceiveMessage"
],
"Resource": {
"Fn::GetAtt": [
"EventStreamLambda",
"Arn"
]
},
"Condition": {
"ArnEquals": {
"aws:SourceArn": {
"Ref": "EventSNSTopicArn"
}
}
}
}
]
},
"Queues": [
{
"Ref": "EventSNSTopicArn"
}
]
}
},
"EventDeliveryPolicy": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "sqs_delivery_policy",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:PutObject"
],
"Resource": [
{
"Fn::GetAtt": [
"S3DeployBucket",
"Arn"
]
},
{
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"S3DeployBucket",
"Arn"
]
},
"/*"
]
]
}
]
},
{
"Effect": "Allow",
"Action": [
"logs:PutLogEvents"
],
"Resource": {
"Fn::Sub": "arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:${EventStreamLogGroup}:log-stream:${EventLogStreamLogStream}"
}
},
{
"Effect": "Allow",
"Action": [
"kms:Decrypt",
"kms:GenerateDataKey"
],
"Resource": [
{
"Ref": "AuditRecordKeyArn"
}
],
"Condition": {
"StringEquals": {
"kms:ViaService": {
"Fn::Join": [
"",
[
"s3.",
{
"Ref": "AWS::Region"
},
".amazonaws.com"
]
]
}
},
"StringLike": {
"kms:EncryptionContext:aws:s3:arn": {
"Fn::Join": [
"",
[
{
"Fn::GetAtt": [
"S3DeployBucket",
"Arn"
]
},
"/*"
]
]
}
}
}
}
]
},
"Roles": [
{
"Ref": "EventDeliveryRole"
}
]
}
},
"EventStreamLambda": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Handler": "lambda_function.lambda_handler",
"MemorySize": 128,
"Runtime": "python3.8",
"Timeout": 30,
"FunctionName": "sqs_s3_pipeline_job",
"Role": {
"Fn::GetAtt": [
"SQSLambdaExecutionRole",
"Arn"
]
},
"Code": {
"S3Bucket": {
"Ref": "S3DeployBucket"
},
"S3Key": {
"Ref": "S3DeployBucket"
}
},
"TracingConfig": {
"Mode": "Active"
}
}
},
"SQSLambdaExecutionRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"lambda.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Policies": [
{
"PolicyName": "StreamLambdaLogs",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:*"
],
"Resource": "arn:aws:logs:*:*:*"
}
]
}
},
{
"PolicyName": "SQSLambdaPolicy",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"sqs:ReceiveMessage",
"sqs:DeleteMessage",
"sqs:GetQueueAttributes",
"sqs:ChangeMessageVisibility"
],
"Resource":"*"
}
]
}
}
]
}
}
},
"Outputs": {
"VpcSubnet3ExportKey": {
"Value": {
"Fn::Sub": "${ParentVPCStack}-privateSubnet3"
}
}
}
}
SubscriptionRoleArn is only for kinesis:
This property applies only to Amazon Kinesis Data Firehose delivery stream subscriptions.

Lambda RDS Proxy connection from different VPC

I have two AWS accounts with VPCs connected with peer connections. I have RDS Proxy on account 1 and Lambda in the private, isolated subnet on account 2.
I cannot figure out how to connect to RDS Proxy. I was trying all possible VPC endpoints and interfaces and whatnot.
The only way I managed to connect was through NAT Gateway, but it's expensive. And to be honest, weird, if I want to keep a private network.
Is it possible to have a PrivateLink or something?
How should I connect to RDS Proxy from Lambda?
I have attached the CF template (I hope I cleaned it from all sensitive data 😑):
{
"Resources": {
"vpcA2121C38": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "10.0.0.0/16",
"EnableDnsHostnames": true,
"EnableDnsSupport": true,
"InstanceTenancy": "default"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/Resource"
}
},
"vpcPrivateSubnet1Subnet934893E8": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"CidrBlock": "10.0.0.0/26",
"VpcId": {
"Ref": "vpcA2121C38"
},
"AvailabilityZone": {
"Fn::Select": [
0,
{
"Fn::GetAZs": ""
}
]
},
"MapPublicIpOnLaunch": false
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/Subnet"
}
},
"vpcPrivateSubnet1RouteTableB41A48CC": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "vpcA2121C38"
}
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/RouteTable"
}
},
"vpcPrivateSubnet1RouteTableAssociation67945127": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"RouteTableId": {
"Ref": "vpcPrivateSubnet1RouteTableB41A48CC"
},
"SubnetId": {
"Ref": "vpcPrivateSubnet1Subnet934893E8"
}
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/RouteTableAssociation"
}
},
"vpcS3CB758969": {
"Type": "AWS::EC2::VPCEndpoint",
"Properties": {
"ServiceName": {
"Fn::Join": [
"",
[
"com.amazonaws.",
{
"Ref": "AWS::Region"
},
".s3"
]
]
},
"VpcId": {
"Ref": "vpcA2121C38"
},
"RouteTableIds": [
{
"Ref": "vpcPrivateSubnet1RouteTableB41A48CC"
}
],
"VpcEndpointType": "Gateway"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/S3/Resource"
}
},
"vpcPeertomainaccount833D3E2C": {
"Type": "AWS::EC2::VPCPeeringConnection",
"Properties": {
"PeerVpcId": "vpc-f04b939b",
"VpcId": {
"Ref": "vpcA2121C38"
},
"PeerOwnerId": "XXXX__ACCOINT_1__XXXXXX",
"PeerRoleArn": "arn:aws:iam::XXXX__ACCOINT_1__XXXXXX:role/VPCPeerConnection"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/Peer to main account"
}
},
"producerServiceRoleEBCB54D0": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "lambda.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
]
]
},
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
]
]
}
]
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/ServiceRole/Resource"
}
},
"producerServiceRoleDefaultPolicyEA5B80A1": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"xray:PutTraceSegments",
"xray:PutTelemetryRecords"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"s3:GetObject*",
"s3:GetBucket*",
"s3:List*"
],
"Effect": "Allow",
"Resource": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":s3:::",
{
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
}
]
]
},
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":s3:::",
{
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"/*"
]
]
}
]
}
],
"Version": "2012-10-17"
},
"PolicyName": "producerServiceRoleDefaultPolicyEA5B80A1",
"Roles": [
{
"Ref": "producerServiceRoleEBCB54D0"
}
]
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/ServiceRole/DefaultPolicy/Resource"
}
},
"producerSecurityGroup9AA1BE28": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Automatic security group for Lambda Function Mwsproducer416E938A",
"SecurityGroupEgress": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Allow all outbound traffic by default",
"IpProtocol": "-1"
}
],
"VpcId": {
"Ref": "vpcA2121C38"
}
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/SecurityGroup/Resource"
}
},
"producerAD962441": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Code": {
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"S3Key": "dc05df325f7034421a587e7ee47aed301c7472d001152e686053dd9d5c45c164.zip"
},
"Role": {
"Fn::GetAtt": [
"producerServiceRoleEBCB54D0",
"Arn"
]
},
"Description": "Produces MWS customer orders messages",
"Environment": {
"Variables": {
"DB_HOST": "mws-rds-proxy.proxy-XXXXXXXXX.eu-central-1.rds.amazonaws.com",
"DB_NAME": "dev"
}
},
"Handler": "lambda/mws_producer.php",
"Layers": [
"arn:aws:lambda:eu-central-1:209497400698:layer:php-80:18"
],
"MemorySize": 1024,
"Runtime": "provided.al2",
"Timeout": 900,
"TracingConfig": {
"Mode": "Active"
},
"VpcConfig": {
"SecurityGroupIds": [
{
"Fn::GetAtt": [
"producerSecurityGroup9AA1BE28",
"GroupId"
]
}
],
"SubnetIds": [
{
"Ref": "vpcPrivateSubnet1Subnet934893E8"
}
]
}
},
"DependsOn": [
"producerServiceRoleDefaultPolicyEA5B80A1",
"producerServiceRoleEBCB54D0"
]
}
}
}
Devil as we know hidden in detail. In this case, it's a Route Tables.
When we create a Peering connection between VPCs we also need to give to know our subnets how to use it. Basically, just add peering connection id (pcx-XXX) as a target for peering network.
Took me several days to realize it. Happy happy happy!

Connecting a SSL cert to a CloudFront CDN in CloudFormation

So far I have this to create the resources.
"staticFileBucketPolicy": {
"Type": "AWS::S3::BucketPolicy",
"DependsOn": "staticFileBucket",
"Properties": {
"Bucket": { "Ref": "staticFileBucket" },
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "staticFileBucket" } , "/*" ]]}
}]
}
}
},
"certificate": {
"Type": "AWS::CertificateManager::Certificate",
"Properties": {
"DomainName": { "Ref": "Domain" },
"SubjectAlternativeNames": [
{ "Fn::Join": ["", [ "*.", { "Ref": "Domain" } ]] }
],
"DomainValidationOptions" : [{
"DomainName": { "Ref": "Domain" },
"ValidationDomain" : { "Ref": "Domain" }
}],
"Tags": [{
"Key": "CloudFormationStack",
"Value": { "Ref": "AWS::StackName" }
}]
}
},
"staticCDN": {
"Type": "AWS::CloudFront::Distribution",
"DependsOn": "staticFileBucket",
"Properties": {
"DistributionConfig": {
"Comment": "CDN for Sagely static files.",
"Enabled": true,
"DefaultRootObject": "index.html",
"DefaultCacheBehavior": {
"AllowedMethods": [ "HEAD", "GET", "OPTIONS" ],
"TargetOriginId": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"ForwardedValues": {
"QueryString": false,
"Headers": [ "Access-Control-Request-Headers", "Access-Control-Request-Method", "Origin" ]
},
"ViewerProtocolPolicy": "redirect-to-https"
},
"Origins": [{
"DomainName": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" }, ".s3.amazonaws.com" ]] },
"Id": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"S3OriginConfig": { }
}]
}
}
},
The CDN works through my custom domain. But how to I connect the SSL certificate to the CDN?
You want to have a ViewerCertificate property on your DistributionConfig. It should be something like:
"ViewerCertificate": {
"AcmCertificateArn": { "Ref": "certificate" },
"SslSupportMethod": "sni-only"
}
Based on your code, probably want to update your staticCDN to something like:
"staticCDN": {
"Type": "AWS::CloudFront::Distribution",
"DependsOn": "staticFileBucket",
"Properties": {
"DistributionConfig": {
"Comment": "CDN for Sagely static files.",
"Enabled": true,
"DefaultRootObject": "index.html",
"DefaultCacheBehavior": {
"AllowedMethods": [ "HEAD", "GET", "OPTIONS" ],
"TargetOriginId": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"ForwardedValues": {
"QueryString": false,
"Headers": [ "Access-Control-Request-Headers", "Access-Control-Request-Method", "Origin" ]
},
"ViewerProtocolPolicy": "redirect-to-https"
},
"Origins": [{
"DomainName": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" }, ".s3.amazonaws.com" ]] },
"Id": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"S3OriginConfig": { }
}],
"ViewerCertificate": {
"AcmCertificateArn": { "Ref": "certificate" },
"SslSupportMethod": "sni-only"
}
}
}
},
You are missing the ViewerCertificate property.
This should be a Ref to the certificate, since ref returns the ARN of the certificate.

How to get AvailabilityZone given a SubnetId Parameter, or vice versa, in a CloudFormation template?

I extracted a CloudFormation template from a stack created by cfncluster. I'm trying to simplify and group the parameters so that the mandatory parameters are grouped together and are non-redundant. The default subnet-related parameters include MasterSubnetId, AvailabilityZone, ComputeSubnetId, and ComputeSubnetCidr. The template uses these to calculate several conditions and also to set AvailabilityZone properties on several resources:
"CreateComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
]
},
"UseComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
}
]
}
]
},
"UseMasterSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
},
I want to simplify the template to require either MasterSubnetId or AvailabilityZone, but not both. I don't need the option to provide one or the other, it's fine for me to require one or the other. The main problem is I can't figure out how to do that. The various Resources used by the template seem to require both, even though they should be related. I can't use Fn::GetAtt like I want because the first argument cannot come from functions, like this:
"Fn::GetAtt" : [ { "Ref": "MasterSubnetId" }, "AvailabilityZone" ]
Also, I want the compute fleet to always get its own subnet.
Here are the subnet-related parts of the template :
{
...snip...
"Parameters": {
"ComputeSubnetId": {
"Description": "ID of the Subnet you want to provision the Compute Servers into. Set to NONE to use the same subnet as Master Server.",
"Type": "String",
"Default": "NONE"
},
"ComputeSubnetCidr": {
"Description": "CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited list and can support multiple CIDR ranges for a multi-AZ cluster. The order and length of this list MUST match the AvailabilityZones parameter. Set to NONE to use the same subnet as Master Server.",
"Type": "String",
"ConstraintDescription": "must be a valid CIDR range of the form x.x.x.x/x.",
"AllowedPattern": "(NONE|(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2}))",
"Default": "NONE"
},
"MasterSubnetId": {
"Description": "ID of the Subnet you want to provision the Master server into",
"Type": "AWS::EC2::Subnet::Id"
},
"AvailabilityZone": {
"Description": "Availability Zone the cluster will launch into. THIS IS REQUIRED",
"Type": "AWS::EC2::AvailabilityZone::Name"
},
"VPCId": {
"Description": "ID of the VPC you want to provision cluster into. Only used with UseVPCBase=false",
"Type": "AWS::EC2::VPC::Id"
},
"UsePublicIps": {
"Description": "Boolean flag to use public IP's for instances. If false, the VPC must be correctly setup to use NAT for all traffic.",
"Type": "String",
"Default": "true",
"ConstraintDescription": "true/false",
"AllowedValues": [
"true",
"false"
]
}
},
"Conditions": {
"CreateComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
]
},
"UseComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
}
]
}
]
},
"UseMasterSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
},
"Mappings": {
"AWSInstanceType2Capabilites": {
...snip...
},
"AWSRegionOS2AMI": {
...snip...
},
"OSFeatures": {
...snip...
},
"CfnClusterVersions": {
"default": {
"cfncluster": "cfncluster-1.1.0",
"cookbook": "cfncluster-cookbook-1.1.0",
"chef": "12.4.3",
"ridley": "4.3.2",
"berkshelf": "4.0.1",
"ami": "201602192042"
}
},
"AWSRegion2Capabilites": {
...snip...
}
},
"Resources": {
...snip...
"CfnClusterPolicies": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "cfncluster",
"PolicyDocument": {
"Statement": [
{
"Sid": "EC2",
"Action": [
"ec2:AttachVolume",
"ec2:DescribeInstanceAttribute",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
...snip...
]
},
"Roles": [
{
"Ref": "RootRole"
}
]
},
"Condition": "CreateEC2IAMRole",
},
"MasterEIP": {
"Type": "AWS::EC2::EIP",
"Properties": {
"Domain": "vpc"
},
"Condition": "MasterPublicIp",
},
"MasterServer": {
"Type": "AWS::EC2::Instance",
"Properties": {
"InstanceType": {
"Ref": "MasterInstanceType"
},
"BlockDeviceMappings": [
...snip...
],
"KeyName": {
"Ref": "KeyName"
},
"Tags": [
{
"Key": "Application",
"Value": {
"Ref": "AWS::StackName"
}
},
{
"Key": "Name",
"Value": "Master"
}
],
"NetworkInterfaces": [
{
"NetworkInterfaceId": {
"Ref": "MasterENI"
},
"DeviceIndex": "0"
}
],
"ImageId": {
"Fn::If": [
"UseCustomAMI",
{
"Ref": "CustomAMI"
},
{
"Fn::FindInMap": [
"AWSRegionOS2AMI",
{
"Ref": "AWS::Region"
},
{
"Ref": "BaseOS"
}
]
}
]
},
"UserData": {
...snip...
}
}
},
"Metadata": {
"Comment": "cfncluster Master server",
"AWS::CloudFormation::Init": {
"configSets": {
"default": [
"deployConfigFiles",
"getCookbooks",
"chefPrepEnv",
"shellRunPreInstall",
"chefConfig",
"shellRunPostInstall",
"shellForkClusterReadyInstall"
]
},
"deployConfigFiles": {
"files": {
"/tmp/dna.json": {
"mode": "000644",
"owner": "root",
"group": "root",
"content": {
"cfncluster": {
"stack_name": {
"Ref": "AWS::StackName"
},
"cfn_preinstall": {
"Ref": "PreInstallScript"
},
"cfn_preinstall_args": {
"Ref": "PreInstallArgs"
},
"cfn_postinstall": {
"Ref": "PostInstallScript"
},
"cfn_postinstall_args": {
"Ref": "PostInstallArgs"
},
"cfn_region": {
"Ref": "AWS::Region"
},
"cfn_volume": {
"Fn::If": [
"UseExistingEBSVolume",
{
"Ref": "EBSVolumeId"
},
{
"Ref": "SharedVolume"
}
]
},
"cfn_scheduler": {
"Ref": "Scheduler"
},
"cfn_encrypted_ephemeral": {
"Ref": "EncryptedEphemeral"
},
"cfn_ephemeral_dir": {
"Ref": "EphemeralDir"
},
"cfn_shared_dir": {
"Ref": "SharedDir"
},
"cfn_proxy": {
"Ref": "ProxyServer"
},
"cfn_node_type": "MasterServer",
"cfn_cluster_user": {
"Fn::FindInMap": [
"OSFeatures",
{
"Ref": "BaseOS"
},
"User"
]
},
"cfn_ddb_table": {
"Ref": "DynamoDBTable"
},
"cfn_sqs_queue": {
"Fn::GetAtt": [
"SQS",
"QueueName"
]
}
},
"run_list": {
"Fn::If": [
"UseCustomRunList",
{
"Ref": "CustomChefRunList"
},
{
"Fn::Join": [
"",
[
"recipe[cfncluster::",
{
"Ref": "Scheduler"
},
"_config]"
]
]
}
]
}
}
},
...snip...
},
"commands": {
...snip...
}
},
...snip...
},
},
"CreationPolicy": {
"ResourceSignal": {
"Count": "1",
"Timeout": "PT30M"
}
}
},
"ComputeFleet": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"MaxSize": {
"Ref": "MaxQueueSize"
},
"AvailabilityZones": [
{
"Ref": "AvailabilityZone"
}
],
"VPCZoneIdentifier": [
{
"Fn::If": [
"UseMasterSubnetForCompute",
{
"Ref": "MasterSubnetId"
},
{
"Fn::If": [
"CreateComputeSubnetForCompute",
{
"Ref": "ComputeSubnet"
},
{
"Ref": "ComputeSubnetId"
}
]
}
]
}
],
...snip...
},
"DependsOn": "MasterServer",
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT30M",
"Count": {
"Ref": "ComputeWaitConditionCount"
}
}
},
},
"ComputeServerLaunchConfig": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"SecurityGroups": [
{
"Fn::If": [
"CreateSecurityGroups",
{
"Ref": "ComputeSecurityGroup"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"AddAdditionalSG",
{
"Ref": "AdditionalSG"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"UseExistingSecurityGroup",
{
"Ref": "VPCSecurityGroupId"
},
{
"Ref": "AWS::NoValue"
}
]
}
],
...snip...
},
"Metadata": {
"Comment": "cfncluster Compute server",
"AWS::CloudFormation::Init": {
"configSets": {
"default": [
"deployConfigFiles",
"getCookbooks",
"chefPrepEnv",
"shellRunPreInstall",
"chefConfig",
"shellRunPostInstall",
"shellForkClusterReadyInstall",
"signalComputeReady"
]
},
"deployConfigFiles": {
"files": {
"/tmp/dna.json": {
"mode": "000644",
"owner": "root",
"group": "root",
"content": {
"cfncluster": {
"stack_name": {
"Ref": "AWS::StackName"
},
"cfn_preinstall": {
"Ref": "PreInstallScript"
},
"cfn_preinstall_args": {
"Ref": "PreInstallArgs"
},
"cfn_postinstall": {
"Ref": "PostInstallScript"
},
"cfn_postinstall_args": {
"Ref": "PostInstallArgs"
},
"cfn_region": {
"Ref": "AWS::Region"
},
"cfn_scheduler": {
"Ref": "Scheduler"
},
"cfn_encrypted_ephemeral": {
"Ref": "EncryptedEphemeral"
},
"cfn_ephemeral_dir": {
"Ref": "EphemeralDir"
},
"cfn_shared_dir": {
"Ref": "SharedDir"
},
"cfn_proxy": {
"Ref": "ProxyServer"
},
"cfn_sqs_queue": {
"Ref": "SQS"
},
"cfn_master": {
"Fn::GetAtt": [
"MasterServer",
"PrivateDnsName"
]
},
"cfn_node_type": "ComputeFleet",
"cfn_cluster_user": {
"Fn::FindInMap": [
"OSFeatures",
{
"Ref": "BaseOS"
},
"User"
]
}
},
"run_list": {
"Fn::If": [
"UseCustomRunList",
{
"Ref": "CustomChefRunList"
},
{
"Fn::Join": [
"",
[
"recipe[cfncluster::",
{
"Ref": "Scheduler"
},
"_config]"
]
]
}
]
}
}
},
...snip...
},
"commands": {
...snip...
}
},
...snip...
}
},
...snip...
"ComputeSubnet": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": {
"Ref": "VPCId"
},
"CidrBlock": {
"Ref": "ComputeSubnetCidr"
},
"Tags": [
{
"Key": "Network",
"Value": "ComputeSubnet"
}
],
"AvailabilityZone": {
"Ref": "AvailabilityZone"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeRouteTable": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "VPCId"
},
"Tags": [
{
"Key": "Application",
"Value": {
"Ref": "AWS::StackName"
}
},
{
"Key": "Network",
"Value": "ComputeSubnet"
}
]
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeRoute": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {
"Ref": "ComputeRouteTable"
},
"DestinationCidrBlock": "0.0.0.0/0",
"NetworkInterfaceId": {
"Ref": "MasterENI"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeSubnetRouteTableAssociation": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {
"Ref": "ComputeSubnet"
},
"RouteTableId": {
"Ref": "ComputeRouteTable"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"MasterSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Enable access to the Master host",
"VpcId": {
"Ref": "VPCId"
},
"SecurityGroupIngress": [
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": {
"Ref": "AccessFrom"
}
},
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": {
"Ref": "AccessFrom"
}
}
]
},
"Condition": "CreateSecurityGroups",
},
"MasterSecurityGroupIngress": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535",
"SourceSecurityGroupId": {
"Ref": "ComputeSecurityGroup"
},
"GroupId": {
"Ref": "MasterSecurityGroup"
}
},
"Condition": "CreateSecurityGroups",
},
"ComputeSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Allow access to resources in subnets behind front",
"VpcId": {
"Ref": "VPCId"
},
"SecurityGroupIngress": [
{
"SourceSecurityGroupId": {
"Ref": "MasterSecurityGroup"
},
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535"
}
]
},
"Condition": "CreateSecurityGroups",
},
"ComputeSecurityGroupIngress": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535",
"SourceSecurityGroupId": {
"Ref": "ComputeSecurityGroup"
},
"GroupId": {
"Ref": "ComputeSecurityGroup"
}
},
"Condition": "CreateSecurityGroups"
},
"MasterENI": {
"Type": "AWS::EC2::NetworkInterface",
"Properties": {
"Description": "cfncluster Master Server",
"SubnetId": {
"Ref" : "MasterSubnetId"
},
"SourceDestCheck": "false",
"GroupSet": [
{
"Fn::If": [
"CreateSecurityGroups",
{
"Ref": "MasterSecurityGroup"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"AddAdditionalSG",
{
"Ref": "AdditionalSG"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"UseExistingSecurityGroup",
{
"Ref": "VPCSecurityGroupId"
},
{
"Ref": "AWS::NoValue"
}
]
}
]
},
},
"SharedVolume": {
"Type": "AWS::EC2::Volume",
"Properties": {
"AvailabilityZone": {
"Ref": "AvailabilityZone"
},
"VolumeType": {
"Ref": "VolumeType"
},
"Size": {
"Fn::If": [
"UseEBSSnapshot",
{
"Ref": "AWS::NoValue"
},
{
"Ref": "VolumeSize"
}
]
},
...snip...
},
...snip...
},
...snip...