Lambda RDS Proxy connection from different VPC - amazon-web-services

I have two AWS accounts with VPCs connected with peer connections. I have RDS Proxy on account 1 and Lambda in the private, isolated subnet on account 2.
I cannot figure out how to connect to RDS Proxy. I was trying all possible VPC endpoints and interfaces and whatnot.
The only way I managed to connect was through NAT Gateway, but it's expensive. And to be honest, weird, if I want to keep a private network.
Is it possible to have a PrivateLink or something?
How should I connect to RDS Proxy from Lambda?
I have attached the CF template (I hope I cleaned it from all sensitive data 😑):
{
"Resources": {
"vpcA2121C38": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "10.0.0.0/16",
"EnableDnsHostnames": true,
"EnableDnsSupport": true,
"InstanceTenancy": "default"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/Resource"
}
},
"vpcPrivateSubnet1Subnet934893E8": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"CidrBlock": "10.0.0.0/26",
"VpcId": {
"Ref": "vpcA2121C38"
},
"AvailabilityZone": {
"Fn::Select": [
0,
{
"Fn::GetAZs": ""
}
]
},
"MapPublicIpOnLaunch": false
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/Subnet"
}
},
"vpcPrivateSubnet1RouteTableB41A48CC": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "vpcA2121C38"
}
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/RouteTable"
}
},
"vpcPrivateSubnet1RouteTableAssociation67945127": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"RouteTableId": {
"Ref": "vpcPrivateSubnet1RouteTableB41A48CC"
},
"SubnetId": {
"Ref": "vpcPrivateSubnet1Subnet934893E8"
}
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/PrivateSubnet1/RouteTableAssociation"
}
},
"vpcS3CB758969": {
"Type": "AWS::EC2::VPCEndpoint",
"Properties": {
"ServiceName": {
"Fn::Join": [
"",
[
"com.amazonaws.",
{
"Ref": "AWS::Region"
},
".s3"
]
]
},
"VpcId": {
"Ref": "vpcA2121C38"
},
"RouteTableIds": [
{
"Ref": "vpcPrivateSubnet1RouteTableB41A48CC"
}
],
"VpcEndpointType": "Gateway"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/S3/Resource"
}
},
"vpcPeertomainaccount833D3E2C": {
"Type": "AWS::EC2::VPCPeeringConnection",
"Properties": {
"PeerVpcId": "vpc-f04b939b",
"VpcId": {
"Ref": "vpcA2121C38"
},
"PeerOwnerId": "XXXX__ACCOINT_1__XXXXXX",
"PeerRoleArn": "arn:aws:iam::XXXX__ACCOINT_1__XXXXXX:role/VPCPeerConnection"
},
"Metadata": {
"aws:cdk:path": "Mws/vpc/Peer to main account"
}
},
"producerServiceRoleEBCB54D0": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "lambda.amazonaws.com"
}
}
],
"Version": "2012-10-17"
},
"ManagedPolicyArns": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
]
]
},
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
]
]
}
]
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/ServiceRole/Resource"
}
},
"producerServiceRoleDefaultPolicyEA5B80A1": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"xray:PutTraceSegments",
"xray:PutTelemetryRecords"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Action": [
"s3:GetObject*",
"s3:GetBucket*",
"s3:List*"
],
"Effect": "Allow",
"Resource": [
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":s3:::",
{
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
}
]
]
},
{
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition"
},
":s3:::",
{
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"/*"
]
]
}
]
}
],
"Version": "2012-10-17"
},
"PolicyName": "producerServiceRoleDefaultPolicyEA5B80A1",
"Roles": [
{
"Ref": "producerServiceRoleEBCB54D0"
}
]
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/ServiceRole/DefaultPolicy/Resource"
}
},
"producerSecurityGroup9AA1BE28": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Automatic security group for Lambda Function Mwsproducer416E938A",
"SecurityGroupEgress": [
{
"CidrIp": "0.0.0.0/0",
"Description": "Allow all outbound traffic by default",
"IpProtocol": "-1"
}
],
"VpcId": {
"Ref": "vpcA2121C38"
}
},
"Metadata": {
"aws:cdk:path": "Mws/producer/producer/SecurityGroup/Resource"
}
},
"producerAD962441": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Code": {
"S3Bucket": {
"Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}"
},
"S3Key": "dc05df325f7034421a587e7ee47aed301c7472d001152e686053dd9d5c45c164.zip"
},
"Role": {
"Fn::GetAtt": [
"producerServiceRoleEBCB54D0",
"Arn"
]
},
"Description": "Produces MWS customer orders messages",
"Environment": {
"Variables": {
"DB_HOST": "mws-rds-proxy.proxy-XXXXXXXXX.eu-central-1.rds.amazonaws.com",
"DB_NAME": "dev"
}
},
"Handler": "lambda/mws_producer.php",
"Layers": [
"arn:aws:lambda:eu-central-1:209497400698:layer:php-80:18"
],
"MemorySize": 1024,
"Runtime": "provided.al2",
"Timeout": 900,
"TracingConfig": {
"Mode": "Active"
},
"VpcConfig": {
"SecurityGroupIds": [
{
"Fn::GetAtt": [
"producerSecurityGroup9AA1BE28",
"GroupId"
]
}
],
"SubnetIds": [
{
"Ref": "vpcPrivateSubnet1Subnet934893E8"
}
]
}
},
"DependsOn": [
"producerServiceRoleDefaultPolicyEA5B80A1",
"producerServiceRoleEBCB54D0"
]
}
}
}

Devil as we know hidden in detail. In this case, it's a Route Tables.
When we create a Peering connection between VPCs we also need to give to know our subnets how to use it. Basically, just add peering connection id (pcx-XXX) as a target for peering network.
Took me several days to realize it. Happy happy happy!

Related

Cloud Formation: S3 linked to Lambda gives The ARN is not well formed

I'm trying to use CloudFormation to deploy an S3 bucket that on ObjectCreate invokes a Lambda function.
Here are my resources:
"ExampleFunction": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Handler": "index.lambda_handler",
"Code": {
"S3Bucket": "bucketname",
"S3Key": "something.zip"
},
"Runtime": "python3.6",
"Role": {
"Fn::GetAtt": [
"LambdaExecutionRole",
"Arn"
]
}
}
},
"InputDataBucket": {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": "input-data",
"NotificationConfiguration": {
"LambdaConfigurations": [
{
"Function": {
"Ref": "ExampleFunction"
},
"Event": "s3:ObjectCreated:*",
"Filter": {
"S3Key": {
"Rules": [
{
"Name": "suffix",
"Value": "zip"
}
]
}
}
}
]
}
}
},
"LambdaInvokePermission": {
"Type": "AWS::Lambda::Permission",
"Properties": {
"Action": "lambda:InvokeFunction",
"FunctionName": {
"Fn::GetAtt": [
"ExampleFunction",
"Arn"
]
},
"Principal": "s3.amazonaws.com",
"SourceAccount": {
"Ref": "AWS::AccountId"
},
"SourceArn": {
"Fn::Join": [
":",
[
"arn",
"aws",
"s3",
"",
"",
{
"Ref": "InputDataBucket"
}
]
]
}
}
}
I've tried to follow the documentation of the Notification Configuration, that says that there can be a circular dependency. However, if I follow the instructions I get the same error. Reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-s3-bucket-notificationconfig.html
When I try to create the stack, the S3 always breaks it with error "The ARN is not well formed"
I've tried many things, but I always receive this same error.
I can get this to work as long as I know the S3 bucket name in advance (mybucketname below). If you don't know the bucket name in advance, then you can enhance this to request the bucket name as a stack parameter and it should still work. If you need the bucket name to be auto-generated (so you can't predict the name in advance) then this will not work and you'll have to go the create/update route.
Key thing here is to manually create the S3 bucket ARN from the known bucket name, rather than relying on "Ref": "InputDataBucket" to get the bucket name for you.
Also worth reading this support article.
{
"AWSTemplateFormatVersion": "2010-09-09",
"Description": "stackoverflow-48037497",
"Resources" : {
"ExampleFunction": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Handler": "index.lambda_handler",
"Code": {
"S3Bucket": "bucketname",
"S3Key": "something.zip"
},
"Runtime": "python3.6",
"Role": {
"Fn::GetAtt": [
"LambdaExecutionRole",
"Arn"
]
}
}
},
"LambdaInvokePermission": {
"Type": "AWS::Lambda::Permission",
"DependsOn": [ "ExampleFunction" ],
"Properties": {
"Action": "lambda:InvokeFunction",
"FunctionName": {
"Fn::GetAtt": [
"ExampleFunction",
"Arn"
]
},
"Principal": "s3.amazonaws.com",
"SourceAccount": {
"Ref": "AWS::AccountId"
},
"SourceArn": "arn:aws:s3:::mybucketname"
}
},
"InputDataBucket": {
"Type": "AWS::S3::Bucket",
"DependsOn": [ "ExampleFunction", "LambdaInvokePermission" ],
"Properties": {
"BucketName": "mybucketname",
"NotificationConfiguration": {
"LambdaConfigurations": [
{
"Function": { "Fn::GetAtt" : [ "ExampleFunction", "Arn" ] },
"Event": "s3:ObjectCreated:*"
}
]
}
}
}
}
}

Connecting a SSL cert to a CloudFront CDN in CloudFormation

So far I have this to create the resources.
"staticFileBucketPolicy": {
"Type": "AWS::S3::BucketPolicy",
"DependsOn": "staticFileBucket",
"Properties": {
"Bucket": { "Ref": "staticFileBucket" },
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [{
"Sid": "AddPerm",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "staticFileBucket" } , "/*" ]]}
}]
}
}
},
"certificate": {
"Type": "AWS::CertificateManager::Certificate",
"Properties": {
"DomainName": { "Ref": "Domain" },
"SubjectAlternativeNames": [
{ "Fn::Join": ["", [ "*.", { "Ref": "Domain" } ]] }
],
"DomainValidationOptions" : [{
"DomainName": { "Ref": "Domain" },
"ValidationDomain" : { "Ref": "Domain" }
}],
"Tags": [{
"Key": "CloudFormationStack",
"Value": { "Ref": "AWS::StackName" }
}]
}
},
"staticCDN": {
"Type": "AWS::CloudFront::Distribution",
"DependsOn": "staticFileBucket",
"Properties": {
"DistributionConfig": {
"Comment": "CDN for Sagely static files.",
"Enabled": true,
"DefaultRootObject": "index.html",
"DefaultCacheBehavior": {
"AllowedMethods": [ "HEAD", "GET", "OPTIONS" ],
"TargetOriginId": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"ForwardedValues": {
"QueryString": false,
"Headers": [ "Access-Control-Request-Headers", "Access-Control-Request-Method", "Origin" ]
},
"ViewerProtocolPolicy": "redirect-to-https"
},
"Origins": [{
"DomainName": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" }, ".s3.amazonaws.com" ]] },
"Id": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"S3OriginConfig": { }
}]
}
}
},
The CDN works through my custom domain. But how to I connect the SSL certificate to the CDN?
You want to have a ViewerCertificate property on your DistributionConfig. It should be something like:
"ViewerCertificate": {
"AcmCertificateArn": { "Ref": "certificate" },
"SslSupportMethod": "sni-only"
}
Based on your code, probably want to update your staticCDN to something like:
"staticCDN": {
"Type": "AWS::CloudFront::Distribution",
"DependsOn": "staticFileBucket",
"Properties": {
"DistributionConfig": {
"Comment": "CDN for Sagely static files.",
"Enabled": true,
"DefaultRootObject": "index.html",
"DefaultCacheBehavior": {
"AllowedMethods": [ "HEAD", "GET", "OPTIONS" ],
"TargetOriginId": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"ForwardedValues": {
"QueryString": false,
"Headers": [ "Access-Control-Request-Headers", "Access-Control-Request-Method", "Origin" ]
},
"ViewerProtocolPolicy": "redirect-to-https"
},
"Origins": [{
"DomainName": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" }, ".s3.amazonaws.com" ]] },
"Id": { "Fn::Join": ["", [ { "Ref": "SubDomain" }, "-static.", { "Ref": "Domain" } ]] },
"S3OriginConfig": { }
}],
"ViewerCertificate": {
"AcmCertificateArn": { "Ref": "certificate" },
"SslSupportMethod": "sni-only"
}
}
}
},
You are missing the ViewerCertificate property.
This should be a Ref to the certificate, since ref returns the ARN of the certificate.

Setup Lambda to trigger from CloudWatch using CloudFormation

I want to use CloudFormation to trigger Lambda when my CloudWatch function is called. I have the below, but it does not work.
CloudWatch rule created fine
"CloudWatchNewEc2": {
"Type": "AWS::Events::Rule",
"DependsOn": ["LambdaNewEc2"],
"Properties": {
"Description": "Triggered on new EC2 instances",
"EventPattern": {
"source": [
"aws.ec2"
],
"detail-type": [
"AWS API Call via CloudTrail"
],
"detail": {
"eventSource": [
"ec2.amazonaws.com"
],
"eventName": [
"RunInstances"
]
}
},
"Targets": [
{
"Arn": {
"Fn::GetAtt": ["LambdaNewEc2", "Arn"]
},
"Id": "NewEc2AutoTag"
}
]
}
},
Lambda created but is not triggered
"LambdaNewEc2": {
"Type": "AWS::Lambda::Function",
"DependsOn": ["S3Lambda", "IAMRoleLambda"],
"Properties": {
"Code": {
"S3Bucket": {"Ref": "LambdaBucketName"},
"S3Key": "skynet-lambda.zip"
},
"Description": "When new EC2 instances are created, auto tag them",
"FunctionName": "newEc2AutoTag",
"Handler": "index.newEc2_autoTag",
"Role": {"Fn::GetAtt": ["IAMRoleLambda", "Arn"]},
"Runtime": "nodejs6.10",
"Timeout": "30"
}
}
},
It seems like CloudWatch Target is not sufficient?
UPDATE (Full CloudFormation template)
{
"Parameters": {
"Environment": {
"Type": "String",
"Default": "Staging",
"AllowedValues": [
"Testing",
"Staging",
"Production"
],
"Description": "Environment name"
},
"BucketName": {
"Type": "String",
"Default": "skynet-staging",
"Description": "Bucket Name"
},
"LambdaBucketName": {
"Type": "String",
"Default": "skynet-lambda",
"Description": "Lambda Bucket Name"
},
"Owner": {
"Type": "String",
"Description": "Owner"
}
},
"Resources": {
"S3Web": {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": {
"Ref": "BucketName"
},
"WebsiteConfiguration": {
"IndexDocument": "index.html",
"RoutingRules": [
{
"RedirectRule": {
"ReplaceKeyPrefixWith": "#"
},
"RoutingRuleCondition": {
"HttpErrorCodeReturnedEquals": "404"
}
}
]
},
"AccessControl": "PublicRead",
"Tags": [
{
"Key": "Cost Center",
"Value": "Skynet"
},
{
"Key": "Environment",
"Value": {
"Ref": "Environment"
}
},
{
"Key": "Owner",
"Value": {
"Ref": "Owner"
}
}
]
}
},
"S3Lambda": {
"Type": "AWS::S3::Bucket",
"Properties": {
"BucketName": {
"Ref": "LambdaBucketName"
},
"VersioningConfiguration": {
"Status": "Enabled"
},
"Tags": [
{
"Key": "Cost Center",
"Value": "Skynet"
},
{
"Key": "Owner",
"Value": {
"Ref": "Owner"
}
}
]
}
},
"CloudWatchNewEc2": {
"Type": "AWS::Events::Rule",
"DependsOn": ["LambdaNewEc2"],
"Properties": {
"Description": "Triggered on new EC2 instances",
"EventPattern": {
"source": [
"aws.ec2"
],
"detail-type": [
"AWS API Call via CloudTrail"
],
"detail": {
"eventSource": [
"ec2.amazonaws.com"
],
"eventName": [
"RunInstances"
]
}
},
"Targets": [
{
"Arn": {
"Fn::GetAtt": ["LambdaNewEc2", "Arn"]
},
"Id": "NewEc2AutoTag"
}
]
}
},
"IAMRoleLambda": {
"Type": "AWS::IAM::Role",
"Properties": {
"RoleName": "skynet-lambda-role",
"AssumeRolePolicyDocument": {
"Version" : "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [ "lambda.amazonaws.com" ]
},
"Action": [ "sts:AssumeRole" ]
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/AmazonEC2FullAccess",
"arn:aws:iam::aws:policy/AWSLambdaFullAccess",
"arn:aws:iam::aws:policy/AWSXrayWriteOnlyAccess",
"arn:aws:iam::aws:policy/CloudWatchLogsFullAccess"
]
}
},
"LambdaNewEc2": {
"Type": "AWS::Lambda::Function",
"DependsOn": ["S3Lambda", "IAMRoleLambda"],
"Properties": {
"Code": {
"S3Bucket": {"Ref": "LambdaBucketName"},
"S3Key": "skynet-lambda.zip"
},
"Description": "When new EC2 instances are created, auto tag them",
"FunctionName": "newEc2AutoTag",
"Handler": "index.newEc2_autoTag",
"Role": {"Fn::GetAtt": ["IAMRoleLambda", "Arn"]},
"Runtime": "nodejs6.10",
"Timeout": "30"
}
}
},
"Outputs": {
"WebUrl": {
"Value": {
"Fn::GetAtt": [
"S3Web",
"WebsiteURL"
]
},
"Description": "S3 bucket for web files"
}
}
}
I managed to deploy your template into a CloudFormation stack (by removing the LambdaBucket and pointing to my own zip file). It seems to create all resource correctly.
It took about 10 minutes for the RunInstances event to appear in CloudTrail. It then successfully triggered the Rule, but the CloudWatch metrics for my rule showed a failed invocation because I faked a Lambda function for your template.
Once I edited the rule to point to a better function and re-tested, it worked fine.
Bottom line: Seems to work!

Creating an ALB Target Group in CloudFormation

I'm trying to create an Application Load Balancer in CloudFormation, with a target group that forwards traffic to EC2 instances. Here is the relevant snippet, where ELBSubnets, ECSCluster, taskdefinition, and VpcId are passed in as parameters:
"EcsElasticLoadBalancer" : {
"Type" : "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties" : {
"Subnets" : { "Ref" : "ELBSubnets" },
"SecurityGroups": [
{ "Ref": "ELBAccessSecurityGroup" }
]
}
},
"LoadBalancerListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": { "Ref": "TargetGroup" }
}],
"LoadBalancerArn": { "Ref": "EcsElasticLoadBalancer" },
"Port": 80,
"Protocol": "HTTP"
}
},
"TargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "TargetGroup" ] ] },
"Port": 80,
"Protocol": "HTTP",
"VpcId": { "Ref": "VpcId" }
},
"DependsOn": [ "EcsElasticLoadBalancer" ]
},
"service": {
"Type": "AWS::ECS::Service",
"Properties" : {
"Cluster": { "Ref": "ECSCluster" },
"DesiredCount": "1",
"LoadBalancers": [
{
"ContainerName": "main-app",
"ContainerPort": 3000,
"TargetGroupArn": { "Ref": "TargetGroup" }
}
],
"Role" : {"Ref":"ECSServiceRole"},
"TaskDefinition" : {"Ref":"taskdefinition"}
}
},
"ECSServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ecs.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "ecs-service",
"PolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:Describe*",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"ec2:Describe*",
"ec2:AuthorizeSecurityGroupIngress"
],
"Resource": "*"
}
]
}
}
]
}
}
I get the following error when creating the service:
The target group with targetGroupArn arn:aws:elasticloadbalancing:us-east-1:xxxxxxxx:targetgroup/AlbServiceStack-TargetGroup/6ba9c037c26cdb36 does not have an associated load balancer.
What am I missing? In the documentation there doesn't seem to be a way to specify a load balancer for the target group.
Got it working - the problem was twofold:
The following lines were missing from the Role PolicyDocument:
"elasticloadbalancing:DeregisterTargets"
"elasticloadbalancing:RegisterTargets"
The service needed "DependsOn": [ "LoadBalancerListener" ] as an additional attribute.
Updated template looks like this:
"EcsElasticLoadBalancer" : {
"Type" : "AWS::ElasticLoadBalancingV2::LoadBalancer",
"Properties" : {
"Subnets" : { "Ref" : "ELBSubnets" },
"SecurityGroups": [
{ "Ref": "ELBAccessSecurityGroup" }
]
}
},
"LoadBalancerListener": {
"Type": "AWS::ElasticLoadBalancingV2::Listener",
"Properties": {
"DefaultActions": [{
"Type": "forward",
"TargetGroupArn": { "Ref": "TargetGroup" }
}],
"LoadBalancerArn": { "Ref": "EcsElasticLoadBalancer" },
"Port": 80,
"Protocol": "HTTP"
}
},
"TargetGroup": {
"Type": "AWS::ElasticLoadBalancingV2::TargetGroup",
"Properties": {
"Name": { "Fn::Join": [ "-", [ { "Ref": "AWS::StackName" }, "TargetGroup" ] ] },
"Port": 80,
"Protocol": "HTTP",
"VpcId": { "Ref": "VpcId" }
},
"DependsOn": [ "EcsElasticLoadBalancer" ]
},
"service": {
"Type": "AWS::ECS::Service",
"DependsOn": [ "LoadBalancerListener" ],
"Properties" : {
"Cluster": { "Ref": "ECSCluster" },
"DesiredCount": "1",
"LoadBalancers": [
{
"ContainerName": "main-app",
"ContainerPort": 3000,
"TargetGroupArn": { "Ref": "TargetGroup" }
}
],
"Role" : {"Ref":"ECSServiceRole"},
"TaskDefinition" : {"Ref":"taskdefinition"}
}
},
"ECSServiceRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": [
"ecs.amazonaws.com"
]
},
"Action": [
"sts:AssumeRole"
]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "ecs-service",
"PolicyDocument": {
"Statement": [
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:Describe*",
"elasticloadbalancing:DeregisterInstancesFromLoadBalancer",
"elasticloadbalancing:RegisterInstancesWithLoadBalancer",
"ec2:Describe*",
"ec2:AuthorizeSecurityGroupIngress",
"elasticloadbalancing:DeregisterTargets",
"elasticloadbalancing:RegisterTargets"
],
"Resource": "*"
}
]
}
}
]
}
}

How to get AvailabilityZone given a SubnetId Parameter, or vice versa, in a CloudFormation template?

I extracted a CloudFormation template from a stack created by cfncluster. I'm trying to simplify and group the parameters so that the mandatory parameters are grouped together and are non-redundant. The default subnet-related parameters include MasterSubnetId, AvailabilityZone, ComputeSubnetId, and ComputeSubnetCidr. The template uses these to calculate several conditions and also to set AvailabilityZone properties on several resources:
"CreateComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
]
},
"UseComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
}
]
}
]
},
"UseMasterSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
},
I want to simplify the template to require either MasterSubnetId or AvailabilityZone, but not both. I don't need the option to provide one or the other, it's fine for me to require one or the other. The main problem is I can't figure out how to do that. The various Resources used by the template seem to require both, even though they should be related. I can't use Fn::GetAtt like I want because the first argument cannot come from functions, like this:
"Fn::GetAtt" : [ { "Ref": "MasterSubnetId" }, "AvailabilityZone" ]
Also, I want the compute fleet to always get its own subnet.
Here are the subnet-related parts of the template :
{
...snip...
"Parameters": {
"ComputeSubnetId": {
"Description": "ID of the Subnet you want to provision the Compute Servers into. Set to NONE to use the same subnet as Master Server.",
"Type": "String",
"Default": "NONE"
},
"ComputeSubnetCidr": {
"Description": "CIDR(s) for new backend subnet(s) i.e. 10.0.100.0/24. This is a comma-delimited list and can support multiple CIDR ranges for a multi-AZ cluster. The order and length of this list MUST match the AvailabilityZones parameter. Set to NONE to use the same subnet as Master Server.",
"Type": "String",
"ConstraintDescription": "must be a valid CIDR range of the form x.x.x.x/x.",
"AllowedPattern": "(NONE|(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2}))",
"Default": "NONE"
},
"MasterSubnetId": {
"Description": "ID of the Subnet you want to provision the Master server into",
"Type": "AWS::EC2::Subnet::Id"
},
"AvailabilityZone": {
"Description": "Availability Zone the cluster will launch into. THIS IS REQUIRED",
"Type": "AWS::EC2::AvailabilityZone::Name"
},
"VPCId": {
"Description": "ID of the VPC you want to provision cluster into. Only used with UseVPCBase=false",
"Type": "AWS::EC2::VPC::Id"
},
"UsePublicIps": {
"Description": "Boolean flag to use public IP's for instances. If false, the VPC must be correctly setup to use NAT for all traffic.",
"Type": "String",
"Default": "true",
"ConstraintDescription": "true/false",
"AllowedValues": [
"true",
"false"
]
}
},
"Conditions": {
"CreateComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
]
},
"UseComputeSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
},
{
"Fn::Not": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
}
]
}
]
},
"UseMasterSubnetForCompute": {
"Fn::And": [
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetId"
},
"NONE"
]
},
{
"Fn::Equals": [
{
"Ref": "ComputeSubnetCidr"
},
"NONE"
]
}
]
}
},
"Mappings": {
"AWSInstanceType2Capabilites": {
...snip...
},
"AWSRegionOS2AMI": {
...snip...
},
"OSFeatures": {
...snip...
},
"CfnClusterVersions": {
"default": {
"cfncluster": "cfncluster-1.1.0",
"cookbook": "cfncluster-cookbook-1.1.0",
"chef": "12.4.3",
"ridley": "4.3.2",
"berkshelf": "4.0.1",
"ami": "201602192042"
}
},
"AWSRegion2Capabilites": {
...snip...
}
},
"Resources": {
...snip...
"CfnClusterPolicies": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyName": "cfncluster",
"PolicyDocument": {
"Statement": [
{
"Sid": "EC2",
"Action": [
"ec2:AttachVolume",
"ec2:DescribeInstanceAttribute",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances"
],
"Effect": "Allow",
"Resource": [
"*"
]
},
...snip...
]
},
"Roles": [
{
"Ref": "RootRole"
}
]
},
"Condition": "CreateEC2IAMRole",
},
"MasterEIP": {
"Type": "AWS::EC2::EIP",
"Properties": {
"Domain": "vpc"
},
"Condition": "MasterPublicIp",
},
"MasterServer": {
"Type": "AWS::EC2::Instance",
"Properties": {
"InstanceType": {
"Ref": "MasterInstanceType"
},
"BlockDeviceMappings": [
...snip...
],
"KeyName": {
"Ref": "KeyName"
},
"Tags": [
{
"Key": "Application",
"Value": {
"Ref": "AWS::StackName"
}
},
{
"Key": "Name",
"Value": "Master"
}
],
"NetworkInterfaces": [
{
"NetworkInterfaceId": {
"Ref": "MasterENI"
},
"DeviceIndex": "0"
}
],
"ImageId": {
"Fn::If": [
"UseCustomAMI",
{
"Ref": "CustomAMI"
},
{
"Fn::FindInMap": [
"AWSRegionOS2AMI",
{
"Ref": "AWS::Region"
},
{
"Ref": "BaseOS"
}
]
}
]
},
"UserData": {
...snip...
}
}
},
"Metadata": {
"Comment": "cfncluster Master server",
"AWS::CloudFormation::Init": {
"configSets": {
"default": [
"deployConfigFiles",
"getCookbooks",
"chefPrepEnv",
"shellRunPreInstall",
"chefConfig",
"shellRunPostInstall",
"shellForkClusterReadyInstall"
]
},
"deployConfigFiles": {
"files": {
"/tmp/dna.json": {
"mode": "000644",
"owner": "root",
"group": "root",
"content": {
"cfncluster": {
"stack_name": {
"Ref": "AWS::StackName"
},
"cfn_preinstall": {
"Ref": "PreInstallScript"
},
"cfn_preinstall_args": {
"Ref": "PreInstallArgs"
},
"cfn_postinstall": {
"Ref": "PostInstallScript"
},
"cfn_postinstall_args": {
"Ref": "PostInstallArgs"
},
"cfn_region": {
"Ref": "AWS::Region"
},
"cfn_volume": {
"Fn::If": [
"UseExistingEBSVolume",
{
"Ref": "EBSVolumeId"
},
{
"Ref": "SharedVolume"
}
]
},
"cfn_scheduler": {
"Ref": "Scheduler"
},
"cfn_encrypted_ephemeral": {
"Ref": "EncryptedEphemeral"
},
"cfn_ephemeral_dir": {
"Ref": "EphemeralDir"
},
"cfn_shared_dir": {
"Ref": "SharedDir"
},
"cfn_proxy": {
"Ref": "ProxyServer"
},
"cfn_node_type": "MasterServer",
"cfn_cluster_user": {
"Fn::FindInMap": [
"OSFeatures",
{
"Ref": "BaseOS"
},
"User"
]
},
"cfn_ddb_table": {
"Ref": "DynamoDBTable"
},
"cfn_sqs_queue": {
"Fn::GetAtt": [
"SQS",
"QueueName"
]
}
},
"run_list": {
"Fn::If": [
"UseCustomRunList",
{
"Ref": "CustomChefRunList"
},
{
"Fn::Join": [
"",
[
"recipe[cfncluster::",
{
"Ref": "Scheduler"
},
"_config]"
]
]
}
]
}
}
},
...snip...
},
"commands": {
...snip...
}
},
...snip...
},
},
"CreationPolicy": {
"ResourceSignal": {
"Count": "1",
"Timeout": "PT30M"
}
}
},
"ComputeFleet": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"MaxSize": {
"Ref": "MaxQueueSize"
},
"AvailabilityZones": [
{
"Ref": "AvailabilityZone"
}
],
"VPCZoneIdentifier": [
{
"Fn::If": [
"UseMasterSubnetForCompute",
{
"Ref": "MasterSubnetId"
},
{
"Fn::If": [
"CreateComputeSubnetForCompute",
{
"Ref": "ComputeSubnet"
},
{
"Ref": "ComputeSubnetId"
}
]
}
]
}
],
...snip...
},
"DependsOn": "MasterServer",
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT30M",
"Count": {
"Ref": "ComputeWaitConditionCount"
}
}
},
},
"ComputeServerLaunchConfig": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Properties": {
"SecurityGroups": [
{
"Fn::If": [
"CreateSecurityGroups",
{
"Ref": "ComputeSecurityGroup"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"AddAdditionalSG",
{
"Ref": "AdditionalSG"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"UseExistingSecurityGroup",
{
"Ref": "VPCSecurityGroupId"
},
{
"Ref": "AWS::NoValue"
}
]
}
],
...snip...
},
"Metadata": {
"Comment": "cfncluster Compute server",
"AWS::CloudFormation::Init": {
"configSets": {
"default": [
"deployConfigFiles",
"getCookbooks",
"chefPrepEnv",
"shellRunPreInstall",
"chefConfig",
"shellRunPostInstall",
"shellForkClusterReadyInstall",
"signalComputeReady"
]
},
"deployConfigFiles": {
"files": {
"/tmp/dna.json": {
"mode": "000644",
"owner": "root",
"group": "root",
"content": {
"cfncluster": {
"stack_name": {
"Ref": "AWS::StackName"
},
"cfn_preinstall": {
"Ref": "PreInstallScript"
},
"cfn_preinstall_args": {
"Ref": "PreInstallArgs"
},
"cfn_postinstall": {
"Ref": "PostInstallScript"
},
"cfn_postinstall_args": {
"Ref": "PostInstallArgs"
},
"cfn_region": {
"Ref": "AWS::Region"
},
"cfn_scheduler": {
"Ref": "Scheduler"
},
"cfn_encrypted_ephemeral": {
"Ref": "EncryptedEphemeral"
},
"cfn_ephemeral_dir": {
"Ref": "EphemeralDir"
},
"cfn_shared_dir": {
"Ref": "SharedDir"
},
"cfn_proxy": {
"Ref": "ProxyServer"
},
"cfn_sqs_queue": {
"Ref": "SQS"
},
"cfn_master": {
"Fn::GetAtt": [
"MasterServer",
"PrivateDnsName"
]
},
"cfn_node_type": "ComputeFleet",
"cfn_cluster_user": {
"Fn::FindInMap": [
"OSFeatures",
{
"Ref": "BaseOS"
},
"User"
]
}
},
"run_list": {
"Fn::If": [
"UseCustomRunList",
{
"Ref": "CustomChefRunList"
},
{
"Fn::Join": [
"",
[
"recipe[cfncluster::",
{
"Ref": "Scheduler"
},
"_config]"
]
]
}
]
}
}
},
...snip...
},
"commands": {
...snip...
}
},
...snip...
}
},
...snip...
"ComputeSubnet": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": {
"Ref": "VPCId"
},
"CidrBlock": {
"Ref": "ComputeSubnetCidr"
},
"Tags": [
{
"Key": "Network",
"Value": "ComputeSubnet"
}
],
"AvailabilityZone": {
"Ref": "AvailabilityZone"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeRouteTable": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "VPCId"
},
"Tags": [
{
"Key": "Application",
"Value": {
"Ref": "AWS::StackName"
}
},
{
"Key": "Network",
"Value": "ComputeSubnet"
}
]
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeRoute": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {
"Ref": "ComputeRouteTable"
},
"DestinationCidrBlock": "0.0.0.0/0",
"NetworkInterfaceId": {
"Ref": "MasterENI"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"ComputeSubnetRouteTableAssociation": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": {
"Ref": "ComputeSubnet"
},
"RouteTableId": {
"Ref": "ComputeRouteTable"
}
},
"Condition": "CreateComputeSubnetForCompute",
},
"MasterSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Enable access to the Master host",
"VpcId": {
"Ref": "VPCId"
},
"SecurityGroupIngress": [
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": {
"Ref": "AccessFrom"
}
},
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": {
"Ref": "AccessFrom"
}
}
]
},
"Condition": "CreateSecurityGroups",
},
"MasterSecurityGroupIngress": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535",
"SourceSecurityGroupId": {
"Ref": "ComputeSecurityGroup"
},
"GroupId": {
"Ref": "MasterSecurityGroup"
}
},
"Condition": "CreateSecurityGroups",
},
"ComputeSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Allow access to resources in subnets behind front",
"VpcId": {
"Ref": "VPCId"
},
"SecurityGroupIngress": [
{
"SourceSecurityGroupId": {
"Ref": "MasterSecurityGroup"
},
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535"
}
]
},
"Condition": "CreateSecurityGroups",
},
"ComputeSecurityGroupIngress": {
"Type": "AWS::EC2::SecurityGroupIngress",
"Properties": {
"IpProtocol": "-1",
"FromPort": "0",
"ToPort": "65535",
"SourceSecurityGroupId": {
"Ref": "ComputeSecurityGroup"
},
"GroupId": {
"Ref": "ComputeSecurityGroup"
}
},
"Condition": "CreateSecurityGroups"
},
"MasterENI": {
"Type": "AWS::EC2::NetworkInterface",
"Properties": {
"Description": "cfncluster Master Server",
"SubnetId": {
"Ref" : "MasterSubnetId"
},
"SourceDestCheck": "false",
"GroupSet": [
{
"Fn::If": [
"CreateSecurityGroups",
{
"Ref": "MasterSecurityGroup"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"AddAdditionalSG",
{
"Ref": "AdditionalSG"
},
{
"Ref": "AWS::NoValue"
}
]
},
{
"Fn::If": [
"UseExistingSecurityGroup",
{
"Ref": "VPCSecurityGroupId"
},
{
"Ref": "AWS::NoValue"
}
]
}
]
},
},
"SharedVolume": {
"Type": "AWS::EC2::Volume",
"Properties": {
"AvailabilityZone": {
"Ref": "AvailabilityZone"
},
"VolumeType": {
"Ref": "VolumeType"
},
"Size": {
"Fn::If": [
"UseEBSSnapshot",
{
"Ref": "AWS::NoValue"
},
{
"Ref": "VolumeSize"
}
]
},
...snip...
},
...snip...
},
...snip...