How can i ensure NAT is up before instances - amazon-web-services

Question

My instances when created by my cloudformation template do not run yum update or install aws-cfn-bootstrap. I see a timeout in the logs however it works after I login and they have fully booted.
Logs show me they are unable to connect at boot, I think because the natgw hasn't been built yet. It was working yesterday i have been tweaking since but cannot seem to get it to load anymore.
"Parameters": {
"ONtestenv": {
"Description": "env name",
"Type": "String"
},
"ONcidr": {
"Description": "subs for vpc",
"Type": "String",
"Default": "10.0.0.0/16"
},
"pubONsubnet": {
"Description": "pub sub block",
"Type": "String",
"Default": "10.0.0.0/24"
},
"privONsubnet": {
"Description": "priv subn clok",
"Type": "String",
"Default": "10.0.1.0/24"
},
"KeyName": {
"Description": "Name of an existing EC2 KeyPair to enable SSH access to the instances",
"Type": "AWS::EC2::KeyPair::KeyName",
"ConstraintDescription": "must be the name of an existing EC2 KeyPair."
},
"SSHLocation": {
"Description": "The IP address range that can be used to SSH to the EC2 instances",
"Type": "String",
"MinLength": "9",
"MaxLength": "18",
"Default": "0.0.0.0/0",
"AllowedPattern": "(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})/(\\d{1,2})",
"ConstraintDescription": "must be a valid IP CIDR range of the form x.x.x.x/x."
},
"InstanceType": {
"Description": "WebServer EC2 instance type",
"Type": "String",
"Default": "t2.micro",
"AllowedValues": [
"t2.micro",
"m1.small",
"m1.medium",
"m1.large",
"m1.xlarge",
"m2.xlarge",
"m2.2xlarge",
"m2.4xlarge"
],
"ConstraintDescription": "must be a valid EC2 instance type."
}
},
"Mappings": {
"Region2Examples": {
"us-east-1": {
"Examples": "https://s3.amazonaws.com/cloudformation-examples-us-east-1"
},
"ca-central-1": {
"Examples": "https://s3.amazonaws.com/cloudformation-examples-us-east-1"
},
"us-west-2": {
"Examples": "https://s3-us-west-2.amazonaws.com/cloudformation-examples-us-west-2"
},
"us-west-1": {
"Examples": "https://s3-us-west-1.amazonaws.com/cloudformation-examples-us-west-1"
},
"eu-west-1": {
"Examples": "https://s3-eu-west-1.amazonaws.com/cloudformation-examples-eu-west-1"
},
"eu-central-1": {
"Examples": "https://s3-eu-central-1.amazonaws.com/cloudformation-examples-eu-central-1"
},
"ap-southeast-1": {
"Examples": "https://s3-ap-southeast-1.amazonaws.com/cloudformation-examples-ap-southeast-1"
},
"us-east-2": {
"Examples": "https://s3-us-east-2.amazonaws.com/cloudformation-examples-us-east-2"
},
"sa-east-1": {
"Examples": "https://s3-sa-east-1.amazonaws.com/cloudformation-examples-sa-east-1"
},
"cn-north-1": {
"Examples": "https://s3.cn-north-1.amazonaws.com.cn/cloudformation-examples-cn-north-1"
}
},
"AWSInstanceType2Arch": {
"t2.micro": {
"Arch": "64"
},
"m1.small": {
"Arch": "64"
},
"m1.medium": {
"Arch": "64"
},
"m1.large": {
"Arch": "64"
},
"m1.xlarge": {
"Arch": "64"
},
"m2.xlarge": {
"Arch": "64"
},
"m2.2xlarge": {
"Arch": "64"
},
"m2.4xlarge": {
"Arch": "64"
},
"c1.medium": {
"Arch": "64"
},
"c1.xlarge": {
"Arch": "64"
},
"cc1.4xlarge": {
"Arch": "64HVM"
},
"cc2.8xlarge": {
"Arch": "64HVM"
},
"cg1.4xlarge": {
"Arch": "64HVM"
}
},
"AWSRegionArch2AMI": {
"us-east-1": {
"32": "ami-31814f58",
"64": "ami-1b814f72",
"64HVM": "ami-0da96764"
},
"ca-central-1": {
"32": "ami-31814f58",
"64": "ami-b61b96d2",
"64HVM": "ami-b61b96d2"
},
"us-west-2": {
"32": "ami-38fe7308",
"64": "ami-30fe7300",
"64HVM": "NOT_YET_SUPPORTED"
},
"us-west-1": {
"32": "ami-11d68a54",
"64": "ami-1bd68a5e",
"64HVM": "NOT_YET_SUPPORTED"
},
"eu-west-1": {
"32": "ami-973b06e3",
"64": "ami-953b06e1",
"64HVM": "NOT_YET_SUPPORTED"
},
"ap-southeast-1": {
"32": "ami-b4b0cae6",
"64": "ami-beb0caec",
"64HVM": "NOT_YET_SUPPORTED"
},
"ap-northeast-1": {
"32": "ami-0644f007",
"64": "ami-0a44f00b",
"64HVM": "NOT_YET_SUPPORTED"
},
"sa-east-1": {
"32": "ami-3e3be423",
"64": "ami-3c3be421",
"64HVM": "NOT_YET_SUPPORTED"
}
}
},
"Resources": {
"ONtestVPC": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": {
"Ref": "ONcidr"
},
"Tags": [
{
"Key": "Name",
"Value": {
"Ref": "ONtestenv"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "3321d2b3-88cd-4e8f-bef1-b5d0b853ca46"
}
}
},
"ONIG": {
"Type": "AWS::EC2::InternetGateway",
"Properties": {
"Tags": [
{
"Key": "Name",
"Value": {
"Ref": "ONtestenv"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "d5878b9e-87f5-4088-8401-1a60d827a01a"
}
}
},
"ONgatewayattach": {
"Type": "AWS::EC2::VPCGatewayAttachment",
"Properties": {
"InternetGatewayId": {
"Ref": "ONIG"
},
"VpcId": {
"Ref": "ONtestVPC"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "b654db30-aa3f-4ffe-ab5c-27b9a14be28e"
}
}
},
"natGW": {
"Type": "AWS::EC2::NatGateway",
"Properties": {
"AllocationId": {
"Fn::GetAtt": [
"natEIP",
"AllocationId"
]
},
"SubnetId": {
"Ref": "pubsub"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "b030b414-e088-4733-8d0a-bbe426610828"
}
}
},
"pubsub": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": {
"Ref": "ONtestVPC"
},
"AvailabilityZone": {
"Fn::Select": [
0,
{
"Fn::GetAZs": ""
}
]
},
"CidrBlock": {
"Ref": "pubONsubnet"
},
"MapPublicIpOnLaunch": false,
"Tags": [
{
"Key": "Name",
"Value": {
"Fn::Sub": "${ONtestenv} pub sub"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "dd0e3e42-3b94-4ffe-a8e3-85690934c839"
}
}
},
"privsub": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"VpcId": {
"Ref": "ONtestVPC"
},
"AvailabilityZone": {
"Fn::Select": [
0,
{
"Fn::GetAZs": ""
}
]
},
"CidrBlock": {
"Ref": "privONsubnet"
},
"MapPublicIpOnLaunch": false,
"Tags": [
{
"Key": "Name",
"Value": {
"Fn::Sub": "${ONtestenv} priv sub"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "4cfc646e-acb2-45ea-a075-596b7453e7d7"
}
}
},
"natEIP": {
"Type": "AWS::EC2::EIP",
"DependsOn": "ONgatewayattach",
"Properties": {
"Domain": "vpc"
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "c3501a25-dec8-4d5b-a8af-4c8ddc2b8c48"
}
}
},
"pubroutes": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "ONtestVPC"
},
"Tags": [
{
"Key": "Name",
"Value": {
"Fn::Sub": "${ONtestenv} pub routes"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "15610e2e-2838-4b07-9ed0-3339a8ee2c6b"
}
}
},
"defaultpubroute": {
"Type": "AWS::EC2::Route",
"DependsOn": "ONgatewayattach",
"Properties": {
"RouteTableId": {
"Ref": "pubroutes"
},
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": {
"Ref": "ONIG"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "2ccd0372-a83b-42cf-8d24-4bf2937f9db2"
}
}
},
"pubsubrtassoc": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"RouteTableId": {
"Ref": "pubroutes"
},
"SubnetId": {
"Ref": "pubsub"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "871f2e97-ff70-4bcf-a707-07cd7629a070"
}
}
},
"privroutetable": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": {
"Ref": "ONtestVPC"
},
"Tags": [
{
"Key": "Name",
"Value": {
"Fn::Sub": "${ONtestenv} priv routes"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "e3959861-54ef-41eb-8732-644b3302f1a2"
}
}
},
"defaultprivroute": {
"Type": "AWS::EC2::Route",
"Properties": {
"RouteTableId": {
"Ref": "privroutetable"
},
"DestinationCidrBlock": "0.0.0.0/0",
"NatGatewayId": {
"Ref": "natGW"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "0bd7af21-8d8c-4bcb-ac8d-b7a0c1bcc7f1"
}
}
},
"privsubrtassoc": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"RouteTableId": {
"Ref": "privroutetable"
},
"SubnetId": {
"Ref": "privsub"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "e4a0e22d-b70f-498f-8269-7569a2a260cc"
}
}
},
"SG": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Enable ping and ssh access via port 22 and ALL from VPC CIDR",
"VpcId": {
"Ref": "ONtestVPC"
},
"SecurityGroupIngress": [
{
"IpProtocol": "icmp",
"FromPort": "8",
"ToPort": "-1",
"CidrIp": "0.0.0.0/0"
},
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": "0.0.0.0/0"
},
{
"IpProtocol": "icmp",
"FromPort": "-1",
"ToPort": "-1",
"CidrIp": "10.0.0.0/16"
},
{
"IpProtocol": "udp",
"FromPort": "1",
"ToPort": "65535",
"CidrIp": "10.0.0.0/16"
},
{
"IpProtocol": "tcp",
"FromPort": "1",
"ToPort": "65535",
"CidrIp": "10.0.0.0/16"
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "64176529-142e-41de-a97d-b4306dd2c445"
}
}
},
"webAutoscalingGroup": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"LaunchConfigurationName": {
"Ref": "LaunchConfig"
},
"LoadBalancerNames": [
{
"Ref": "ElasticLoadBalancer"
}
],
"MaxSize": 4,
"MinSize": 2,
"VPCZoneIdentifier": [
{
"Ref": "pubsub"
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "a331154a-b1ca-416c-80d3-651425c8ad8e"
}
}
},
"webAutoscalePolicy": {
"Type": "AWS::AutoScaling::ScalingPolicy",
"Description": "A policy to expand the size of the pool by 1 instance",
"Properties": {
"AutoScalingGroupName": {
"Ref": "webAutoscalingGroup"
},
"AdjustmentType": "ChangeInCapacity",
"Cooldown": 300,
"ScalingAdjustment": 1
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "77ce9e6f-f50d-4f96-a229-76ffe3bfc32a"
}
}
},
"webAutoScaleAlarm": {
"Type": "AWS::CloudWatch::Alarm",
"Description": "When average bandwidth for the instances exceeds a threshold trigger\nthe policy (increasing instance count by 1)\n",
"Properties": {
"AlarmName": {
"Fn::Join": [
"-",
[
{
"Ref": "AWS::StackName"
},
"bytes out alarm"
]
]
},
"AlarmDescription": "This metric monitors network utilization",
"ComparisonOperator": "GreaterThanOrEqualToThreshold",
"EvaluationPeriods": 2,
"MetricName": "NetworkIn",
"Namespace": "AWS/EC2",
"Period": 60,
"Statistic": "Average",
"Threshold": 5000,
"Dimensions": [
{
"Name": "AutoScalingGroupName",
"Value": {
"Ref": "webAutoscalingGroup"
}
}
]
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "e04f759b-93b9-4072-8966-484a4e953230"
}
}
},
"LBSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "SSH and HTTP inbound, egress to VPC",
"SecurityGroupIngress": [
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": "0.0.0.0/0"
},
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": {
"Ref": "SSHLocation"
}
}
],
"SecurityGroupEgress": [
{
"IpProtocol": "tcp",
"FromPort": 0,
"ToPort": 65535,
"CidrIp": "0.0.0.0/0"
}
],
"VpcId": {
"Ref": "ONtestVPC"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "f6b443e1-6c3f-4e3c-bcc4-fbccb527b1f1"
}
}
},
"ElasticLoadBalancer": {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"Subnets": [
{
"Ref": "pubsub"
}
],
"CrossZone": "true",
"Listeners": [
{
"LoadBalancerPort": "80",
"InstancePort": "80",
"Protocol": "HTTP"
}
],
"HealthCheck": {
"Target": "HTTP:80/",
"HealthyThreshold": "3",
"UnhealthyThreshold": "5",
"Interval": "30",
"Timeout": "5"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "33772321-0e4d-4a0e-bfc6-e4196b7cdead"
}
}
},
"InstanceSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "SSH inbound, port 80 inbound from the load balancer",
"SecurityGroupIngress": [
{
"IpProtocol": "tcp",
"FromPort": "80",
"ToPort": "80",
"CidrIp": {
"Ref": "SSHLocation"
}
},
{
"IpProtocol": "tcp",
"FromPort": "22",
"ToPort": "22",
"CidrIp": {
"Ref": "SSHLocation"
}
}
],
"VpcId": {
"Ref": "ONtestVPC"
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "9bed003f-7101-4f65-a41c-175bfa579c08"
}
}
},
"LaunchConfig": {
"Type": "AWS::AutoScaling::LaunchConfiguration",
"Metadata": {
"Comment": "Install a simple application",
"AWS::CloudFormation::Init": {
"config": {
"packages": {
"yum": {
"httpd": []
}
},
"files": {
"/var/www/html/index.html": {
"content": {
"Fn::Join": [
"\n",
[
"<img src=\"",
{
"Fn::FindInMap": [
"Region2Examples",
{
"Ref": "AWS::Region"
},
"Examples"
]
},
"/cloudformation_graphic.png\" alt=\"AWS CloudFormation Logo\"/>",
"<h1>Congratulations, you have successfully launched the AWS CloudFormation sample.</h1>"
]
]
},
"mode": "000644",
"owner": "root",
"group": "root"
},
"/etc/cfn/cfn-hup.conf": {
"content": {
"Fn::Join": [
"",
[
"[main]\n",
"stack=",
{
"Ref": "AWS::StackId"
},
"\n",
"region=",
{
"Ref": "AWS::Region"
},
"\n"
]
]
},
"mode": "000400",
"owner": "root",
"group": "root"
},
"/etc/cfn/hooks.d/cfn-auto-reloader.conf": {
"content": {
"Fn::Join": [
"",
[
"[cfn-auto-reloader-hook]\n",
"triggers=post.update\n",
"path=Resources.LaunchConfig.Metadata.AWS::CloudFormation::Init\n",
"action=/opt/aws/bin/cfn-init -v ",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource LaunchConfig ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n",
"runas=root\n"
]
]
}
}
},
"services": {
"sysvinit": {
"httpd": {
"enabled": "true",
"ensureRunning": "true"
},
"cfn-hup": {
"enabled": "true",
"ensureRunning": "true",
"files": [
"/etc/cfn/cfn-hup.conf",
"/etc/cfn/hooks.d/cfn-auto-reloader.conf"
]
}
}
}
}
},
"AWS::CloudFormation::Designer": {
"id": "fb6eab98-3a1b-426a-945f-14c25bb99862"
}
},
"Properties": {
"KeyName": {
"Ref": "KeyName"
},
"ImageId": {
"Fn::FindInMap": [
"AWSRegionArch2AMI",
{
"Ref": "AWS::Region"
},
{
"Fn::FindInMap": [
"AWSInstanceType2Arch",
{
"Ref": "InstanceType"
},
"Arch"
]
}
]
},
"SecurityGroups": [
{
"Ref": "InstanceSecurityGroup"
}
],
"InstanceType": {
"Ref": "InstanceType"
},
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -xe\n",
"yum update -y aws-cfn-bootstrap\n",
"/opt/aws/bin/cfn-init -v ",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource LaunchConfig ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource WebServerGroup ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n"
]
]
}
}
}
}
},
"Outputs": {
"VPC": {
"Description": "A reference to the created VPC",
"Value": {
"Ref": "ONtestVPC"
}
},
"PublicSubnet": {
"Description": "public subnet",
"Value": {
"Fn::Join": [
",",
[
{
"Ref": "pubsub"
}
]
]
}
},
"PrivateSubnet": {
"Description": "private subnet",
"Value": {
"Fn::Join": [
",",
[
{
"Ref": "privsub"
}
]
]
}
}
}
}```
I am hoping it will install the simple httpd server but it does not. I am able to run the exact same command ```yum update -y``` after it's booted and I login to it.

Answer 1

You can put a DependsOn condition on the Auto Scaling group referencing the natGW, so that it won't launch instances until the NAT Gateway is ready.
See: DependsOn Attribute - AWS CloudFormation
CloudFormation normally figures out "depends on" linkages automatically, based on references from one resource to another (eg a Subnet references a VPC, so CloudFormation waits for the VPC to be ready before creating the Subnet).
However, not all relationships are obvious such as between your Auto Scaling group and the NAT Gateway. For these situations, you can manually add a DependsOn.

Answer 2

It can also be beneficial to use multiple stacks. This reduces the number of resource dependencies that need to be specified.
Key things like a NAT Gateway should be in one of the first stacks and then if you put things like an autoscaling group in a stack added later the dependency doesn't have to be specified as the NAT Gateway will already be up from the earlier stack.