AWS Lambda in VPC doesn't have internet access behind NAT - amazon-web-services

Question

My problem is that a Lambda function that I run behind NAT inside a VPC with an IGW doesn't have access to anything on the Internet.
What I'm trying to do is creating a VPC that has:
Internet Gateway;
2 private subnets (PrivateA and PrivateB) in availability zones A and B respectively;
1 public subnet (PublicA) in availability zone A
NAT Gateway in PublicA subnet
PrivateA and PrivateB have a route table that routes 0.0.0.0/0 to the NAT Gateway.
PublicA has a route table that routes 0.0.0.0/0 to the Internet Gateway.
Private subnets, as well as the public subnet have Access Control Lists allowing all Ingress and Egress traffic.
That part kind of works.
Next, I want to create a Lambda function inside the VPC. I put it into PrivateA and PrivateB and assign to it a Security Group that allows all Egress and Ingress traffic.
Below is a self-contained example (the whole template) that reproduces the issue. I've read all the possible docs and articles on the Internet so I would very much appreciate it if anybody could point me in the right direction.
{
"AWSTemplateFormatVersion": "2010-09-09",
"Resources": {
"Vpc": {
"Type": "AWS::EC2::VPC",
"Properties": {
"CidrBlock": "10.0.0.0/16",
"EnableDnsSupport": true,
"EnableDnsHostnames": true,
"InstanceTenancy": "default"
}
},
"InternetGateway": {
"Type": "AWS::EC2::InternetGateway"
},
"VpcGatewayAttachment": {
"Type": "AWS::EC2::VPCGatewayAttachment",
"Properties": {
"VpcId": { "Ref": "Vpc" },
"InternetGatewayId": { "Ref": "InternetGateway" }
}
},
"ElasticIP":{
"Type": "AWS::EC2::EIP",
"Properties": {
"Domain": "vpc"
}
},
"NatGateway": {
"Type": "AWS::EC2::NatGateway",
"DependsOn": [ "VpcGatewayAttachment" ],
"Properties": {
"AllocationId": { "Fn::GetAtt": [ "ElasticIP", "AllocationId" ] },
"SubnetId": { "Ref": "SubnetAPublic" }
}
},
"SubnetAPublic": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": { "Fn::Select" : [ "0", { "Fn::GetAZs" : "" } ] },
"CidrBlock": "10.0.0.0/19",
"MapPublicIpOnLaunch": true,
"VpcId": { "Ref": "Vpc" }
}
},
"SubnetAPrivate": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": { "Fn::Select" : [ "0", { "Fn::GetAZs" : "" } ] },
"CidrBlock": "10.0.64.0/19",
"VpcId": { "Ref": "Vpc" }
}
},
"SubnetBPrivate": {
"Type": "AWS::EC2::Subnet",
"Properties": {
"AvailabilityZone": { "Fn::Select" : [ "1", { "Fn::GetAZs" : "" } ] },
"CidrBlock": "10.0.96.0/19",
"VpcId": { "Ref": "Vpc" }
}
},
"RouteTablePublic": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": { "Ref": "Vpc" }
}
},
"RouteTablePrivate": {
"Type": "AWS::EC2::RouteTable",
"Properties": {
"VpcId": { "Ref": "Vpc" }
}
},
"RouteTableAssociationAPublic": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": { "Ref": "SubnetAPublic" },
"RouteTableId": { "Ref": "RouteTablePublic" }
}
},
"RouteTableAssociationAPrivate": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": { "Ref": "SubnetAPrivate" },
"RouteTableId": { "Ref": "RouteTablePrivate" }
}
},
"RouteTableAssociationBPrivate": {
"Type": "AWS::EC2::SubnetRouteTableAssociation",
"Properties": {
"SubnetId": { "Ref": "SubnetBPrivate" },
"RouteTableId": { "Ref": "RouteTablePrivate" }
}
},
"RouteTablePrivateInternetRoute": {
"Type": "AWS::EC2::Route",
"DependsOn": [ "VpcGatewayAttachment" ],
"Properties": {
"RouteTableId": { "Ref": "RouteTablePrivate" },
"DestinationCidrBlock": "0.0.0.0/0",
"NatGatewayId": { "Ref": "NatGateway" }
}
},
"RouteTablePublicInternetRoute": {
"Type": "AWS::EC2::Route",
"DependsOn": [ "VpcGatewayAttachment" ],
"Properties": {
"RouteTableId": { "Ref": "RouteTablePublic" },
"DestinationCidrBlock": "0.0.0.0/0",
"GatewayId": { "Ref": "InternetGateway" }
}
},
"NetworkAclPublic": {
"Type": "AWS::EC2::NetworkAcl",
"Properties": {
"VpcId": { "Ref": "Vpc" }
}
},
"NetworkAclPrivate": {
"Type": "AWS::EC2::NetworkAcl",
"Properties": {
"VpcId": { "Ref": "Vpc" }
}
},
"SubnetNetworkAclAssociationAPublic": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties":{
"SubnetId": { "Ref": "SubnetAPublic" },
"NetworkAclId": { "Ref": "NetworkAclPublic" }
}
},
"SubnetNetworkAclAssociationAPrivate": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties":{
"SubnetId": { "Ref": "SubnetAPrivate" },
"NetworkAclId": { "Ref": "NetworkAclPrivate" }
}
},
"SubnetNetworkAclAssociationBPrivate": {
"Type": "AWS::EC2::SubnetNetworkAclAssociation",
"Properties": {
"SubnetId": { "Ref": "SubnetBPrivate" },
"NetworkAclId": { "Ref": "NetworkAclPrivate" }
}
},
"NetworkAclEntryInPublicAllowAll": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": { "Ref": "NetworkAclPublic" },
"RuleNumber": 99,
"Protocol": -1,
"RuleAction": "allow",
"Egress": false,
"CidrBlock": "0.0.0.0/0"
}
},
"NetworkAclEntryOutPublicAllowAll": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": { "Ref": "NetworkAclPublic" },
"RuleNumber": 99,
"Protocol": -1,
"RuleAction": "allow",
"Egress": true,
"CidrBlock": "0.0.0.0/0"
}
},
"NetworkAclEntryInPrivateAllowVpc": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": { "Ref": "NetworkAclPrivate" },
"RuleNumber": 99,
"Protocol": -1,
"RuleAction": "allow",
"Egress": false,
"CidrBlock": "0.0.0.0/16"
}
},
"NetworkAclEntryOutPrivateAllowVpc": {
"Type": "AWS::EC2::NetworkAclEntry",
"Properties": {
"NetworkAclId": { "Ref": "NetworkAclPrivate" },
"RuleNumber": 99,
"Protocol": -1,
"RuleAction": "allow",
"Egress": true,
"CidrBlock": "0.0.0.0/0"
}
},
"LambdasSecurityGroup": {
"Type": "AWS::EC2::SecurityGroup",
"Properties": {
"GroupDescription": "Lambdas security group",
"SecurityGroupEgress": [
{ "CidrIp": "0.0.0.0/0", "IpProtocol": "-1" }
],
"SecurityGroupIngress": [
{ "CidrIp": "0.0.0.0/0", "IpProtocol": "-1" }
],
"VpcId": { "Ref": "Vpc" }
}
},
"LambdaFunctionExecutionRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": { "Service": "lambda.amazonaws.com" },
"Action": "sts:AssumeRole"
}
]
},
"ManagedPolicyArns": [
"arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole",
"arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
]
}
},
"LambdaFunction": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Handler": "index.lambda_handler",
"Runtime": "python2.7",
"Role": {
"Fn::GetAtt": ["LambdaFunctionExecutionRole", "Arn"]
},
"Code": {
"ZipFile": {
"Fn::Join": ["\n", [
"import urllib2",
"def lambda_handler(event, context):",
"\tresponse = urllib2.urlopen('http://python.org/')",
"\treturn response.read()"
]]
}
},
"VpcConfig": {
"SecurityGroupIds": [
{ "Fn::GetAtt": [ "LambdasSecurityGroup", "GroupId"] }
],
"SubnetIds": [
{ "Ref": "SubnetAPrivate" },
{ "Ref": "SubnetBPrivate" }
]
}
}
}
}
}

Answer 1

The cause of the failed connectivity lies within your ACL config for "NetworkAclEntryInPrivateAllowVpc" and "NetworkAclEntryOutPrivateAllowVpc".
If you open that CIDR block from "0.0.0.0/16" to "0.0.0.0/0", Lambda can access the internet.
I'm not that knowledgeable about NAT, but it seems that the NAT traffic is blocked by that ACL rule.