ECS Cluster problem: no container instance met all of its requirements - amazon-web-services

Question

Roles:
resource "aws_iam_role" "ecs-ec2-role" {
name = "${var.app_name}-ecs-ec2-role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": [
"ecs.amazonaws.com",
"ecs-tasks.amazonaws.com"
]
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
}
resource "aws_iam_instance_profile" "ecs-ec2-role" {
name = "${var.app_name}-ecs-ec2-role"
role = "${aws_iam_role.ecs-ec2-role.name}"
}
resource "aws_iam_role_policy" "ecs-ec2-role-policy" {
name = "${var.app_name}-ecs-ec2-role-policy"
role = "${aws_iam_role.ecs-ec2-role.id}"
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ecs:CreateCluster",
"ecs:DeregisterContainerInstance",
"ecs:DiscoverPollEndpoint",
"ecs:Poll",
"ecs:RegisterContainerInstance",
"ecs:StartTelemetrySession",
"ecs:Submit*",
"ecs:StartTask",
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:DescribeLogStreams"
],
"Resource": [
"arn:aws:logs:*:*:*"
]
}
]
}
EOF
}
# ecs service role
resource "aws_iam_role" "ecs-service-role" {
name = "${var.app_name}-ecs-service-role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": [
"ecs.amazonaws.com"
]
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
}
resource "aws_iam_role_policy_attachment" "ecs-service-attach" {
role = "${aws_iam_role.ecs-service-role.name}"
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceRole"
}
data "aws_iam_policy_document" "aws_secrets_policy" {
version = "2012-10-17"
statement {
sid = ""
effect = "Allow"
actions = ["secretsmanager:GetSecretValue"]
resources = [
var.aws_secrets
]
}
}
resource "aws_iam_policy" "aws_secrets_policy" {
name = "aws_secrets_policy"
policy = "${data.aws_iam_policy_document.aws_secrets_policy.json}"
}
resource "aws_iam_role_policy_attachment" "aws_secrets_policy" {
role = aws_iam_role.ecs-ec2-role.name
policy_arn = aws_iam_policy.aws_secrets_policy.arn
}
ECS:
resource "aws_ecs_cluster" "main" {
name = "${var.app_name}-cluster"
}
data "template_file" "app" {
template = file("./templates/ecs/app.json.tpl")
vars = {
app_name = var.app_name
app_image = var.app_image
app_host = var.app_host
endpoint_protocol = var.endpoint_protocol
app_port = var.app_port
container_cpu = var.container_cpu
container_memory = var.container_memory
aws_region = var.aws_region
aws_secrets = var.aws_secrets
}
}
resource "aws_ecs_task_definition" "app" {
family = "${var.app_name}-task"
execution_role_arn = aws_iam_role.ecs-ec2-role.arn
cpu = var.container_cpu
memory = var.container_memory
container_definitions = data.template_file.app.rendered
}
resource "aws_ecs_service" "main" {
name = "${var.app_name}-service"
cluster = aws_ecs_cluster.main.id
task_definition = aws_ecs_task_definition.app.arn
desired_count = var.app_count
iam_role = aws_iam_role.ecs-service-role.arn
depends_on = [aws_iam_role_policy_attachment.ecs-service-attach]
load_balancer {
target_group_arn = aws_lb_target_group.app.id
container_name = var.app_name
container_port = var.app_port
}
}
Autoscaling:
data "aws_ami" "latest_ecs" {
most_recent = true
filter {
name = "name"
values = ["*amazon-ecs-optimized"]
}
filter {
name = "virtualization-type"
values = ["hvm"]
}
owners = ["591542846629"] # AWS
}
resource "aws_launch_configuration" "ecs-launch-configuration" {
// name = "${var.app_name}-launch-configuration"
image_id = data.aws_ami.latest_ecs.id
instance_type = var.instance_type
iam_instance_profile = aws_iam_instance_profile.ecs-ec2-role.id
security_groups = [aws_security_group.ecs_tasks.id]
root_block_device {
volume_type = "standard"
volume_size = 100
delete_on_termination = true
}
lifecycle {
create_before_destroy = true
}
associate_public_ip_address = "false"
key_name = "backend-dev"
#
# register the cluster name with ecs-agent which will in turn coord
# with the AWS api about the cluster
#
user_data = data.template_file.autoscaling_user_data.rendered
}
data "template_file" "autoscaling_user_data" {
template = file("./templates/ecs/autoscaling_user_data.tpl")
vars = {
ecs_cluster = aws_ecs_cluster.main.name
}
}
#
# need an ASG so we can easily add more ecs host nodes as necessary
#
resource "aws_autoscaling_group" "ecs-autoscaling-group" {
name = "${var.app_name}-autoscaling-group"
max_size = "4"
min_size = "2"
health_check_grace_period = 300
desired_capacity = "2"
vpc_zone_identifier = [aws_subnet.private[0].id, aws_subnet.private[1].id]
launch_configuration = aws_launch_configuration.ecs-launch-configuration.name
health_check_type = "ELB"
tag {
key = "Name"
value = var.app_name
propagate_at_launch = true
}
}
resource "aws_autoscaling_policy" "demo-cluster" {
name = "${var.app_name}-ecs-autoscaling-polycy"
policy_type = "TargetTrackingScaling"
estimated_instance_warmup = "90"
adjustment_type = "ChangeInCapacity"
autoscaling_group_name = aws_autoscaling_group.ecs-autoscaling-group.name
target_tracking_configuration {
predefined_metric_specification {
predefined_metric_type = "ASGAverageCPUUtilization"
}
target_value = 40.0
}
}
Cluster name was added to an Instances successfully via User Data:
$ cat /etc/ecs/ecs.config
ECS_CLUSTER=mercure-cluster
But I'm getting an error:
service mercure-service was unable to place a task because no
container instance met all of its requirements. Reason: No Container
Instances were found in your cluster.
ecs-agent.log:
$ grep 'WARN\|ERROR' ecs-agent.log.2019-10-24-10
2019-10-24T10:36:45Z [WARN] Error getting valid credentials (AKID ): NoCredentialProviders: no valid providers in chain. Deprecated.
2019-10-24T10:36:45Z [ERROR] Unable to register as a container instance with ECS: NoCredentialProviders: no valid providers in chain. Deprecated.
2019-10-24T10:36:45Z [ERROR] Error registering: NoCredentialProviders: no valid providers in chain. Deprecated.
ecs-init.log:
$ grep 'WARN\|ERROR' ecs-init.log
2019-10-24T10:36:45Z [WARN] ECS Agent failed to start, retrying in 547.77941ms
2019-10-24T10:36:46Z [WARN] ECS Agent failed to start, retrying in 1.082153551s
2019-10-24T10:36:50Z [WARN] ECS Agent failed to start, retrying in 2.066145821s
2019-10-24T10:36:55Z [WARN] ECS Agent failed to start, retrying in 4.235010051s