UnrecognizedClientException in AWS fargate - amazon-iam

I try to deploy a fargate container in AWS ECS. But I get the following error
error getting rds cred staging/tas:
{
"message": "The security token included in the request is invalid",
"code": "UnrecognizedClientException",
"time": "2020-06-07T06:10:17.324Z",
"requestId": "5a3287f8-8c7b-49f9-b346-9239840f05bd",
"statusCode": 400,
"retryable": false,
"retryDelay": 7.012616197026311
}
as I am using the fargate, the next thing is I check the task role and the task execution role. They use the same IAM which uses the AmazonECSTaskExecutionRolePolicy
{
"Statement": [
{
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"logs:CreateLogStream",
"logs:PutLogEvents",
"secretsmanager:GetSecretValue"
],
"Resource": "*",
"Effect": "Allow"
}
]
}
as requested i add in the task definition
{
"ipcMode": null,
"executionRoleArn": "arn:aws-cn:iam::xxxyyyeeezzz:role/tas-common-resource-ECSTaskExecutionRole-ZEATKLAUUDV8",
"containerDefinitions": [
{
"dnsSearchDomains": [],
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": [],
"options": {
"awslogs-group": "TASStagingLogs",
"awslogs-region": "cn-north-1",
"awslogs-stream-prefix": "web-app-staging"
}
},
"entryPoint": [],
"portMappings": [
{
"hostPort": 8000,
"protocol": "tcp",
"containerPort": 8000
}
],
"command": [
"npm",
"start"
],
"linuxParameters": null,
"cpu": 0,
"environment": [
{
"name": "NODE_ENV",
"value": "staging"
},
{
"name": "RDS_SECRET_NAME",
"value": "staging/tas"
}
],
"resourceRequirements": null,
"ulimits": [],
"dnsServers": [],
"mountPoints": [],
"workingDirectory": null,
"secrets": [],
"dockerSecurityOptions": [],
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"stopTimeout": null,
"image": "xxxyyyeeezzz.dkr.ecr.cn-north-1.amazonaws.com.cn/tas/master-server",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": [],
"hostname": null,
"extraHosts": [],
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": {},
"systemControls": [],
"privileged": null,
"name": "web-app"
}
],
"placementConstraints": [],
"memory": "4096",
"taskRoleArn": "arn:aws-cn:iam::xxxyyyeeezzz:role/tas-common-resource-ECSTaskExecutionRole-ZEATKLAUUDV8",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws-cn:ecs:cn-north-1:xxxyyyeeezzz:task-definition/master-web-staging-WebTaskDef-ZE50JXHI06U1:1",
"family": "master-web-staging-WebTaskDef-ZE50JXHI06U1",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.17"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
}
],
"pidMode": null,
"requiresCompatibilities": [
"EC2",
"FARGATE"
],
"networkMode": "awsvpc",
"cpu": "2048",
"revision": 1,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": []
}
But it has read access to all resources in secretsmanager, so i wonder what could have gone wrong. Please help. thanks

Hi the problem has been fixed. The issue is that the region is wrong. It is hardcoded in the app as ap-southeast-1 but the app was deployed in cn-north-1

Related

AWS ECS Fargate files upload without S3

I want to ask about any feature that allows normal files uploads.
I tried to use the AWS EFS, but it keeps showing errors.
this is the error I'm getting:
ResourceInitializationError: failed to invoke EFS utils commands to set up EFS volumes: stderr: Failed to resolve "fs-0d3a6954788af5d3c.efs.us-east-1.amazonaws.com" - check that your file system ID is correct, and ensure that the VPC has an EFS mount target for this file system ID. See https://docs.aws.amazon.com/console/efs/mount-dns-name for more detail. Attempting to lookup mount target ip address using botocore. Failed to import necessary dependency botocore, please install botocore first. : unsuccessful EFS utils command execution; code: 1
And this is the task definition in JSON
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::ACCOUNTID:role/ecsTaskExecutionRole",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "/ecs/laravel-test",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "ecs"
}
},
"entryPoint": null,
"portMappings": [
{
"hostPort": 80,
"protocol": "tcp",
"containerPort": 80
}
],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"resourceRequirements": null,
"ulimits": null,
"dnsServers": null,
"mountPoints": [
{
"readOnly": null,
"containerPath": "/var/www/storage/app/public",
"sourceVolume": "storage"
}
],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"stopTimeout": null,
"image": "ACCOUNTID.dkr.ecr.us-east-1.amazonaws.com/laravel-s3",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "laravel-test"
}
],
"placementConstraints": [],
"memory": "1024",
"taskRoleArn": null,
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws:ecs:us-east-1:ACCOUNTID:task-definition/laravel-test:1",
"family": "laravel-test",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.efsAuth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.efs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.25"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
}
],
"pidMode": null,
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"runtimePlatform": {
"operatingSystemFamily": "LINUX",
"cpuArchitecture": null
},
"cpu": "512",
"revision": 1,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": [
{
"fsxWindowsFileServerVolumeConfiguration": null,
"efsVolumeConfiguration": {
"transitEncryptionPort": null,
"fileSystemId": "fs-0d3a6954788af5d3c",
"authorizationConfig": {
"iam": "DISABLED",
"accessPointId": null
},
"transitEncryption": "DISABLED",
"rootDirectory": "/data"
},
"name": "storage",
"host": null,
"dockerVolumeConfiguration": null
}
]
}
The final solution for me is to use ECS with EC2 on demand and EPS but I really want it to be without any EC2.
Is there any suggestion for me or any idea how to use the EFS without any errors?
I just found this solution and it's about defining the security group on creation
ECS and EFS connection issue
At the end still facing errors, I'm moving files to S3

ECS Task Denied access to S3 ENV file

I found a couple of posts like this one which I thought would solve my issue for me but I am still getting the failed to download env file error.
My workflow:
ECS Cluster running a FARGATE task definition
Task definition: network mode awsvpc on Linux OS. Has a role with AmazonECSTaskExecutionRolePolicy and this in-line policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Envfile",
"Effect": "Allow",
"Action": "s3:GetObject",
"Resource": "arn:aws:s3:::my-bucket/docker-assets/.env"
},
{
"Sid": "EnvFolder",
"Effect": "Allow",
"Action": "s3:GetBucketLocation",
"Resource": "arn:aws:s3:::my-bucket/docker-assets"
}
]
}
ECS Task Definition:
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::my-account:role/ecsTaskExecutionRole",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": [
{
"value": "arn:aws:s3:::my-bucket/docker-assets/.env",
"type": "s3"
}
],
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "/ecs/My-Task",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "ecs"
}
},
"entryPoint": null,
"portMappings": [
{
"hostPort": 3000,
"protocol": "tcp",
"containerPort": 3000
}
],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"resourceRequirements": null,
"ulimits": null,
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"stopTimeout": null,
"image": "my-account.dkr.ecr.us-east-1.amazonaws.com/my-app-v1:v1",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "my-app"
}
],
"placementConstraints": [],
"memory": "512",
"taskRoleArn": "arn:aws:iam::my-account:role/ecsTaskExecutionRole",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws:ecs:us-east-1:my-account:task-definition/My-Task:5",
"family": "My-Task",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.env-files.s3"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
}
],
"pidMode": null,
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"runtimePlatform": null,
"cpu": "256",
"revision": 5,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": []
}
The task definition also lists the s3 ARN of the env file.
I am running with a default VPC and subnet
The automatic IP is enabled
My s3 bucket was created with block all public access
Please let me know if any further information might required to help with the answer. Thanks in Advance.
I updated my ecsTaskExecutionRole's in-line policy to cover the bucket (with and without slash) and everything under it. This seemed to work:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject"
],
"Resource": [
"arn:aws:s3:::my-bucket/docker-assets/.env"
]
},
{
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::my-bucket",
"arn:aws:s3:::my-bucket/",
"arn:aws:s3:::my-bucket/*"
]
}
]
}

Cannot See Workflows while Running Temporal on AWS ECS with AWS Elasticsearch

I deployed Temporal along with Temporal Web on AWS ECS. I replicated the configuration using this Docker Compose file: https://github.com/temporalio/docker-compose/blob/main/docker-compose.yml
I used this configuration file: https://github.com/temporalio/docker-compose/blob/main/dynamicconfig/development_es.yaml
For Elasticsearch, I used a single node password enabled AWS Elasticsearch domain having ES v7.10.
For Postgres, I used a single instance password enabled AWS RDS database having Postgres v13.3.
My task definition for Temporal is as follows:
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::999999999999:role/dev-ecs-task-execution-role",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "awslogs-dev",
"awslogs-region": "eu-central-1",
"awslogs-create-group": "true",
"awslogs-stream-prefix": "awslogs-temporal"
}
},
"entryPoint": [],
"portMappings": [
{
"hostPort": 7233,
"protocol": "tcp",
"containerPort": 7233
}
],
"command": [],
"linuxParameters": null,
"cpu": 32,
"environment": [
{
"name": "DB",
"value": "postgresql"
},
{
"name": "DB_PORT",
"value": "5432"
},
{
"name": "DYNAMIC_CONFIG_FILE_PATH",
"value": "/temporal/development_es.yaml"
},
{
"name": "ENABLE_ES",
"value": "true"
},
{
"name": "ES_PORT",
"value": "443"
},
{
"name": "ES_SCHEME",
"value": "https"
},
{
"name": "ES_VERSION",
"value": "v7"
},
{
"name": "LOG_LEVEL",
"value": "info"
},
{
"name": "SERVICES",
"value": "history,matching,frontend,worker"
}
],
"resourceRequirements": null,
"ulimits": [],
"dnsServers": null,
"mountPoints": [
{
"readOnly": true,
"containerPath": "/temporal",
"sourceVolume": "configVolume"
}
],
"workingDirectory": null,
"secrets": [
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_POSTGRES_USERNAME",
"name": "POSTGRES_USER"
},
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_POSTGRES_PASSWORD",
"name": "POSTGRES_PWD"
},
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_POSTGRES_HOST",
"name": "POSTGRES_SEEDS"
},
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_ES_HOST",
"name": "ES_SEEDS"
},
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_ES_USERNAME",
"name": "ES_USER"
},
{
"valueFrom": "arn:aws:ssm:eu-central-1:999999999999:parameter/docker/dev/temporal_ES_PASSWORD",
"name": "ES_PWD"
}
],
"dockerSecurityOptions": null,
"memory": 256,
"memoryReservation": 128,
"volumesFrom": [],
"stopTimeout": 60,
"image": "temporalio/auto-setup:1.15.0",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "temporal"
}
],
"placementConstraints": [],
"memory": null,
"taskRoleArn": null,
"compatibilities": [
"EC2"
],
"taskDefinitionArn": "arn:aws:ecs:eu-central-1:999999999999:task-definition/temporal-dev:36",
"family": "temporal-dev",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.efsAuth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.efs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.container-ordering"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.25"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.secrets.ssm.environment-variables"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
}
],
"pidMode": null,
"requiresCompatibilities": [
"EC2"
],
"networkMode": "awsvpc",
"runtimePlatform": null,
"cpu": null,
"revision": 36,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": [
{
"fsxWindowsFileServerVolumeConfiguration": null,
"efsVolumeConfiguration": {
"transitEncryptionPort": null,
"fileSystemId": "fs-99999999",
"authorizationConfig": null,
"transitEncryption": null,
"rootDirectory": "temporal"
},
"name": "configVolume",
"host": null,
"dockerVolumeConfiguration": null
}
]
}
My task definition for Temporal Web is as follows:
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::999999999999:role/dev-ecs-task-execution-role",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "awslogs-dev",
"awslogs-region": "eu-central-1",
"awslogs-create-group": "true",
"awslogs-stream-prefix": "awslogs-temporal-web"
}
},
"entryPoint": [],
"portMappings": [
{
"hostPort": 8088,
"protocol": "tcp",
"containerPort": 8088
}
],
"command": [],
"linuxParameters": null,
"cpu": 32,
"environment": [
{
"name": "TEMPORAL_GRPC_ENDPOINT",
"value": "temporal.dev-local:7233"
},
{
"name": "TEMPORAL_PERMIT_WRITE_API",
"value": "true"
}
],
"resourceRequirements": null,
"ulimits": [],
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"secrets": [],
"dockerSecurityOptions": null,
"memory": 256,
"memoryReservation": 128,
"volumesFrom": [],
"stopTimeout": 60,
"image": "temporalio/web:1.13.0",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "temporal-web"
}
],
"placementConstraints": [],
"memory": null,
"taskRoleArn": null,
"compatibilities": [
"EC2"
],
"taskDefinitionArn": "arn:aws:ecs:eu-central-1:999999999999:task-definition/temporal-web-dev:7",
"family": "temporal-web-dev",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.container-ordering"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
}
],
"pidMode": null,
"requiresCompatibilities": [
"EC2"
],
"networkMode": "awsvpc",
"runtimePlatform": null,
"cpu": null,
"revision": 7,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": []
}
I'm able to run those 2 containers successfully and temporal connects to and writes data on Postgres and Elasticsearch successfully.
The issue is, I'm not able to see any workflows on the Temporal Web UI neither in my created namespace nor under the default namespace but I can see data on Elasticsearch.
I'm not able to find any error logs, I only see info logs under AWS CloudWatch for both Temporal and Temporal Web UI.
What did I miss? What's the actual issue? Is there any tutorial or guide available to make it work on AWS ECS with AWS RDS Postgres and AWS Elasticsearch using dynamic configuration?
Note: When I run it locally using docker-compose, it works.
I checked the temporal-system namespace here and I was able to see the workflows: https://temporal-dev.example.com/namespaces/temporal-system/workflows?range=last-30-days&status=ALL
I can also see workflows in our namespaces.
I did the following things:
I enabled authentication (username and password) on Elasticsearch for successful connectivity.
I deleted the temporal index from Elasticsearch.
I upgraded the temporal version to v1.15.0 by using the following image: temporalio/auto-setup:1.15.0
It was 1.13.1 before.
I deleted the temporal databases from Postgres.
I restarted the temporal ECS services and the services that create namespaces.
It worked.

AWS ECS FARGATE - unable to pull image from docker private repository

I'm trying to create an ECS Cluster with a task pulling image from my private docker repository.
I created a secret in AWS SecretsManager.
I created policy with following parameters.
"Action": "secretsmanager:GetSecretValue",
"Resource": "arn:aws:secretsmanager:eu-west-1:123456789:secret:docker_private_repo-123456"
I added new policy to "ecsTaskExecutionRole" that is created by ECS Task Definition process.
But unfortunately, task always stopped.
I tried to do everything in following tutorials.
https://docs.aws.amazon.com/AmazonECS/latest/developerguide/private-auth.html
https://aws.amazon.com/blogs/compute/introducing-private-registry-authentication-support-for-aws-fargate/
I'm getting this error.
"Stopped reason ResourceInitializationError: unable to pull secrets or registry auth: execution resource retrieval failed: unable to get registry auth from asm: service call has been retried 1 time(s): unable to unmarshal secret value of authorization data from asm: i..."
Launch type FARGATE
Platform version 1.4.0
Can anyone help me, please.
Thank you..
Task Definition:
{
"ipcMode": null,
"executionRoleArn": "arn:aws:iam::123456789:role/ecsTaskExecutionRole",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"environmentFiles": null,
"logConfiguration": {
"logDriver": "awslogs",
"secretOptions": null,
"options": {
"awslogs-group": "/ecs/WebFTask",
"awslogs-region": "eu-west-1",
"awslogs-stream-prefix": "ecs"
}
},
"entryPoint": null,
"portMappings": [
{
"hostPort": 80,
"protocol": "tcp",
"containerPort": 80
}
],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"resourceRequirements": null,
"ulimits": null,
"repositoryCredentials": {
"credentialsParameter": "arn:aws:secretsmanager:eu-west-1:123456789:secret:docker_private_repo-123456"
},
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"secrets": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": 512,
"volumesFrom": [],
"stopTimeout": null,
"image": "docker.io/username/imageName:latest",
"startTimeout": null,
"firelensConfiguration": null,
"dependsOn": null,
"disableNetworking": null,
"interactive": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"pseudoTerminal": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"systemControls": null,
"privileged": null,
"name": "WebContariner"
}
],
"placementConstraints": [],
"memory": "2048",
"taskRoleArn": "arn:aws:iam::123456789:role/ecsTaskExecutionRole",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws:ecs:eu-west-1:123456789:task-definition/WebFTask:6",
"family": "WebFTask",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.private-registry-authentication.secretsmanager"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
}
],
"pidMode": null,
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"cpu": "1024",
"revision": 6,
"status": "ACTIVE",
"inferenceAccelerators": null,
"proxyConfiguration": null,
"volumes": []
}
ecsTaskExecutionRole:
Policy 1:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"ecr:GetAuthorizationToken",
"ecr:BatchCheckLayerAvailability",
"ecr:GetDownloadUrlForLayer",
"ecr:BatchGetImage",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": "*"
}
]
}
Policy 2:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "VisualEditor0",
"Effect": "Allow",
"Action": "secretsmanager:GetSecretValue",
"Resource": "arn:aws:secretsmanager:eu-west-1:123456789:secret:docker_private_repo-123456c"
}
]
}
Your task definition shows lack of task role:
"taskRoleArn": null,
Seems you created it, but you haven't assigned it to the task.

Attach taskRoleArn to AWS Fargate does not work

I'm trying to attach the IAM role to AWS Fargate container. There is no error and the container could execute. However, the container could not call the AWS API.
From the container:
When run aws command, I got the Unable to locate credentials
error
curl to http://169.254.169.254/latest/meta-data/iam/info does not success
My task definition:
{
"executionRoleArn": "arn:aws:iam::my-account-id:role/test-ecs-role",
"containerDefinitions": [
{
"dnsSearchDomains": null,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "Fargate",
"awslogs-region": "us-west-2",
"awslogs-stream-prefix": "my-app"
}
},
"entryPoint": null,
"portMappings": [],
"command": null,
"linuxParameters": null,
"cpu": 0,
"environment": [],
"ulimits": null,
"dnsServers": null,
"mountPoints": [],
"workingDirectory": null,
"dockerSecurityOptions": null,
"memory": null,
"memoryReservation": null,
"volumesFrom": [],
"image": "my-account-id.dkr.ecr.us-west-2.amazonaws.com/app/submit_data:3e87860f128a286d9b557c90664ad99c",
"disableNetworking": null,
"healthCheck": null,
"essential": true,
"links": null,
"hostname": null,
"extraHosts": null,
"user": null,
"readonlyRootFilesystem": null,
"dockerLabels": null,
"privileged": null,
"name": "my-app"
}
],
"placementConstraints": [],
"memory": "2048",
"taskRoleArn": "arn:aws:iam::my-account-id:role/MasterFargate",
"compatibilities": [
"EC2",
"FARGATE"
],
"taskDefinitionArn": "arn:aws:ecs:us-west-2:my-account-id:task-definition/my-app:10",
"family": "my-app",
"requiresAttributes": [
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-ecr-pull"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.task-eni"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.ecr-auth"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "ecs.capability.execution-role-awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"targetId": null,
"targetType": null,
"value": null,
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
}
],
"requiresCompatibilities": [
"FARGATE"
],
"networkMode": "awsvpc",
"cpu": "256",
"revision": 10,
"status": "ACTIVE",
"volumes": []
}
What should I do to get it work? Thanks.
With the same symptoms, my issue was resolved by installing the latest awscli via pip rather than using the packaged version.
Older versions (for example one available in ubuntu trusty) do not support being run from Fargate.