Deploy Multiple full-stack React + Express applications with path routing on ELB - amazon-web-services

I currently have four containerized React + Express applications (port 3001 exposed) sitting on four individual ECS instances with four different CNAMEs. They each sit behind their own nginx service as a reverse proxy.
Given that the number of applications may increase, I'm hoping to re-deploy this on ECS with an ELB, but I'm running into problems with the path routing. My goal is to have a system where <url_name>/service-1 will route traffic to the container referenced in service-1 and so on.
Currently, the services are all in a running state, but the routing provides a bunch of console errors saying the static js and css files produced by the react build command cannot be found at <url_name>/. Has anyone found a way to run multiple full-stack React + Express applications with path routing on ELB or a workaround of adding in an Nginx service or updating the React homepage to a fixed value?
# container_definition
[
{
"name": "service-1",
"image": "<image-name>:latest",
"cpu": 256,
"memory": 256,
"portMappings": [
{
"containerPort": 3001,
"hostPort": 3001
}
],
"essential": true
}
]
# rule.json
{
"ListenerArn": "placeholder",
"Conditions": [
{
"Field": "path-pattern",
"Values": [
"/service-1*"
]
}
],
"Priority": 1,
"Actions": [
{
"Type": "forward",
"TargetGroupArn": "placeholder"
}
]
}
# server.js
const express = require('express'),
path = require('path');
const createServer = function () {
const port = 3001;
// Create a new Express server
const app = express(),
subpath = express();
// Ensure every api route is prefixed by /api
app.use('/api', subpath);
// All routes related to access points
const users = require('./routes/users');
subpath.get('/users/:id', users.getUserById);
// serve up static assets, i.e. HTML, CSS, and JS from /build
app.use(express.static('build'));
if (process.env.NODE_ENV)
app.get('*', (req, res) => res.sendFile(path.join(__dirname + '/build/index.html')));
// Start the HTTP listener using the given port
return app.listen(port, () => console.log(`Express server (HTTP) listening on port ${port}`))
};
module.exports = createServer;

Related

AWS ECS does not start new tasks

AWS ECS cluster services do not start new tasks.
Already checked:
ECS EC2 instances are registered, active, full CPU and memory available, ECS agent is connected.
there are no events in ECS service "Events" tab, nothing about registering, starting, stopping, no errors, it's just empty.
Registered EC2 instances are set up correctly, in other cluster the same AMI is working perfect.
Task definition is correct, it used to work a day before and since then no changes happened.
Checked Service role contains all relevant policies
Querying ECS with AWS CLI aws ecs describe-services --services my-service --cluster my-cluster yields that deployment rollout is constantly IN_PROGRESS and stays like this.
Full response with configuration is here (I've substituted real names and IDs):
{
"serviceArn": "arn:aws:ecs:eu-central-1:my-account-id:service/my-cluster/my-service",
"serviceName": "my-service",
"clusterArn": "arn:aws:ecs:eu-central-1:my-account-id:cluster/my-cluster",
"loadBalancers": [
{
"targetGroupArn": "arn:aws:elasticloadbalancing:eu-central-1:my-account-id:targetgroup/my-service-lb/load-balancer-id",
"containerName": "my-service",
"containerPort": 8065
}
],
"serviceRegistries": [
{
"registryArn": "arn:aws:servicediscovery:eu-central-1:my-account-id:service/srv-srv_id",
"containerName": "my-service",
"containerPort": 8065
}
],
"status": "ACTIVE",
"desiredCount": 1,
"runningCount": 0,
"pendingCount": 0,
"launchType": "EC2",
"taskDefinition": "arn:aws:ecs:eu-central-1:my-account-id:task-definition/my-service:76",
"deploymentConfiguration": {
"deploymentCircuitBreaker": {
"enable": false,
"rollback": false
},
"maximumPercent": 200,
"minimumHealthyPercent": 100
},
"deployments": [
{
"id": "ecs-svc/deployment_id",
"status": "PRIMARY",
"taskDefinition": "arn:aws:ecs:eu-central-1:my-account-id:task-definition/my-service:76",
"desiredCount": 1,
"pendingCount": 0,
"runningCount": 0,
"failedTasks": 0,
"createdAt": "2022-06-28T09:15:08.241000+02:00",
"updatedAt": "2022-06-28T09:15:08.241000+02:00",
"launchType": "EC2",
"rolloutState": "IN_PROGRESS",
"rolloutStateReason": "ECS deployment ecs-svc/deployment_id in progress."
}
],
"roleArn": "arn:aws:iam::my-account-id:role/aws-service-role/ecs.amazonaws.com/AWSServiceRoleForECS",
"events": [],
"createdAt": "2022-06-28T09:15:08.241000+02:00",
"placementConstraints": [],
"placementStrategy": [
{
"type": "spread",
"field": "attribute:ecs.availability-zone"
}
],
"healthCheckGracePeriodSeconds": 120,
"schedulingStrategy": "REPLICA",
"createdBy": "arn:aws:iam::my-account-id:role/my-role",
"enableECSManagedTags": false,
"propagateTags": "NONE",
"enableExecuteCommand": false
}
The ECS service and service discovery entry is created using Terraform, and the service definition is
resource "aws_service_discovery_service" "ecs_discovery_service" {
name = var.service_name
dns_config {
namespace_id = var.service_discovery_hosted_zone_id
dns_records {
ttl = 10
type = "SRV"
}
}
health_check_custom_config {
failure_threshold = 1
}
}
resource "aws_ecs_service" "ecs_service" {
name = var.service_name
cluster = var.ecs_cluster_id
task_definition = var.task_definition_arn
desired_count = var.desired_count
deployment_minimum_healthy_percent = 100
deployment_maximum_percent = 200
health_check_grace_period_seconds = var.health_check_grace_period_seconds
target_group_arn = aws_lb_target_group.target_group.arn
container_name = var.service_name
container_port = var.service_container_port
ordered_placement_strategy {
type = "spread"
field = "attribute:ecs.availability-zone"
}
service_registries {
registry_arn = aws_service_discovery_service.ecs_discovery_service.arn
container_name = var.service_name
container_port = var.service_container_port
}
}
This code used to work pretty fine, and without any changes in infrastructure, after destroying and applying the infrastructure code, ECS does not start any new tasks.
I could narrow problem to the service discovery, as if I remove the service_registries section, the tasks are started as normal.
Removing the service discovery solves the issue, however it's not the proper solution and I don't understand what is the reason of the problem.
Again, the Service Role has the permissions for the service discovery.
"servicediscovery:DeregisterInstance",
"servicediscovery:Get*",
"servicediscovery:List*",
"servicediscovery:RegisterInstance",
"servicediscovery:UpdateInstanceCustomHealthStatus"
I can't find any ways to trace this strange behaviour and want to ask you guys for help:
could you give me any hints what / where I could check. I've checked multiple troubleshooting guides, however all of them rely on events in ECS service and I don't have any there, anything else I had in mind is checked.
maybe you know what could be the problem that the service discovery blocks the ECS to start new tasks? I thought ECS adds a SRV record to the registry when it starts the container and the container is healthy, however I could not see that any containers have been started at all.
I would be very thankful for any hints and let me know if you need any details.
Have a nice day and best regards.

cannot connect redis sentinels on django

I'm trying to connect sentinels, but every time we got the same error
Exception: Could not connect to any sentinel
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [
{ "sentinels": [("redis-cluster.local.svc.cluster.local", 26379, )]
, "master_name": "mymaster"}
]}
},
}
I can't figure out where to put the password key and db key.
And do I need to put in the url the sentinels url's ? or service is enough?
note: when trying to connect redis/sentinels without channels we do not have any issue at all
From the channels_redis readme:
hosts
The server(s) to connect to, as either URIs, (host, port) tuples, or dicts conforming to create_connection. Defaults to ['localhost', 6379]. Pass multiple hosts to enable sharding, but note that changing the host list will lose some sharded data.
Sentinel connections require dicts conforming to create_sentinel with an additional master_name key specifying the Sentinel master set. Plain Redis and Sentinel connections can be mixed and matched if sharding.
(emphasis mine)
From what I read it doesn't seem possible to use URIs for sentinel connections, so if you want to set the db and password keys you need to add the relevant keys in the hosts list item:
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [
{
"sentinels": [
("redis-cluster.local.svc.cluster.local", 26379, )
],
"master_name": "mymaster",
"db": 0,
"password": "your_password"
}
]
}
}
}

Access Django admin from Firebase

I have a website which has a React frontend hosted on Firebase and a Django backend which is hosted on Google Cloud Run. I have a Firebase rewrite rule which points all my API calls to the Cloud Run instance. However, I am unable to use the Django admin panel from my custom domain which points to Firebase.
I have tried two different versions of rewrite rules -
"rewrites": [
{
"source": "/**",
"run": {
"serviceId": "serviceId",
"region": "europe-west1"
}
},
{
"source": "**",
"destination": "/index.html"
}
]
--- AND ---
"rewrites": [
{
"source": "/api/**",
"run": {
"serviceId": "serviceId",
"region": "europe-west1"
}
},
{
"source": "/admin/**",
"run": {
"serviceId": "serviceId",
"region": "europe-west1"
}
},
{
"source": "**",
"destination": "/index.html"
}
]
I am able to see the log in page when I go to url.com/admin/, however I am unable to go any further. It just refreshes the page with empty email/password fields and no error message. Just as an FYI, it is not to do with my username and password as I have tested the admin panel and it works fine when accessing it directly using the Cloud Run url.
Any help will be much appreciated.
I didn't actually find an answer to why the admin login page was just refreshing when I was trying to log in using the Firebase rewrite rule, however I thought of an alternative way to access the admin panel using my custom domain.
I have added a custom domain to the Cloud Run instance so that is uses a subdomain of my site domain and I can access the admin panel by using admin.customUrl.com rather than customUrl.com/admin/.

Problem connecting redis and django in GCP

I have deployed a Redis instance using GCP Memorystore.
I also have a django app deployed using App Engine. However, I am facing problems connecting these 2. Both are deployed in the same timezone.
The package that I'm using is django_redis. When I try to login to admin page I face a connection error.
The error is:
Exception Value: Error 110 connecting to <Redis instance IP>:6379. Connection timed out.
Exception Location: /env/lib/python3.7/site-packages/redis/connection.py in connect, line 557
In settings.py I use:
CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [("<Redis instance IP>", 6379)],
},
},
}
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": 'redis://<Redis instance IP>/0',
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient"
}
}
}
Note: With locally installed Redis and set to localhost, everything works fine.
In order to connect to Memorystore, you have to set up a VPC Network for your application, and add that connection into app.yaml into property vpc_access_connector. It's described here in docs: Connecting to a VPC network

Nginx internal dns resolve issue

I have nginx container in AWS that does reverse proxy for my website e.g. https://example.com. I have backend services that automatically register in local DNS - aws.local (this is done by AWS ECS Auto-Discovery).
The problem I have is that nginx is only resolving name to IP during start, so when service container is rebooted and gets new IP, nginx still tries old IP and I have "502 Bad Gateway" error.
Here is a code that I am running:
worker_processes 1;
events { worker_connections 1024; }
http {
sendfile on;
include /etc/nginx/mime.types;
log_format graylog2_json '{ "timestamp": "$time_iso8601", '
'"remote_addr": "$remote_addr", '
'"body_bytes_sent": $body_bytes_sent, '
'"request_time": $request_time, '
'"response_status": $status, '
'"request": "$request", '
'"request_method": "$request_method", '
'"host": "$host",'
'"upstream_cache_status": "$upstream_cache_status",'
'"upstream_addr": "$upstream_addr",'
'"http_x_forwarded_for": "$http_x_forwarded_for",'
'"http_referrer": "$http_referer", '
'"http_user_agent": "$http_user_agent" }';
upstream service1 {
server service1.aws.local:8070;
}
upstream service2 {
server service2.aws.local:8080;
}
resolver 10.0.0.2 valid=10s;
server {
listen 443 http2 ssl;
server_name example.com;
location /main {
proxy_pass http://service1;
}
location /auth {
proxy_pass http://service2;
}
I find advices to change nginx config to resolve names per request, but then I see my browser tries to open "service2.aws.local:8070" and fails since its AWS local DNS name. I should see https://example.com/auth" on my browser.
server {
set $main service1.aws.local:2000;
set $auth service2.aws.local:8070;
location /main {
proxy_http_version 1.1;
proxy_pass http://$main;
}
location /auth {
proxy_http_version 1.1;
proxy_pass http://$auth;
}
Can you help me fixing it?
Thanks !!!
TL;DR
resolver 169.254.169.253;
set $upstream "service1.aws.local";
proxy_pass http://$upstream:8070;
Just like with ECS, I experienced the same issue when using Docker Compose.
According to six8's comment on GitHub
nginx only resolves hostnames on startup. You can use variables with
proxy_pass to get it to use the resolver for runtime lookups.
See:
https://forum.nginx.org/read.php?2,215830,215832#msg-215832
https://www.ruby-forum.com/topic/4407628
It's quite annoying.
One of the links above provides an example
resolver 127.0.0.1;
set $backend "foo.example.com";
proxy_pass http://$backend;
The resolver part is necessary. And we can't refer to the defined upstreams here.
According to Ivan Frolov's answer on StackExchange, the resolver's address should be set to 169.254.169.253
What is the TTL for your CloudMap Service Discovery records? If you do an NS lookup from the NGINX container (assuming EC2 mode and you can exec into the container) does it return the new record? Without more information, it's hard to say, but I'd venture to say this is a TTL issue and not an NGINX/Service Discovery problem.
Lower the TTL to 1 second and see if that works.
AWS CloudMap API Reference DNS Record
I found perfectly solution of this issue.
Nginx "proxy_pass" can't use "etc/hosts" information.
I wanna sugguest you use HA-Proxy reverse proxy in ECS.
I tried nginx reverse proxy, but failed. And success with HA-Proxy.
It is more simple than nginx configuration.
First, use "links" option of Docker and setting "environment variables" (eg. LINK_APP, LINK_PORT).
Second, fill this "environment variables" into haproxy.cfg.
Also, I recommend you use "dynamic port mapping" to ALB. it makes more flexible works.
taskdef.json :
# taskdef.json
{
"executionRoleArn": "arn:aws:iam::<AWS_ACCOUNT_ID>:role/<APP_NAME>_ecsTaskExecutionRole",
"containerDefinitions": [
{
"name": "<APP_NAME>-rp",
"image": "gnokoheat/ecs-reverse-proxy:latest",
"essential": true,
"memoryReservation": <MEMORY_RESV>,
"portMappings": [
{
"hostPort": 0,
"containerPort": 80,
"protocol": "tcp"
}
],
"links": [
"<APP_NAME>"
],
"environment": [
{
"name": "LINK_PORT",
"value": "<SERVICE_PORT>"
},
{
"name": "LINK_APP",
"value": "<APP_NAME>"
}
]
},
{
"name": "<APP_NAME>",
"image": "<IMAGE_NAME>",
"essential": true,
"memoryReservation": <MEMORY_RESV>,
"portMappings": [
{
"protocol": "tcp",
"containerPort": <SERVICE_PORT>
}
],
"environment": [
{
"name": "PORT",
"value": "<SERVICE_PORT>"
},
{
"name": "APP_NAME",
"value": "<APP_NAME>"
}
]
}
],
"requiresCompatibilities": [
"EC2"
],
"networkMode": "bridge",
"family": "<APP_NAME>"
}
haproxy.cfg :
# haproxy.cfg
global
daemon
pidfile /var/run/haproxy.pid
defaults
log global
mode http
retries 3
timeout connect 5000
timeout client 50000
timeout server 50000
frontend http
bind *:80
http-request set-header X-Forwarded-Host %[req.hdr(Host)]
compression algo gzip
compression type text/css text/javascript text/plain application/json application/xml
default_backend app
backend app
server static "${LINK_APP}":"${LINK_PORT}"
Dockerfile(haproxy) :
FROM haproxy:1.7
USER root
COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
See :
Github : https://github.com/gnokoheat/ecs-reverse-proxy
Docker image : gnokoheat/ecs-reverse-proxy:latest