Configsets config commands are not executing sequencially cloudformation - amazon-web-services

This is metadata for cloudformationinit
{
"Metadata": {
"AWS::CloudFormation::Init": {
"configSets": {
"setup": [
"setup_docker",
"setup_redis"
]
},
"setup_docker": {
"commands": {
"install_docker": {
"command": {
"Fn::Join": ["", ["amazon-linux-extras install docker"]]
}
},
"start_docker": {
"command": { "Fn::Join": ["", ["service docker start"]] }
},
"start_on_restart": {
"command": { "Fn::Join": ["", ["systemctl enable docker"]] }
}
}
},
"setup_redis": {
"commands": {
"install_redis": {
"command": {
"Fn::Join": ["", ["amazon-linux-extras install redis6"]]
}
},
"change_redis_6379_protected_mode": {
"command": {
"Fn::Join": [
"",
[
"sed -i 's/protected-mode yes/protected-mode no/' /etc/redis/redis.conf"
]
]
}
},
"change_redis_6379_daemonize": {
"command": {
"Fn::Join": [
"",
["sed -i 's/daemonize no/daemonize yes/' /etc/redis/redis.conf"]
]
}
},
"start_redis": {
"command": { "Fn::Join": ["", ["systemctl start redis"]] }
}
}
}
}
}
}
When I execute this template, setup_docker commands got executed in order, first it will execute install_docker, then start_docker, then start_on_restart command, but setup_redis commands are not executing, first two commands got skipped, third command got executed and it throws an error /etc/redis/redis.conf file not found
Why this is happening? commands are wrapped inside setup_redis object only and what is the solution for this?
Thanks in advance
Update
If I separate out the installation part of redis into separate config, it is working properly, does this mean command will not execute sequentially? Why commands are not executing sequentially?
{
"Metadata": {
"AWS::CloudFormation::Init": {
"configSets": {
"setup": ["setup_redis", "modify_files"]
},
"setup_redis": {
"commands": {
"install_redis": {
"command": {
"Fn::Join": ["", ["amazon-linux-extras install redis6"]]
}
}
}
},
"modify_files": {
"commands": {
"change_redis_6379_protected_mode": {
"command": {
"Fn::Join": [
"",
[
"sed -i 's/protected-mode yes/protected-mode no/' /etc/redis/redis.conf"
]
]
}
},
"change_redis_6379_daemonize": {
"command": {
"Fn::Join": [
"",
["sed -i 's/daemonize no/daemonize yes/' /etc/redis/redis.conf"]
]
}
},
"start_redis": {
"command": { "Fn::Join": ["", ["systemctl start redis"]] }
}
}
}
}
},
"Properties": {
"ImageId": {
"Ref": "AMI"
},
"KeyName": {
"Ref": "Key"
},
"InstanceType": {
"Ref": "InstanceTypeParameter"
},
"SubnetId": {
"Ref": "SubnetIdParameter"
},
"SecurityGroupIds": {
"Ref": "SecurityGroupIdsParameter"
},
"PrivateIpAddress": "10.0.0.57",
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -xe",
"\n",
"yum update -y",
"\n",
"/opt/aws/bin/cfn-init -v",
" --stack ",
{ "Ref": "AWS::StackName" },
" --resource DevInstance",
" --configsets setup",
" --region ",
{ "Ref": "AWS::Region" },
"\n",
"/opt/aws/bin/cfn-signal -e $?",
" --stack ",
{ "Ref": "AWS::StackName" },
" --resource DevInstance",
" --region ",
{ "Ref": "AWS::Region" },
"\n"
]
]
}
}
}
}

Commands are processed in alphabetical order by name. So in your original configuration, they were going to be executed in this order:
change_redis_6379_daemonize
change_redis_6379_protected_mode
install_redis
start_redis
Separating out the installation part to make it execute first is the correct solution.
Reference: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-init.html#aws-resource-init-commands

Related

Error in Userdata Scripts in Cloud-init of EC2 Cloudformation Template

I am executing the below code to create stack. Instance got created but received this error:
Loaded plugins: amazon-id, rhui-lb, search-disabled-repos
No Match for argument: aws-cfn-bootstrap
No package aws-cfn-bootstrap available.
No packages marked for update
/var/lib/cloud/instance/scripts/part-001: line 4: cd: /opt/aws/bin/cfn-init: No such file or directory
My CloudFormation template is:
{
"Resources": {
"MyInstance": {
"Type": "AWS::EC2::Instance",
"Properties": {
"ImageId": "ami-011b3ccf1bd6db744",
"InstanceType": "t2.micro",
"KeyName": "EC2KeyPair",
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash",
"\n",
"exec > /tmp/user_data.log 2>&1 \n",
"yum update -y\n",
"yum install -y epel-release\n",
"yum install -y https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.amzn1.noarch.rpm \n",
"ln -s /usr/local/lib/python2.7/site-packages/cfnbootstrap /usr/lib/python2.7/site-packages/cfnbootstrap\n",
"/opt/aws/bin/cfn-init -v",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource MyInstance",
" --configsets scripts ",
" --region ",
{
"Ref": "AWS::Region"
},
"\n"
]
]
}
}
},
"Metadata": {
"AWS::CloudFormation::Init": {
"configSets": {
"scripts": [
"configure_cfn",
"pythonInstallation"
]
},
"configure_cfn": {
"files": {
"/etc/cfn/hooks.d/cfn-auto-reloader.conf": {
"content": {
"Fn::Join": [
"",
[
"[cfn-auto-reloader-hook]\n",
"triggers=post.update\n",
"path=Resources.EC2.Metadata.AWS::CloudFormation::Init\n",
"action=/opt/aws/bin/cfn-init -v",
" --stack ",
{
"Ref": "AWS::StackName"
},
" --resource EC2",
" --configsets wordpress",
" --region ",
{
"Ref": "AWS::Region"
},
"\n"
]
]
},
"mode": "000400",
"owner": "root",
"group": "root"
},
"/etc/cfn/cfn-hup.conf": {
"content": {
"Fn::Join": [
"",
[
"[main]\n",
"stack=",
{
"Ref": "AWS::StackId"
},
"\n",
"region=",
{
"Ref": "AWS::Region"
},
"\n",
"verbose=true\n",
"interval=5\n"
]
]
},
"mode": "000400",
"owner": "root",
"group": "root"
},
"/var/www/html/index2.html": {
"content": "Hi"
}
},
"services": {
"sysvinit": {
"cfn-hup": {
"enabled": "true",
"ensureRunning": "true",
"files": [
"/etc/cfn/cfn-hup.conf",
"/etc/cfn/hooks.d/cfn-auto-reloader.conf"
]
}
}
}
},
"pythonInstallation": {
"packages": {
"yum": {
"wget": [],
"unzip": [],
"gcc-c++": [],
"zlib-devel": [],
"libffi-devel": [],
"httpd": []
}
},
"sources": {
"usr/src/": "https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tgz"
},
"commands": {
"python1": {
"command": "tar xzf Python-3.7.2.tgz",
"cwd": "/usr/src/"
},
"python2": {
"command": {
"comm1": "./configure --enable-optimizations",
"comm2": "make altinstall",
"comm3": "rm /usr/src/Python-3.7.2.tgz"
},
"cwd": "/usr/src/Python-3.7.2"
}
},
"files": {
"/var/www/html/index.php": {
"content": {
"Fn::Join": [
"",
[
"<html>\n",
" <head>\n",
" <title>AWS CloudFormation PHP Sample</title>\n",
" <meta http-equiv=\"Content-Type\" content=\"text/html; charset=ISO-8859-1\">\n",
" </head>\n",
" <body>\n",
" <h1>Welcome to the AWS CloudFormation PHP Sample</h1>\n",
" </body>\n",
"</html>\n"
]
]
}
}
}
}
}
}
}
}
}
It would appear that Redhat does not have cfn-init pre-loaded.
You could either use an Amazon Linux AMI instead, or see How to install aws-cfn-bootstrap/cfn-init package in Redhat using CloudFormation? - Stack Overflow for tips on how to load cfn-init as part of the boot process.

Cloudformation Init config files not writing the files

I am using AWS Cloudformation scripts to bring up a Auto-scaling Ec2 Instance - sample code provided below
"GatewayLabAutoScalingGroup": {
"Metadata": {
"AWS::CloudFormation::Init": {
"config": {
"commands": {
"a_install_pip": {
"command": "pip install requests boto3"
},
"c_restart_cron": {
"command": "service crond restart"
},
"d_restart_cfn_hup": {
"command": "service cfn-hup restart"
}
},
"files": {
"/etc/cfn/cfn-hup.conf": {
"content": {
"Fn::Join": [
"",
[
"[main]\nstack=",
{
"Ref": "AWS::StackName"
},
"\nregion=",
{
"Ref": "AWS::Region"
},
"\nverbose=true\ninterval=1\n"
]
]
},
"group": "root",
"mode": "000644",
"owner": "root"
},
"/usr/local/sbin/join_ad_script.sh": {
"content": {
"Fn::Join": [
"",
[
"sudo yum -y update\nsudo yum -y install sssd realmd krb5-workstation\nsudo realm leave\n\nDOMAIN=\"",
{
"Ref": "SimpleADDomain"
},
"\"\n\ncat <<EOF > /etc/resolv.conf\nnameserver ",
{
"Fn::Select": [
0,
{
"Fn::GetAtt": [
"WorkspacesSimplead",
"DnsIpAddresses"
]
}
]
},
"\nnameserver ",
{
"Fn::Select": [
1,
{
"Fn::GetAtt": [
"WorkspacesSimplead",
"DnsIpAddresses"
]
}
]
},
"\nEOF\n\n# empty all current sssd cache\nsss_cache -E\n\necho ",
{
"Ref": "SimpleADPassword"
},
" | sudo realm join -U Administrator#${DOMAIN^^} ${DOMAIN^^} --verbose\nsudo sed -re 's/^(PasswordAuthentication)([[:space:]]+)no/\\1\\2yes/' -i.`date -I` /etc/ssh/sshd_config\necho \"enumerate=true\" >> /etc/sssd/sssd.conf\nsudo service sssd restart\nsudo service sshd restart\n\n# empty all current sssd cache\nsss_cache -E\n"
]
]
},
"group": "root",
"mode": "000755",
"owner": "root"
}
}
}
}
},
"Properties": {
"AvailabilityZones": [
{
"Fn::Select": [
0,
{
"Fn::GetAZs": ""
}
]
}
],
"HealthCheckGracePeriod": 300,
"HealthCheckType": "EC2",
"LaunchConfigurationName": {
"Ref": "GatewayLabLaunchConfiguration"
},
"LoadBalancerNames": [
],
"MaxSize": 2,
"MinSize": 1,
"Tags": [
{
"Key": "Name",
"PropagateAtLaunch": true,
"Value": "hub-autoscaling"
}
}
],
"VPCZoneIdentifier": [
{
"Ref": "EC2SubnetSubnet1"
}
]
},
"Type": "AWS::AutoScaling::AutoScalingGroup",
"UpdatePolicy": {
"AutoScalingRollingUpdate": {
"MaxBatchSize": 1,
"MinInstancesInService": 1,
"PauseTime": "PT60S"
}
}
}
The files are not written to in the instance
The instance is coming up in a Private VPC
We have a proxy configured on port 8080
The works fine when the instance is connected to a NAT Gateway without a proxy
I do have ports 80, 22 & 443 opened up
the userdata statements are run initially
They then call the cfn-init scripts
There were some errors in the scripts and they never completed; one of the problems as mentioned above was my instance behind a proxy
Getting the proxy configuration in as part of the UserData helped

CloudFormation template failing to create "files" under MetaData Section

Could somebody help me point what wrong am I doing here. I am failing to understand how the meta data part uses the authentication resource, In the AWS::CloudFormation::Authentication part I've mentioned the role same as the one attached to the instance, Yet I'm unable to create the file "some.txt"
{
"Parameters": {
"SecurityGroupId": {
"Description": "Security group for instance",
"Type": "AWS::EC2::SecurityGroup::Id"
}
},
"Resources": {
"MyInstance": {
"Type": "AWS::EC2::Instance",
"Metadata": {
"AWS::CloudFormation::Init": {
"configsets": {
"InstallIt": ["config1"]
},
"config1": {
"files": {
"/home/ec2-user/some.txt": {
"content": "This is my name ",
"encoding": "base64",
"mode": "000644",
"owner": "root",
"group": "root"
}
}
}
},
"AWS::CloudFormation::Authentication": {
"HelpMe": {
"type": "S3",
"buckets": "poc-bucket",
"roleName": "EC2andS3"
}
}
},
"Properties": {
"KeyName": "GoldenImage-NV-Anant",
"DisableApiTermination": "false",
"ImageId": "ami-0b33d91d",
"InstanceType": "t2.micro",
"Monitoring": "false",
"SubnetId": "subnet-73487a59",
"SecurityGroupIds": [{
"Ref": "SecurityGroupId"
}],
"IamInstanceProfile": {
"Ref": "MyInstanceProfile"
},
"Tags": [{
"Key": "Name",
"Value": "GeicoUserDataPocInstance"
}],
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"", [
"#!/bin/bash -ex \n",
"echo \"hello dudes\" > /home/ec2-user/hello.txt \n",
"yum update -y aws-cfn-bootstrap\n",
"/opt/aws/bin/cfn-init -v",
" --stack ", {
"Ref": "AWS::StackId"
},
" --resource MyInstance ",
" --configsets InstallIt ",
" --region ", {
"Ref": "AWS::Region"
}, "\n",
"echo \"bye dudes\" > /home/ec2-user/bye.txt", "\n",
"/opt/aws/bin/cfn-signal -e $? ",
" --stack ", {
"Ref": "AWS::StackId"
},
" --resource MyInstance ",
" --region ", {
"Ref": "AWS::Region"
}, "\n"
]
]
}
}
},
"CreationPolicy": {
"ResourceSignal": {
"Timeout": "PT90M",
"Count": "1"
}
}
},
"MyInstanceProfile": {
"Description": "Instance profile for the instance",
"Type": "AWS::IAM::InstanceProfile",
"Properties": {
"Path": "/",
"Roles": ["EC2andS3"]
}
}
}
}
configsets should be configSets with capital S:
"configSets": {
"InstallIt": ["config1"]
},
buckets property needs to be a list of strings (this might not be necessary, the documentation is a bit unclear):
"buckets": ["poc-bucket"]
AWS::CloudFormation::Authentication resource shouldn't be necessary unless the source of your file is an S3 bucket. Even then, it still shouldn't be necessary when using an attached instance profile, since it will use the instance profile for authentication by default.

AWS::CloudFormation::Init not executing commands in metadata

This is my template with a test command in the metadata. I can't figure out why the command doesn't get executed. It's supposed to save a string to a file in /tmp.
The machine is Ubuntu 16.04 with cloud-init installed. In UserData I install the helper scripts and execute cfn-init.
Thanks for your help.
{
"AWSTemplateFormatVersion": "2010-09-09",
"Metadata": {
"AWS::CloudFormation::Designer": {
"2af5b799-f6bf-7f20-a6eb-943274f18373": {
"size": {
"width": 60,
"height": 60
},
"position": {
"x": 326,
"y": 118
},
"z": 0,
"embeds": []
}
}
},
"Resources": {
"EC2I3WADD": {
"Type": "AWS::EC2::Instance",
"Properties": {
"ImageId": "ami-c60b90d1",
"KeyName": "CF-KEY",
"InstanceType": "t2.micro",
"UserData": {
"Fn::Base64": {
"Fn::Join": [
"",
[
"#!/bin/bash -xe\n",
"apt-get -y install python-setuptools\n",
"mkdir aws-cfn-bootstrap-latest\n",
"curl https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz | tar xz -C aws-cfn-bootstrap-latest --strip-components 1\n",
"easy_install aws-cfn-bootstrap-latest\n",
"/usr/local/bin/cfn-init --stack ",
{
"Ref": "AWS::StackName"
},
" --resource WebServer",
" --region ",
{
"Ref": "AWS::Region"
},
"\n"
]
]
}
}
},
"Metadata": {
"AWS::CloudFormation::Designer": {
"id": "2af5b799-f6bf-7f20-a6eb-943274f18373"
},
"AWS::CloudFormation::Init": {
"config": {
"commands": {
"test": {
"command": "echo \"$MAGIC\" > /tmp/test.txt",
"env": {
"MAGIC": "I come from the environment!"
}
}
}
}
}
}
}
}
}
The issue is likely being cause by an incorrect --resource argument being passed to the cfn-init command within your UserData. This argument should match the logical resource name of the resource containing the MetaData, in your case EC2I3WADD.
"/usr/local/bin/cfn-init --stack ", { "Ref": "AWS::StackName" }, " --resource EC2I3WADD", " --region ", { "Ref": "AWS::Region" }
https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-init.html

CloudFormation WaitCondition requires public access

I have a CloudFormation script which deploys a single instance in the VPC.
The script only works when the instance has a public IP. Without a public IP address, the template deployment fails at WaitCondition stage. I guess thats because the WaitCondition requires instance has access to the Internet?
Instead of assigning a public IP, will it work if I set the proxy address on the server? So the instance access the Internet via web proxy. If that's the case, what's the web URLs need to be whitelisted on the proxy? and can I add the add proxy address command in the userdata?
Below is my template:
"Mappings": {
"AWSRegion2AMI": {
"ap-southeast-2": {
"Windows2008r2": "ami-27b39a44",
"Windows2012r2": "ami-83b198e0"
}
}
},
"Resources": {
"DomainController": {
"Type": "AWS::EC2::Instance",
"Metadata": {
"AWS::CloudFormation::Init": {
"config": {
"files": {
"c:\\cfn\\cfn-hup.conf": {
"content": {
"Fn::Join": ["", [
"[main]\n",
"stack=", {
"Ref": "AWS::StackId"
}, "\n",
"region=", {
"Ref": "AWS::Region"
}, "\n"
]]
}
},
"c:\\cfn\\hooks.d\\cfn-auto-reloader.conf": {
"content": {
"Fn::Join": ["", [
"[cfn-auto-reloader-hook]\n",
"triggers=post.update\n",
"path=Resources.DomainController.Metadata.AWS::CloudFormation::Init\n",
"action=cfn-init.exe -v -s ", {
"Ref": "AWS::StackId"
},
" -r DomainController",
" --region ", {
"Ref": "AWS::Region"
}, "\n"
]]
}
},
"C:\\cfn\\RunCommand.bat": {
"content": "%~1\nIF %ERRORLEVEL% GTR 10 ( exit /b 1 ) else ( exit /b 0 )"
}
},
"commands": {
"1-run-dcpromo": {
"command": {
"Fn::Join": ["", [
"C:\\cfn\\RunCommand.bat \"dcpromo /unattend /ReplicaOrNewDomain:Domain /NewDomain:Forest /NewDomainDNSName:", {
"Ref": "DomainDNSName"
},
" /ForestLevel:4 /DomainNetbiosName:", {
"Ref": "DomainNetBIOSName"
},
" /DomainLevel:4 /InstallDNS:Yes /ConfirmGc:Yes /CreateDNSDelegation:No /DatabasePath:\"C:\\Windows\\NTDS\" /LogPath:\"C:\\Windows\\NTDS\" /SYSVOLPath:\"C:\\Windows\\SYSVOL\" /SafeModeAdminPassword=", {
"Ref": "RestoreModePassword"
},
" /RebootOnCompletion:Yes\""
]]
},
"waitAfterCompletion": "forever"
},
"2-signal-success": {
"command": {
"Fn::Join": ["", [
"cfn-signal.exe -e 0 \"", {
"Fn::Base64": {
"Ref": "DomainControllerWaitHandle"
}
}, "\""
]]
}
}
},
"services": {
"windows": {
"cfn-hup": {
"enabled": "true",
"ensureRunning": "true",
"files": ["c:\\cfn\\cfn-hup.conf", "c:\\cfn\\hooks.d\\cfn-auto-reloader.conf"]
}
}
}
}
}
},
"Properties": {
"ImageId": {
"Fn::FindInMap": ["AWSRegion2AMI", {
"Ref": "AWS::Region"
}, "Windows2008r2"]
},
"InstanceType": {
"Ref": "InstanceType"
},
"NetworkInterfaces": [{
"AssociatePublicIpAddress": "false",
"DeviceIndex": "0",
"SubnetId": {
"Ref": "SubnetId"
}
}],
"KeyName": {
"Ref": "KeyName"
},
"UserData": {
"Fn::Base64": {
"Fn::Join": ["", [
"<script>\n",
"cfn-init.exe -v -s ", {
"Ref": "AWS::StackId"
},
" -r DomainController ",
" --region ", {
"Ref": "AWS::Region"
}, "\n",
"</script>"
]]
}
}
}
},
"DomainControllerWaitCondition": {
"Type": "AWS::CloudFormation::WaitCondition",
"DependsOn": "DomainController",
"Properties": {
"Handle": {
"Ref": "DomainControllerWaitHandle"
},
"Timeout": "1500"
}
},
"DomainControllerWaitHandle": {
"Type": "AWS::CloudFormation::WaitConditionHandle"
}
Pass in the proxy by adding on of those parameters to cfn-signal.exe:
--http-proxy
An HTTP proxy (non-SSL). Use the following format: http://user:password#host:port
--https-proxy
An HTTPS proxy. Use the following format: https://user:password#host:port
The signal is sent to an S3 Bucket. You can whitelist something like:
https://cloudformation-waitcondition-*.s3.amazonaws.com
Note: You can also use the S3 VPC endpoint feature to allow resources inside private subnets to access S3. http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-endpoints.html