TF project:
main.tf
inputs.tf
The contents are:
main.tf
locals {
common_tags = {
SECRET_MGR_HOST = "${var.SECRET_MGR_HOST}",
SECRET_MGR_SAFE = "${var.SECRET_MGR_SAFE}",
SECRET_MGR_SECRET_KEY_NAME = "${var.SECRET_MGR_SECRET_KEY_NAME}",
SECRET_MGR_USER_NAME = "${var.SECRET_MGR_USER_NAME}",
LOGON_URL = "${var.LOGON_URL}",
PLATFORM_SECRET_NAME = "${var.PLATFORM_SECRET_NAME}"
}
vpc_config_vars = {
subnet_ids = "${var.SUBNET_IDS}",
security_group_ids = "${var.SECURITY_GROUP_IDS}"
}
}
module "lambda" {
source = "git::https://corpsource.io/corp-cloud-platform-team/corpcloudv2/terraform/lambda-modules.git?ref=dev"
lambda_name = var.name
lambda_role = "arn:aws:iam::${var.ACCOUNT}:role/${var.lambda_role}"
lambda_handler = var.handler
lambda_runtime = var.runtime
default_lambda_timeout = var.timeout
ACCOUNT = var.ACCOUNT
vpc_config_vars = merge(
local.vpc_config_vars
)
env = merge(
local.common_tags,
{ DEFAULT_ROLE = "corp-platform" }
)
}
module "lambda_iam" {
source = "git::https://corpsource.io/corp-cloud-platform-team/corpcloudv2/terraform/iam-modules/lambda-iam.git?ref=dev"
lambda_policy = var.lambda_policy
ACCOUNT = var.ACCOUNT
lambda_role = var.lambda_role
}
and inputs.tf
variable "handler" {
type = string
default = "handler.lambda_handler"
}
variable "runtime" {
type = string
default = "python3.8"
}
variable "name" {
type = string
default = "create-SECRET_MGR-entry"
}
variable "timeout"{
type = string
default = "120"
}
variable "lambda_role" {
type = string
default = "create-SECRET_MGR-entry-role"
}
variable "ACCOUNT" {
type = string
default = ""
}
variable "SECRET_MGR_HOST" {
type = string
default = ""
}
variable "SECRET_MGR_SAFE" {
type = string
default = ""
}
variable "SUBNET_IDS" {
type = string
default = ""
}
variable "subnet_ids" {
type = string
default = ""
}
variable "security_group_ids" {
type = string
default = ""
}
variable "SECURITY_GROUP_IDS" {
type = string
default = ""
}
variable "SECRET_MGR_SECRET_KEY_NAME" {
type = string
default = ""
}
variable "SECRET_MGR_USER_NAME" {
type = string
default = ""
}
variable "LOGON_URL" {
type = string
default = ""
}
variable "PLATFORM_SECRET_NAME" {
type = string
default = ""
}
variable "lambda_policy" {
default = "{\"Version\": \"2012-10-17\",\"Statement\": [{\"Sid\":\"VisualEditor0\",\"Effect\":\"Allow\",\"Action\":[\"logs:CreateLogStream\",\"logs:CreateLogGroup\"],\"Resource\":\"*\"},{\"Sid\":\"UseKMSKey\",\"Effect\":\"Allow\",\"Action\":\"kms:Decrypt\",\"Resource\":\"*\"},{\"Sid\":\"GetSecret\",\"Effect\":\"Allow\",\"Action\":\"secretsmanager:GetSecretValue\",\"Resource\":\"*\"},{\"Sid\":\"ConnectToVPC\",\"Effect\":\"Allow\",\"Action\":[\"ec2:CreateNetworkInterface\",\"ec2:DescribeNetworkInterfaces\",\"ec2:DeleteNetworkInterface\"],\"Resource\":\"*\"},{\"Sid\":\"VisualEditor1\",\"Effect\":\"Allow\",\"Action\":\"logs:PutLogEvents\",\"Resource\":\"*\"},{\"Effect\": \"Allow\",\"Action\": [\"logs:*\"],\"Resource\": \"arn:aws:logs:*:*:*\"},{\"Effect\": \"Allow\",\"Action\": [\"s3:GetObject\",\"s3:PutObject\"],\"Resource\": \"arn:aws:s3:::*\"}]}"
}
As you see, main.tf references a module in another project referenced via source argument. The structure of the module project is also:
main.tf
inputs.tf
main.tf
data "archive_file" "lambda_handler" {
type = "zip"
output_path = "lambda_package.zip"
source_dir = "lambda_code/"
}
resource "aws_lambda_function" "lambda_function" {
filename = "lambda_package.zip"
function_name = var.lambda_name
role = var.lambda_role
handler = var.lambda_handler
runtime = var.lambda_runtime
memory_size = 256
timeout = var.default_lambda_timeout
source_code_hash = filebase64sha256("lambda_code/lambda_package.zip")
dynamic "vpc_config" {
for_each = length(keys(var.vpc_config_vars)) == 0 ? [] : [true]
content {
variables = var.vpc_config_vars
}
}
dynamic "environment" {
for_each = length(keys(var.env)) == 0 ? [] : [true]
content {
variables = var.env
}
}
}
inputs.tf
variable "lambda_name" {
type = string
}
variable "lambda_runtime" {
type = string
}
variable "lambda_role" {
type = string
}
variable "default_lambda_timeout" {
type = string
}
variable "lambda_handler" {
type = string
}
variable "vpc_config_vars" {
type = map(string)
default = {}
}
variable "env" {
type = map(string)
default = {}
}
variable "tags" {
default = {
blc = "1539"
costcenter = "54111"
itemid = "obfuscated"
owner = "cloudengineer#company.com"
}
}
variable "ACCOUNT" {
type = string
}
Error when my pipeline runs the project:
Error: Missing required argument
(and 7 more similar warnings elsewhere)
on .terraform/modules/lambda/main.tf line 18, in resource "aws_lambda_function" "lambda_function":
18: content {
The argument "subnet_ids" is required, but no definition was found.
Error: Missing required argument
on .terraform/modules/lambda/main.tf line 18, in resource "aws_lambda_function" "lambda_function":
18: content {
The argument "security_group_ids" is required, but no definition was found.
Error: Unsupported argument
on .terraform/modules/lambda/main.tf line 19, in resource "aws_lambda_function" "lambda_function":
19: variables = var.vpc_config_vars
An argument named "variables" is not expected here.
Oh and I'm passing in the value for subnet_ids and security_group_ids as an environment variable using my gitlab ci file. And log statements confirm that those values are defined.
What is wrong? thank you
You need to pass the required arguments for the vpc_config child block, which are subnet_ids and security_group_ids. You cannot use the entire map variable as it is inside the nested content block. You need to use the equals sign "=" to introduce the argument value.
Try the below code snippet
###################
# Root Module
###################
locals {
vpc_config_vars = {
vpc_config = {
subnet_ids = ["subnet-072297c000a32e200"],
security_group_ids = ["sg-05d06431bd25870b4"]
}
}
}
module "lambda" {
source = "./modules"
...
......
vpc_config_vars = local.vpc_config_vars
}
###################
# Child Module
###################
variable "vpc_config_vars" {
default = {}
}
resource "aws_lambda_function" "lambda_function" {
filename = "lambda_package.zip"
function_name = var.lambda_name
role = var.lambda_role
handler = var.lambda_handler
runtime = var.lambda_runtime
memory_size = 256
timeout = var.default_lambda_timeout
source_code_hash = filebase64sha256("lambda_code/lambda_package.zip")
dynamic "vpc_config" {
for_each = var.vpc_config_vars != {} ? var.vpc_config_vars : {}
content {
subnet_ids = vpc_config.value["subnet_ids"]
security_group_ids = vpc_config.value["security_group_ids"]
}
}
}
Related
I need to create TF CloudWatch Metrices only of the env is QAT and PROD. Currently using TF modules to create those in all env.
module "aws_cloudwatch_log_metric_filter" {
source = "https://github.com/modules.git//aws-cloudwatch-log-metric-filter"
log_group_name = "/aws/lambda/${var.lambda_name}"
pattern = "{$.message = \"---------------- Message ----------------\"}"
}
locals {
base_tags = {
environment = var.environment
}
}
Main resource where I am calling module.
resource "aws_cloudwatch_log_metric_filter" "log_metric" {
count = var.count
name = "Metric"
pattern = var.pattern
log_group_name = var.log_group_name
metric_transformation {
name = "name"
namespace = "namespace"
value = "1"
default_value = "0"
}
}
You could do this in the module:
resource "aws_cloudwatch_log_metric_filter" "log_metric" {
count = var.environment == "QAT" || var.environment == "PROD" ? 1 : 0
name = "Metric"
pattern = var.pattern
log_group_name = "${var.environment}-${var.log_group_name}"
metric_transformation {
name = "name"
namespace = "namespace"
value = "1"
default_value = "0"
}
}
The log_group_name = "${var.environment}-${var.log_group_name}" will ensure you don't get issues with the same log group name.
A simple but good solution :
locals {
cw_env = ["QUA", "PROD"]
}
resource "aws_cloudwatch_log_metric_filter" "log_metric" {
count = contains(local.cw_env, var.env) ? 1 : 0
name = "Metric"
pattern = var.pattern
log_group_name = var.log_group_name
metric_transformation {
name = "name"
namespace = "namespace"
value = "1"
default_value = "0"
}
}
Note you can also use it at the above level :
module "aws_cloudwatch_log_metric_filter" {
source = "https://github.com/modules.git//aws-cloudwatch-log-metric-filter"
count = contains(local.cw_env, local.base_tags.environment) ? 1 : 0
log_group_name = "/aws/lambda/${var.lambda_name}"
pattern = "{$.message = \"---------------- Message ----------------\"}"
}
locals {
base_tags = {
environment = var.environment
}
cw_env = ["QUA", "PROD"]
}
Im trying to add retention policy but I want to enable it conditionally, as you can see from the code
buckets.tf
locals {
team_buckets = {
arc = { app_id = "20390", num_buckets = 2, retention_period = null }
ana = { app_id = "25402", num_buckets = 2, retention_period = 631139040 }
cha = { app_id = "20391", num_buckets = 2, retention_period = 631139040 } #20 year
}
}
module "team_bucket" {
source = "../../../../modules/gcs_bucket"
for_each = {
for bucket in flatten([
for product_name, bucket_info in local.team_buckets : [
for i in range(bucket_info.num_buckets) : {
name = format("%s-%02d", product_name, i + 1)
team = "ei_${product_name}"
app_id = bucket_info.app_id
retention_period = bucket_info.retention_period
}
]
]) : bucket.name => bucket
}
project_id = var.project
name = "teambucket-${each.value.name}"
app_id = each.value.app_id
team = each.value.team
retention_period = each.value.retention_period
}
root module is defined as follows
main.tf
resource "google_storage_bucket" "bucket" {
project = var.project_id
name = "${var.project_id}-${var.name}"
location = var.location
labels = {
app_id = var.app_id
ei_team = var.team
cost_center = var.cost_center
}
uniform_bucket_level_access = var.uniform_bucket_level_access
dynamic "retention_policy" {
for_each = var.retention_policy == null ? [] : [var.retention_period]
content {
retention_period = var.retention_period
}
}
}
but I can't seem to make the code pick up the value,
for example as you see below the value doesn't get implemented
~ resource "google_storage_bucket" "bucket" {
id = "teambucket-cha-02"
name = "teambucket-cha-02"
# (11 unchanged attributes hidden)
- retention_policy {
- is_locked = false -> null
- retention_period = 3155760000 -> null
}
}
variables.tf for retention policy is as follows
variable "retention_policy" {
description = "Configuation of the bucket's data retention policy for how long objects in the bucket should be retained"
type = any
default = null
}
variable "retention_period" {
default = null
}
Your var.retention_policy is always null, as its default value. You are not changing the default value at all. Probably you wanted the following:
for_each = var.retention_period == null ? [] : [var.retention_period]
instead of
for_each = var.retention_policy == null ? [] : [var.retention_period]
I'm trying to create an IP whitelist in nonprod for load testing, the WAF is dynamically created in prod and nonprod based on the envname/envtype:
resource "aws_waf_ipset" "pwa_cloudfront_ip_restricted" {
name = "${var.envname}-pwa-cloudfront-whitelist"
dynamic "ip_set_descriptors" {
for_each = var.cloudfront_ip_restricted_waf_cidr_whitelist
content {
type = ip_set_descriptors.value.type
value = ip_set_descriptors.value.value
}
}
}
resource "aws_waf_rule" "pwa_cloudfront_ip_restricted" {
depends_on = [aws_waf_ipset.pwa_cloudfront_ip_restricted]
name = "${var.envname}-pwa-cloudfront-whitelist"
metric_name = "${var.envname}PWACloudfrontWhitelist"
predicates {
data_id = aws_waf_ipset.pwa_cloudfront_ip_restricted.id
negated = false
type = "IPMatch"
}
}
resource "aws_waf_ipset" "pwa_cloudfront_ip_restricted_load_testing" {
name = "${var.envname}-pwa-cloudfront-whitelist_load_testing"
count = var.envtype == "nonprod" ? 1 : 0
dynamic "ip_set_descriptors" {
for_each = var.cloudfront_ip_restricted_waf_cidr_whitelist_load_testing
content {
type = ip_set_descriptors.value.type
value = ip_set_descriptors.value.value
}
}
}
resource "aws_waf_rule" "pwa_cloudfront_ip_restricted_load_testing" {
depends_on = [aws_waf_ipset.pwa_cloudfront_ip_restricted_load_testing]
count = var.envtype == "nonprod" ? 1 : 0
name = "${var.envname}-pwa-cloudfront-whitelist-load_testing"
metric_name = "${var.envname}PWACloudfrontWhitelistload_testing"
predicates {
data_id = aws_waf_ipset.pwa_cloudfront_ip_restricted_load_testing[count.index].id
negated = false
type = "IPMatch"
}
}
resource "aws_waf_web_acl" "pwa_cloudfront_ip_restricted" {
name = "${var.envname}-pwa-cloudfront-whitelist"
metric_name = "${var.envname}PWACloudfrontWhitelist"
default_action {
type = "BLOCK"
}
rules {
action {
type = "ALLOW"
}
priority = 1
rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted.id
type = "REGULAR"
}
rules {
action {
type = "ALLOW"
}
priority = 2
rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing.id
type = "REGULAR"
}
}
The second rules block throws and error in the terraform plan:
Error: Missing resource instance key
on waf.tf line 73, in resource "aws_waf_web_acl" "pwa_cloudfront_ip_restricted":
73: rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing.id
Because aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing has "count" set,
its attributes must be accessed on specific instances.
For example, to correlate with indices of a referring resource, use:
aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing[count.index]
However if I add [count.index] :
Error: Reference to "count" in non-counted context
on waf.tf line 73, in resource "aws_waf_web_acl" "pwa_cloudfront_ip_restricted":
73: rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing[count.index].id
The "count" object can only be used in "module", "resource", and "data"
blocks, and only when the "count" argument is set.
Is there a way to do this that doesn't use the count param? Or am I missing something in the way that I am using it?
Since there is difference between the prod and non-prod environment, the way this should be tackled is by using dynamic [1] and for_each meta-argument [2]:
resource "aws_waf_web_acl" "pwa_cloudfront_ip_restricted" {
name = "${var.envname}-pwa-cloudfront-whitelist"
metric_name = "${var.envname}PWACloudfrontWhitelist"
default_action {
type = "BLOCK"
}
dynamic "rules" {
for_each = var.envtype == "nonprod" ? [1] : []
content {
action {
type = "ALLOW"
}
priority = 1
rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted[0].id
type = "REGULAR"
}
}
dynamic "rules" {
for_each = var.envtype == "nonprod" ? [1] : []
content {
action {
type = "ALLOW"
}
priority = 2
rule_id = aws_waf_rule.pwa_cloudfront_ip_restricted_load_testing[0].id
type = "REGULAR"
}
}
}
[1] https://developer.hashicorp.com/terraform/language/expressions/dynamic-blocks
[2] https://developer.hashicorp.com/terraform/language/expressions/for
I am trying to pass the values s3 name and create_user into local block in main.tf so that both of them have the value in list and then I am passing list_of_bucket in local block in module s3 to create the buckets and looping of user_to_create in module s3_user to create the user if the boolean is set to true. All of these values are passed to variable.tf and then to main.tf
dev.tfvars
wea-nonprod = {
services = {
s3 = [
sthree = {
create_user = true,
}
sfour = {
create_user = true,
}
sfive = {
create_user = true,
}
]
}
}
variable.tf
variable "s3_buckets" {
type = list(map)
}
main.tf
locals {
users_to_create = ""
list_of_buckets = ""
}
module "s3" {
source = "../../s3"
name = join("-", [var.name_prefix, "s3"])
tags = merge(var.tags, {Name = join("-", [var.name_prefix, "s3"])})
buckets = list_of_buckets
sse_algorithm = "AES256"
access_log_bucket_name = var.access_log_bucket_name
}
module "s3_user" {
for_each = local.users_to_create
source = "./service-s3-bucket-user"
name = join("-", [var.name_prefix, each.key])
tags = var.tags
bucket_arn = module.s3.bucket_arns[each.key]
depends_on = [module.s3]
}
Just iterate over your wea-nonprod map:
locals {
users_to_create = [ for name in var.wea-nonprod.services.s3 if name.create_user == true ]
list_of_buckets = [ for bucket in var.wea-nonprod.services.s3 ]
}
And a few changes to your module blocks:
module "s3" {
source = "../../s3"
name = "${var.name_prefix}-s3"
tags = merge(var.tags, { Name = "${var.name_prefix}-s3" })
buckets = local.list_of_buckets
sse_algorithm = "AES256"
access_log_bucket_name = var.access_log_bucket_name
}
module "s3_user" {
count = length(local.users_to_create)
source = "./service-s3-bucket-user"
name = "${var.name_prefix}${local.users_to_create[count.index]}"
tags = var.tags
bucket_arn = module.s3.bucket_arns[local.users_to_create[count.index]]
depends_on = [module.s3]
}
Im trying to iterate through a variable type map and i'm not sure how to
This is what i have so far
In my main.tf:
resource "aws_route_53_record" "proxy_dns" {
count = "${length(var.account_name)}"
zone_id = "${infrastructure.zone_id}"
name = "proxy-${element(split(",", var.account_name), count.index)}-dns
type = CNAME
ttl = 60
records = ["{records.dns_name}"]
}
And in my variables.tf
variable "account_name" {
type = "map"
default = {
"account1" = "accountA"
"account2" = "accountB"
}
}
I want to be able to create multiple resources with the different account names
If you are using Terraform 0.12.6 or later then you can use for_each instead of count to produce one instance for each element in your map:
resource "aws_route53_record" "proxy_dns" {
for_each = var.account_name
zone_id = infrastructure.zone_id
name = "proxy-${each.value}-dns"
# ... etc ...
}
The primary advantage of for_each over count is that Terraform will identify the instances by the key in the map, so you'll get instances like aws_route53_record.proxy_dns["account1"] instead of aws_route53_record.proxy_dns[0], and so you can add and remove elements from your map in future with Terraform knowing which specific instance belongs to each element.
each.key and each.value in the resource type arguments replace count.index when for_each is used. They evaluate to the key and value of the current map element, respectively.
You can use a combination of map, keys function,index function, and count. This terraform creates 3 acls with various rules.
The names of the acl's are determined by the keys.
The number of acl's is determined by the count of the keys.
The index of each rule (priority) is determined by the index function
The name of each rule is from the CONTAINS_WORD or CONTAINS property in the map
=>
variable "acls" {
type = map(any)
default = {
"acl1" = {
"CONTAINS_WORD" = ["api","aaa", "bbb", "ccc"]
"CONTAINS" = ["xxx","yyy"]
}
"acl2" = {
"CONTAINS_WORD" = [ "url1,"url2","url3"]
"CONTAINS" = ["url4"]
}
"acl3" = {
"CONTAINS_WORD" = ["xxx"]
"CONTAINS" = []
}
}
}
resource "aws_wafv2_web_acl" "acl" {
name = keys(var.acls)[count.index]
scope = "REGIONAL"
count = length(keys(var.acls))
default_action {
block {}
}
dynamic "rule" {
for_each = toset(var.acls[keys(var.acls)[count.index]].CONTAINS_WORD)
content {
name = rule.key
priority = index(var.acls[keys(var.acls)[count.index]].CONTAINS_WORD, rule.key)
action {
allow {}
}
statement {
#https://docs.aws.amazon.com/waf/latest/APIReference/API_ByteMatchStatement.html
byte_match_statement {
positional_constraint = "CONTAINS_WORD"
search_string = lower(rule.key)
field_to_match {
uri_path {}
}
text_transformation {
priority = 0
type = "LOWERCASE"
}
}
}
visibility_config {
cloudwatch_metrics_enabled = true
metric_name = "waf-${keys(var.acls)[count.index]}-${rule.key}"
sampled_requests_enabled = true
}
}
}
dynamic "rule" {
for_each = toset(var.acls[keys(var.acls)[count.index]].CONTAINS)
content {
name = replace(rule.key, ".", "_")
priority = index(var.acls[keys(var.acls)[count.index]].CONTAINS, rule.key) + length(var.acls[keys(var.acls)[count.index]].CONTAINS_WORD)
action {
allow {}
}
statement {
#https://docs.aws.amazon.com/waf/latest/APIReference/API_ByteMatchStatement.html
byte_match_statement {
positional_constraint = "CONTAINS"
search_string = lower(rule.key)
field_to_match {
uri_path {}
}
text_transformation {
priority = 0
type = "LOWERCASE"
}
}
}
visibility_config {
cloudwatch_metrics_enabled = true
metric_name = "waf-${keys(var.acls)[count.index]}-${replace(rule.key, ".", "_")}"
sampled_requests_enabled = true
}
}
}
visibility_config {
cloudwatch_metrics_enabled = true
metric_name = "waf-${keys(var.acls)[count.index]}"
sampled_requests_enabled = true
}
}
Make the variable a list instead of a map. Maps are used to reference a name to a value. Lists are better for iterating over via a count method.
variable "account_name" {
type = "list"
default = {"accountA","accountB"}
}
resource "aws_route_53_record" "proxy_dns" {
count = "${length(var.account_name)}"
zone_id = "${infrastructure.zone_id}"
name = "proxy-${element(var.account_name, count.index)}-dns
type = CNAME
ttl = 60
records = ["{records.dns_name}"]
}