Say that I have two modules that require the same source (append_only_history_bq_tables). Their difference is only the from_dwh_core variable which the default set to false
module "dwh_core_append_only_history_tables" {
source = "../append_only_history_bq_tables"
gcp_project = var.gcp_project
bq_location = var.bq_location
dataset_id = google_bigquery_dataset.jago_pagerduty.dataset_id
module_base_path = path.module
tables = {
"list_incident" = { "business_key_columns" = "id", "partition_type" = "DAY" },
"raw_incident_information" = { "business_key_columns" = "id", "partition_type" = "DAY" }
}
}
module "daily_closing_balance" {
source = "../append_only_history_bq_tables"
gcp_project = var.gcp_project
bq_location = var.bq_location
dataset_id = google_bigquery_dataset.dwh_core_dataset.dataset_id
module_base_path = path.module
use_source_ord = false
from_dwh_core = true
tables = {
"daily_closing_balance" = { "business_key_columns" = "full_date,account_number,customer_id", "partition_type" = "DAY" }
}
depends_on = [google_bigquery_dataset.dwh_core_dataset]
}
The append_only_history_bq_tables contains this resource
resource "google_bigquery_table" "dwh_core_snapshot_daily" {
for_each = var.tables
dataset_id = var.dataset_id
project = var.gcp_project
table_id = "${each.key}_snapshot_current_daily"
schema = file("${local.schema_def_folder}/${each.key}.json")
deletion_protection=false
}
How to conditionally call the dwh_core_snapshot_daily resource only if the from_dwh_core variable is set to true?
Thank you for your help
You can use Conditional Expression:
resource "google_bigquery_table" "dwh_core_snapshot_daily" {
for_each = var.from_dwh_core == true ? var.tables : {}
dataset_id = var.dataset_id
project = var.gcp_project
table_id = "${each.key}_snapshot_current_daily"
schema = file("${local.schema_def_folder}/${each.key}.json")
deletion_protection=false
}
Related
Im trying to add retention policy but I want to enable it conditionally, as you can see from the code
buckets.tf
locals {
team_buckets = {
arc = { app_id = "20390", num_buckets = 2, retention_period = null }
ana = { app_id = "25402", num_buckets = 2, retention_period = 631139040 }
cha = { app_id = "20391", num_buckets = 2, retention_period = 631139040 } #20 year
}
}
module "team_bucket" {
source = "../../../../modules/gcs_bucket"
for_each = {
for bucket in flatten([
for product_name, bucket_info in local.team_buckets : [
for i in range(bucket_info.num_buckets) : {
name = format("%s-%02d", product_name, i + 1)
team = "ei_${product_name}"
app_id = bucket_info.app_id
retention_period = bucket_info.retention_period
}
]
]) : bucket.name => bucket
}
project_id = var.project
name = "teambucket-${each.value.name}"
app_id = each.value.app_id
team = each.value.team
retention_period = each.value.retention_period
}
root module is defined as follows
main.tf
resource "google_storage_bucket" "bucket" {
project = var.project_id
name = "${var.project_id}-${var.name}"
location = var.location
labels = {
app_id = var.app_id
ei_team = var.team
cost_center = var.cost_center
}
uniform_bucket_level_access = var.uniform_bucket_level_access
dynamic "retention_policy" {
for_each = var.retention_policy == null ? [] : [var.retention_period]
content {
retention_period = var.retention_period
}
}
}
but I can't seem to make the code pick up the value,
for example as you see below the value doesn't get implemented
~ resource "google_storage_bucket" "bucket" {
id = "teambucket-cha-02"
name = "teambucket-cha-02"
# (11 unchanged attributes hidden)
- retention_policy {
- is_locked = false -> null
- retention_period = 3155760000 -> null
}
}
variables.tf for retention policy is as follows
variable "retention_policy" {
description = "Configuation of the bucket's data retention policy for how long objects in the bucket should be retained"
type = any
default = null
}
variable "retention_period" {
default = null
}
Your var.retention_policy is always null, as its default value. You are not changing the default value at all. Probably you wanted the following:
for_each = var.retention_period == null ? [] : [var.retention_period]
instead of
for_each = var.retention_policy == null ? [] : [var.retention_period]
I'm trying to create certain BigQuery tables with time_partitioning with the dynamic block and I want to use the values from tfvars in runtime as follows:
./tables/tables.tf:
resource "google_bigquery_table" "tables" {
for_each = var.tables == [] ? [] : toset(var.tables)
dataset_id = var.db_id
deletion_protection = false
table_id = each.key
dynamic "time_partitioning" {
for_each = var.partitioned_tables
content {
type = "DAY"
field = time_partitioning.value.field
}
}
labels = {
environment = var.environment
application = var.application
}
schema = fileexists("${path.module}/${var.db_id}/${each.key}.json") ? file("${path.module}/${var.db_id}/${each.key}.json") : null
}
main.tf:
resource "google_bigquery_dataset" "database" {
count = length(var.dbs)
dataset_id = var.dbs[count.index].db_id
friendly_name = var.dbs[count.index].db_name
description = "TF"
location = "US"
delete_contents_on_destroy = var.delete_contents_on_destroy
labels = {
environment = var.environment
application = var.dbs[count.index].app_name
}
}
module "tables" {
source = "./tables"
count = length(var.dbs)
db_id = google_bigquery_dataset.database[count.index].dataset_id
environment = var.environment
application = var.dbs[count.index].app_name
tables = var.dbs[count.index].tables
partitioned_tables = var.dbs[count.index].partitioned_tables
}
module "iam" {
source = "./iam"
count = length(var.dbs)
db_id = google_bigquery_dataset.database[count.index].dataset_id
iam_members = var.dbs[count.index].iam_members
}
dev.tfvars:
region = "us-central1"
project_id = "some-project"
dbs = [
{
db_id = "dataset1"
db_name = "dataset1"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com",
}
]
tables = ["daily_inventory", "dc_inventory", "products", "daily_sales", "planned_inventory", "stores", "stores_in_program"]
partitioned_tables = [
{
table = "daily_sales"
field = "sales_timestamp"
},
{
table = "daily_inventory"
field = "inventory_timestamp"
}
]
},
{
db_id = "dataset2"
db_name = "dataset2"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com"
}
]
tables = []
}
]
environment = "development"
delete_contents_on_destroy = true
var.dbs is type = list(any)
Getting:
The given value is not suitable for var.dbs declared at
variables.tf:9,1-15: all list elements must have the same type.
Thanks in advance!
list(any) does not mean that you can have elements of "any" type in your list. All elements must have same type, and you can't mix types, as you do now (i.e. second element is missing partitioned_tables). any only means that TF will infer the single type for the elements, but all elements must be of that single type. So you have three choices:
remove type = list(any)
Fully define your type with optional arguments, instead of using any
Add partitioned_tables to the second element:
[
{
db_id = "dataset1"
db_name = "dataset1"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com",
}
]
tables = ["daily_inventory", "dc_inventory", "products", "daily_sales", "planned_inventory", "stores", "stores_in_program"]
partitioned_tables = [
{
table = "daily_sales"
field = "sales_timestamp"
},
{
table = "daily_inventory"
field = "inventory_timestamp"
}
]
},
{
db_id = "dataset2"
db_name = "dataset2"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com"
}
]
partitioned_tables = []
tables = []
}
]
I am creating aws_workspace using the terraform. I am merging the variable in local.tf with the default values if the values are not provided in the variable. Then passing those to module. So inside resource.tf I want to eliminate the for_each loop and assign the values without any loop. Is it possible to do it?
local.tf
locals {
my_defaults = {
root_volume_encryption_enabled = true
user_volume_encryption_enabled = true
volume_encryption_key = "alias/aws/workspaces"
compute_type_name = "VALUE"
user_volume_size_gib = 10
root_volume_size_gib = 80
running_mode = "AUTO_STOP"
running_mode_auto_stop_timeout_in_minutes = 60
}
final_aws_workspace = { for k, v in var.aws_workspace :
k => merge(local.my_defaults, v)
}
}
Module.tf
variable "aws_workspace" {
default = {
user1 = {
user_name = "john.doe"
root_volume_encryption_enabled = true
user_volume_encryption_enabled = true
volume_encryption_key = "alias/aws/workspaces"
compute_type_name = "VALUE"
user_volume_size_gib = 10
root_volume_size_gib = 80
running_mode = "AUTO_STOP"
running_mode_auto_stop_timeout_in_minutes = 60
},
user2 = {
user_name = "wahaj.akmal"
}
}
description = "aws workspace configuration"
}
variable "tags" {
default = ""
description = "tags for the resouces"
}
variable "region" {
default = ""
description = "region on which aws infra is to be deployed"
}
data "aws_workspaces_bundle" "value_windows_10" {
bundle_id = "wsb-bh8rsxt14"
}
module "aws_workspace" {
source = "./modules/aws_workspace"
aws_workspace = local.final_aws_workspace
tags = {
Name = "cloud"
}
bundle_id = data.aws_workspaces_bundle.value_windows_10.id
directory_id = aws_workspaces_directory.example.id
}
Resource.tf
variable "aws_workspace" {
default = ""
description = "configuration of aws workspaces"
}
variable "tags" {
default = ""
description = "tags of the resources"
}
variable "directory_id" {
default = ""
description = "Id of the directory"
}
variable "bundle_id" {
default = ""
description = "id of the bundle"
}
resource "aws_workspaces_workspace" "this" {
directory_id = var.directory_id
bundle_id = var.bundle_id
for_each = var.aws_workspace
user_name = each.value.user_name
root_volume_encryption_enabled = each.value.root_volume_encryption_enabled
user_volume_encryption_enabled = each.value.user_volume_encryption_enabled
volume_encryption_key = each.value.volume_encryption_key
workspace_properties {
compute_type_name = each.value.compute_type_name
user_volume_size_gib = each.value.user_volume_size_gib
root_volume_size_gib = each.value.root_volume_size_gib
running_mode = each.value.running_mode
running_mode_auto_stop_timeout_in_minutes = each.value.running_mode_auto_stop_timeout_in_minutes
}
tags = var.tags
}
Your aws_workspace is a map with different values for each user. You could only change to count, but this poses its own issues, and for_each is preferred in your case.
So you either use for_each as you do know, change it to count. Or if you totally don't want to use any of that, you have to put for_each in your module instead.
Update
The new aws_workspaces_workspace:
resource "aws_workspaces_workspace" "this" {
directory_id = var.directory_id
bundle_id = var.bundle_id
user_name = var.aws_workspace.user_name
root_volume_encryption_enabled = var.aws_workspace.root_volume_encryption_enabled
user_volume_encryption_enabled = var.aws_workspace.user_volume_encryption_enabled
volume_encryption_key = var.aws_workspace.volume_encryption_key
workspace_properties {
compute_type_name = var.aws_workspace.compute_type_name
user_volume_size_gib = var.aws_workspace.user_volume_size_gib
root_volume_size_gib = var.aws_workspace.root_volume_size_gib
running_mode = var.aws_workspace.running_mode
running_mode_auto_stop_timeout_in_minutes = var.aws_workspace.running_mode_auto_stop_timeout_in_minutes
}
and for module:
module "aws_workspace" {
for_each = local.final_aws_workspace
source = "./modules/aws_workspace"
aws_workspace = each.value
tags = {
Name = "cloud"
}
bundle_id = data.aws_workspaces_bundle.value_windows_10.id
directory_id = aws_workspaces_directory.example.id
}
I am trying to pass the values s3 name and create_user into local block in main.tf so that both of them have the value in list and then I am passing list_of_bucket in local block in module s3 to create the buckets and looping of user_to_create in module s3_user to create the user if the boolean is set to true. All of these values are passed to variable.tf and then to main.tf
dev.tfvars
wea-nonprod = {
services = {
s3 = [
sthree = {
create_user = true,
}
sfour = {
create_user = true,
}
sfive = {
create_user = true,
}
]
}
}
variable.tf
variable "s3_buckets" {
type = list(map)
}
main.tf
locals {
users_to_create = ""
list_of_buckets = ""
}
module "s3" {
source = "../../s3"
name = join("-", [var.name_prefix, "s3"])
tags = merge(var.tags, {Name = join("-", [var.name_prefix, "s3"])})
buckets = list_of_buckets
sse_algorithm = "AES256"
access_log_bucket_name = var.access_log_bucket_name
}
module "s3_user" {
for_each = local.users_to_create
source = "./service-s3-bucket-user"
name = join("-", [var.name_prefix, each.key])
tags = var.tags
bucket_arn = module.s3.bucket_arns[each.key]
depends_on = [module.s3]
}
Just iterate over your wea-nonprod map:
locals {
users_to_create = [ for name in var.wea-nonprod.services.s3 if name.create_user == true ]
list_of_buckets = [ for bucket in var.wea-nonprod.services.s3 ]
}
And a few changes to your module blocks:
module "s3" {
source = "../../s3"
name = "${var.name_prefix}-s3"
tags = merge(var.tags, { Name = "${var.name_prefix}-s3" })
buckets = local.list_of_buckets
sse_algorithm = "AES256"
access_log_bucket_name = var.access_log_bucket_name
}
module "s3_user" {
count = length(local.users_to_create)
source = "./service-s3-bucket-user"
name = "${var.name_prefix}${local.users_to_create[count.index]}"
tags = var.tags
bucket_arn = module.s3.bucket_arns[local.users_to_create[count.index]]
depends_on = [module.s3]
}
I am trying to create a simple dynamodb table using following reource modules of terraform.
Get the following error while running terraform:
All attributes must be indexed. Unused attributes: ["pactitle" "ipadress" "Timestamp"].
why do we need to index all attributes ?
How to solve this ?
resource "aws_dynamodb_table" "this" {
count = var.create_table ? 1 : 0
name = var.name
billing_mode = var.billing_mode
hash_key = var.hash_key
range_key = var.range_key
read_capacity = var.read_capacity
write_capacity = var.write_capacity
//stream_enabled = var.stream_enabled
//stream_view_type = var.stream_view_type
dynamic "attribute" {
for_each = var.attributes
content {
name = attribute.value.name
type = attribute.value.type
}
}
server_side_encryption {
enabled = var.server_side_encryption_enabled
kms_key_arn = var.server_side_encryption_kms_key_arn
}
tags = merge(
var.tags,
{
"Name" = format("%s", var.name)
},
)
timeouts {
create = lookup(var.timeouts, "create", null)
delete = lookup(var.timeouts, "delete", null)
update = lookup(var.timeouts, "update", null)
}
}
calling module
module "dynamodb_table" {
source = "./../../../modules/dynamodb"
name = "pack-audit-cert"
hash_key = "id"
create_table= true
read_capacity=5
write_capacity=5
billing_mode = "PROVISIONED"
range_key = "pacid"
attributes = [
{
name = "id"
type = "N"
},
{
name = "pacid"
type = "S"
},
{
name = "pactitle"
type = "S"
},
{
name = "ipadress"
type = "S"
},
{
name = "Timestamp"
type = "S"
}
]
}
Thank you
That error message is a bit misleading. You should only define the indexed attributes when you are creating the table. Since DynamoDB is a schemaless database, it doesn't care about the other attributes at table creation time.