I am trying to create a simple dynamodb table using following reource modules of terraform.
Get the following error while running terraform:
All attributes must be indexed. Unused attributes: ["pactitle" "ipadress" "Timestamp"].
why do we need to index all attributes ?
How to solve this ?
resource "aws_dynamodb_table" "this" {
count = var.create_table ? 1 : 0
name = var.name
billing_mode = var.billing_mode
hash_key = var.hash_key
range_key = var.range_key
read_capacity = var.read_capacity
write_capacity = var.write_capacity
//stream_enabled = var.stream_enabled
//stream_view_type = var.stream_view_type
dynamic "attribute" {
for_each = var.attributes
content {
name = attribute.value.name
type = attribute.value.type
}
}
server_side_encryption {
enabled = var.server_side_encryption_enabled
kms_key_arn = var.server_side_encryption_kms_key_arn
}
tags = merge(
var.tags,
{
"Name" = format("%s", var.name)
},
)
timeouts {
create = lookup(var.timeouts, "create", null)
delete = lookup(var.timeouts, "delete", null)
update = lookup(var.timeouts, "update", null)
}
}
calling module
module "dynamodb_table" {
source = "./../../../modules/dynamodb"
name = "pack-audit-cert"
hash_key = "id"
create_table= true
read_capacity=5
write_capacity=5
billing_mode = "PROVISIONED"
range_key = "pacid"
attributes = [
{
name = "id"
type = "N"
},
{
name = "pacid"
type = "S"
},
{
name = "pactitle"
type = "S"
},
{
name = "ipadress"
type = "S"
},
{
name = "Timestamp"
type = "S"
}
]
}
Thank you
That error message is a bit misleading. You should only define the indexed attributes when you are creating the table. Since DynamoDB is a schemaless database, it doesn't care about the other attributes at table creation time.
Related
Im trying to add retention policy but I want to enable it conditionally, as you can see from the code
buckets.tf
locals {
team_buckets = {
arc = { app_id = "20390", num_buckets = 2, retention_period = null }
ana = { app_id = "25402", num_buckets = 2, retention_period = 631139040 }
cha = { app_id = "20391", num_buckets = 2, retention_period = 631139040 } #20 year
}
}
module "team_bucket" {
source = "../../../../modules/gcs_bucket"
for_each = {
for bucket in flatten([
for product_name, bucket_info in local.team_buckets : [
for i in range(bucket_info.num_buckets) : {
name = format("%s-%02d", product_name, i + 1)
team = "ei_${product_name}"
app_id = bucket_info.app_id
retention_period = bucket_info.retention_period
}
]
]) : bucket.name => bucket
}
project_id = var.project
name = "teambucket-${each.value.name}"
app_id = each.value.app_id
team = each.value.team
retention_period = each.value.retention_period
}
root module is defined as follows
main.tf
resource "google_storage_bucket" "bucket" {
project = var.project_id
name = "${var.project_id}-${var.name}"
location = var.location
labels = {
app_id = var.app_id
ei_team = var.team
cost_center = var.cost_center
}
uniform_bucket_level_access = var.uniform_bucket_level_access
dynamic "retention_policy" {
for_each = var.retention_policy == null ? [] : [var.retention_period]
content {
retention_period = var.retention_period
}
}
}
but I can't seem to make the code pick up the value,
for example as you see below the value doesn't get implemented
~ resource "google_storage_bucket" "bucket" {
id = "teambucket-cha-02"
name = "teambucket-cha-02"
# (11 unchanged attributes hidden)
- retention_policy {
- is_locked = false -> null
- retention_period = 3155760000 -> null
}
}
variables.tf for retention policy is as follows
variable "retention_policy" {
description = "Configuation of the bucket's data retention policy for how long objects in the bucket should be retained"
type = any
default = null
}
variable "retention_period" {
default = null
}
Your var.retention_policy is always null, as its default value. You are not changing the default value at all. Probably you wanted the following:
for_each = var.retention_period == null ? [] : [var.retention_period]
instead of
for_each = var.retention_policy == null ? [] : [var.retention_period]
I'm trying to create a dynamic method to create multiple dynamodb tables with their own attributes. I tried for loops with dynamic blocks, list objects, etc but was not able to iterate attributes for each table. The goal is to have multiple tables with different attributes and global indexes for each table in one go. I have terraform.tfvars and main.tf with the following structure:
Variable declaration:
variable "dynamodb_table" {
type = list(object({
table_name = string
billing_mode = string
read_capacity = optional(number)
write_capacity = optional(string)
hash_key = string
ttl_attribute_name = string
ttl_enabled = string
range_key = optional(string)
attribute = object({
name = string
type = string
})
}))
}
variable "global_secondary_indexes" {
description = "Describe a GSI for the table; subject to the normal limits on the number of GSIs, projected attributes, etc."
type = list(object({
index_name = string
index_projection_type = string
index_range_key = string
index_hash_key = string
index_write_capacity = optional(string)
index_read_capacity = optional(string)
index_non_key_attributes = list(string)
}))
default = []
}
dynamodb_table = [
{
table_name = "devops-test-01",
billing_mode = "PAY_PER_REQUEST",
hash_key = "UserId",
range_key = "GameTitle",
ttl_attribute_name = "ttl_attribute_name",
ttl_enabled = "false"
attribute = [
{
name = "UserId"
type = "S"
},
{
name = "GameTitle"
type = "S"
}
]
},
{
table_name = "devops-test-02",
billing_mode = "PAY_PER_REQUEST",
hash_key = "GameTitle",
ttl_attribute_name = "ttl_attribute_name",
ttl_enabled = "false"
}
]
global_secondary_indexes = [
{
index_name = "TitleIndex"
index_hash_key = "UserId"
index_range_key = "GameTitle"
index_projection_type = "INCLUDE"
index_non_key_attributes = ["Id"]
}
]
default_tags = {
"Environment" = "Dev",
"Owner" = "xxx"
}
resource "aws_dynamodb_table" "basic-dynamodb-table" {
for_each = { for key, value in var.dynamodb_table : key => value }
name = each.value.table_name
billing_mode = each.value.billing_mode
read_capacity = each.value.read_capacity
write_capacity = each.value.write_capacity
hash_key = each.value.hash_key
range_key = each.value.range_key
ttl {
attribute_name = each.value.ttl_attribute_name
enabled = each.value.ttl_enabled
}
dynamic "attribute" {
for_each = { for key, value in var.attributes : key => value }
content {
name = attribute.value.name
type = attribute.value.type
}
}
dynamic "global_secondary_index" {
for_each = var.global_secondary_indexes
content {
name = global_secondary_index.value.index_name
hash_key = global_secondary_index.value.index_hash_key
projection_type = global_secondary_index.value.index_projection_type
range_key = lookup(global_secondary_index.value, "index_range_key", null)
read_capacity = lookup(global_secondary_index.value, "index_read_capacity", null)
write_capacity = lookup(global_secondary_index.value, "index_write_capacity", null)
non_key_attributes = lookup(global_secondary_index.value, "index_non_key_attributes", null)
}
}
tags = merge(
var.default_tags,
{
Name = each.value.table_name
})
}
This code produces the following error:
The given value is not suitable for var.dynamodb_table declared at variable.tf:6,1-26: element 0: attribute │ "attribute": object required
You did not share your attributes variable but I have used attributes in dynamodb_table variable.
Your main problem is attribute property in dynamodb_table variable is requeired but you did not provide any value for it in devops-test-02 table values.
variables.tf
variable "dynamodb_table" {
type = list(object({
table_name = string
billing_mode = string
// read_capacity = optional(number)
//write_capacity = optional(string)
hash_key = string
ttl_attribute_name = string
ttl_enabled = string
//range_key = optional(string)
attribute = list(object({
name = string
type = string
}))
}))
default = [
{
table_name = "devops-test-01",
billing_mode = "PAY_PER_REQUEST",
hash_key = "UserId",
range_key = "GameTitle",
ttl_attribute_name = "ttl_attribute_name",
ttl_enabled = "false"
attribute = [
{
name = "UserId"
type = "S"
},
{
name = "GameTitle"
type = "S"
}
]
},
{
table_name = "devops-test-02",
billing_mode = "PAY_PER_REQUEST",
hash_key = "GameTitle",
ttl_attribute_name = "ttl_attribute_name",
ttl_enabled = "false"
attribute = [
{
name = "UserId"
type = "S"
},
{
name = "GameTitle"
type = "S"
}
]
}
]
}
variable "global_secondary_indexes" {
description = "Describe a GSI for the table; subject to the normal limits on the number of GSIs, projected attributes, etc."
type = list(object({
index_name = string
index_projection_type = string
index_range_key = string
index_hash_key = string
//index_write_capacity = optional(string)
//index_read_capacity = optional(string)
index_non_key_attributes = list(string)
}))
default = [
{
index_name = "TitleIndex"
index_hash_key = "UserId"
index_range_key = "GameTitle"
index_projection_type = "INCLUDE"
index_non_key_attributes = ["Id"]
}
]
}
variable "default_tags" {
default = {
"Environment" = "Dev",
"Owner" = "xxx"
}
}
dynamodb.tf
resource "aws_dynamodb_table" "basic-dynamodb-table" {
for_each = { for key, value in var.dynamodb_table : value.table_name => value }
name = each.value.table_name
billing_mode = each.value.billing_mode
read_capacity = lookup(each.value, "read_capacity", null)
write_capacity = lookup(each.value, "write_capacity", null)
hash_key = each.value.hash_key
range_key = lookup(each.value, "range_key", null)
ttl {
attribute_name = each.value.ttl_attribute_name
enabled = each.value.ttl_enabled
}
dynamic "attribute" {
for_each = { for key, value in each.value.attribute : key => value }
content {
name = attribute.value.name
type = attribute.value.type
}
}
dynamic "global_secondary_index" {
for_each = var.global_secondary_indexes
content {
name = global_secondary_index.value.index_name
hash_key = global_secondary_index.value.index_hash_key
projection_type = global_secondary_index.value.index_projection_type
range_key = lookup(global_secondary_index.value, "index_range_key", null)
read_capacity = lookup(global_secondary_index.value, "index_read_capacity", null)
write_capacity = lookup(global_secondary_index.value, "index_write_capacity", null)
non_key_attributes = lookup(global_secondary_index.value, "index_non_key_attributes", null)
}
}
tags = merge(
var.default_tags,
{
Name = each.value.table_name
})
}
UPDATE 2023-01-17
Add Kinesis streaming destination resource to dynamodb tables.
resource "aws_kinesis_stream" "example" {
for_each = aws_dynamodb_table.basic-dynamodb-table
name = "${each.key}_table_stream"
shard_count = 1
}
resource "aws_dynamodb_kinesis_streaming_destination" "example" {
for_each = aws_dynamodb_table.basic-dynamodb-table
stream_arn = aws_kinesis_stream.example[each.key].arn
table_name = each.key
}
I'm trying to create certain BigQuery tables with time_partitioning with the dynamic block and I want to use the values from tfvars in runtime as follows:
./tables/tables.tf:
resource "google_bigquery_table" "tables" {
for_each = var.tables == [] ? [] : toset(var.tables)
dataset_id = var.db_id
deletion_protection = false
table_id = each.key
dynamic "time_partitioning" {
for_each = var.partitioned_tables
content {
type = "DAY"
field = time_partitioning.value.field
}
}
labels = {
environment = var.environment
application = var.application
}
schema = fileexists("${path.module}/${var.db_id}/${each.key}.json") ? file("${path.module}/${var.db_id}/${each.key}.json") : null
}
main.tf:
resource "google_bigquery_dataset" "database" {
count = length(var.dbs)
dataset_id = var.dbs[count.index].db_id
friendly_name = var.dbs[count.index].db_name
description = "TF"
location = "US"
delete_contents_on_destroy = var.delete_contents_on_destroy
labels = {
environment = var.environment
application = var.dbs[count.index].app_name
}
}
module "tables" {
source = "./tables"
count = length(var.dbs)
db_id = google_bigquery_dataset.database[count.index].dataset_id
environment = var.environment
application = var.dbs[count.index].app_name
tables = var.dbs[count.index].tables
partitioned_tables = var.dbs[count.index].partitioned_tables
}
module "iam" {
source = "./iam"
count = length(var.dbs)
db_id = google_bigquery_dataset.database[count.index].dataset_id
iam_members = var.dbs[count.index].iam_members
}
dev.tfvars:
region = "us-central1"
project_id = "some-project"
dbs = [
{
db_id = "dataset1"
db_name = "dataset1"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com",
}
]
tables = ["daily_inventory", "dc_inventory", "products", "daily_sales", "planned_inventory", "stores", "stores_in_program"]
partitioned_tables = [
{
table = "daily_sales"
field = "sales_timestamp"
},
{
table = "daily_inventory"
field = "inventory_timestamp"
}
]
},
{
db_id = "dataset2"
db_name = "dataset2"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com"
}
]
tables = []
}
]
environment = "development"
delete_contents_on_destroy = true
var.dbs is type = list(any)
Getting:
The given value is not suitable for var.dbs declared at
variables.tf:9,1-15: all list elements must have the same type.
Thanks in advance!
list(any) does not mean that you can have elements of "any" type in your list. All elements must have same type, and you can't mix types, as you do now (i.e. second element is missing partitioned_tables). any only means that TF will infer the single type for the elements, but all elements must be of that single type. So you have three choices:
remove type = list(any)
Fully define your type with optional arguments, instead of using any
Add partitioned_tables to the second element:
[
{
db_id = "dataset1"
db_name = "dataset1"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com",
}
]
tables = ["daily_inventory", "dc_inventory", "products", "daily_sales", "planned_inventory", "stores", "stores_in_program"]
partitioned_tables = [
{
table = "daily_sales"
field = "sales_timestamp"
},
{
table = "daily_inventory"
field = "inventory_timestamp"
}
]
},
{
db_id = "dataset2"
db_name = "dataset2"
app_name = "hello"
iam_members = [
{
role = "roles/bigquery.dataEditor"
member = "serviceAccount:ser-sa#some-project.iam.gserviceaccount.com"
}
]
partitioned_tables = []
tables = []
}
]
I have a used case where,
I need to create 3 dynamodb tables but only the naming convention changes
resource "aws_dynamodb_table" "GB_SKU_COLOR" {
name = "GB_SKU_COLOR_${var.stage}"
billing_mode = "PAY_PER_REQUEST"
hash_key = "PRODUCT_ID"
stream_enabled = true
stream_view_type = "NEW_AND_OLD_IMAGES"
attribute {
name = "PRODUCT_ID"
type = "S"
}
}
I need to create the same table with 2 different names, like MN_SKU_COLOR and CH_SKU_COLOR
Currently, am replicating the resource and giving a new name
resource "aws_dynamodb_table" "MN_SKU_COLOR" {
name = "MN_SKU_COLOR_${var.stage}"
billing_mode = "PAY_PER_REQUEST"
hash_key = "PRODUCT_ID"
stream_enabled = true
stream_view_type = "NEW_AND_OLD_IMAGES"
attribute {
name = "PRODUCT_ID"
type = "S"
}
}
resource "aws_dynamodb_table" "CH_SKU_COLOR" {
name = "CH_SKU_COLOR_${var.stage}"
billing_mode = "PAY_PER_REQUEST"
hash_key = "PRODUCT_ID"
stream_enabled = true
stream_view_type = "NEW_AND_OLD_IMAGES"
attribute {
name = "PRODUCT_ID"
type = "S"
}
}
What is the best way to create the resource 3 times without replicating the code ?
If the only difference is in name, then you can do:
variable "names" {
default = ["MN_SKU_COLOR", "GB_SKU_COLOR", "CH_SKU_COLOR"]
}
resource "aws_dynamodb_table" "table" {
for_each = toset(var.names)
name = "${each.key}_${var.stage}"
billing_mode = "PAY_PER_REQUEST"
hash_key = "PRODUCT_ID"
stream_enabled = true
stream_view_type = "NEW_AND_OLD_IMAGES"
attribute {
name = "PRODUCT_ID"
type = "S"
}
}
Then you refer to the individual tables using their name from variabile, e.g.
aws_dynamodb_table.table["MN_SKU_COLOR"].id
Use for_each along with a set or list of names: https://www.terraform.io/language/meta-arguments/for_each
I wanted to create a for_each loop that loops only over objects in array that have some specific key : value pair.
My input variables are:
inputs = {
names = ["first", "second"]
lifecycle_rules = [
{
name = first
condition = {
age = "1"
}
action = {
type = "Delete"
}
},{
condition = {
age = "2"
}
action = {
type = "Delete"
}
},
{
name = second
condition = {
age = "3"
}
action = {
type = "Delete"
}
},{
condition = {
age = "4"
}
action = {
type = "Delete"
}
}
]
and in my main.tf (btw for deploying gcp bucket for reference), I wanted to separate the lifecycle per bucket and wanted to apply only the rules that have the buckets name in it.
So if anyone has idea how to modify for_each code below to work, I would highly appreciate. I believe only the for_each needs to be changed to loop over the right elements (let's say only objects in that list that have name = first). from the var.lifecycle_rules set
resource "google_storage_bucket" "buckets" {
count = length(var.names)
name = "${lower(element(var.names, count.index))}"
...
dynamic "lifecycle_rule" {
#for_each = length(lookup(lifecycle_rules[lookup(element(var.names, count.index))])
for_each = lifecycle_rules
content {
action {
type = lifecycle_rule.value.action.type
storage_class = lookup(lifecycle_rule.value.action, "storage_class", null)
}
condition {
#age = lifecycle_rule.value.name == element(var.names, count.index) ? lookup(lifecycle_rule.value.condition, "age", null) : null
age = lookup(lifecycle_rule.value.condition, "age", null) : null
...
I think that this "wierd" look can be obtained in two stages.
Reorganize lifecycle_rules into a map based on names
variable "input" {
default = {
names = ["first", "second"],
lifecycle_rules = [
{
name = "first",
condition = {
age = "1"
},
action = {
type = "Delete"
}
},
{
condition = {
age = "2"
},
action = {
type = "Delete"
}
},
{
name = "second",
condition = {
age = "3"
},
action = {
type = "Delete"
}
},
{
condition = {
age = "4"
},
action = {
type = "Delete"
}
}
]
}
}
locals {
new = {
for name in var.input.names:
name => [for rule in var.input.lifecycle_rules:
contains(keys(rule), "name" ) ?
rule.name == name ? rule: null :
null ]
}
}
which will give local.new in the form of:
{
"first" = [
{
"action" = {
"type" = "Delete"
}
"condition" = {
"age" = "1"
}
"name" = "first"
},
null,
null,
null,
]
"second" = [
null,
null,
{
"action" = {
"type" = "Delete"
}
"condition" = {
"age" = "3"
}
"name" = "second"
},
null,
]
}
Perform the for_each
resource "google_storage_bucket" "buckets" {
for_each = toset(var.input.names)
name = each.key
dynamic "lifecycle_rule" {
# iterate for each name skipping null values
for_each = [for v in local.new[each.key]: v if v != null]
content {
action {
type = lifecycle_rule.value["action"].type
storage_class = lookup(lifecycle_rule.value["action"], "storage_class", null)
}
condition {
age = lookup( tag.value["condition"], "age", null)
}
}
}
}
I could only verify the first step and partial second (using aws_autoscaling_group and its multiple tag components). I don't access to google cloud to fully test the code.