Terraform lambda invocation probable timeout - amazon-web-services

Question

After having spent 7 hours on this, I decided to reach out to you.
I need to update credentials within in a terraform flow. Since the secrets shall not be in the state-file, I use an AWS lambda function to update the secret of the RDS instance. The password is passed via CLI.
locals {
db_password = tostring(var.db_user_password)
}
data "aws_lambda_invocation" "credentials_manager" {
function_name = "credentials-manager"
input = <<JSON
{
"secretString": "{\"username\":\"${module.db_instance.db_user}\",\"password\":\"${local.db_password}\",\"dbInstanceIdentifier\":\"${module.db_instance.db_identifier}\"}",
"secretId": "${module.db_instance.db_secret_id}",
"storageId": "${module.db_instance.db_identifier}",
"forcedMod": "${var.forced_mod}"
}
JSON
depends_on = [
module.db_instance.db_secret_id,
]
}
output "result" {
description = "String result of Lambda execution"
value = jsondecode(data.aws_lambda_invocation.credentials_manager.result)
}
In order to make sure that the RDS instance status is 'available' the lambda function also contains a waiter.
When I manually execute the function everything works like a charm.
But within in terraform it does not proceed from here:
data.aws_lambda_invocation.credentials_manager: Refreshing state...
However, when I look into AWS Cloud Watch I can see that the lambda function is being invoked by Terraform over and over again.
This is the lambda policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Stmt1589715377799",
"Action": [
"rds:ModifyDBInstance",
"rds:DescribeDBInstances"
],
"Effect": "Allow",
"Resource": "*"
}
]
}
The lambda function looks like this:
const secretsManager = require('aws-sdk/clients/secretsmanager')
const rds = require('aws-sdk/clients/rds')
const elastiCache = require('aws-sdk/clients/elasticache')
const log = require('loglevel')
/////////////////////////////////////////
// ENVIRONMENT VARIABLES
/////////////////////////////////////////
const logLevel = process.env["LOG_LEVEL"];
const region = process.env["REGION"]
/////////////////////////////////////////
// CONFIGURE LOGGER
log.setLevel(logLevel);
let protocol = []
/////////////////////////////////////////
/////////////////////////////////////////
// DEFINE THE CLIENTS
const SM = new secretsManager({ region })
const RDS = new rds({ region })
const ELC = new elastiCache({region})
/////////////////////////////////////////
/////////////////////////////////////////
// FUNCTION DEFINITIONS
/////////////////////////////////////////
// HELPERS
/**
* #function waitForSeconds
* Set a custom waiter.
*
* #param {int} milseconds - the milliseconds to set as timeout.
*
*/
const waitForSeconds = (ms) => {
return new Promise(resolve => setTimeout(resolve, ms))
}
// AWS SECRETS MANAGER FUNCTIONS
/**
* #function UpdateSecretInSM
* The function updates the secrect value in the corresponding secret.
*
* #param {string} secretId - The id of the secret located in AWS SecretsManager
* #param {string} secretString - The value of the new secret
*
*/
const UpdateSecretInSM = async (secretId,secretString) => {
const params = {SecretId: secretId, SecretString: secretString}
try {
const data = await SM.updateSecret(params).promise()
log.info(`[INFO]: Password for ${secretId} successfully changed in Scecrets Manager!`)
let success = {Timestamp: new Date().toISOString(),Func: 'UpdateSecretInSM', Message: `Secret for ${secretId} successfully changed!`}
protocol.push(success)
return
} catch (err) {
log.debug("[DEBUG]: Error: ", err.stack);
let error = {Timestamp: new Date().toISOString(),Func: 'UpdateSecretInSM', Error: err.stack}
protocol.push(error)
return
}
}
/**
* #function GetSecretFromSM
* The function retrieves the specified secret from AWS SecretsManager.
* Returns the password.
*
* #param {string} secretId - secretId that is available in AWS SecretsManager
*
*/
const GetSecretFromSM = async (secretId) => {
try {
const data = await SM.getSecretValue({SecretId: secretId}).promise()
log.debug("[DEBUG]: Secret: ", data);
let success = {Timestamp: new Date().toISOString(),Func: 'GetSecretFromSM', Message: 'Secret from SecretsManager successfully received!'}
protocol.push(success)
const { SecretString } = data
const password = JSON.parse(SecretString)
return password.password
} catch (err) {
log.debug("[DEBUG]: Error: ", err.stack);
let error = {Timestamp: new Date().toISOString(),Func: 'GetSecretFromSM', Error: err.stack}
protocol.push(error)
return
}
}
// AWS RDS FUNCTIONS
/**
* #function ChangeRDSSecret
* Change the secret of the specified RDS instance.
*
* #param {string} rdsId - id of the RDS instance
* #param {string} password - new password
*
*/
const ChangeRDSSecret = async (rdsId,password) => {
const params = {
DBInstanceIdentifier: rdsId,
MasterUserPassword: password
}
try {
await RDS.modifyDBInstance(params).promise()
log.info(`[INFO]: Password for ${rdsId} successfully changed!`)
let success = {Timestamp: new Date().toISOString(), Func: 'ChangeRDSSecret', Message: `Secret for ${rdsId} successfully changed!`}
protocol.push(success)
return
} catch (err) {
log.debug("[DEBUG]: Error: ", err.stack);
let error = {Timestamp: new Date().toISOString(), Func: 'ChangeRDSSecret', Error: err.stack}
protocol.push(error)
return
}
}
const DescribeRDSInstance = async(id) => {
const params = { DBInstanceIdentifier : id }
const secondsToWait = 10000
try {
let pendingModifications = true
while (pendingModifications == true) {
log.info(`[INFO]: Checking modified values for ${id}`)
let data = await RDS.describeDBInstances(params).promise()
console.log(data)
// Extract the 'PendingModifiedValues' object
let myInstance = data['DBInstances']
myInstance = myInstance[0]
if (myInstance.DBInstanceStatus === "resetting-master-credentials") {
log.info(`[INFO]:Password change is being processed!`)
pendingModifications = false
}
log.info(`[INFO]: Waiting for ${secondsToWait/1000} seconds!`)
await waitForSeconds(secondsToWait)
}
let success = {Timestamp: new Date().toISOString(), Func: 'DescribeRDSInstance', Message: `${id} available again!`}
protocol.push(success)
return
} catch (err) {
log.debug("[DEBUG]: Error: ", err.stack);
let error = {Timestamp: new Date().toISOString(), Func: 'DescribeRDSInstance', Error: err.stack}
protocol.push(error)
return
}
}
const WaitRDSForAvailableState = async(id) => {
/**
* #function WaitRDSForAvailableState
* Wait for the instance to be available again.
*
* #param {string} id - id of the RDS instance
*
*/
const params = { DBInstanceIdentifier: id}
try {
log.info(`[INFO]: Waiting for ${id} to be available again!`)
const data = await RDS.waitFor('dBInstanceAvailable', params).promise()
log.info(`[INFO]: ${id} available again!`)
let success = {Timestamp: new Date().toISOString(), Func: 'WaitRDSForAvailableState', Message: `${id} available again!`}
protocol.push(success)
return
} catch (err) {
log.debug("[DEBUG]: Error: ", err.stack);
let error = {Timestamp: new Date().toISOString(), Func: 'WaitRDSForAvailableState', Error: err.stack}
protocol.push(error)
return
}
}
// AWS ELASTICACHE FUNCTIONS
// ... removed since they follow the same principle like RDS
/////////////////////////////////////////
// Lambda Handler
/////////////////////////////////////////
exports.handler = async (event,context,callback) => {
protocol = []
log.debug("[DEBUG]: Event:", event)
log.debug("[DEBUG]: Context:", context)
// Variable for the final message the lambda function will return
let finalValue
// Get the password and rds from terraform output
const secretString = event.secretString // manual input
const secretId = event.secretId // coming from secretesmanager
const storageId = event.storageId // coming from db identifier
const forcedMod = event.forcedMod // manual input
// Extract the password from the passed secretString to for comparison
const passedSecretStringJSON = JSON.parse(secretString)
const passedSecretString = passedSecretStringJSON.password
const currentSecret = await GetSecretFromSM(secretId)
// Case if the password has already been updated
if (currentSecret !== "ChangeMeViaScriptOrConsole" && passedSecretString === "ChangeMeViaScriptOrConsole") {
log.debug("[DEBUG]: No change necessary.")
finalValue = {timestamp: new Date().toISOString(),
message: 'Lambda function execution finished!',
summary: 'Password already updated. It is not "ChangeMeViaScriptOrConsole."'}
return finalValue
}
// Case if the a new password has not been set yet
if (currentSecret === "ChangeMeViaScriptOrConsole" && passedSecretString === "ChangeMeViaScriptOrConsole") {
finalValue = {timestamp: new Date().toISOString(),
message: 'Lambda function execution finished!',
summary: 'Password still "ChangeMeViaScriptOrConsole". Please change me!'}
return finalValue
}
// Case if the passed password is equal to the stored password and if pw modification is enforced
if (currentSecret === passedSecretString && forcedMod === "no") {
finalValue = {timestamp: new Date().toISOString(),
message: 'Lambda function execution finished!',
summary: 'Stored password is the same as the passed one. No changes made!'}
return finalValue
}
// Case for changing the password
if (passedSecretString !== "ChangeMeViaScriptOrConsole") {
// Update the secret in SM for the specified RDS Instances
await UpdateSecretInSM(secretId,secretString)
log.debug("[DEBUG]: Secret updated for: ", secretId)
// Change the new secret vom SM
const updatedSecret = await GetSecretFromSM(secretId)
log.debug("[DEBUG]: Updated secret: ", updatedSecret)
if (secretId.includes("rds")) {
// Update RDS instance with new secret and wait for it to be available again
await ChangeRDSSecret(storageId, updatedSecret)
await DescribeRDSInstance(storageId)
await WaitRDSForAvailableState(storageId)
} else if (secretId.includes("elasticache")) {
// ... removed since it is analogeous to RDS
} else {
protocol.push(`No corresponding Storage Id exists for ${secretId}. Please check the Secret Id/Name in the terraform configuration.`)
}
finalValue ={timestamp: new Date().toISOString(),
message: 'Lambda function execution finished!',
summary: protocol}
return finalValue
} else {
finalValue = {timestamp: new Date().toISOString(),
message: 'Lambda function execution finished!',
summary: 'Nothing changed'}
return finalValue
}
}
Anyone an idea how to solve or mitigate this behaviour?

Answer 1

Can you please show the iam policy for your lambda function? By my understanding you might be missing this resource aws_lambda_permission for your lambda function. https://www.terraform.io/docs/providers/aws/r/lambda_permission.html