AWS S3 replication fails without explicit error - amazon-web-services

Question

I'm trying to setup one-way replication between two accounts using this guide. Here's the relevant code for the source account:
data "aws_iam_policy_document" "s3-replication-trust" {
statement {
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["s3.amazonaws.com"]
}
effect = "Allow"
}
}
resource "aws_iam_role" "s3-replication-prod" {
count = var.env == "prod" ? 1 : 0 # only apply in prod account
name = "s3-replication-role-prod"
path = "/"
assume_role_policy = data.aws_iam_policy_document.s3-replication-trust.json
}
data "aws_iam_policy_document" "s3-replication-prod" {
count = var.env == "prod" ? 1 : 0 # only apply in prod account
statement {
actions = [
"s3:GetReplicationConfiguration",
"s3:ListBucket"
]
effect = "Allow"
resources = [ aws_s3_bucket.source.arn ]
}
statement {
actions = [
"s3:GetObjectVersionForReplication",
"s3:GetObjectVersion",
"s3:GetObjectVersionAcl",
"s3:GetObjectVersionTagging"
]
effect = "Allow"
resources = [ "${aws_s3_bucket.source.arn}/*" ]
}
statement {
actions = [
"s3:ReplicateObject",
"s3:ReplicateDelete",
"s3:ReplicateTags"
]
effect = "Allow"
resources = [ "${aws_s3_bucket.destination.arn}/*" ]
}
}
resource "aws_iam_policy" "s3-replication-prod" {
count = var.env == "prod" ? 1 : 0 # only apply in prod account
name = "s3-replication"
path = "/"
policy = data.aws_iam_policy_document.s3-replication-prod[0].json
}
resource "aws_iam_role_policy_attachment" "s3-replication-prod" {
count = var.env == "prod" ? 1 : 0 # only apply in prod account
role = aws_iam_role.s3-replication-prod[0].name
policy_arn = aws_iam_policy.s3-replication-prod[0].arn
}
resource "aws_s3_bucket_replication_configuration" "replication" {
count = var.env == "prod" ? 1 : 0 # only apply in prod account
# Must have bucket versioning enabled first
depends_on = [aws_s3_bucket_versioning.source]
role = aws_iam_role.s3-replication-prod[0].arn
bucket = aws_s3_bucket.source.id
rule {
id = "ReplicateToDev"
status = "Enabled"
destination {
bucket = aws_s3_bucket.destination.arn
storage_class = "ONEZONE_IA"
access_control_translation {
owner = "Destination"
}
account = var.destination_account_id
}
}
}
And here's the code relevant to the destination account:
data "aws_iam_policy_document" "destination_bucket_policy" {
statement {
principals {
type = "AWS"
identifiers = [
"arn:aws:iam::${var.prod_account_id}:role/s3-replication-role-prod"
]
}
actions = [
"s3:ReplicateDelete",
"s3:ReplicateObject"
]
resources = ["${aws_s3_bucket.destination.arn}/*"]
}
statement {
principals {
type = "AWS"
identifiers = [
"arn:aws:iam::${var.prod_account_id}:role/s3-replication-role-prod"
]
}
actions = [
"s3:List*",
"s3:GetBucketVersioning",
"s3:PutBucketVersioning"
]
resources = [aws_s3_bucket.destination.arn]
}
}
resource "aws_s3_bucket_policy" "s3-replication-dev" {
count = var.env == "dev" ? 1 : 0 # only apply in dev account
bucket = "${var.app}-dev"
policy = data.aws_iam_policy_document.destination_bucket_policy.json
}
When I try to add any new object to the source bucket, it is unable to replicate. When I navigate to the object's listing in the console, it shows replication status "FAILED".
There's no obvious errors showing up in CloudTrail.
What am I doing wrong here?