From 77f560c605a8f7e2323621b4d70bf1296fe2cbf0 Mon Sep 17 00:00:00 2001 From: Mina Slater Date: Wed, 9 Nov 2022 15:51:13 -0600 Subject: [PATCH 1/7] Adjust admin-login module to handle replica Saves the replica database url to the same secrets so its value can be exposed as env variable when needed. --- rds-postgres/admin-login/main.tf | 26 +++++++++++++------ .../admin-login/rotation/lambda_function.py | 17 +++++++++--- rds-postgres/admin-login/variables.tf | 6 +++++ 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index 4ad1b59..8fe2168 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -8,14 +8,7 @@ module "secret" { resource_tags = var.tags trust_tags = var.trust_tags - initial_value = jsonencode({ - dbname = var.database_name - engine = data.aws_db_instance.this.engine - host = data.aws_db_instance.this.address - password = var.initial_password - port = tostring(data.aws_db_instance.this.port) - username = var.username - }) + initial_value = jsonencode(local.initial_secret_value) } module "rotation" { @@ -77,6 +70,23 @@ data "aws_db_instance" "this" { db_instance_identifier = var.identifier } +data "aws_db_instance" "replica" { + count = var.replica_identifier ? 1 : 0 + + db_instance_identifier = var.replica_identifier +} + locals { full_name = join("-", ["rds-postgres", var.identifier]) + + base_value = { + dbname = var.database_name + engine = data.aws_db_instance.this.engine + host = data.aws_db_instance.this.address + password = var.initial_password + port = tostring(data.aws_db_instance.this.port) + username = var.username + } + + initial_secret_value = var.replica_identifier ? merge(local.base_value, { replica_host = data.aws_db_instance.replica[0].address }) : local.base_value } diff --git a/rds-postgres/admin-login/rotation/lambda_function.py b/rds-postgres/admin-login/rotation/lambda_function.py index 76e621f..4c4e36c 100644 --- a/rds-postgres/admin-login/rotation/lambda_function.py +++ b/rds-postgres/admin-login/rotation/lambda_function.py @@ -31,7 +31,8 @@ def lambda_handler(event, context): 'username': , 'password': , 'dbname': , - 'port': + 'port': , + 'replica_host': } Args: @@ -126,7 +127,11 @@ def create_secret(service_client, arn, token): current_dict['password'] = passwd['RandomPassword'] # Add DATABASE_URL to secret - current_dict['DATABASE_URL'] = dict_to_url(current_dict) + current_dict['DATABASE_URL'] = dict_to_url(current_dict, false) + + if secret['replica_host']: + # Add DATABASE_REPLICA_URL to secret + current_dict['DATABASE_REPLICA_URL'] = dict_to_url(current_dict, true) # Put the secret service_client.put_secret_value(SecretId=arn, ClientRequestToken=token, SecretString=json.dumps(current_dict), VersionStages=['AWSPENDING']) @@ -278,7 +283,7 @@ def finish_secret(service_client, arn, token): service_client.update_secret_version_stage(SecretId=arn, VersionStage="AWSCURRENT", MoveToVersionId=token, RemoveFromVersionId=current_version) logger.info("finishSecret: Successfully set AWSCURRENT stage to version %s for secret %s." % (token, arn)) -def dict_to_url(secret): +def dict_to_url(secret, replica): """Reformats connection details as a URL string Generate a Heroku-style DATABASE_URL with connection details @@ -289,9 +294,13 @@ def dict_to_url(secret): Returns: url: DATABASE_URL-style string """ + if replica: + host = secret['host'] + else: + host = secret['replica_host'] return "postgres://%s:%s@%s:%s/%s" % (secret['username'], - secret['password'], secret['host'], secret['port'], + secret['password'], host, secret['port'], secret['dbname']) def get_connection(secret_dict): diff --git a/rds-postgres/admin-login/variables.tf b/rds-postgres/admin-login/variables.tf index ea01d32..a2c4c14 100644 --- a/rds-postgres/admin-login/variables.tf +++ b/rds-postgres/admin-login/variables.tf @@ -31,6 +31,12 @@ variable "read_principals" { default = null } +variable "replica_identifier" { + description = "Identifier of the database replica" + type = string + default = null +} + variable "secret_name" { description = "Override the name for this secret" type = string From ceaf27643db92e5aa8b1f074964fbff21a772ee2 Mon Sep 17 00:00:00 2001 From: Mina Slater Date: Thu, 10 Nov 2022 13:41:34 -0600 Subject: [PATCH 2/7] use can to eval replica to bool --- rds-postgres/admin-login/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index 8fe2168..f6919a8 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -71,7 +71,7 @@ data "aws_db_instance" "this" { } data "aws_db_instance" "replica" { - count = var.replica_identifier ? 1 : 0 + count = can(var.replica_identifier) ? 1 : 0 db_instance_identifier = var.replica_identifier } @@ -88,5 +88,5 @@ locals { username = var.username } - initial_secret_value = var.replica_identifier ? merge(local.base_value, { replica_host = data.aws_db_instance.replica[0].address }) : local.base_value + initial_secret_value = can(var.replica_identifier) ? merge(local.base_value, { replica_host = data.aws_db_instance.replica[0].address }) : local.base_value } From 3692ea8c34a274817da050f39df50a292c9d926a Mon Sep 17 00:00:00 2001 From: Mina Slater Date: Thu, 10 Nov 2022 15:43:51 -0600 Subject: [PATCH 3/7] Pass in replica hostname The only thing we need to know about the replica is the hostname/address. Doing this will eliminate the need to read from the instance. --- rds-postgres/admin-login/main.tf | 10 +++++----- rds-postgres/admin-login/variables.tf | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index f6919a8..e6f827b 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -70,11 +70,11 @@ data "aws_db_instance" "this" { db_instance_identifier = var.identifier } -data "aws_db_instance" "replica" { - count = can(var.replica_identifier) ? 1 : 0 +# data "aws_db_instance" "replica" { +# count = length(var.replica_identifier[*]) - db_instance_identifier = var.replica_identifier -} +# db_instance_identifier = var.replica_identifier +# } locals { full_name = join("-", ["rds-postgres", var.identifier]) @@ -88,5 +88,5 @@ locals { username = var.username } - initial_secret_value = can(var.replica_identifier) ? merge(local.base_value, { replica_host = data.aws_db_instance.replica[0].address }) : local.base_value + initial_secret_value = can(var.replica_host) ? merge(local.base_value, { replica_host = var.replica_host }) : local.base_value } diff --git a/rds-postgres/admin-login/variables.tf b/rds-postgres/admin-login/variables.tf index a2c4c14..012e784 100644 --- a/rds-postgres/admin-login/variables.tf +++ b/rds-postgres/admin-login/variables.tf @@ -31,8 +31,8 @@ variable "read_principals" { default = null } -variable "replica_identifier" { - description = "Identifier of the database replica" +variable "replica_host" { + description = "Hostname to use when connecting to the database replica" type = string default = null } From 44d9ce6fad9976e4d739f1cd292824dbed3f62d2 Mon Sep 17 00:00:00 2001 From: Olamide Date: Mon, 14 Nov 2022 16:03:57 +0100 Subject: [PATCH 4/7] Update replica url in RDs secret rotation script --- rds-postgres/admin-login/main.tf | 1 + rds-postgres/admin-login/rotation/lambda_function.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index e6f827b..3a4e891 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -33,6 +33,7 @@ module "rotation" { variables = { ALTERNATE_USERNAME = coalesce(var.alternate_username, "${var.username}_alt") PRIMARY_USERNAME = var.username + REPLICA_HOST = can(var.replica_host) ? var.replica_host : "" } } diff --git a/rds-postgres/admin-login/rotation/lambda_function.py b/rds-postgres/admin-login/rotation/lambda_function.py index 4c4e36c..4af5b7c 100644 --- a/rds-postgres/admin-login/rotation/lambda_function.py +++ b/rds-postgres/admin-login/rotation/lambda_function.py @@ -13,6 +13,7 @@ ALTERNATE_USERNAME = os.environ['ALTERNATE_USERNAME'] PRIMARY_USERNAME = os.environ['PRIMARY_USERNAME'] +REPLICA_HOST = os.environ['REPLICA_HOST'] def lambda_handler(event, context): @@ -127,11 +128,14 @@ def create_secret(service_client, arn, token): current_dict['password'] = passwd['RandomPassword'] # Add DATABASE_URL to secret - current_dict['DATABASE_URL'] = dict_to_url(current_dict, false) + current_dict['DATABASE_URL'] = dict_to_url(current_dict, False) - if secret['replica_host']: + if REPLICA_HOST: + current_dict['replica_host'] = REPLICA_HOST + + if current_dict['replica_host']: # Add DATABASE_REPLICA_URL to secret - current_dict['DATABASE_REPLICA_URL'] = dict_to_url(current_dict, true) + current_dict['DATABASE_REPLICA_URL'] = dict_to_url(current_dict, True) # Put the secret service_client.put_secret_value(SecretId=arn, ClientRequestToken=token, SecretString=json.dumps(current_dict), VersionStages=['AWSPENDING']) From 78b9e06ed93dc4bdba9644c339683111c326790f Mon Sep 17 00:00:00 2001 From: Mina Slater Date: Mon, 14 Nov 2022 13:49:47 -0500 Subject: [PATCH 5/7] Pass REPLICA_HOST address in as env var The initial_values for the postgres login secrets doesn't seem to be tracked as changes, so when the replica database host name is changes, terraform doesn't recognize anything has changed. Olamide suggested changing this module to pass replica host as an env var to the python lambda rotation script to correct this. --- rds-postgres/admin-login/main.tf | 34 ++++++++++--------- .../admin-login/rotation/lambda_function.py | 6 +--- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index 3a4e891..20f50ac 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -8,7 +8,15 @@ module "secret" { resource_tags = var.tags trust_tags = var.trust_tags - initial_value = jsonencode(local.initial_secret_value) + # initial_value = jsonencode(local.initial_secret_value) + initial_value = jsonencode({ + dbname = var.database_name + engine = data.aws_db_instance.this.engine + host = data.aws_db_instance.this.address + password = var.initial_password + port = tostring(data.aws_db_instance.this.port) + username = var.username + }) } module "rotation" { @@ -71,23 +79,17 @@ data "aws_db_instance" "this" { db_instance_identifier = var.identifier } -# data "aws_db_instance" "replica" { -# count = length(var.replica_identifier[*]) - -# db_instance_identifier = var.replica_identifier -# } - locals { full_name = join("-", ["rds-postgres", var.identifier]) - base_value = { - dbname = var.database_name - engine = data.aws_db_instance.this.engine - host = data.aws_db_instance.this.address - password = var.initial_password - port = tostring(data.aws_db_instance.this.port) - username = var.username - } + # base_value = { + # dbname = var.database_name + # engine = data.aws_db_instance.this.engine + # host = data.aws_db_instance.this.address + # password = var.initial_password + # port = tostring(data.aws_db_instance.this.port) + # username = var.username + # } - initial_secret_value = can(var.replica_host) ? merge(local.base_value, { replica_host = var.replica_host }) : local.base_value + # initial_secret_value = can(var.replica_host) ? merge(local.base_value, { replica_host = var.replica_host }) : local.base_value } diff --git a/rds-postgres/admin-login/rotation/lambda_function.py b/rds-postgres/admin-login/rotation/lambda_function.py index 4af5b7c..8728e69 100644 --- a/rds-postgres/admin-login/rotation/lambda_function.py +++ b/rds-postgres/admin-login/rotation/lambda_function.py @@ -33,7 +33,6 @@ def lambda_handler(event, context): 'password': , 'dbname': , 'port': , - 'replica_host': } Args: @@ -131,9 +130,6 @@ def create_secret(service_client, arn, token): current_dict['DATABASE_URL'] = dict_to_url(current_dict, False) if REPLICA_HOST: - current_dict['replica_host'] = REPLICA_HOST - - if current_dict['replica_host']: # Add DATABASE_REPLICA_URL to secret current_dict['DATABASE_REPLICA_URL'] = dict_to_url(current_dict, True) @@ -301,7 +297,7 @@ def dict_to_url(secret, replica): if replica: host = secret['host'] else: - host = secret['replica_host'] + host = REPLICA_HOST return "postgres://%s:%s@%s:%s/%s" % (secret['username'], secret['password'], host, secret['port'], From 7ffe6c37e80994f74fab21efeebb13100e6819ff Mon Sep 17 00:00:00 2001 From: Mina Slater Date: Mon, 14 Nov 2022 13:54:31 -0500 Subject: [PATCH 6/7] fixes indentation --- rds-postgres/admin-login/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rds-postgres/admin-login/main.tf b/rds-postgres/admin-login/main.tf index 20f50ac..6e127fe 100644 --- a/rds-postgres/admin-login/main.tf +++ b/rds-postgres/admin-login/main.tf @@ -10,12 +10,12 @@ module "secret" { # initial_value = jsonencode(local.initial_secret_value) initial_value = jsonencode({ - dbname = var.database_name - engine = data.aws_db_instance.this.engine - host = data.aws_db_instance.this.address - password = var.initial_password - port = tostring(data.aws_db_instance.this.port) - username = var.username + dbname = var.database_name + engine = data.aws_db_instance.this.engine + host = data.aws_db_instance.this.address + password = var.initial_password + port = tostring(data.aws_db_instance.this.port) + username = var.username }) } From 2729d2f388890a0be345173c858a378ab4bcfdee Mon Sep 17 00:00:00 2001 From: Olamide <65307752+OlamideOl1@users.noreply.github.com> Date: Tue, 15 Nov 2022 15:31:18 +0100 Subject: [PATCH 7/7] Update lambda_function.py Review dict_to_url function when replica is set --- rds-postgres/admin-login/rotation/lambda_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rds-postgres/admin-login/rotation/lambda_function.py b/rds-postgres/admin-login/rotation/lambda_function.py index 8728e69..b1a6bbd 100644 --- a/rds-postgres/admin-login/rotation/lambda_function.py +++ b/rds-postgres/admin-login/rotation/lambda_function.py @@ -295,9 +295,9 @@ def dict_to_url(secret, replica): url: DATABASE_URL-style string """ if replica: - host = secret['host'] - else: host = REPLICA_HOST + else: + host = secret['host'] return "postgres://%s:%s@%s:%s/%s" % (secret['username'], secret['password'], host, secret['port'],