Cumulus 4694 - Add Iceberg Replication

CHANGELOG.md

@@ -1,6 +1,7 @@
 # CHANGELOG
 
 ## Unreleased
+* Add iceberg target to Makefile to build Iceberg replication components
 * Add validation of deploy_name in env.sh
 * Expose `report_sns_topic_subscriber_arns` variable to cumulus.tf module
 
@@ -35,12 +36,12 @@
 * Update `cumulus` module to allow for optional use of AWS Secrets Manager for `archive_api_url, urs_client_password, metrics_es_password, cmr_password, cmr_username, lzards_launchpad_passphrase, launchpad_passphrase, token_secret` via `configuration_secret` variable
 * Update GH actions `tflint` to v0.61.0, update GH actions `checkout` to v4
 * Add `send_pan_task` to `cumulus` module output
-* Update Docker image to install docker CLI 
+* Update Docker image to install docker CLI
 * Update Makefile to allow docker-in-docker on MacOS hosts
 
 ## v21.0.1.0
 * Upgrade to [Cumulus v21.0.1](https://github.com/nasa/cumulus/releases/tag/v21.0.1)
-* **NOTE** This version of Cumulus requires a manual update to the PostgreSQL database in the production environment. Please follow the instructions in [Update granules to include producer_granule_id](https://nasa.github.io/cumulus/docs/next/upgrade-notes/update-granules-to-include-producer_granule_id) 
+* **NOTE** This version of Cumulus requires a manual update to the PostgreSQL database in the production environment. Please follow the instructions in [Update granules to include producer_granule_id](https://nasa.github.io/cumulus/docs/next/upgrade-notes/update-granules-to-include-producer_granule_id)
 
 ## v20.3.0.0
 * Upgrade to [Cumulus v20.3.0](https://github.com/nasa/cumulus/releases/tag/v20.3.0)

Makefile

@@ -20,7 +20,7 @@
 #  PYTHON_VER:            python3
 
 # ---------------------------
-DOCKER_TAG := v22.2.2.0
+DOCKER_TAG := v22.2.5.0
 export TF_IN_AUTOMATION="true"
 export TF_VAR_MATURITY=${MATURITY}
 export TF_VAR_DEPLOY_NAME=${DEPLOY_NAME}
@@ -83,7 +83,7 @@ container-shell:
 		--name=cirrus-core \
 		cirrus-core:$(DOCKER_TAG) \
 		bash
-		
+
 .PHONY: shell
 shell:
 		DAAC_DIR="${DAAC_DIR}" \
@@ -342,6 +342,7 @@ all: \
 	tf \
 	daac \
 	data-persistence \
+	iceberg \
 	cumulus \
 	workflows
 
@@ -351,5 +352,6 @@ initial-deploy: \
 	daac \
 	rds \
 	data-persistence \
+	iceberg \
 	cumulus \
 	workflows

README.md

@@ -209,6 +209,7 @@ If you want to deploy everything besides the `rds` module you can run the comman
         $ make daac
         $ make rds
         $ make data-persistence
+        $ make iceberg
         $ make cumulus
         $ make workflows
 

cumulus/locals.tf

@@ -1,5 +1,6 @@
 locals {
   prefix = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}"
+  iceberg_namespace = var.iceberg_namespace != null ? var.iceberg_namespace : replace("${local.prefix}_iceberg", "-", "_")
 
   buckets = data.terraform_remote_state.daac.outputs.bucket_map
 
@@ -53,10 +54,10 @@ locals {
   lzards_launchpad_passphrase = sensitive(lookup(local.configuration_secret_values, "lzards_launchpad_passphrase", var.lzards_launchpad_passphrase))
   token_secret                = sensitive(lookup(local.configuration_secret_values, "token_secret", var.token_secret))
   urs_client_id               = sensitive(lookup(local.configuration_secret_values, "urs_client_id", var.urs_client_id))
-  
+
   urs_tea_client_id       = var.urs_tea_client_id != null ? var.urs_tea_client_id : local.urs_client_id
   urs_tea_client_password = var.urs_tea_client_password != null ? var.urs_tea_client_password : local.urs_client_password
-  
+
   throttled_queues = [
     for q in var.dynamic_throttled_queues : {
       url             = "https://sqs.${data.aws_region.current.name}.amazonaws.com/${data.aws_caller_identity.current.account_id}/${local.prefix}-${q.queue_name}"

cumulus/main.tf

@@ -1,5 +1,5 @@
 module "cumulus" {
-  source = "https://github.com/nasa/cumulus/releases/download/v22.2.2/terraform-aws-cumulus.zip//tf-modules/cumulus"
+  source = "https://github.com/nasa/cumulus/releases/download/v22.2.5/terraform-aws-cumulus.zip//tf-modules/cumulus"
 
   cumulus_message_adapter_lambda_layer_version_arn = data.terraform_remote_state.daac.outputs.cma_layer_arn
 
@@ -21,8 +21,6 @@ module "cumulus" {
   ecs_cluster_instance_type               = var.ecs_cluster_instance_type
   ecs_cluster_instance_docker_volume_size = var.ecs_cluster_instance_docker_volume_size
 
-  ecs_include_docker_cleanup_cronjob = var.ecs_include_docker_cleanup_cronjob
-
   key_name = var.key_name
 
   rds_security_group         = data.terraform_remote_state.data_persistence.outputs.rds_security_group_id
@@ -91,7 +89,7 @@ module "cumulus" {
   tea_internal_api_endpoint     = module.thin_egress_app.internal_api_endpoint
   tea_external_api_endpoint     = module.thin_egress_app.api_endpoint
   tea_distribution_url_per_cmr_provider = var.tea_distribution_url_per_cmr_provider
-  
+
   sts_credentials_lambda_function_arn   = data.aws_lambda_function.sts_credentials.arn
   sts_policy_helper_lambda_function_arn = data.aws_lambda_function.sts_policy_helper.arn
   cmr_acl_based_credentials             = var.cmr_acl_based_credentials
@@ -116,6 +114,22 @@ module "cumulus" {
 
   archive_records_config = var.archive_records_config
   report_sns_topic_subscriber_arns = var.report_sns_topic_subscriber_arns
+  ecs_include_docker_cleanup_cronjob = var.ecs_include_docker_cleanup_cronjob
+
+  # Iceberg API configuration
+  deploy_iceberg_api                        = var.deploy_iceberg_api
+  iceberg_api_cpu                           = var.iceberg_api_cpu
+  iceberg_api_memory                        = var.iceberg_api_memory
+  cumulus_iceberg_api_image_version         = var.cumulus_iceberg_api_image_version
+  cumulus_iceberg_api_image_repository_url  = var.cumulus_iceberg_api_image_repository_url
+  api_service_autoscaling_min_capacity      = var.api_service_autoscaling_min_capacity
+  api_service_autoscaling_max_capacity      = var.api_service_autoscaling_max_capacity
+  api_service_autoscaling_target_cpu        = var.api_service_autoscaling_target_cpu
+  iceberg_s3_bucket                         = var.iceberg_s3_bucket
+  iceberg_namespace                         = local.iceberg_namespace
+  iceberg_health_check_grace_period_seconds = var.iceberg_health_check_grace_period_seconds
+  duckdb_max_pool_size                      = var.duckdb_max_pool_size
+  duckdb_pool_rebuild_interval_seconds      = var.duckdb_pool_rebuild_interval_seconds
 }
 
 resource "aws_security_group" "no_ingress_all_egress" {

cumulus/variables.tf

@@ -542,6 +542,7 @@ variable "allow_provider_mismatch_on_rule_filter" {
   type = bool
   default = false
 }
+
 variable "dynamic_throttled_queues" {
   type = list(object({
     queue_name      = string
@@ -562,3 +563,80 @@ variable "report_sns_topic_subscriber_arns" {
   default     = null
   description = "Account ARNs to supply to report SNS topics policy with subscribe action"
 }
+
+variable "deploy_iceberg_api" {
+  type        = bool
+  default     = false
+  description = "Whether to deploy the Iceberg API (hosted in ECS with limited endpoints)"
+}
+
+variable "iceberg_api_cpu" {
+  type        = number
+  default     = 512
+  description = "The amount of CPU units to reserve for the Iceberg API Fargate Task"
+}
+
+variable "iceberg_api_memory" {
+  type        = number
+  default     = 1024
+  description = "The amount of memory in MB to reserve for the Iceberg API Fargate Task"
+}
+
+variable "duckdb_max_pool_size" {
+  type        = number
+  default     = 3
+  description = "Maximum number of DuckDB connections in the connection pool"
+}
+
+variable "duckdb_pool_rebuild_interval_seconds" {
+  type        = number
+  default     = 18000
+  description = "Seconds between preemptive DuckDB idle-pool rebuilds"
+}
+
+variable "api_service_autoscaling_min_capacity" {
+  type        = number
+  default     = 1
+  description = "Minimum number of API service tasks to run"
+}
+
+variable "api_service_autoscaling_max_capacity" {
+  type        = number
+  default     = 2
+  description = "Maximum number of API service tasks to run"
+}
+
+variable "api_service_autoscaling_target_cpu" {
+  type        = number
+  default     = 70
+  description = "Target CPU utilization percentage for API service autoscaling"
+}
+
+variable "iceberg_health_check_grace_period_seconds" {
+  description = "Seconds to ignore failing load balancer health checks on newly instantiated ECS tasks"
+  type        = number
+  default     = 180
+}
+
+variable "cumulus_iceberg_api_image_version" {
+  description = "The version of the Cumulus Iceberg API image to use"
+  type        = string
+  default     = "latest"
+}
+
+variable "cumulus_iceberg_api_image_repository_url" {
+  description = "The repository URL of the Cumulus Iceberg API image to use"
+  type        = string
+  default     = null
+}
+
+variable "iceberg_s3_bucket" {
+  description = "Name of the S3 bucket the Iceberg API task needs read access to"
+  type        = string
+}
+
+variable "iceberg_namespace" {
+  description = "AWS Glue schema (database) name containing the Iceberg tables. This should be set for developer stacks that are enabling the Iceberg API, but left unset otherwise (it will be computed if needed)"
+  type        = string
+  default     = null
+}

data-persistence/main.tf

@@ -1,5 +1,5 @@
 module "data_persistence" {
-  source = "https://github.com/nasa/cumulus/releases/download/v22.2.2/terraform-aws-cumulus.zip//tf-modules/data-persistence"
+  source = "https://github.com/nasa/cumulus/releases/download/v22.2.5/terraform-aws-cumulus.zip//tf-modules/data-persistence"
 
   prefix                = local.prefix
   subnet_ids            = data.aws_subnets.subnet_ids.ids