diff --git a/CHANGELOG.md b/CHANGELOG.md index aa5e8ce..b9da5fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # CHANGELOG ## Unreleased +* Add iceberg target to Makefile to build Iceberg replication components * Add validation of deploy_name in env.sh * Expose `report_sns_topic_subscriber_arns` variable to cumulus.tf module @@ -35,12 +36,12 @@ * Update `cumulus` module to allow for optional use of AWS Secrets Manager for `archive_api_url, urs_client_password, metrics_es_password, cmr_password, cmr_username, lzards_launchpad_passphrase, launchpad_passphrase, token_secret` via `configuration_secret` variable * Update GH actions `tflint` to v0.61.0, update GH actions `checkout` to v4 * Add `send_pan_task` to `cumulus` module output -* Update Docker image to install docker CLI +* Update Docker image to install docker CLI * Update Makefile to allow docker-in-docker on MacOS hosts ## v21.0.1.0 * Upgrade to [Cumulus v21.0.1](https://github.com/nasa/cumulus/releases/tag/v21.0.1) -* **NOTE** This version of Cumulus requires a manual update to the PostgreSQL database in the production environment. Please follow the instructions in [Update granules to include producer_granule_id](https://nasa.github.io/cumulus/docs/next/upgrade-notes/update-granules-to-include-producer_granule_id) +* **NOTE** This version of Cumulus requires a manual update to the PostgreSQL database in the production environment. Please follow the instructions in [Update granules to include producer_granule_id](https://nasa.github.io/cumulus/docs/next/upgrade-notes/update-granules-to-include-producer_granule_id) ## v20.3.0.0 * Upgrade to [Cumulus v20.3.0](https://github.com/nasa/cumulus/releases/tag/v20.3.0) diff --git a/Makefile b/Makefile index 8849d03..464cbc9 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ # PYTHON_VER: python3 # --------------------------- -DOCKER_TAG := v22.2.2.0 +DOCKER_TAG := v22.2.5.0 export TF_IN_AUTOMATION="true" export TF_VAR_MATURITY=${MATURITY} export TF_VAR_DEPLOY_NAME=${DEPLOY_NAME} @@ -83,7 +83,7 @@ container-shell: --name=cirrus-core \ cirrus-core:$(DOCKER_TAG) \ bash - + .PHONY: shell shell: DAAC_DIR="${DAAC_DIR}" \ @@ -342,6 +342,7 @@ all: \ tf \ daac \ data-persistence \ + iceberg \ cumulus \ workflows @@ -351,5 +352,6 @@ initial-deploy: \ daac \ rds \ data-persistence \ + iceberg \ cumulus \ workflows diff --git a/README.md b/README.md index 8933ae2..d9addeb 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,7 @@ If you want to deploy everything besides the `rds` module you can run the comman $ make daac $ make rds $ make data-persistence + $ make iceberg $ make cumulus $ make workflows diff --git a/cumulus/locals.tf b/cumulus/locals.tf index cb9b8a9..a104f6e 100644 --- a/cumulus/locals.tf +++ b/cumulus/locals.tf @@ -1,5 +1,6 @@ locals { prefix = "${var.DEPLOY_NAME}-cumulus-${var.MATURITY}" + iceberg_namespace = var.iceberg_namespace != null ? var.iceberg_namespace : replace("${local.prefix}_iceberg", "-", "_") buckets = data.terraform_remote_state.daac.outputs.bucket_map @@ -53,10 +54,10 @@ locals { lzards_launchpad_passphrase = sensitive(lookup(local.configuration_secret_values, "lzards_launchpad_passphrase", var.lzards_launchpad_passphrase)) token_secret = sensitive(lookup(local.configuration_secret_values, "token_secret", var.token_secret)) urs_client_id = sensitive(lookup(local.configuration_secret_values, "urs_client_id", var.urs_client_id)) - + urs_tea_client_id = var.urs_tea_client_id != null ? var.urs_tea_client_id : local.urs_client_id urs_tea_client_password = var.urs_tea_client_password != null ? var.urs_tea_client_password : local.urs_client_password - + throttled_queues = [ for q in var.dynamic_throttled_queues : { url = "https://sqs.${data.aws_region.current.name}.amazonaws.com/${data.aws_caller_identity.current.account_id}/${local.prefix}-${q.queue_name}" diff --git a/cumulus/main.tf b/cumulus/main.tf index e0bb67d..3feeb42 100644 --- a/cumulus/main.tf +++ b/cumulus/main.tf @@ -1,5 +1,5 @@ module "cumulus" { - source = "https://github.com/nasa/cumulus/releases/download/v22.2.2/terraform-aws-cumulus.zip//tf-modules/cumulus" + source = "https://github.com/nasa/cumulus/releases/download/v22.2.5/terraform-aws-cumulus.zip//tf-modules/cumulus" cumulus_message_adapter_lambda_layer_version_arn = data.terraform_remote_state.daac.outputs.cma_layer_arn @@ -21,8 +21,6 @@ module "cumulus" { ecs_cluster_instance_type = var.ecs_cluster_instance_type ecs_cluster_instance_docker_volume_size = var.ecs_cluster_instance_docker_volume_size - ecs_include_docker_cleanup_cronjob = var.ecs_include_docker_cleanup_cronjob - key_name = var.key_name rds_security_group = data.terraform_remote_state.data_persistence.outputs.rds_security_group_id @@ -91,7 +89,7 @@ module "cumulus" { tea_internal_api_endpoint = module.thin_egress_app.internal_api_endpoint tea_external_api_endpoint = module.thin_egress_app.api_endpoint tea_distribution_url_per_cmr_provider = var.tea_distribution_url_per_cmr_provider - + sts_credentials_lambda_function_arn = data.aws_lambda_function.sts_credentials.arn sts_policy_helper_lambda_function_arn = data.aws_lambda_function.sts_policy_helper.arn cmr_acl_based_credentials = var.cmr_acl_based_credentials @@ -116,6 +114,22 @@ module "cumulus" { archive_records_config = var.archive_records_config report_sns_topic_subscriber_arns = var.report_sns_topic_subscriber_arns + ecs_include_docker_cleanup_cronjob = var.ecs_include_docker_cleanup_cronjob + + # Iceberg API configuration + deploy_iceberg_api = var.deploy_iceberg_api + iceberg_api_cpu = var.iceberg_api_cpu + iceberg_api_memory = var.iceberg_api_memory + cumulus_iceberg_api_image_version = var.cumulus_iceberg_api_image_version + cumulus_iceberg_api_image_repository_url = var.cumulus_iceberg_api_image_repository_url + api_service_autoscaling_min_capacity = var.api_service_autoscaling_min_capacity + api_service_autoscaling_max_capacity = var.api_service_autoscaling_max_capacity + api_service_autoscaling_target_cpu = var.api_service_autoscaling_target_cpu + iceberg_s3_bucket = var.iceberg_s3_bucket + iceberg_namespace = local.iceberg_namespace + iceberg_health_check_grace_period_seconds = var.iceberg_health_check_grace_period_seconds + duckdb_max_pool_size = var.duckdb_max_pool_size + duckdb_pool_rebuild_interval_seconds = var.duckdb_pool_rebuild_interval_seconds } resource "aws_security_group" "no_ingress_all_egress" { diff --git a/cumulus/variables.tf b/cumulus/variables.tf index 22ec9ff..1ab1039 100644 --- a/cumulus/variables.tf +++ b/cumulus/variables.tf @@ -542,6 +542,7 @@ variable "allow_provider_mismatch_on_rule_filter" { type = bool default = false } + variable "dynamic_throttled_queues" { type = list(object({ queue_name = string @@ -562,3 +563,80 @@ variable "report_sns_topic_subscriber_arns" { default = null description = "Account ARNs to supply to report SNS topics policy with subscribe action" } + +variable "deploy_iceberg_api" { + type = bool + default = false + description = "Whether to deploy the Iceberg API (hosted in ECS with limited endpoints)" +} + +variable "iceberg_api_cpu" { + type = number + default = 512 + description = "The amount of CPU units to reserve for the Iceberg API Fargate Task" +} + +variable "iceberg_api_memory" { + type = number + default = 1024 + description = "The amount of memory in MB to reserve for the Iceberg API Fargate Task" +} + +variable "duckdb_max_pool_size" { + type = number + default = 3 + description = "Maximum number of DuckDB connections in the connection pool" +} + +variable "duckdb_pool_rebuild_interval_seconds" { + type = number + default = 18000 + description = "Seconds between preemptive DuckDB idle-pool rebuilds" +} + +variable "api_service_autoscaling_min_capacity" { + type = number + default = 1 + description = "Minimum number of API service tasks to run" +} + +variable "api_service_autoscaling_max_capacity" { + type = number + default = 2 + description = "Maximum number of API service tasks to run" +} + +variable "api_service_autoscaling_target_cpu" { + type = number + default = 70 + description = "Target CPU utilization percentage for API service autoscaling" +} + +variable "iceberg_health_check_grace_period_seconds" { + description = "Seconds to ignore failing load balancer health checks on newly instantiated ECS tasks" + type = number + default = 180 +} + +variable "cumulus_iceberg_api_image_version" { + description = "The version of the Cumulus Iceberg API image to use" + type = string + default = "latest" +} + +variable "cumulus_iceberg_api_image_repository_url" { + description = "The repository URL of the Cumulus Iceberg API image to use" + type = string + default = null +} + +variable "iceberg_s3_bucket" { + description = "Name of the S3 bucket the Iceberg API task needs read access to" + type = string +} + +variable "iceberg_namespace" { + description = "AWS Glue schema (database) name containing the Iceberg tables. This should be set for developer stacks that are enabling the Iceberg API, but left unset otherwise (it will be computed if needed)" + type = string + default = null +} \ No newline at end of file diff --git a/data-persistence/main.tf b/data-persistence/main.tf index 25f3135..0f9f081 100644 --- a/data-persistence/main.tf +++ b/data-persistence/main.tf @@ -1,5 +1,5 @@ module "data_persistence" { - source = "https://github.com/nasa/cumulus/releases/download/v22.2.2/terraform-aws-cumulus.zip//tf-modules/data-persistence" + source = "https://github.com/nasa/cumulus/releases/download/v22.2.5/terraform-aws-cumulus.zip//tf-modules/data-persistence" prefix = local.prefix subnet_ids = data.aws_subnets.subnet_ids.ids