From 3cf5c1d775b08c703cc0302834f3097514be0eed Mon Sep 17 00:00:00 2001 From: Harsh Date: Fri, 26 Jun 2026 15:05:37 +0000 Subject: [PATCH 1/2] feat: automate snapshot creation and scheduling Signed-off-by: Harsh --- backend/apps/owasp/Makefile | 6 + .../commands/owasp_create_snapshot.py | 145 +++++++++++++ .../commands/owasp_create_snapshot_test.py | 190 ++++++++++++++++++ infrastructure/modules/tasks/README.md | 3 + infrastructure/modules/tasks/main.tf | 33 +++ infrastructure/modules/tasks/variables.tf | 12 ++ 6 files changed, 389 insertions(+) create mode 100644 backend/apps/owasp/management/commands/owasp_create_snapshot.py create mode 100644 backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py diff --git a/backend/apps/owasp/Makefile b/backend/apps/owasp/Makefile index 72ca53001f..8f47f49f5c 100644 --- a/backend/apps/owasp/Makefile +++ b/backend/apps/owasp/Makefile @@ -50,6 +50,12 @@ owasp-generate-community-snapshot-video: nest-snapshot-video \ python manage.py owasp_generate_community_snapshot_video $(snapshot_key) /home/owasp/generated_videos +SNAPSHOT_FREQUENCY ?= weekly + +owasp-create-snapshot: + @echo "Creating OWASP community snapshot" + @CMD="python manage.py owasp_create_snapshot --frequency $(SNAPSHOT_FREQUENCY)" $(MAKE) exec-backend-command + owasp-process-snapshots: @echo "Processing OWASP snapshots" @CMD="python manage.py owasp_process_snapshots" $(MAKE) exec-backend-command diff --git a/backend/apps/owasp/management/commands/owasp_create_snapshot.py b/backend/apps/owasp/management/commands/owasp_create_snapshot.py new file mode 100644 index 0000000000..21f712b252 --- /dev/null +++ b/backend/apps/owasp/management/commands/owasp_create_snapshot.py @@ -0,0 +1,145 @@ +"""Django management command for creating OWASP community snapshots.""" + +import logging +from datetime import UTC, datetime, timedelta + +from django.core.management.base import BaseCommand + +from apps.owasp.models.snapshot import Snapshot + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """Command to create a community snapshot with automatic date range calculation.""" + + help = "Create a community snapshot for a given frequency (weekly or monthly)" + + def add_arguments(self, parser): + """Add command-line arguments. + + Args: + parser (argparse.ArgumentParser): The argument parser instance. + + """ + parser.add_argument( + "--frequency", + type=str, + choices=[Snapshot.Frequency.WEEKLY, Snapshot.Frequency.MONTHLY], + default=Snapshot.Frequency.WEEKLY, + help="Snapshot frequency: weekly (default) or monthly.", + ) + + def handle(self, *args, **options): + """Handle command execution. + + Args: + *args: Variable length argument list. + **options: Arbitrary keyword arguments containing command options. + + """ + frequency = options["frequency"] + start_at, end_at = self.calculate_date_range(frequency) + + self.stdout.write(f"Creating {frequency} snapshot") + self.stdout.write(f"Period: {start_at.date()} to {end_at.date()}") + logger.info( + "Creating %s snapshot from %s to %s", + frequency, + start_at.date(), + end_at.date(), + ) + + # Generate key to check for duplicates before creating. + key = self.generate_key(start_at, frequency) + if Snapshot.objects.filter(key=key).exists(): + self.stdout.write( + self.style.WARNING(f"Snapshot with key '{key}' already exists, skipping creation") + ) + logger.info("Snapshot with key '%s' already exists, skipping", key) + return + + snapshot = Snapshot.objects.create( + key=key, + start_at=start_at, + end_at=end_at, + title=self.generate_title(start_at, frequency), + status=Snapshot.Status.PENDING, + ) + + self.stdout.write( + self.style.SUCCESS( + f"Snapshot created successfully (ID: {snapshot.id}, key: {snapshot.key})" + ) + ) + logger.info( + "Snapshot created: id=%s, key=%s, frequency=%s", + snapshot.id, + snapshot.key, + frequency, + ) + + @staticmethod + def calculate_date_range(frequency): + """Calculate the start and end dates for the snapshot period. + + Args: + frequency (str): The snapshot frequency (weekly or monthly). + + Returns: + tuple[datetime, datetime]: A tuple of (start_at, end_at) datetimes. + + """ + current = datetime.now(tz=UTC) + + if frequency == Snapshot.Frequency.WEEKLY: + # Last week: Monday 00:00:00 to Sunday 23:59:59 + days_since_monday = current.weekday() + last_monday = current - timedelta(days=days_since_monday + 7) + start_at = last_monday.replace(hour=0, minute=0, second=0, microsecond=0) + end_at = start_at + timedelta(days=6, hours=23, minutes=59, seconds=59) + else: + # Last month: 1st 00:00:00 to last day 23:59:59 + first_of_current_month = current.replace( + day=1, hour=0, minute=0, second=0, microsecond=0 + ) + end_at = first_of_current_month - timedelta(seconds=1) + start_at = end_at.replace(day=1, hour=0, minute=0, second=0, microsecond=0) + + return start_at, end_at + + @staticmethod + def generate_key(start_at, frequency): + """Generate the snapshot key based on the start date and frequency. + + This mirrors the logic in the Snapshot model's save() method. + + Args: + start_at (datetime): The start date of the snapshot period. + frequency (str): The snapshot frequency (weekly or monthly). + + Returns: + str: The generated snapshot key. + + """ + if frequency == Snapshot.Frequency.WEEKLY: + iso_year, iso_week, _ = start_at.isocalendar() + return f"{iso_year}-W{iso_week:02d}" + return start_at.strftime("%Y-%m") + + @staticmethod + def generate_title(start_at, frequency): + """Generate a human-readable title for the snapshot. + + Args: + start_at (datetime): The start date of the snapshot period. + frequency (str): The snapshot frequency (weekly or monthly). + + Returns: + str: The generated snapshot title. + + """ + if frequency == Snapshot.Frequency.WEEKLY: + iso_year, iso_week, _ = start_at.isocalendar() + return f"Week {iso_week} {iso_year} OWASP Community Snapshot" + return f"{start_at.strftime('%B %Y')} OWASP Community Snapshot" diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py new file mode 100644 index 0000000000..9e3dba5e51 --- /dev/null +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py @@ -0,0 +1,190 @@ +from datetime import UTC, datetime +from unittest import mock + +import pytest + +from apps.owasp.management.commands.owasp_create_snapshot import Command +from apps.owasp.models.snapshot import Snapshot + + +class TestCreateSnapshot: + """Tests for owasp_create_snapshot management command.""" + + def test_handle_creates_snapshot_weekly(self): + """Test that handle creates a weekly snapshot successfully.""" + command = Command() + mock_snapshot = mock.MagicMock() + mock_snapshot.id = 1 + mock_snapshot.key = "2026-W24" + + with ( + mock.patch.object( + command, + "calculate_date_range", + return_value=( + datetime(2026, 6, 8, tzinfo=UTC), + datetime(2026, 6, 14, 23, 59, 59, tzinfo=UTC), + ), + ), + mock.patch.object(command, "generate_key", return_value="2026-W24"), + mock.patch.object( + command, "generate_title", return_value="Week 24 2026 OWASP Community Snapshot" + ), + mock.patch("apps.owasp.models.snapshot.Snapshot.objects") as mock_objects, + ): + mock_objects.filter.return_value.exists.return_value = False + mock_objects.create.return_value = mock_snapshot + + command.handle(frequency="weekly") + + mock_objects.create.assert_called_once_with( + key="2026-W24", + start_at=datetime(2026, 6, 8, tzinfo=UTC), + end_at=datetime(2026, 6, 14, 23, 59, 59, tzinfo=UTC), + title="Week 24 2026 OWASP Community Snapshot", + status=Snapshot.Status.PENDING, + ) + + def test_handle_creates_snapshot_monthly(self): + """Test that handle creates a monthly snapshot successfully.""" + command = Command() + mock_snapshot = mock.MagicMock() + mock_snapshot.id = 1 + mock_snapshot.key = "2026-05" + + with ( + mock.patch.object( + command, + "calculate_date_range", + return_value=( + datetime(2026, 5, 1, tzinfo=UTC), + datetime(2026, 5, 31, 23, 59, 59, tzinfo=UTC), + ), + ), + mock.patch.object(command, "generate_key", return_value="2026-05"), + mock.patch.object( + command, "generate_title", return_value="May 2026 OWASP Community Snapshot" + ), + mock.patch("apps.owasp.models.snapshot.Snapshot.objects") as mock_objects, + ): + mock_objects.filter.return_value.exists.return_value = False + mock_objects.create.return_value = mock_snapshot + + command.handle(frequency="monthly") + + mock_objects.create.assert_called_once_with( + key="2026-05", + start_at=datetime(2026, 5, 1, tzinfo=UTC), + end_at=datetime(2026, 5, 31, 23, 59, 59, tzinfo=UTC), + title="May 2026 OWASP Community Snapshot", + status=Snapshot.Status.PENDING, + ) + + def test_handle_skips_duplicate_snapshot(self): + """Test that handle skips creation when a snapshot with the same key exists.""" + command = Command() + + with ( + mock.patch.object( + command, + "calculate_date_range", + return_value=( + datetime(2026, 6, 8, tzinfo=UTC), + datetime(2026, 6, 14, 23, 59, 59, tzinfo=UTC), + ), + ), + mock.patch.object(command, "generate_key", return_value="2026-W24"), + mock.patch("apps.owasp.models.snapshot.Snapshot.objects") as mock_objects, + mock.patch( + "apps.owasp.management.commands.owasp_create_snapshot.logger" + ) as mock_logger, + ): + mock_objects.filter.return_value.exists.return_value = True + + command.handle(frequency="weekly") + + mock_objects.create.assert_not_called() + mock_logger.info.assert_any_call( + "Snapshot with key '%s' already exists, skipping", "2026-W24" + ) + + @pytest.mark.parametrize( + ("frozen_now", "expected_start", "expected_end"), + [ + pytest.param( + datetime(2026, 6, 15, 12, 0, 0, tzinfo=UTC), + datetime(2026, 6, 8, 0, 0, 0, tzinfo=UTC), + datetime(2026, 6, 14, 23, 59, 59, tzinfo=UTC), + id="monday-calculates-last-week", + ), + pytest.param( + datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC), + datetime(2026, 6, 1, 0, 0, 0, tzinfo=UTC), + datetime(2026, 6, 7, 23, 59, 59, tzinfo=UTC), + id="thursday-calculates-last-week", + ), + ], + ) + def test_calculate_date_range_weekly(self, frozen_now, expected_start, expected_end): + """Test weekly date range calculation.""" + with mock.patch( + "apps.owasp.management.commands.owasp_create_snapshot.datetime" + ) as mock_datetime: + mock_datetime.now.return_value = frozen_now + mock_datetime.side_effect = datetime + start_at, end_at = Command.calculate_date_range("weekly") + + assert start_at == expected_start + assert end_at == expected_end + + def test_calculate_date_range_monthly(self): + """Test monthly date range calculation.""" + frozen_now = datetime(2026, 6, 15, 12, 0, 0, tzinfo=UTC) + + with mock.patch( + "apps.owasp.management.commands.owasp_create_snapshot.datetime" + ) as mock_datetime: + mock_datetime.now.return_value = frozen_now + mock_datetime.side_effect = datetime + start_at, end_at = Command.calculate_date_range("monthly") + + assert start_at == datetime(2026, 5, 1, 0, 0, 0, tzinfo=UTC) + assert end_at == datetime(2026, 5, 31, 23, 59, 59, tzinfo=UTC) + + def test_generate_key_weekly(self): + """Test key generation for weekly snapshots.""" + start_at = datetime(2026, 6, 8, tzinfo=UTC) + key = Command.generate_key(start_at, "weekly") + assert key == "2026-W24" + + def test_generate_key_monthly(self): + """Test key generation for monthly snapshots.""" + start_at = datetime(2026, 5, 1, tzinfo=UTC) + key = Command.generate_key(start_at, "monthly") + assert key == "2026-05" + + def test_generate_title_weekly(self): + """Test title generation for weekly snapshots.""" + start_at = datetime(2026, 6, 8, tzinfo=UTC) + title = Command.generate_title(start_at, "weekly") + assert title == "Week 24 2026 OWASP Community Snapshot" + + def test_generate_title_monthly(self): + """Test title generation for monthly snapshots.""" + start_at = datetime(2026, 5, 1, tzinfo=UTC) + title = Command.generate_title(start_at, "monthly") + assert title == "May 2026 OWASP Community Snapshot" + + def test_generate_key_matches_model_save_weekly(self): + """Test that generate_key produces the same key as the model's save method.""" + start_at = datetime(2026, 1, 5, tzinfo=UTC) + key = Command.generate_key(start_at, "weekly") + iso_year, iso_week, _ = start_at.isocalendar() + expected = f"{iso_year}-W{iso_week:02d}" + assert key == expected + + def test_generate_key_matches_model_save_monthly(self): + """Test that generate_key produces the same key as the model's save method.""" + start_at = datetime(2026, 12, 1, tzinfo=UTC) + key = Command.generate_key(start_at, "monthly") + assert key == "2026-12" diff --git a/infrastructure/modules/tasks/README.md b/infrastructure/modules/tasks/README.md index fe6a8df84f..9483005142 100644 --- a/infrastructure/modules/tasks/README.md +++ b/infrastructure/modules/tasks/README.md @@ -20,6 +20,7 @@ | [load\_data\_task](#module\_load\_data\_task) | ./modules/task | n/a | | [mentorship\_sync\_modules\_data](#module\_mentorship\_sync\_modules\_data) | ./modules/task | n/a | | [migrate\_task](#module\_migrate\_task) | ./modules/task | n/a | +| [owasp\_create\_snapshot\_task](#module\_owasp\_create\_snapshot\_task) | ./modules/task | n/a | | [owasp\_update\_project\_health\_metrics\_task](#module\_owasp\_update\_project\_health\_metrics\_task) | ./modules/task | n/a | | [owasp\_update\_project\_health\_scores\_task](#module\_owasp\_update\_project\_health\_scores\_task) | ./modules/task | n/a | | [slack\_sync\_data\_task](#module\_slack\_sync\_data\_task) | ./modules/task | n/a | @@ -53,6 +54,8 @@ | [aws\_region](#input\_aws\_region) | The AWS region. | `string` | n/a | yes | | [common\_tags](#input\_common\_tags) | A map of common tags to apply to all resources. | `map(string)` | `{}` | no | | [container\_parameters\_arns](#input\_container\_parameters\_arns) | Map of environment variable names to the ARNs of all SSM parameters. | `map(string)` | `{}` | no | +| [create\_snapshot\_task\_cpu](#input\_create\_snapshot\_task\_cpu) | The CPU for the create-snapshot task. | `string` | `"256"` | no | +| [create\_snapshot\_task\_memory](#input\_create\_snapshot\_task\_memory) | The memory for the create-snapshot task. | `string` | `"1024"` | no | | [ecr\_repository\_arn](#input\_ecr\_repository\_arn) | The ARN of the ECR repository for the backend image. | `string` | n/a | yes | | [ecr\_repository\_url](#input\_ecr\_repository\_url) | The URL of the ECR repository for the backend image. | `string` | n/a | yes | | [ecs\_sg\_id](#input\_ecs\_sg\_id) | The ID of the security group for the ECS tasks. | `string` | n/a | yes | diff --git a/infrastructure/modules/tasks/main.tf b/infrastructure/modules/tasks/main.tf index 42cf9959a7..da5c28d59b 100644 --- a/infrastructure/modules/tasks/main.tf +++ b/infrastructure/modules/tasks/main.tf @@ -12,6 +12,7 @@ terraform { data "aws_caller_identity" "current" {} locals { + create_snapshot_schedule_expression = var.enable_cron_tasks ? "cron(0 06 ? * MON *)" : null mentorship_sync_modules_data_schedule_expression = var.enable_cron_tasks ? "cron(30 06 * * ? *)" : null slack_sync_data_schedule_expression = var.enable_cron_tasks ? "cron(0 */6 ? * MON-FRI *)" : null sync_data_schedule_expression = var.enable_cron_tasks ? "cron(17 05 * * ? *)" : null @@ -219,6 +220,38 @@ resource "aws_iam_role_policy_attachment" "event_bridge_policy_attachment" { role = aws_iam_role.event_bridge_role.name } +module "owasp_create_snapshot_task" { + source = "./modules/task" + + assign_public_ip = var.assign_public_ip + aws_region = var.aws_region + command = [ + "/bin/sh", + "-c", + <<-EOT + set -e + EXEC_MODE=direct make owasp-create-snapshot + EXEC_MODE=direct make owasp-process-snapshots + EOT + ] + common_tags = var.common_tags + container_parameters_arns = var.container_parameters_arns + cpu = var.create_snapshot_task_cpu + ecs_cluster_arn = aws_ecs_cluster.main.arn + ecs_tasks_execution_role_arn = aws_iam_role.ecs_tasks_execution_role.arn + environment = var.environment + event_bridge_role_arn = aws_iam_role.event_bridge_role.arn + image_url = "${var.ecr_repository_url}:${var.image_tag}" + kms_key_arn = var.kms_key_arn + memory = var.create_snapshot_task_memory + project_name = var.project_name + schedule_expression = local.create_snapshot_schedule_expression + security_group_ids = [var.ecs_sg_id] + subnet_ids = var.subnet_ids + task_name = "owasp-create-snapshot" + use_fargate_spot = var.use_fargate_spot +} + module "sync_data_task" { source = "./modules/task" diff --git a/infrastructure/modules/tasks/variables.tf b/infrastructure/modules/tasks/variables.tf index 556130352a..137e755939 100644 --- a/infrastructure/modules/tasks/variables.tf +++ b/infrastructure/modules/tasks/variables.tf @@ -21,6 +21,18 @@ variable "container_parameters_arns" { default = {} } +variable "create_snapshot_task_cpu" { + description = "The CPU for the create-snapshot task." + type = string + default = "256" +} + +variable "create_snapshot_task_memory" { + description = "The memory for the create-snapshot task." + type = string + default = "1024" +} + variable "ecr_repository_arn" { description = "The ARN of the ECR repository for the backend image." type = string From c1581e0ebddc2401a126b366eb4091d1d2b1b012 Mon Sep 17 00:00:00 2001 From: Harsh Date: Fri, 26 Jun 2026 18:47:14 +0000 Subject: [PATCH 2/2] update code Signed-off-by: Harsh --- backend/apps/owasp/management/commands/owasp_create_snapshot.py | 1 + .../owasp/management/commands/owasp_create_snapshot_test.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/apps/owasp/management/commands/owasp_create_snapshot.py b/backend/apps/owasp/management/commands/owasp_create_snapshot.py index 21f712b252..cf6b6727ab 100644 --- a/backend/apps/owasp/management/commands/owasp_create_snapshot.py +++ b/backend/apps/owasp/management/commands/owasp_create_snapshot.py @@ -61,6 +61,7 @@ def handle(self, *args, **options): snapshot = Snapshot.objects.create( key=key, + frequency=frequency, start_at=start_at, end_at=end_at, title=self.generate_title(start_at, frequency), diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py index 9e3dba5e51..76ed426a73 100644 --- a/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_create_snapshot_test.py @@ -39,6 +39,7 @@ def test_handle_creates_snapshot_weekly(self): mock_objects.create.assert_called_once_with( key="2026-W24", + frequency="weekly", start_at=datetime(2026, 6, 8, tzinfo=UTC), end_at=datetime(2026, 6, 14, 23, 59, 59, tzinfo=UTC), title="Week 24 2026 OWASP Community Snapshot", @@ -74,6 +75,7 @@ def test_handle_creates_snapshot_monthly(self): mock_objects.create.assert_called_once_with( key="2026-05", + frequency="monthly", start_at=datetime(2026, 5, 1, tzinfo=UTC), end_at=datetime(2026, 5, 31, 23, 59, 59, tzinfo=UTC), title="May 2026 OWASP Community Snapshot",