From cf7ab34c57e6ae2470d637bac9dd312f6a8fe49b Mon Sep 17 00:00:00 2001 From: Julien Cornebise Date: Tue, 28 Jul 2026 01:52:28 +0200 Subject: [PATCH] =?UTF-8?q?Clojure->Python=20CUTOVER:=20Step=203=20?= =?UTF-8?q?=E2=80=94=20decommission=20the=20Clojure=20math=20service=20eve?= =?UTF-8?q?rywhere?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the flip (Step 2) has run stable for the agreed period, this removes the Clojure math service from every place that could still start or build it. The Python engine becomes the one and only math service. The Clojure SOURCE CODE stays in the repo until Step 4. ## What's in this PR - `docker-compose.yml`: the `math` (Clojure) service is deleted. `math-python` now starts by default with `make start` (no longer opt-in) and reads the SAME setting the server reads — `MATH_ENV=${MATH_ENV:-prod}`, exactly the line the Clojure service had. The transitional setting `MATH_PYTHON_ENV` (used for the flip) stops existing here. - `docker-compose.dev.yml`: the Clojure REPL developer override deleted. - `docker-compose.test.yml` (what CI runs): Clojure math replaced with a `math-python` service, built with `USE_CPU_TORCH=true` so CI reuses the already-built image layers instead of building a second multi-GB image with GPU libraries nothing uses. - `.github/workflows/deploy-alpha-aws.yml`: the lines that built and pushed the Clojure math image are deleted — with the service gone they would fail and block EVERY deploy. (That registry copy was never used by deploys anyway; instances build locally.) - Deploy scripts (US + the forgotten euro copy): the math host starts `math-python`; `down --remove-orphans` added so a leftover Clojure container cannot survive a deploy. - `example.env` / `test.env`: dead Clojure-only settings removed; the poller section rewritten. - `math/README.md`: banner saying the service is retired and the source tree goes away at Step 4. ## The settings handover At the moment this deploys, the instance `.env` still contains `MATH_PYTHON_ENV=prod` (from Step 2) and `MATH_ENV=prod` (the server's) — after this PR the poller simply reads `MATH_ENV=prod`. Same value, no behavior change. Optional cleanup afterwards (Julien): delete `MATH_PYTHON_ENV` from the secret. ## Merge gate Only after the Step 2 soak. Note the rollback cost changes here: bringing Clojure math back stops being a single compose command. ## Rollback 1. First point the Python engine at a different `math_env` than the server's (secret `MATH_PYTHON_ENV='python'`, propagated via a deploy or the manual `.env` route — see Step 2's note that restarts do not pick up new settings), so the returning Clojure engine is the only writer of the server's rows. 2. Revert this PR and redeploy. 3. Run `docker compose up -d math` yourself — the revert+redeploy alone still starts only `math-python`. The old Clojure rows were never deleted; the engine resumes from the rows as they are. Series: Step 0 = #2685. Nothing merges without Julien's explicit go. commit-id:1f679952 --- .github/workflows/deploy-alpha-aws.yml | 6 ++-- docker-compose.dev.yml | 11 ------- docker-compose.test.yml | 21 +++++++++---- docker-compose.yml | 43 ++++++-------------------- example.env | 17 +++++----- math/README.md | 10 +++++- scripts-euro/after_install.sh | 6 ++-- scripts/after_install.sh | 28 +++++++++-------- test.env | 1 - 9 files changed, 65 insertions(+), 78 deletions(-) diff --git a/.github/workflows/deploy-alpha-aws.yml b/.github/workflows/deploy-alpha-aws.yml index 6dd26a8dc1..7aafca10d8 100644 --- a/.github/workflows/deploy-alpha-aws.yml +++ b/.github/workflows/deploy-alpha-aws.yml @@ -38,8 +38,10 @@ jobs: docker-compose build server docker push $ECR_REGISTRY/polis/server:latest - docker-compose build math - docker push $ECR_REGISTRY/polis/math:latest + # The Clojure math image is gone (cutover Step 3). No math-side + # push is needed here: instances build locally in + # after_install.sh (`docker-compose up --build`); the ECR tag + # was never pulled by the deploy path. # ===================================================== # == JOB 2: DEPLOY TO PRODUCTION (US) == diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 23a2062cc9..1a6ee6ccb9 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -29,17 +29,6 @@ services: # Tell Node.js to trust the mkcert CA for HTTPS connections - NODE_EXTRA_CA_CERTS=/root/.simulacrum/certs/rootCA.pem - math: - # command: clojure -X:dev-poller - volumes: - - ./math:/app - ports: - # nrepl port - - "18975:18975" - environment: - - CHOKIDAR_USEPOLLING=true - - DD_ENABLED=false - postgres: restart: no ports: diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 9548185ce8..f908cb4ed3 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -55,18 +55,27 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - math: + # Python math poller — THE math engine (the Clojure `math` service was + # decommissioned at cutover Step #3). Same env var + default the old clj + # test service used. + math-python: build: - context: ./math + context: ./delphi + target: final + # Share every layer with the delphi service's test build (CPU torch); + # without this the default CUDA wheels build a second multi-GB image + # in CI for no benefit — the poller needs no torch at runtime. + args: + USE_CPU_TORCH: "true" + command: ["python", "scripts/math_poller.py"] labels: polis_tag: test environment: - DATABASE_URL=${DATABASE_URL} - - DD_ENABLED=false - - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn} + - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable} + - POSTGRES_CONNECT_TIMEOUT=${POSTGRES_CONNECT_TIMEOUT:-5} + - LOG_LEVEL=${DELPHI_LOG_LEVEL:-INFO} - MATH_ENV=${MATH_ENV:-dev} - - WEBSERVER_USERNAME=${WEBSERVER_USERNAME} - - WEBSERVER_PASS=${WEBSERVER_PASS} networks: - polis-test restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index b7bd0b52cb..90395eccd4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,26 +76,6 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - math: - image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/math:latest - build: - context: ./math - labels: - polis_tag: ${TAG:-dev} - environment: - - DATABASE_URL=${DATABASE_URL} - - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn} - - MATH_ENV=${MATH_ENV:-prod} - - WEBSERVER_USERNAME=${WEBSERVER_USERNAME} - - WEBSERVER_PASS=${WEBSERVER_PASS} - # Database connection pool configuration for math service - - DATABASE_POOL_SIZE=${MATH_DATABASE_POOL_SIZE:-10} - networks: - - "polis-net" - restart: unless-stopped - extra_hosts: - - "host.docker.internal:host-gateway" - delphi: image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest build: @@ -145,13 +125,11 @@ services: cpus: ${DELPHI_CONTAINER_CPUS:-2} restart: unless-stopped - # Python math poller — the eventual replacement for the Clojure `math` - # container. Profile-gated so it only runs when explicitly requested - # (`--profile math-python`). Reuses the delphi build target and overrides the - # command to run the poller CLI. Defaults to SHADOW mode: writes under a - # DISTINCT math_env (`python`) so its rows are invisible to the prod server - # (UNIQUE(zid, math_env)) — zero production risk while parity is validated. - # See delphi/docs/MATH_POLLER_DESIGN.md §4. + # Python math poller — THE math engine (the Clojure `math` container was + # decommissioned at cutover Step #3). Reuses the delphi build target and + # overrides the command to run the poller CLI. Reads/writes the same + # MATH_ENV the server reads, exactly as the Clojure service did. + # See delphi/docs/MATH_POLLER_DESIGN.md §4 + docs/CUTOVER_RUNBOOK.md. math-python: image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest build: @@ -165,8 +143,9 @@ services: - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable} - POSTGRES_CONNECT_TIMEOUT=${POSTGRES_CONNECT_TIMEOUT:-30} - LOG_LEVEL=${DELPHI_LOG_LEVEL:-INFO} - # Shadow-mode math_env (distinct from the Clojure math service's MATH_ENV). - - MATH_ENV=${MATH_PYTHON_ENV:-python} + # Same env var + default the Clojure math service used — the server + # reads the rows written under this math_env. + - MATH_ENV=${MATH_ENV:-prod} # Poll cadences (ms) and boot window (days). - POLL_VOTE_INTERVAL_MS=${POLL_VOTE_INTERVAL_MS:-1000} - POLL_MOD_INTERVAL_MS=${POLL_MOD_INTERVAL_MS:-1000} @@ -189,15 +168,13 @@ services: - "host.docker.internal:host-gateway" # Hard memory backstop (mirrors the delphi service). The in-memory conv cache # never evicted by default (Clojure's 4h reboot was the de-facto cap, which we - # dropped); this limit bounds a long shadow soak. Set MATH_CONV_CACHE_CAP>0 to - # LRU-evict cold conversations before hitting it. + # dropped); this limit bounds any long-running deployment. Set + # MATH_CONV_CACHE_CAP>0 to LRU-evict cold conversations before hitting it. deploy: resources: limits: memory: ${DELPHI_POLLER_CONTAINER_MEMORY:-16g} restart: unless-stopped - profiles: - - math-python postgres: restart: always diff --git a/example.env b/example.env index 548ac0a216..37047df034 100644 --- a/example.env +++ b/example.env @@ -9,7 +9,6 @@ GIT_HASH= # Options: prod, preprod, dev: MATH_ENV=dev # Options: debug, info, warn, error, fatal. Default is warn. -MATH_LOG_LEVEL= # Optionally give the server container a distinct env_file. Useful for CI tests. SERVER_ENV_FILE=.env # Used by winston via server/utils/logger. Defaults to "warn". @@ -53,15 +52,15 @@ LOCAL_SERVICES_DOCKER=true # Leave empty for autodetection on AWS deployment. See delphi/DELPHI_AUTOSCALING_SETUP.md for configuring instance size in production. INSTANCE_SIZE=dev -###### PYTHON MATH POLLER (math-python, --profile math-python) ###### -# The Python replacement for the Clojure `math` container. Runs in SHADOW mode by -# default: writes math_main/math_bidtopid/math_ptptstats under a DISTINCT math_env -# so its rows stay invisible to the prod server (UNIQUE(zid, math_env)) while -# parity is validated. See delphi/docs/MATH_POLLER_DESIGN.md. +###### PYTHON MATH POLLER (math-python) ###### +# THE math engine (the Clojure `math` container was decommissioned at +# cutover Step #3). Writes math_main/math_bidtopid/math_ptptstats under +# the SAME MATH_ENV the server reads (set above). See +# delphi/docs/MATH_POLLER_DESIGN.md + delphi/docs/CUTOVER_RUNBOOK.md. # -# math_env the poller writes under. Keep distinct from the Clojure MATH_ENV while -# shadowing; set equal to the server's MATH_ENV to cut over. Default: python -# MATH_PYTHON_ENV=python +# Conv-cache LRU cap (0 = unlimited). Set >0 for any long-running +# deployment — the cache never evicts by default. Default: 0 +# MATH_CONV_CACHE_CAP=64 # Watermark boot window: start polling from N days ago. Default 10 # POLL_FROM_DAYS_AGO=10 # Poll cadences in ms. Defaults 1000/1000 diff --git a/math/README.md b/math/README.md index 85d5090fec..928072afe9 100644 --- a/math/README.md +++ b/math/README.md @@ -1,4 +1,12 @@ -# polismath +# polismath — ARCHIVED (decommissioned at Clojure->Python cutover Step #3) + +> **This Clojure service no longer runs anywhere.** The math engine is the +> Python poller (`math-python` service; code under `delphi/polismath/` — moving +> to top-level `math/` at cutover Step #4, which removes this tree). Its +> Clojure-exact behavior was certified against THIS tree (battery 20/20 + +> live poller equivalence — see `delphi/docs/CUTOVER_RUNBOOK.md`). After +> Step #4 this tree remains available in git history as the certification +> oracle (`git log -- math/`). The real-time machine learning system powering Polis. diff --git a/scripts-euro/after_install.sh b/scripts-euro/after_install.sh index 066826717b..8e714fe4da 100644 --- a/scripts-euro/after_install.sh +++ b/scripts-euro/after_install.sh @@ -106,8 +106,10 @@ if [ "$SERVICE_FROM_FILE" == "server" ]; then sudo systemctl stop nginx sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha --build --force-recreate elif [ "$SERVICE_FROM_FILE" == "math" ]; then - echo "Starting docker-compose up for 'math' service" - sudo /usr/local/bin/docker-compose up -d math --build --force-recreate + # The Python math poller is THE math engine (Clojure math service + # removed from compose at cutover Step 3) — mirrors scripts/after_install.sh. + echo "Starting docker-compose up for 'math-python' service" + sudo /usr/local/bin/docker-compose up -d math-python --build --force-recreate elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then echo "Starting docker-compose up for 'delphi' service" echo "Fetching Ollama Service URL for Delphi..." diff --git a/scripts/after_install.sh b/scripts/after_install.sh index eef2829616..1f4bb26840 100644 --- a/scripts/after_install.sh +++ b/scripts/after_install.sh @@ -85,7 +85,11 @@ echo "DEBUG: Service type read from /etc/app-info/service_type.txt: [$SERVICE_FR # Original Docker cleanup/start logic echo "Stopping and removing existing Docker containers..." -sudo /usr/local/bin/docker-compose down || true +# --remove-orphans: containers whose service left the compose file (e.g. +# the Clojure `math` container after cutover Step 3) must not survive a +# deploy. The blanket `docker rm -f` below also catches them, but only +# because CodeDeploy runs as root — belt and braces. +sudo /usr/local/bin/docker-compose down --remove-orphans || true sudo docker rm -f $(docker ps -aq) || true echo "Docker containers stopped and removed." @@ -104,18 +108,16 @@ if [ "$SERVICE_FROM_FILE" == "server" ]; then echo "Starting docker-compose up for 'server', 'nginx-proxy', and 'client-participation-alpha' services" sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha --build --force-recreate elif [ "$SERVICE_FROM_FILE" == "math" ]; then - # Cutover Step 2 (flip, clean cut): the Python math poller is THE math - # engine. Its MATH_ENV comes from the instance .env, which THIS SCRIPT - # rewrites from the polis-web-app-env-vars secret on every deploy (key - # MATH_PYTHON_ENV, 'prod' post-flip) — container restarts do NOT pick - # up secret edits. The Clojure `math` service is deliberately NOT - # started anymore: two writers must never share a math_env. Its - # definition stays in docker-compose.yml until Step 3, so an emergency - # manual restart is `docker-compose up -d math` — but only AFTER - # taking the poller off 'prod'; the normal rollback is revert-this-PR - # + redeploy. Compose profiles gate DEV only; prod starts services BY - # NAME here. - echo "Starting docker-compose up for 'math-python' service (cutover Step 2)" + # The Python math poller is THE math engine (Clojure math service + # removed from compose at cutover Step 3). It reads the same MATH_ENV + # the server reads, from the instance .env — which is materialized + # from Secrets Manager (polis-web-app-env-vars) ONLY by this script: + # a secret edit takes effect on the NEXT deploy, never on a container + # restart. Rolling back to Clojure math = revert the Step 3 PR AND + # redeploy AND `docker-compose up -d math` (a revert-redeploy alone + # starts only math-python), with the poller taken out of the server's + # math_env first. + echo "Starting docker-compose up for 'math-python' service" sudo /usr/local/bin/docker-compose up -d math-python --build --force-recreate elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then echo "Starting docker-compose up for 'delphi' service" diff --git a/test.env b/test.env index 7fbafacca0..6ae510ef58 100644 --- a/test.env +++ b/test.env @@ -24,7 +24,6 @@ POSTGRES_USER=postgres DATABASE_URL=postgres://postgres:PdwPNS2mDN73Vfbc@postgres:5432/polis-test MATH_ENV=dev -MATH_LOG_LEVEL=warn WEBSERVER_PASS=ws-pass WEBSERVER_USERNAME=ws-user