Clojure->Python CUTOVER: Step 3 — decommission the Clojure math service everywhere

.github/workflows/deploy-alpha-aws.yml

@@ -38,8 +38,10 @@ jobs:
           docker-compose build server
           docker push $ECR_REGISTRY/polis/server:latest
 
-          docker-compose build math
-          docker push $ECR_REGISTRY/polis/math:latest
+          # The Clojure math image is gone (cutover Step 3). No math-side
+          # push is needed here: instances build locally in
+          # after_install.sh (`docker-compose up --build`); the ECR tag
+          # was never pulled by the deploy path.
 
   # =====================================================
   # ==      JOB 2: DEPLOY TO PRODUCTION (US)           ==

docker-compose.dev.yml

@@ -29,17 +29,6 @@ services:
       # Tell Node.js to trust the mkcert CA for HTTPS connections
       - NODE_EXTRA_CA_CERTS=/root/.simulacrum/certs/rootCA.pem
 
-  math:
-    # command: clojure -X:dev-poller
-    volumes:
-      - ./math:/app
-    ports:
-      # nrepl port
-      - "18975:18975"
-    environment:
-      - CHOKIDAR_USEPOLLING=true
-      - DD_ENABLED=false
-
   postgres:
     restart: no
     ports:

docker-compose.test.yml

@@ -55,18 +55,27 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  math:
+  # Python math poller — THE math engine (the Clojure `math` service was
+  # decommissioned at cutover Step #3). Same env var + default the old clj
+  # test service used.
+  math-python:
     build:
-      context: ./math
+      context: ./delphi
+      target: final
+      # Share every layer with the delphi service's test build (CPU torch);
+      # without this the default CUDA wheels build a second multi-GB image
+      # in CI for no benefit — the poller needs no torch at runtime.
+      args:
+        USE_CPU_TORCH: "true"
+    command: ["python", "scripts/math_poller.py"]
     labels:
       polis_tag: test
     environment:
       - DATABASE_URL=${DATABASE_URL}
-      - DD_ENABLED=false
-      - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn}
+      - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable}
+      - POSTGRES_CONNECT_TIMEOUT=${POSTGRES_CONNECT_TIMEOUT:-5}
+      - LOG_LEVEL=${DELPHI_LOG_LEVEL:-INFO}
       - MATH_ENV=${MATH_ENV:-dev}
-      - WEBSERVER_USERNAME=${WEBSERVER_USERNAME}
-      - WEBSERVER_PASS=${WEBSERVER_PASS}
     networks:
       - polis-test
     restart: unless-stopped

docker-compose.yml

@@ -76,26 +76,6 @@ services:
     extra_hosts:
       - "host.docker.internal:host-gateway"
 
-  math:
-    image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/math:latest
-    build:
-      context: ./math
-    labels:
-      polis_tag: ${TAG:-dev}
-    environment:
-      - DATABASE_URL=${DATABASE_URL}
-      - LOGGING_LEVEL=${MATH_LOG_LEVEL:-warn}
-      - MATH_ENV=${MATH_ENV:-prod}
-      - WEBSERVER_USERNAME=${WEBSERVER_USERNAME}
-      - WEBSERVER_PASS=${WEBSERVER_PASS}
-      # Database connection pool configuration for math service
-      - DATABASE_POOL_SIZE=${MATH_DATABASE_POOL_SIZE:-10}
-    networks:
-      - "polis-net"
-    restart: unless-stopped
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-
   delphi:
     image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest
     build:
@@ -145,13 +125,11 @@ services:
           cpus: ${DELPHI_CONTAINER_CPUS:-2}
     restart: unless-stopped
 
-  # Python math poller — the eventual replacement for the Clojure `math`
-  # container.  Profile-gated so it only runs when explicitly requested
-  # (`--profile math-python`).  Reuses the delphi build target and overrides the
-  # command to run the poller CLI.  Defaults to SHADOW mode: writes under a
-  # DISTINCT math_env (`python`) so its rows are invisible to the prod server
-  # (UNIQUE(zid, math_env)) — zero production risk while parity is validated.
-  # See delphi/docs/MATH_POLLER_DESIGN.md §4.
+  # Python math poller — THE math engine (the Clojure `math` container was
+  # decommissioned at cutover Step #3).  Reuses the delphi build target and
+  # overrides the command to run the poller CLI.  Reads/writes the same
+  # MATH_ENV the server reads, exactly as the Clojure service did.
+  # See delphi/docs/MATH_POLLER_DESIGN.md §4 + docs/CUTOVER_RUNBOOK.md.
   math-python:
     image: 050917022930.dkr.ecr.us-east-1.amazonaws.com/polis/delphi:latest
     build:
@@ -165,8 +143,9 @@ services:
       - DATABASE_SSL_MODE=${DATABASE_SSL_MODE:-disable}
       - POSTGRES_CONNECT_TIMEOUT=${POSTGRES_CONNECT_TIMEOUT:-30}
       - LOG_LEVEL=${DELPHI_LOG_LEVEL:-INFO}
-      # Shadow-mode math_env (distinct from the Clojure math service's MATH_ENV).
-      - MATH_ENV=${MATH_PYTHON_ENV:-python}
+      # Same env var + default the Clojure math service used — the server
+      # reads the rows written under this math_env.
+      - MATH_ENV=${MATH_ENV:-prod}
       # Poll cadences (ms) and boot window (days).
       - POLL_VOTE_INTERVAL_MS=${POLL_VOTE_INTERVAL_MS:-1000}
       - POLL_MOD_INTERVAL_MS=${POLL_MOD_INTERVAL_MS:-1000}
@@ -189,15 +168,13 @@ services:
       - "host.docker.internal:host-gateway"
     # Hard memory backstop (mirrors the delphi service). The in-memory conv cache
     # never evicted by default (Clojure's 4h reboot was the de-facto cap, which we
-    # dropped); this limit bounds a long shadow soak. Set MATH_CONV_CACHE_CAP>0 to
-    # LRU-evict cold conversations before hitting it.
+    # dropped); this limit bounds any long-running deployment. Set
+    # MATH_CONV_CACHE_CAP>0 to LRU-evict cold conversations before hitting it.
     deploy:
       resources:
         limits:
           memory: ${DELPHI_POLLER_CONTAINER_MEMORY:-16g}
     restart: unless-stopped
-    profiles:
-      - math-python
 
   postgres:
     restart: always

example.env

@@ -9,7 +9,6 @@ GIT_HASH=
 # Options: prod, preprod, dev:
 MATH_ENV=dev
 # Options: debug, info, warn, error, fatal. Default is warn.
-MATH_LOG_LEVEL=
 # Optionally give the server container a distinct env_file. Useful for CI tests.
 SERVER_ENV_FILE=.env
 # Used by winston via server/utils/logger. Defaults to "warn".
@@ -53,15 +52,15 @@ LOCAL_SERVICES_DOCKER=true
 # Leave empty for autodetection on AWS deployment. See delphi/DELPHI_AUTOSCALING_SETUP.md for configuring instance size in production.
 INSTANCE_SIZE=dev
 
-###### PYTHON MATH POLLER (math-python, --profile math-python) ######
-# The Python replacement for the Clojure `math` container. Runs in SHADOW mode by
-# default: writes math_main/math_bidtopid/math_ptptstats under a DISTINCT math_env
-# so its rows stay invisible to the prod server (UNIQUE(zid, math_env)) while
-# parity is validated. See delphi/docs/MATH_POLLER_DESIGN.md.
+###### PYTHON MATH POLLER (math-python) ######
+# THE math engine (the Clojure `math` container was decommissioned at
+# cutover Step #3). Writes math_main/math_bidtopid/math_ptptstats under
+# the SAME MATH_ENV the server reads (set above). See
+# delphi/docs/MATH_POLLER_DESIGN.md + delphi/docs/CUTOVER_RUNBOOK.md.
 #
-# math_env the poller writes under. Keep distinct from the Clojure MATH_ENV while
-# shadowing; set equal to the server's MATH_ENV to cut over. Default: python
-# MATH_PYTHON_ENV=python
+# Conv-cache LRU cap (0 = unlimited). Set >0 for any long-running
+# deployment — the cache never evicts by default. Default: 0
+# MATH_CONV_CACHE_CAP=64
 # Watermark boot window: start polling from N days ago. Default 10
 # POLL_FROM_DAYS_AGO=10
 # Poll cadences in ms. Defaults 1000/1000

math/README.md

@@ -1,4 +1,12 @@
-# polismath
+# polismath — ARCHIVED (decommissioned at Clojure->Python cutover Step #3)
+
+> **This Clojure service no longer runs anywhere.** The math engine is the
+> Python poller (`math-python` service; code under `delphi/polismath/` — moving
+> to top-level `math/` at cutover Step #4, which removes this tree). Its
+> Clojure-exact behavior was certified against THIS tree (battery 20/20 +
+> live poller equivalence — see `delphi/docs/CUTOVER_RUNBOOK.md`). After
+> Step #4 this tree remains available in git history as the certification
+> oracle (`git log -- math/`).
 
 The real-time machine learning system powering Polis.
 

scripts-euro/after_install.sh

@@ -106,8 +106,10 @@ if [ "$SERVICE_FROM_FILE" == "server" ]; then
   sudo systemctl stop nginx
   sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha  --build --force-recreate
 elif [ "$SERVICE_FROM_FILE" == "math" ]; then
-  echo "Starting docker-compose up for 'math' service"
-  sudo /usr/local/bin/docker-compose up -d math --build --force-recreate
+  # The Python math poller is THE math engine (Clojure math service
+  # removed from compose at cutover Step 3) — mirrors scripts/after_install.sh.
+  echo "Starting docker-compose up for 'math-python' service"
+  sudo /usr/local/bin/docker-compose up -d math-python --build --force-recreate
 elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then
   echo "Starting docker-compose up for 'delphi' service"
   echo "Fetching Ollama Service URL for Delphi..."

scripts/after_install.sh

@@ -85,7 +85,11 @@ echo "DEBUG: Service type read from /etc/app-info/service_type.txt: [$SERVICE_FR
 
 # Original Docker cleanup/start logic
 echo "Stopping and removing existing Docker containers..."
-sudo /usr/local/bin/docker-compose down || true
+# --remove-orphans: containers whose service left the compose file (e.g.
+# the Clojure `math` container after cutover Step 3) must not survive a
+# deploy. The blanket `docker rm -f` below also catches them, but only
+# because CodeDeploy runs as root — belt and braces.
+sudo /usr/local/bin/docker-compose down --remove-orphans || true
 sudo docker rm -f $(docker ps -aq) || true
 echo "Docker containers stopped and removed."
 
@@ -104,18 +108,16 @@ if [ "$SERVICE_FROM_FILE" == "server" ]; then
   echo "Starting docker-compose up for 'server', 'nginx-proxy', and 'client-participation-alpha' services"
   sudo /usr/local/bin/docker-compose up -d server nginx-proxy client-participation-alpha --build --force-recreate
 elif [ "$SERVICE_FROM_FILE" == "math" ]; then
-  # Cutover Step 2 (flip, clean cut): the Python math poller is THE math
-  # engine. Its MATH_ENV comes from the instance .env, which THIS SCRIPT
-  # rewrites from the polis-web-app-env-vars secret on every deploy (key
-  # MATH_PYTHON_ENV, 'prod' post-flip) — container restarts do NOT pick
-  # up secret edits. The Clojure `math` service is deliberately NOT
-  # started anymore: two writers must never share a math_env. Its
-  # definition stays in docker-compose.yml until Step 3, so an emergency
-  # manual restart is `docker-compose up -d math` — but only AFTER
-  # taking the poller off 'prod'; the normal rollback is revert-this-PR
-  # + redeploy. Compose profiles gate DEV only; prod starts services BY
-  # NAME here.
-  echo "Starting docker-compose up for 'math-python' service (cutover Step 2)"
+  # The Python math poller is THE math engine (Clojure math service
+  # removed from compose at cutover Step 3). It reads the same MATH_ENV
+  # the server reads, from the instance .env — which is materialized
+  # from Secrets Manager (polis-web-app-env-vars) ONLY by this script:
+  # a secret edit takes effect on the NEXT deploy, never on a container
+  # restart. Rolling back to Clojure math = revert the Step 3 PR AND
+  # redeploy AND `docker-compose up -d math` (a revert-redeploy alone
+  # starts only math-python), with the poller taken out of the server's
+  # math_env first.
+  echo "Starting docker-compose up for 'math-python' service"
   sudo /usr/local/bin/docker-compose up -d math-python --build --force-recreate
 elif [ "$SERVICE_FROM_FILE" == "delphi" ]; then
   echo "Starting docker-compose up for 'delphi' service"

test.env

@@ -24,7 +24,6 @@ POSTGRES_USER=postgres
 DATABASE_URL=postgres://postgres:PdwPNS2mDN73Vfbc@postgres:5432/polis-test
 
 MATH_ENV=dev
-MATH_LOG_LEVEL=warn
 WEBSERVER_PASS=ws-pass
 WEBSERVER_USERNAME=ws-user