Add -verify-storage to purge checksum-mismatched modules from storage (#2145)

cmd/proxy/actions/verify.go

@@ -0,0 +1,65 @@
+package actions
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/paths"
+	"github.com/gomods/athens/pkg/verify"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
+)
+
+// defaultSumDB is used when no SumDBs are configured.
+const defaultSumDB = "https://sum.golang.org"
+
+// ValidateVerifyFlags enforces that -purge is only meaningful together with
+// -verify-storage. -purge is not a general purge; it only acts on the mismatch
+// set produced by the verify pass.
+func ValidateVerifyFlags(verifyStorage, purge bool) error {
+	if purge && !verifyStorage {
+		return fmt.Errorf("-purge requires -verify-storage")
+	}
+	return nil
+}
+
+// RunVerify builds the configured storage backend and sweeps it for zips that
+// disagree with the checksum database, writing a report to out. When purge is
+// true, mismatches are deleted. It does not start the server.
+func RunVerify(conf *config.Config, purge bool, out io.Writer) error {
+	const op errors.Op = "actions.RunVerify"
+
+	client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}
+	store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	sweepStore, ok := store.(verify.Store)
+	if !ok {
+		return errors.E(op, fmt.Errorf("storage backend %q does not support cataloging; verify-storage is unavailable", conf.StorageType))
+	}
+
+	sumdbURL := defaultSumDB
+	if len(conf.SumDBs) > 0 && conf.SumDBs[0] != "" {
+		sumdbURL = conf.SumDBs[0]
+	}
+	oracle := verify.NewSumDBOracle(sumdbURL, client)
+
+	patterns := conf.NoSumPatterns
+	skip := func(mod string) bool {
+		for _, p := range patterns {
+			if paths.MatchesPattern(p, mod) {
+				return true
+			}
+		}
+		return false
+	}
+
+	if _, err := verify.Sweep(context.Background(), sweepStore, oracle, skip, purge, out); err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}

cmd/proxy/actions/verify_test.go

@@ -0,0 +1,28 @@
+package actions
+
+import "testing"
+
+func TestValidateVerifyFlags(t *testing.T) {
+	cases := []struct {
+		name          string
+		verifyStorage bool
+		purge         bool
+		wantErr       bool
+	}{
+		{"neither", false, false, false},
+		{"verify only", true, false, false},
+		{"verify and purge", true, true, false},
+		{"purge without verify", false, true, true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := ValidateVerifyFlags(tc.verifyStorage, tc.purge)
+			if tc.wantErr && err == nil {
+				t.Fatalf("expected error, got nil")
+			}
+			if !tc.wantErr && err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+		})
+	}
+}

cmd/proxy/main.go

@@ -22,12 +22,17 @@ import (
 )
 
 var (
-	configFile = flag.String("config_file", "", "The path to the config file")
-	version    = flag.Bool("version", false, "Print version information and exit")
+	configFile    = flag.String("config_file", "", "The path to the config file")
+	version       = flag.Bool("version", false, "Print version information and exit")
+	verifyStorage = flag.Bool("verify-storage", false, "Verify stored module zips against the checksum database and report mismatches, then exit (does not start the server)")
+	purge         = flag.Bool("purge", false, "With -verify-storage, delete the mismatched module versions (requires -verify-storage)")
 )
 
 func main() {
 	flag.Parse()
+	if err := actions.ValidateVerifyFlags(*verifyStorage, *purge); err != nil {
+		stdlog.Fatalf("%v", err)
+	}
 	if *version {
 		fmt.Println(build.String())
 		os.Exit(0)
@@ -57,6 +62,13 @@ func main() {
 			"run it under an init such as tini or `docker/podman run --init` to avoid zombie processes")
 	}
 
+	if *verifyStorage {
+		if err := actions.RunVerify(conf, *purge, os.Stdout); err != nil {
+			logger.Fatalf("verify-storage failed: %v", err)
+		}
+		return
+	}
+
 	handler, cleanup, err := actions.App(logger, conf)
 	if err != nil {
 		logger.Fatalf("Could not create App: %v", err)

docs/content/configuration/storage.md

@@ -539,3 +539,6 @@ single option with which it can be customized:
         # Threshold for how long to wait in seconds for an in-progress GCP upload to
         # be considered to have failed to unlock.
         StaleThreshold = 120
+
+> If Athens served zips built by an old, buggy Go toolchain, see
+> [Verifying stored modules]({{< ref "verify-storage" >}}) to detect and purge them.

docs/content/configuration/sumdb.md

@@ -67,3 +67,6 @@ The trusted checksums are all stored in `sum.golang.org`, and that server is cen
 Athens does its best to respect and use the trusted checksums while also ensuring that your private names don't get leaked to the public server. In some cases, it has to choose whether to fail your build or leak information, so it chooses to fail your build. That's why everybody using that Athens server needs to set up their `GONOSUMDB` environment variable.
 
 We believe that along with good documentation - which we hope this is! - we have struck the right balance between convenience and privacy.
+
+> If Athens served zips built by an old, buggy Go toolchain, see
+> [Verifying stored modules]({{< ref "verify-storage" >}}) to detect and purge them.

docs/content/configuration/verify-storage.md

@@ -0,0 +1,115 @@
+---
+title: Verifying stored modules
+description: Detect and purge stored module zips that disagree with the Go checksum database (issue #2145).
+weight: 6
+---
+
+Athens builds module zips itself on a cache miss and stores them. If Athens ever
+ran with a Go toolchain that built a zip incorrectly, that wrong zip stays in
+storage and is re-served on every cache hit — a toolchain upgrade alone does not
+fix it, because a cache hit never rebuilds the module.
+
+`athens-proxy -verify-storage` finds stored module versions whose zip bytes
+disagree with the Go **checksum database** (`sum.golang.org` by default) and,
+with `-purge`, deletes them so the next request rebuilds them correctly.
+
+## Am I affected?
+
+You only need this if **all three** are true:
+
+1. You ran **Athens `v0.12.0`–`v0.15.1`** (these bundle Go 1.20.x). `v0.15.2`+
+   bundle Go ≥ 1.22 and are unaffected:
+
+   | Bundled Go | Athens releases | Affected? |
+   | --- | --- | --- |
+   | 1.20.x | `v0.12.0` … `v0.15.1` | **Yes** |
+   | 1.22 | `v0.15.2` … `v0.15.4` | No |
+   | 1.23.5 | `v0.16.0` … `v0.16.1` | No |
+   | 1.25.1 | `v0.16.2` | No |
+   | 1.26.2 | `v0.17.0` … (current) | No |
+
+2. That instance was **still running in ~2025 or later** — the bug only produces
+   a mismatch for modules that declare `go >= 1.24`, which do not exist before the
+   Go 1.24 release (Feb 2025). An instance retired or upgraded to `v0.15.2`+
+   before then holds no poison.
+
+3. You proxied **public** modules that declare `go >= 1.24` and ship files under a
+   `vendor/` directory nested below the module root (or a root
+   `vendor/modules.txt`) — e.g. `github.com/onsi/ginkgo/v2@v2.32.0`.
+
+Symptom your users saw: a `checksum mismatch` (or unexpected `404`) for specific
+module versions that download and verify fine directly from `proxy.golang.org`.
+
+### What your users must do
+
+- **Clients using the default checksum database** were never silently poisoned;
+  they just got errors. Once you purge (so Athens serves canonical bytes) they
+  recover automatically on the next `go` command. No action needed.
+- **Clients that disabled checksum verification** (`GONOSUMDB` / `GOSUMDB=off` /
+  `GOPRIVATE`) for the affected module may have committed the wrong hash into
+  their `go.sum`. After the fix they must refresh it: delete the stale line and
+  run `go mod tidy` (or `GOFLAGS=-mod=mod`).
+
+## Running it
+
+`-verify-storage` runs as a one-shot process using the same image and config as
+your deployment, and **exits without starting the server**. It is safe to run
+against a live deployment: it is a separate process, and it only ever deletes
+versions it can prove are wrong (and therefore re-fetchable).
+
+Report only (deletes nothing):
+
+```bash
+athens-proxy -verify-storage -config_file=/config/config.toml
+```
+
+Verify and delete the mismatches:
+
+```bash
+athens-proxy -verify-storage -purge -config_file=/config/config.toml
+```
+
+`-purge` on its own is rejected — it only ever acts on the mismatches found by
+the verify pass, never as a general cache purge.
+
+### As a Kubernetes Job
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: athens-verify-storage
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: verify
+          image: gomods/athens:latest # same image as your Deployment
+          args: ["-verify-storage", "-config_file=/config/config.toml"]
+          envFrom:
+            - secretRef:
+                name: athens-storage-credentials # same storage config as the Deployment
+          volumeMounts:
+            - { name: config, mountPath: /config }
+      volumes:
+        - { name: config, configMap: { name: athens-config } }
+```
+
+Add `-purge` to `args` once you have reviewed the report. For the **disk**
+storage driver, run it where the storage volume is mounted (mount the same PVC in
+the Job, or `kubectl exec` into the running pod).
+
+## Prerequisites and limitations
+
+- **Checksum-DB access.** The sweep needs outbound access to your configured
+  `SumDBs` (default `https://sum.golang.org`). Air-gapped with no such access →
+  it verifies nothing and safely does nothing.
+- **Private modules are skipped.** Anything matching `NoSumPatterns`
+  (`ATHENS_GONOSUM_PATTERNS`) is skipped *before* any lookup — both because there
+  is no public checksum to compare against and to avoid leaking private module
+  names. Private modules therefore cannot be auto-verified; if you suspect one is
+  poisoned, delete that version manually so it refetches.
+- **Prefer this over emptying storage.** Emptying storage also destroys private
+  modules whose upstream may no longer exist — Athens is often their last copy.
+  The targeted sweep only removes what it can prove wrong *and* replaceable.

pkg/verify/helpers_test.go

@@ -0,0 +1,77 @@
+package verify_test
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"crypto/md5"
+	"os"
+	"testing"
+
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/gomods/athens/pkg/storage/fs"
+	"github.com/gomods/athens/pkg/verify"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/mod/sumdb/dirhash"
+)
+
+// fakeOracle returns canonical hashes from a static map; unknown keys are
+// reported as ErrNotInSumDB.
+type fakeOracle struct{ hashes map[string]string }
+
+func (o fakeOracle) CanonicalHash(_ context.Context, mod, ver string) (string, error) {
+	if h, ok := o.hashes[mod+"@"+ver]; ok {
+		return h, nil
+	}
+	return "", verify.ErrNotInSumDB
+}
+
+// writeZip builds an in-memory zip from name->body entries.
+func writeZip(t *testing.T, files map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for name, body := range files {
+		w, err := zw.Create(name)
+		require.NoError(t, err)
+		_, err = w.Write([]byte(body))
+		require.NoError(t, err)
+	}
+	require.NoError(t, zw.Close())
+	return buf.Bytes()
+}
+
+// hashOf returns the dirhash h1: of a zip's bytes.
+func hashOf(t *testing.T, zipBytes []byte) string {
+	t.Helper()
+	f, err := os.CreateTemp(t.TempDir(), "*.zip")
+	require.NoError(t, err)
+	_, err = f.Write(zipBytes)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	h, err := dirhash.HashZip(f.Name(), dirhash.Hash1)
+	require.NoError(t, err)
+	return h
+}
+
+// newTestStore returns an in-memory fs-backed storage backend (implements
+// Cataloger/Getter/Saver/Deleter).
+func newTestStore(t *testing.T) storage.Backend {
+	t.Helper()
+	memfs := afero.NewMemMapFs()
+	err := memfs.Mkdir("/athens", 0755)
+	require.NoError(t, err)
+	b, err := fs.NewStorage("/athens", memfs)
+	require.NoError(t, err)
+	return b
+}
+
+// saveModule seeds a module version into the backend.
+func saveModule(t *testing.T, ctx context.Context, b storage.Backend, mod, ver string, zipBytes []byte) {
+	t.Helper()
+	sum := md5.Sum(zipBytes)
+	info := []byte(`{"Version":"` + ver + `","Time":"2025-01-01T00:00:00Z"}`)
+	err := b.Save(ctx, mod, ver, []byte("module "+mod+"\n"), bytes.NewReader(zipBytes), sum[:], info)
+	require.NoError(t, err)
+}