diff --git a/cmd/proxy/actions/verify.go b/cmd/proxy/actions/verify.go new file mode 100644 index 000000000..57b9626b1 --- /dev/null +++ b/cmd/proxy/actions/verify.go @@ -0,0 +1,65 @@ +package actions + +import ( + "context" + "fmt" + "io" + "net/http" + + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/paths" + "github.com/gomods/athens/pkg/verify" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +// defaultSumDB is used when no SumDBs are configured. +const defaultSumDB = "https://sum.golang.org" + +// ValidateVerifyFlags enforces that -purge is only meaningful together with +// -verify-storage. -purge is not a general purge; it only acts on the mismatch +// set produced by the verify pass. +func ValidateVerifyFlags(verifyStorage, purge bool) error { + if purge && !verifyStorage { + return fmt.Errorf("-purge requires -verify-storage") + } + return nil +} + +// RunVerify builds the configured storage backend and sweeps it for zips that +// disagree with the checksum database, writing a report to out. When purge is +// true, mismatches are deleted. It does not start the server. +func RunVerify(conf *config.Config, purge bool, out io.Writer) error { + const op errors.Op = "actions.RunVerify" + + client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)} + store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client) + if err != nil { + return errors.E(op, err) + } + sweepStore, ok := store.(verify.Store) + if !ok { + return errors.E(op, fmt.Errorf("storage backend %q does not support cataloging; verify-storage is unavailable", conf.StorageType)) + } + + sumdbURL := defaultSumDB + if len(conf.SumDBs) > 0 && conf.SumDBs[0] != "" { + sumdbURL = conf.SumDBs[0] + } + oracle := verify.NewSumDBOracle(sumdbURL, client) + + patterns := conf.NoSumPatterns + skip := func(mod string) bool { + for _, p := range patterns { + if paths.MatchesPattern(p, mod) { + return true + } + } + return false + } + + if _, err := verify.Sweep(context.Background(), sweepStore, oracle, skip, purge, out); err != nil { + return errors.E(op, err) + } + return nil +} diff --git a/cmd/proxy/actions/verify_test.go b/cmd/proxy/actions/verify_test.go new file mode 100644 index 000000000..1b30f7b6b --- /dev/null +++ b/cmd/proxy/actions/verify_test.go @@ -0,0 +1,28 @@ +package actions + +import "testing" + +func TestValidateVerifyFlags(t *testing.T) { + cases := []struct { + name string + verifyStorage bool + purge bool + wantErr bool + }{ + {"neither", false, false, false}, + {"verify only", true, false, false}, + {"verify and purge", true, true, false}, + {"purge without verify", false, true, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateVerifyFlags(tc.verifyStorage, tc.purge) + if tc.wantErr && err == nil { + t.Fatalf("expected error, got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 105f912bb..c8d5252d7 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -22,12 +22,17 @@ import ( ) var ( - configFile = flag.String("config_file", "", "The path to the config file") - version = flag.Bool("version", false, "Print version information and exit") + configFile = flag.String("config_file", "", "The path to the config file") + version = flag.Bool("version", false, "Print version information and exit") + verifyStorage = flag.Bool("verify-storage", false, "Verify stored module zips against the checksum database and report mismatches, then exit (does not start the server)") + purge = flag.Bool("purge", false, "With -verify-storage, delete the mismatched module versions (requires -verify-storage)") ) func main() { flag.Parse() + if err := actions.ValidateVerifyFlags(*verifyStorage, *purge); err != nil { + stdlog.Fatalf("%v", err) + } if *version { fmt.Println(build.String()) os.Exit(0) @@ -57,6 +62,13 @@ func main() { "run it under an init such as tini or `docker/podman run --init` to avoid zombie processes") } + if *verifyStorage { + if err := actions.RunVerify(conf, *purge, os.Stdout); err != nil { + logger.Fatalf("verify-storage failed: %v", err) + } + return + } + handler, cleanup, err := actions.App(logger, conf) if err != nil { logger.Fatalf("Could not create App: %v", err) diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index c53fd18c8..ac8e48e07 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -539,3 +539,6 @@ single option with which it can be customized: # Threshold for how long to wait in seconds for an in-progress GCP upload to # be considered to have failed to unlock. StaleThreshold = 120 + +> If Athens served zips built by an old, buggy Go toolchain, see +> [Verifying stored modules]({{< ref "verify-storage" >}}) to detect and purge them. diff --git a/docs/content/configuration/sumdb.md b/docs/content/configuration/sumdb.md index 9f54d18cd..77154f0d4 100644 --- a/docs/content/configuration/sumdb.md +++ b/docs/content/configuration/sumdb.md @@ -67,3 +67,6 @@ The trusted checksums are all stored in `sum.golang.org`, and that server is cen Athens does its best to respect and use the trusted checksums while also ensuring that your private names don't get leaked to the public server. In some cases, it has to choose whether to fail your build or leak information, so it chooses to fail your build. That's why everybody using that Athens server needs to set up their `GONOSUMDB` environment variable. We believe that along with good documentation - which we hope this is! - we have struck the right balance between convenience and privacy. + +> If Athens served zips built by an old, buggy Go toolchain, see +> [Verifying stored modules]({{< ref "verify-storage" >}}) to detect and purge them. diff --git a/docs/content/configuration/verify-storage.md b/docs/content/configuration/verify-storage.md new file mode 100644 index 000000000..628c63ac9 --- /dev/null +++ b/docs/content/configuration/verify-storage.md @@ -0,0 +1,115 @@ +--- +title: Verifying stored modules +description: Detect and purge stored module zips that disagree with the Go checksum database (issue #2145). +weight: 6 +--- + +Athens builds module zips itself on a cache miss and stores them. If Athens ever +ran with a Go toolchain that built a zip incorrectly, that wrong zip stays in +storage and is re-served on every cache hit — a toolchain upgrade alone does not +fix it, because a cache hit never rebuilds the module. + +`athens-proxy -verify-storage` finds stored module versions whose zip bytes +disagree with the Go **checksum database** (`sum.golang.org` by default) and, +with `-purge`, deletes them so the next request rebuilds them correctly. + +## Am I affected? + +You only need this if **all three** are true: + +1. You ran **Athens `v0.12.0`–`v0.15.1`** (these bundle Go 1.20.x). `v0.15.2`+ + bundle Go ≥ 1.22 and are unaffected: + + | Bundled Go | Athens releases | Affected? | + | --- | --- | --- | + | 1.20.x | `v0.12.0` … `v0.15.1` | **Yes** | + | 1.22 | `v0.15.2` … `v0.15.4` | No | + | 1.23.5 | `v0.16.0` … `v0.16.1` | No | + | 1.25.1 | `v0.16.2` | No | + | 1.26.2 | `v0.17.0` … (current) | No | + +2. That instance was **still running in ~2025 or later** — the bug only produces + a mismatch for modules that declare `go >= 1.24`, which do not exist before the + Go 1.24 release (Feb 2025). An instance retired or upgraded to `v0.15.2`+ + before then holds no poison. + +3. You proxied **public** modules that declare `go >= 1.24` and ship files under a + `vendor/` directory nested below the module root (or a root + `vendor/modules.txt`) — e.g. `github.com/onsi/ginkgo/v2@v2.32.0`. + +Symptom your users saw: a `checksum mismatch` (or unexpected `404`) for specific +module versions that download and verify fine directly from `proxy.golang.org`. + +### What your users must do + +- **Clients using the default checksum database** were never silently poisoned; + they just got errors. Once you purge (so Athens serves canonical bytes) they + recover automatically on the next `go` command. No action needed. +- **Clients that disabled checksum verification** (`GONOSUMDB` / `GOSUMDB=off` / + `GOPRIVATE`) for the affected module may have committed the wrong hash into + their `go.sum`. After the fix they must refresh it: delete the stale line and + run `go mod tidy` (or `GOFLAGS=-mod=mod`). + +## Running it + +`-verify-storage` runs as a one-shot process using the same image and config as +your deployment, and **exits without starting the server**. It is safe to run +against a live deployment: it is a separate process, and it only ever deletes +versions it can prove are wrong (and therefore re-fetchable). + +Report only (deletes nothing): + +```bash +athens-proxy -verify-storage -config_file=/config/config.toml +``` + +Verify and delete the mismatches: + +```bash +athens-proxy -verify-storage -purge -config_file=/config/config.toml +``` + +`-purge` on its own is rejected — it only ever acts on the mismatches found by +the verify pass, never as a general cache purge. + +### As a Kubernetes Job + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: athens-verify-storage +spec: + template: + spec: + restartPolicy: Never + containers: + - name: verify + image: gomods/athens:latest # same image as your Deployment + args: ["-verify-storage", "-config_file=/config/config.toml"] + envFrom: + - secretRef: + name: athens-storage-credentials # same storage config as the Deployment + volumeMounts: + - { name: config, mountPath: /config } + volumes: + - { name: config, configMap: { name: athens-config } } +``` + +Add `-purge` to `args` once you have reviewed the report. For the **disk** +storage driver, run it where the storage volume is mounted (mount the same PVC in +the Job, or `kubectl exec` into the running pod). + +## Prerequisites and limitations + +- **Checksum-DB access.** The sweep needs outbound access to your configured + `SumDBs` (default `https://sum.golang.org`). Air-gapped with no such access → + it verifies nothing and safely does nothing. +- **Private modules are skipped.** Anything matching `NoSumPatterns` + (`ATHENS_GONOSUM_PATTERNS`) is skipped *before* any lookup — both because there + is no public checksum to compare against and to avoid leaking private module + names. Private modules therefore cannot be auto-verified; if you suspect one is + poisoned, delete that version manually so it refetches. +- **Prefer this over emptying storage.** Emptying storage also destroys private + modules whose upstream may no longer exist — Athens is often their last copy. + The targeted sweep only removes what it can prove wrong *and* replaceable. diff --git a/pkg/verify/helpers_test.go b/pkg/verify/helpers_test.go new file mode 100644 index 000000000..0b4112b54 --- /dev/null +++ b/pkg/verify/helpers_test.go @@ -0,0 +1,77 @@ +package verify_test + +import ( + "archive/zip" + "bytes" + "context" + "crypto/md5" + "os" + "testing" + + "github.com/gomods/athens/pkg/storage" + "github.com/gomods/athens/pkg/storage/fs" + "github.com/gomods/athens/pkg/verify" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + "golang.org/x/mod/sumdb/dirhash" +) + +// fakeOracle returns canonical hashes from a static map; unknown keys are +// reported as ErrNotInSumDB. +type fakeOracle struct{ hashes map[string]string } + +func (o fakeOracle) CanonicalHash(_ context.Context, mod, ver string) (string, error) { + if h, ok := o.hashes[mod+"@"+ver]; ok { + return h, nil + } + return "", verify.ErrNotInSumDB +} + +// writeZip builds an in-memory zip from name->body entries. +func writeZip(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + for name, body := range files { + w, err := zw.Create(name) + require.NoError(t, err) + _, err = w.Write([]byte(body)) + require.NoError(t, err) + } + require.NoError(t, zw.Close()) + return buf.Bytes() +} + +// hashOf returns the dirhash h1: of a zip's bytes. +func hashOf(t *testing.T, zipBytes []byte) string { + t.Helper() + f, err := os.CreateTemp(t.TempDir(), "*.zip") + require.NoError(t, err) + _, err = f.Write(zipBytes) + require.NoError(t, err) + require.NoError(t, f.Close()) + h, err := dirhash.HashZip(f.Name(), dirhash.Hash1) + require.NoError(t, err) + return h +} + +// newTestStore returns an in-memory fs-backed storage backend (implements +// Cataloger/Getter/Saver/Deleter). +func newTestStore(t *testing.T) storage.Backend { + t.Helper() + memfs := afero.NewMemMapFs() + err := memfs.Mkdir("/athens", 0755) + require.NoError(t, err) + b, err := fs.NewStorage("/athens", memfs) + require.NoError(t, err) + return b +} + +// saveModule seeds a module version into the backend. +func saveModule(t *testing.T, ctx context.Context, b storage.Backend, mod, ver string, zipBytes []byte) { + t.Helper() + sum := md5.Sum(zipBytes) + info := []byte(`{"Version":"` + ver + `","Time":"2025-01-01T00:00:00Z"}`) + err := b.Save(ctx, mod, ver, []byte("module "+mod+"\n"), bytes.NewReader(zipBytes), sum[:], info) + require.NoError(t, err) +} diff --git a/pkg/verify/oracle.go b/pkg/verify/oracle.go new file mode 100644 index 000000000..2a49f9364 --- /dev/null +++ b/pkg/verify/oracle.go @@ -0,0 +1,79 @@ +package verify + +import ( + "bufio" + "context" + "errors" + "fmt" + "net/http" + "strings" + + "golang.org/x/mod/module" +) + +// ErrNotInSumDB indicates the module version is absent from the checksum +// database (e.g. a private module). Callers must treat this as "cannot +// verify" and keep the module. +var ErrNotInSumDB = errors.New("module version not found in checksum database") + +// Oracle returns the canonical h1: zip hash for a module version. +type Oracle interface { + CanonicalHash(ctx context.Context, mod, ver string) (string, error) +} + +// sumdbOracle looks up the canonical hash from a Go checksum database +// (e.g. https://sum.golang.org) via its /lookup endpoint. +type sumdbOracle struct { + baseURL string + client *http.Client +} + +// NewSumDBOracle returns an Oracle backed by the checksum database at baseURL. +func NewSumDBOracle(baseURL string, client *http.Client) Oracle { + return &sumdbOracle{baseURL: strings.TrimRight(baseURL, "/"), client: client} +} + +func (o *sumdbOracle) CanonicalHash(ctx context.Context, mod, ver string) (string, error) { + escMod, err := module.EscapePath(mod) + if err != nil { + return "", fmt.Errorf("verify: escape path %q: %w", mod, err) + } + escVer, err := module.EscapeVersion(ver) + if err != nil { + return "", fmt.Errorf("verify: escape version %q: %w", ver, err) + } + url := o.baseURL + "/lookup/" + escMod + "@" + escVer + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", fmt.Errorf("verify: build request: %w", err) + } + resp, err := o.client.Do(req) + if err != nil { + return "", fmt.Errorf("verify: lookup %s: %w", url, err) + } + defer resp.Body.Close() + + switch resp.StatusCode { + case http.StatusOK: + // fall through to parse + case http.StatusNotFound, http.StatusGone: + return "", fmt.Errorf("verify: %s@%s: %w", mod, ver, ErrNotInSumDB) + default: + return "", fmt.Errorf("verify: lookup %s: unexpected status %d", url, resp.StatusCode) + } + + // Response lines: " h1:...=" and " /go.mod h1:...=". + // We want the zip line (second field == the plain version). + sc := bufio.NewScanner(resp.Body) + for sc.Scan() { + f := strings.Fields(sc.Text()) + if len(f) == 3 && f[0] == mod && f[1] == ver { + return f[2], nil + } + } + if err := sc.Err(); err != nil { + return "", fmt.Errorf("verify: read lookup body: %w", err) + } + return "", fmt.Errorf("verify: %s@%s: %w", mod, ver, ErrNotInSumDB) +} diff --git a/pkg/verify/oracle_test.go b/pkg/verify/oracle_test.go new file mode 100644 index 000000000..f61c884d6 --- /dev/null +++ b/pkg/verify/oracle_test.go @@ -0,0 +1,45 @@ +package verify_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gomods/athens/pkg/verify" + "github.com/stretchr/testify/require" +) + +func TestSumDBOracle(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/lookup/example.com/m@v1.0.0": + fmt.Fprint(w, "12345\nexample.com/m v1.0.0 h1:GOOD=\nexample.com/m v1.0.0/go.mod h1:MOD=\n") + case "/lookup/example.com/missing@v1.0.0": + http.Error(w, "not found", http.StatusNotFound) + case "/lookup/example.com/gone@v1.0.0": + http.Error(w, "gone", http.StatusGone) + case "/lookup/example.com/nozip@v1.0.0": + fmt.Fprint(w, "12345\nexample.com/nozip v1.0.0/go.mod h1:MOD=\n") + default: + http.Error(w, "unexpected", http.StatusInternalServerError) + } + })) + defer srv.Close() + + o := verify.NewSumDBOracle(srv.URL, srv.Client()) + + h, err := o.CanonicalHash(context.Background(), "example.com/m", "v1.0.0") + require.NoError(t, err) + require.Equal(t, "h1:GOOD=", h) + + _, err = o.CanonicalHash(context.Background(), "example.com/missing", "v1.0.0") + require.ErrorIs(t, err, verify.ErrNotInSumDB) + + _, err = o.CanonicalHash(context.Background(), "example.com/gone", "v1.0.0") + require.ErrorIs(t, err, verify.ErrNotInSumDB) + + _, err = o.CanonicalHash(context.Background(), "example.com/nozip", "v1.0.0") + require.ErrorIs(t, err, verify.ErrNotInSumDB) +} diff --git a/pkg/verify/sweep.go b/pkg/verify/sweep.go new file mode 100644 index 000000000..438d1c7f2 --- /dev/null +++ b/pkg/verify/sweep.go @@ -0,0 +1,82 @@ +package verify + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/gomods/athens/pkg/storage" +) + +// pageSize is how many catalog entries to fetch per page. +const pageSize = 1000 + +// Store is the subset of a storage backend the sweep needs. +type Store interface { + storage.Cataloger + storage.Getter + storage.Deleter +} + +// Report tallies the outcome of a sweep. +type Report struct { + Matched int // stored hash == canonical + Mismatched int // stored hash != canonical + Purged int // mismatches deleted (purge mode) + SkippedPrivate int // matched a NoSumPattern, never looked up + SkippedUnknown int // absent from the checksum DB (ErrNotInSumDB) + Unverified int // transient/storage/oracle error; kept + DeleteErrors int // Delete failed in purge mode +} + +// Sweep verifies every catalogued module version against the oracle. skip +// reports whether a module is private and MUST be consulted before any oracle +// lookup (to avoid leaking private module paths). When purge is true, +// mismatches are deleted; otherwise they are only reported. +func Sweep(ctx context.Context, s Store, o Oracle, skip func(mod string) bool, purge bool, out io.Writer) (Report, error) { + var rep Report + token := "" + for { + page, next, err := s.Catalog(ctx, token, pageSize) + if err != nil { + return rep, fmt.Errorf("verify: catalog: %w", err) + } + for _, mv := range page { + if skip(mv.Module) { + rep.SkippedPrivate++ + continue + } + mismatch, err := Poisoned(ctx, s, o, mv.Module, mv.Version) + switch { + case errors.Is(err, ErrNotInSumDB): + rep.SkippedUnknown++ + continue + case err != nil: + rep.Unverified++ + fmt.Fprintf(out, "unverified\t%s@%s\t%v\n", mv.Module, mv.Version, err) + continue + case !mismatch: + rep.Matched++ + continue + } + rep.Mismatched++ + fmt.Fprintf(out, "MISMATCH\t%s@%s\n", mv.Module, mv.Version) + if purge { + if err := s.Delete(ctx, mv.Module, mv.Version); err != nil { + rep.DeleteErrors++ + fmt.Fprintf(out, "delete-failed\t%s@%s\t%v\n", mv.Module, mv.Version, err) + continue + } + rep.Purged++ + } + } + if next == "" { + break + } + token = next + } + fmt.Fprintf(out, "\nmatched=%d mismatched=%d purged=%d skipped_private=%d skipped_unknown=%d unverified=%d delete_errors=%d\n", + rep.Matched, rep.Mismatched, rep.Purged, rep.SkippedPrivate, rep.SkippedUnknown, rep.Unverified, rep.DeleteErrors) + return rep, nil +} diff --git a/pkg/verify/sweep_test.go b/pkg/verify/sweep_test.go new file mode 100644 index 000000000..be01bb80f --- /dev/null +++ b/pkg/verify/sweep_test.go @@ -0,0 +1,81 @@ +package verify_test + +import ( + "context" + "io" + "strings" + "testing" + + "github.com/gomods/athens/pkg/verify" + "github.com/stretchr/testify/require" +) + +func TestSweep(t *testing.T) { + ctx := context.Background() + b := newTestStore(t) + store := b.(verify.Store) + + goodZip := writeZip(t, map[string]string{"example.com/good@v1.0.0/go.mod": "module example.com/good\n"}) + badZip := writeZip(t, map[string]string{"example.com/bad@v1.0.0/go.mod": "module example.com/bad\n", "example.com/bad@v1.0.0/x.go": "package bad"}) + privZip := writeZip(t, map[string]string{"corp.internal/secret@v1.0.0/go.mod": "module corp.internal/secret\n"}) + + saveModule(t, ctx, b, "example.com/good", "v1.0.0", goodZip) + saveModule(t, ctx, b, "example.com/bad", "v1.0.0", badZip) + saveModule(t, ctx, b, "corp.internal/secret", "v1.0.0", privZip) + + oracle := fakeOracle{hashes: map[string]string{ + "example.com/good@v1.0.0": hashOf(t, goodZip), // matches stored + "example.com/bad@v1.0.0": "h1:CANONICAL-DIFFERENT=", // mismatches stored + // corp.internal/secret intentionally absent (also skipped by NoSumPatterns) + }} + skip := func(mod string) bool { return strings.HasPrefix(mod, "corp.internal/") } + + // Report-only: detects the mismatch but deletes nothing. + rep, err := verify.Sweep(ctx, store, oracle, skip, false, io.Discard) + require.NoError(t, err) + require.Equal(t, 1, rep.Matched) + require.Equal(t, 1, rep.Mismatched) + require.Equal(t, 0, rep.Purged) + require.Equal(t, 1, rep.SkippedPrivate) + + _, err = store.Zip(ctx, "example.com/bad", "v1.0.0") + require.NoError(t, err, "report mode must not delete") + + // Purge: deletes only the mismatch. + rep, err = verify.Sweep(ctx, store, oracle, skip, true, io.Discard) + require.NoError(t, err) + require.Equal(t, 1, rep.Purged) + + _, err = store.Zip(ctx, "example.com/bad", "v1.0.0") + require.Error(t, err, "purged version must be gone") + _, err = store.Zip(ctx, "example.com/good", "v1.0.0") + require.NoError(t, err, "matching version must remain") + _, err = store.Zip(ctx, "corp.internal/secret", "v1.0.0") + require.NoError(t, err, "private (skipped) version must remain") +} + +func TestSweepSkippedUnknown(t *testing.T) { + ctx := context.Background() + b := newTestStore(t) + store := b.(verify.Store) + + unknownZip := writeZip(t, map[string]string{"example.com/unknown@v1.0.0/go.mod": "module example.com/unknown\n"}) + saveModule(t, ctx, b, "example.com/unknown", "v1.0.0", unknownZip) + + // oracle has no entry for example.com/unknown@v1.0.0, so CanonicalHash + // returns ErrNotInSumDB; the module is not private (skip never matches it). + oracle := fakeOracle{hashes: map[string]string{}} + skip := func(mod string) bool { return strings.HasPrefix(mod, "corp.internal/") } + + rep, err := verify.Sweep(ctx, store, oracle, skip, true, io.Discard) + require.NoError(t, err) + require.Equal(t, 1, rep.SkippedUnknown) + require.Equal(t, 0, rep.Matched) + require.Equal(t, 0, rep.Mismatched) + require.Equal(t, 0, rep.Purged) + require.Equal(t, 0, rep.SkippedPrivate) + require.Equal(t, 0, rep.Unverified) + + _, err = store.Zip(ctx, "example.com/unknown", "v1.0.0") + require.NoError(t, err, "unknown (not in sumdb) version must not be deleted") +} diff --git a/pkg/verify/verify.go b/pkg/verify/verify.go new file mode 100644 index 000000000..022838c8d --- /dev/null +++ b/pkg/verify/verify.go @@ -0,0 +1,51 @@ +package verify + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/gomods/athens/pkg/storage" + "golang.org/x/mod/sumdb/dirhash" +) + +// Poisoned reports whether the stored zip for mod@ver disagrees with the +// canonical hash from the oracle. Any error obtaining the stored zip or the +// canonical hash is returned to the caller, which MUST treat it as +// "cannot verify -> keep" (never delete on uncertainty). +func Poisoned(ctx context.Context, g storage.Getter, o Oracle, mod, ver string) (bool, error) { + rc, err := g.Zip(ctx, mod, ver) + if err != nil { + return false, fmt.Errorf("verify: read stored zip %s@%s: %w", mod, ver, err) + } + defer rc.Close() + + stored, err := hashZipReader(rc) + if err != nil { + return false, fmt.Errorf("verify: hash stored zip %s@%s: %w", mod, ver, err) + } + canonical, err := o.CanonicalHash(ctx, mod, ver) + if err != nil { + return false, err + } + return stored != canonical, nil +} + +// hashZipReader spools a zip to a temp file (dirhash.HashZip needs a path) and +// returns its h1: hash. +func hashZipReader(r io.Reader) (string, error) { + f, err := os.CreateTemp("", "athens-verify-*.zip") + if err != nil { + return "", err + } + defer os.Remove(f.Name()) + if _, err := io.Copy(f, r); err != nil { + _ = f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + return dirhash.HashZip(f.Name(), dirhash.Hash1) +} diff --git a/pkg/verify/verify_test.go b/pkg/verify/verify_test.go new file mode 100644 index 000000000..ea682526e --- /dev/null +++ b/pkg/verify/verify_test.go @@ -0,0 +1,31 @@ +package verify_test + +import ( + "context" + "testing" + + "github.com/gomods/athens/pkg/verify" + "github.com/stretchr/testify/require" +) + +func TestPoisoned(t *testing.T) { + ctx := context.Background() + b := newTestStore(t) + zipBytes := writeZip(t, map[string]string{"example.com/m@v1.0.0/go.mod": "module example.com/m\n"}) + saveModule(t, ctx, b, "example.com/m", "v1.0.0", zipBytes) + stored := hashOf(t, zipBytes) + + // Stored zip matches the canonical hash -> not poisoned. + bad, err := verify.Poisoned(ctx, b, fakeOracle{map[string]string{"example.com/m@v1.0.0": stored}}, "example.com/m", "v1.0.0") + require.NoError(t, err) + require.False(t, bad) + + // Canonical differs -> poisoned. + bad, err = verify.Poisoned(ctx, b, fakeOracle{map[string]string{"example.com/m@v1.0.0": "h1:DIFFERENT="}}, "example.com/m", "v1.0.0") + require.NoError(t, err) + require.True(t, bad) + + // Oracle can't find it -> error propagated (caller keeps the module). + _, err = verify.Poisoned(ctx, b, fakeOracle{map[string]string{}}, "example.com/m", "v1.0.0") + require.ErrorIs(t, err, verify.ErrNotInSumDB) +}