diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml index 460c360d..0a4b3126 100644 --- a/.github/workflows/ci-go.yml +++ b/.github/workflows/ci-go.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "1.26.4" + go-version: "1.26.5" - name: make test run: make test TEST_WAIT_SHORT=20s TEST_WAIT_LONG=80s - name: Run Proto Generation diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b03d7347..d1ae89b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: - go-version: "1.26.4" + go-version: "1.26.5" - name: Install zip uses: montudor/action-zip@v1 - name: Build standalone binaries diff --git a/.govulncheck-ignore.yaml b/.govulncheck-ignore.yaml index 8a9a12ab..01324bc0 100644 --- a/.govulncheck-ignore.yaml +++ b/.govulncheck-ignore.yaml @@ -19,3 +19,12 @@ - id: GO-2026-4883 module: github.com/docker/docker reason: "No fix available — Moby off-by-one error in plugin privilege validation. Pulled in only via testcontainers-go from test/env/components.go (test infra); not linked into product binaries." +- id: GO-2026-5617 + module: github.com/docker/docker + reason: "No fix available — Moby docker cp race condition allows bind mount redirection to host path. Pulled in only via testcontainers-go from test/env/components.go (test infra); not linked into product binaries." +- id: GO-2026-5668 + module: github.com/docker/docker + reason: "No fix available — Moby docker cp race condition allows creation of arbitrary empty files on the host via symlink swap. Pulled in only via testcontainers-go from test/env/components.go (test infra); not linked into product binaries." +- id: GO-2026-5746 + module: github.com/docker/docker + reason: "No fix available — Moby PUT /containers/{id}/archive executes container binary on the host. Pulled in only via testcontainers-go from test/env/components.go (test infra); not linked into product binaries." diff --git a/Dockerfile b/Dockerfile index 84b42f37..1feed6d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.4 AS builder +FROM golang:1.26.5 AS builder ARG GIT_COMMIT='not set' ARG GIT_TAG=development ENV GIT_COMMIT=$GIT_COMMIT diff --git a/deploy/chorus/Chart.yaml b/deploy/chorus/Chart.yaml index 6fc2c3ed..07b667cc 100644 --- a/deploy/chorus/Chart.yaml +++ b/deploy/chorus/Chart.yaml @@ -4,7 +4,7 @@ description: Helm chart for Chorus S3 management software. keywords: ["S3", "Backup", "Replication", "Migration"] home: https://github.com/clyso/chorus type: application -version: 0.2.0 +version: 0.3.0 appVersion: "v0.6.1" dependencies: - name: redis diff --git a/deploy/chorus/README.md b/deploy/chorus/README.md index cc8694f5..f5741df0 100644 --- a/deploy/chorus/README.md +++ b/deploy/chorus/README.md @@ -45,7 +45,10 @@ storage: ### Credentials -Stored in Kubernetes Secret, separate from storage config: +Stored in Kubernetes Secrets, separate from storage config. Worker and proxy +use different credential models: + +**Worker** (`credentials`) - one credential per user per storage: ```yaml credentials: @@ -56,6 +59,28 @@ credentials: secretAccessKey: "..." ``` +**Proxy** (`proxyCredentials`) - multiple credentials per user, keyed by alias +(`user -> alias -> credential`). The proxy authenticates S3 clients by alias +access key and re-signs forwarded requests with the target storage's credential +of the same alias: + +```yaml +proxyCredentials: + storages: + main: + user1: + laptop: + accessKeyID: "..." + secretAccessKey: "..." + ci: + accessKeyID: "..." + secretAccessKey: "..." +``` + +Required when the proxy is enabled with `proxy.config.auth.useStorage`. Routing +and replication policies stay keyed by user; aliases only affect proxy +authentication and request signing. + ### Dynamic Credentials Manage credentials via API instead of config files: diff --git a/deploy/chorus/examples/values-dynamic-credentials.yaml b/deploy/chorus/examples/values-dynamic-credentials.yaml index 3cdd1901..7dd58c51 100644 --- a/deploy/chorus/examples/values-dynamic-credentials.yaml +++ b/deploy/chorus/examples/values-dynamic-credentials.yaml @@ -44,12 +44,25 @@ credentials: accessKeyID: "AKIAIOSFODNN7EXAMPLE" secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +# Initial proxy credentials (user -> alias -> credential) +proxyCredentials: + storages: + main: + initial-admin: + default: + accessKeyID: "BKIAIOSFODNN7EXAMPLE" + secretAccessKey: "xJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + # After deployment, add more credentials via chorctl: # -# Add S3 credentials: +# Add S3 worker credentials (used by the worker for replication): # chorctl set-user --storage main --user new-user --type s3 \ # --access-key NEWAKEYID --secret-key NEWSECRETKEY # +# Add S3 proxy alias credentials (accepted by the proxy s3 endpoint): +# chorctl set-user --storage main --user new-user --type s3 --alias laptop \ +# --access-key ALIASKEYID --secret-key ALIASSECRETKEY +# # Add Swift credentials: # chorctl set-user --storage swift-main --user projectId123 --type swift \ # --swift-username myuser --swift-password mypassword \ diff --git a/deploy/chorus/examples/values-external-redis.yaml b/deploy/chorus/examples/values-external-redis.yaml index 44e56b52..0e553cf1 100644 --- a/deploy/chorus/examples/values-external-redis.yaml +++ b/deploy/chorus/examples/values-external-redis.yaml @@ -38,6 +38,7 @@ storage: provider: Ceph isSecure: true +# Worker credentials credentials: storages: main: @@ -45,6 +46,20 @@ credentials: accessKeyID: "AKIAIOSFODNN7EXAMPLE" secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +# Proxy credentials (user -> alias -> credential) +proxyCredentials: + storages: + main: + user1: + default: + accessKeyID: "BKIAIOSFODNN7EXAMPLE" + secretAccessKey: "xJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +proxy: + config: + auth: + useStorage: main + --- # Example with Redis Sentinel # externalRedis: diff --git a/deploy/chorus/examples/values-s3.yaml b/deploy/chorus/examples/values-s3.yaml index b4c6e3e4..701bbfba 100644 --- a/deploy/chorus/examples/values-s3.yaml +++ b/deploy/chorus/examples/values-s3.yaml @@ -22,7 +22,8 @@ storage: isSecure: true defaultRegion: eu-west-1 -# Storage credentials (stored in Kubernetes Secret) +# Worker storage credentials (stored in Kubernetes Secret) +# One credential per user per storage, used by the worker for replication credentials: storages: main: @@ -42,13 +43,39 @@ credentials: accessKeyID: "BKIAI44QH8DHBEXAMPLE" secretAccessKey: "ke7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" +# Proxy storage credentials (stored in a separate Kubernetes Secret) +# user -> alias -> credential: the proxy authenticates S3 clients by alias +# access key and re-signs forwarded requests with the target storage's +# credential of the same alias +proxyCredentials: + storages: + main: + admin: + default: + accessKeyID: "CKIAIOSFODNN7EXAMPLE" + secretAccessKey: "yJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + backup-service: + default: + accessKeyID: "CKIAI44QH8DHBEXAMPLE" + secretAccessKey: "le7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + + follower: + admin: + default: + accessKeyID: "DKIAIOSFODNN7EXAMPLE" + secretAccessKey: "zJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + backup-service: + default: + accessKeyID: "DKIAI44QH8DHBEXAMPLE" + secretAccessKey: "me7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + # Proxy configuration proxy: enabled: true replicas: 2 config: auth: - # Use credentials from main storage for S3 proxy authentication + # Authenticate S3 clients against main storage proxy credentials useStorage: main # Worker configuration diff --git a/deploy/chorus/examples/values-swift.yaml b/deploy/chorus/examples/values-swift.yaml index 949363d5..426cf14e 100644 --- a/deploy/chorus/examples/values-swift.yaml +++ b/deploy/chorus/examples/values-swift.yaml @@ -28,7 +28,7 @@ storage: provider: Ceph isSecure: true -# Credentials (stored in Kubernetes Secret) +# Worker credentials (stored in Kubernetes Secret) credentials: storages: # Swift credentials - key must be OpenStack Project ID @@ -53,18 +53,27 @@ credentials: accessKeyID: "AKIAIOSFODNN7EXAMPLE" secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" +# Proxy S3 credentials (stored in a separate Kubernetes Secret) +# user -> alias -> credential. Only needed for the S3 storage: for Swift the +# proxy does not perform authentication itself +proxyCredentials: + storages: + s3-backup: + swift-admin: + default: + accessKeyID: "BKIAIOSFODNN7EXAMPLE" + secretAccessKey: "xJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + # Proxy configuration proxy: enabled: true replicas: 2 config: auth: - # For Swift, you typically use custom credentials or disable auth - # since Swift handles its own authentication - custom: - swift-admin: - accessKeyID: "custom-access-key" - secretAccessKey: "custom-secret-key" + # S3 clients authenticate against the S3 storage proxy credentials. + # Swift requests are not authenticated by the proxy - Swift handles + # its own authentication + useStorage: s3-backup # Worker configuration worker: diff --git a/deploy/chorus/templates/_helpers.tpl b/deploy/chorus/templates/_helpers.tpl index c658bfcc..725bb901 100644 --- a/deploy/chorus/templates/_helpers.tpl +++ b/deploy/chorus/templates/_helpers.tpl @@ -70,6 +70,17 @@ Credentials secret name {{- end }} {{- end }} +{{/* +Proxy credentials secret name +*/}} +{{- define "chorus.proxyCredentialsSecretName" -}} +{{- if .Values.existingProxyCredentialsSecret }} +{{- .Values.existingProxyCredentialsSecret }} +{{- else }} +{{- printf "%s-proxy-credentials" (include "chorus.fullname" .) }} +{{- end }} +{{- end }} + {{/* Redis secret name */}} @@ -121,6 +132,15 @@ true {{- end }} {{- end }} +{{/* +Check if proxy credentials secret should be created +*/}} +{{- define "chorus.createProxyCredentialsSecret" -}} +{{- if and (not .Values.existingProxyCredentialsSecret) .Values.proxyCredentials.storages }} +true +{{- end }} +{{- end }} + {{/* Dynamic credentials secret name */}} diff --git a/deploy/chorus/templates/proxy/deployment-proxy.yaml b/deploy/chorus/templates/proxy/deployment-proxy.yaml index a7931fca..a8cf318f 100644 --- a/deploy/chorus/templates/proxy/deployment-proxy.yaml +++ b/deploy/chorus/templates/proxy/deployment-proxy.yaml @@ -19,7 +19,7 @@ spec: app.kubernetes.io/component: proxy annotations: checksum/config: {{ include (print $.Template.BasePath "/proxy/config.yaml") . | sha256sum }} - {{- if include "chorus.createCredentialsSecret" . }} + {{- if include "chorus.createProxyCredentialsSecret" . }} checksum/secret: {{ include (print $.Template.BasePath "/secrets/secret.yaml") . | sha256sum }} {{- end }} {{- if .Values.proxy.config.metrics.enabled }} @@ -95,7 +95,7 @@ spec: - name: config mountPath: /bin/config/config.yaml subPath: config.yaml - {{- if or .Values.existingCredentialsSecret (include "chorus.createCredentialsSecret" .) }} + {{- if or .Values.existingProxyCredentialsSecret (include "chorus.createProxyCredentialsSecret" .) }} - name: credentials mountPath: /bin/config/override.yaml subPath: config.yaml @@ -104,10 +104,10 @@ spec: - name: config configMap: name: {{ include "chorus.fullname" . }}-proxy - {{- if or .Values.existingCredentialsSecret (include "chorus.createCredentialsSecret" .) }} + {{- if or .Values.existingProxyCredentialsSecret (include "chorus.createProxyCredentialsSecret" .) }} - name: credentials secret: - secretName: {{ include "chorus.credentialsSecretName" . }} + secretName: {{ include "chorus.proxyCredentialsSecretName" . }} {{- end }} {{- $nodeSelector := include "chorus.nodeSelector" (list .Values.proxy.nodeSelector .Values.global.nodeSelector) }} {{- if $nodeSelector }} diff --git a/deploy/chorus/templates/secrets/secret.yaml b/deploy/chorus/templates/secrets/secret.yaml index ac6e2581..0b8ed8f9 100644 --- a/deploy/chorus/templates/secrets/secret.yaml +++ b/deploy/chorus/templates/secrets/secret.yaml @@ -1,4 +1,4 @@ -{{- /* Storage credentials secret */}} +{{- /* Worker storage credentials secret (user -> credential) */}} {{- if include "chorus.createCredentialsSecret" . }} apiVersion: v1 kind: Secret @@ -20,6 +20,28 @@ stringData: {{- end }} {{- end }} --- +{{- /* Proxy storage credentials secret (user -> alias -> credential) */}} +{{- if include "chorus.createProxyCredentialsSecret" . }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "chorus.proxyCredentialsSecretName" . }} + labels: + {{- include "chorus.labels" . | nindent 4 }} +type: Opaque +stringData: + config.yaml: | + {{- if .Values.proxyCredentials.storages }} + storage: + storages: + {{- range $storageName, $users := .Values.proxyCredentials.storages }} + {{ $storageName }}: + credentials: + {{- $users | toYaml | nindent 12 }} + {{- end }} + {{- end }} +{{- end }} +--- {{- /* Dynamic credentials master password secret */}} {{- if include "chorus.createDynamicCredentialsSecret" . }} apiVersion: v1 diff --git a/deploy/chorus/values.yaml b/deploy/chorus/values.yaml index 2fd706f4..421e76ef 100644 --- a/deploy/chorus/values.yaml +++ b/deploy/chorus/values.yaml @@ -81,9 +81,10 @@ storage: # ============================================================================= # CREDENTIALS (Secret) # ============================================================================= -# Storage credentials - stored in Kubernetes Secret +# Worker storage credentials - stored in Kubernetes Secret +# One credential per user per storage. Used by the worker for replication. credentials: - # Per-storage credentials + # Per-storage credentials: user -> credential storages: {} # S3 credentials example: # main: @@ -100,9 +101,38 @@ credentials: # tenantName: admin # Use existing secret instead of creating one -# Secret must have 'config' key with YAML content matching override.yaml format +# Secret must have 'config.yaml' key with YAML content matching worker override.yaml format existingCredentialsSecret: "" +# ============================================================================= +# PROXY CREDENTIALS (Secret) +# ============================================================================= +# Proxy S3 storage credentials - stored in a separate Kubernetes Secret. +# Unlike the worker, the proxy supports multiple credentials per user: +# user -> alias -> credential. The proxy authenticates callers by alias access +# key, keeps routing/replication policy keyed by user, and re-signs forwarded +# requests with the target storage's credential of the same alias name. +proxyCredentials: + # Per-storage S3 credentials: user -> alias -> credential + storages: {} + # main: + # user1: + # laptop: # alias name, joins credentials across storages + # accessKeyID: "AKIAIOSFODNN7EXAMPLE" + # secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + # ci: + # accessKeyID: "AKIAI44QH8DHBEXAMPLE" + # secretAccessKey: "je7MtGbClwBF/2Zp9Utk/h3yCo8nvbEXAMPLEKEY" + # follower: + # user1: + # laptop: + # accessKeyID: "BKIAIOSFODNN7EXAMPLE" + # secretAccessKey: "xJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + +# Use existing secret instead of creating one +# Secret must have 'config.yaml' key with YAML content matching proxy override.yaml format +existingProxyCredentialsSecret: "" + # ============================================================================= # FEATURES # ============================================================================= @@ -218,11 +248,13 @@ proxy: auth: allowV2Signature: false - useStorage: "" # Use credentials from this storage - custom: {} # Or define custom credentials + useStorage: "" # Use proxy credentials of this storage (see proxyCredentials) + custom: {} # Or define custom credentials: user -> alias -> credential. + # Each (user, alias) pair must exist in the main storage proxy credentials. # user1: - # accessKeyID: xxx - # secretAccessKey: xxx + # alias1: + # accessKeyID: xxx + # secretAccessKey: xxx # ============================================================================= # WORKER diff --git a/docker-compose/README.md b/docker-compose/README.md index 35c299e3..f20c13ff 100644 --- a/docker-compose/README.md +++ b/docker-compose/README.md @@ -22,7 +22,8 @@ │   ├── s3cmd-follower.conf # s3cmd credentials for follower storage │   ├── s3cmd-main.conf # s3cmd credentials for main storage │   ├── s3cmd-proxy.conf # s3cmd credentials for proxy storage -│   ├── s3-credentials.yaml # chorus common config with S3 credentials +│   ├── s3-credentials.yaml # chorus worker config with S3 credentials +│   ├── proxy-s3-credentials.yaml # chorus proxy config with S3 alias credentials │   └── worker-conf.yaml # example config for chorus-worker ``` @@ -113,7 +114,7 @@ Try to add more worker instances to speed up replication process and observe how ``` The same can be done for `worker3`, `worker4`, etc. And for `proxy` service. -Replace S3 credentials in [./s3-credentials.yaml](./s3-credentials.yaml) with your own s3 storages and start docker-compose without fake backends: +Replace S3 credentials in [./s3-credentials.yaml](./s3-credentials.yaml) and [./proxy-s3-credentials.yaml](./proxy-s3-credentials.yaml) with your own s3 storages and start docker-compose without fake backends: ```shell docker-compose -f ./docker-compose/docker-compose.yml --profile proxy up ``` diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 0c1ff7b7..8aade3c5 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -55,9 +55,9 @@ services: - type: bind source: ./proxy-conf.yaml target: /bin/config/config.yaml - # common config: s3 credentials + # proxy s3 credentials (user -> alias -> credential) - type: bind - source: ./s3-credentials.yaml + source: ./proxy-s3-credentials.yaml target: /bin/config/override.yaml ports: - "9669:9669" # expose proxy s3 api diff --git a/docker-compose/proxy-s3-credentials.yaml b/docker-compose/proxy-s3-credentials.yaml new file mode 100644 index 00000000..ad793faa --- /dev/null +++ b/docker-compose/proxy-s3-credentials.yaml @@ -0,0 +1,35 @@ +storage: + main: main + storages: + main: # yaml key with some handy storage name + type: S3 + address: "http://fake-s3-main:9000" + credentials: # user -> alias -> credential + user1: + default: # alias name for the credential, e.g. laptop or ci + accessKeyID: fakeKey + secretAccessKey: fakeSecret + provider: Other # # S3 provider name. Required to use provider-specific APIs, like setting versionID. Use 'Other' if your storage is S3 compatible but not listed. + healthCheckInterval: 10s + httpTimeout: 1m + isSecure: false #set false for http address + defaultRegion: "" + rateLimit: + enable: true + rpm: 600 + follower: # yaml key with some handy storage name + type: S3 + address: "http://fake-s3-follower:9000" + credentials: # user -> alias -> credential. Alias names join credentials across storages + user1: + default: + accessKeyID: fakeKey2 + secretAccessKey: fakeSecret2 + provider: Other # # S3 provider name. Required to use provider-specific APIs, like setting versionID. Use 'Other' if your storage is S3 compatible but not listed. + healthCheckInterval: 10s + httpTimeout: 1m + isSecure: false + defaultRegion: "" + rateLimit: + enable: true + rpm: 600 diff --git a/docs/dynamic-creds.md b/docs/dynamic-creds.md index e7d54d5a..dff3789d 100644 --- a/docs/dynamic-creds.md +++ b/docs/dynamic-creds.md @@ -55,4 +55,31 @@ Pull-based approach: 2. If `version` is greater than cached `version`, unseal new credentials and update cached credentials and `version`. 3. If `version` is equal to cached `version`, do nothing. +## Storage format + +All dynamic credentials live in a single redis HASH `chorus:dynamic_creds` (see the `credsSvc` +doc comment in [pkg/objstore/credentials.go](../pkg/objstore/credentials.go) - the authoritative +format description). Besides the `version` and `salt` fields, each credential is one hash field: + +- `::` - worker (per-user) credential, e.g. `S3:my_storage:alice` +- `S3:::` - proxy alias credential, e.g. `S3:my_storage:alice:laptop` + +Storage, user, and alias names must not contain `:`. + +## Proxy alias credentials + +The proxy allows many S3 credentials per user. Each is an **alias**: one S3 access key +belonging to a user. The proxy authenticates callers by alias access key, applies routing and +replication policy by user only, and re-signs forwarded requests with the target storage's +credential of the same alias name. + +Alias credentials are set via the `SetUserCredentials` management API with the optional `alias` +field (`chorctl set-user --alias ...`). The `alias` field is supported only for S3 +credentials; combining it with Swift credentials is an error. + +Mode semantics: +- The **worker** uses only per-user entries and ignores alias entries. +- The **proxy** is alias-only: it indexes static nested config plus dynamic alias entries. + Dynamic per-user entries (set without `alias`) do not authenticate at the proxy. + diff --git a/go.mod b/go.mod index acb07a3e..81f54141 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/clyso/chorus -go 1.26.4 +go 1.26.5 replace ( github.com/hibiken/asynq => github.com/clyso/asynq v0.0.0-20251202163730-20cb0d89aa76 diff --git a/pkg/api/chorus_handlers.go b/pkg/api/chorus_handlers.go index d09b48f0..1f7c8263 100644 --- a/pkg/api/chorus_handlers.go +++ b/pkg/api/chorus_handlers.go @@ -83,15 +83,25 @@ func (h *chorusHandlers) SetUserCredentials(ctx context.Context, req *pb.SetUser if req.S3Cred == nil { return nil, fmt.Errorf("%w: s3 credentials are required for storage type S3", dom.ErrInvalidArg) } - err := h.credsSvc.SetS3Credentials(ctx, req.Storage, req.User, s3.CredentialsV4{ + cred := s3.CredentialsV4{ AccessKeyID: req.S3Cred.AccessKey, SecretAccessKey: req.S3Cred.SecretKey, - }) + } + var err error + if req.GetAlias() != "" { + // proxy alias credential + err = h.credsSvc.SetS3AliasCredentials(ctx, req.Storage, req.User, req.GetAlias(), cred) + } else { + err = h.credsSvc.SetS3Credentials(ctx, req.Storage, req.User, cred) + } if err != nil { return nil, err } return &emptypb.Empty{}, nil case dom.Swift: + if req.GetAlias() != "" { + return nil, fmt.Errorf("%w: alias credentials are supported only for S3 storages", dom.ErrInvalidArg) + } if req.SwiftCred == nil { return nil, fmt.Errorf("%w: swift credentials are required for storage type Swift", dom.ErrInvalidArg) } diff --git a/pkg/ctx/context.go b/pkg/ctx/context.go index d93c8ac9..2d05d317 100644 --- a/pkg/ctx/context.go +++ b/pkg/ctx/context.go @@ -36,6 +36,7 @@ type storageKey struct{} type flowKey struct{} type traceKey struct{} type userKey struct{} +type aliasKey struct{} type routingPolicyKey struct{} type replicationsKey struct{} type inProgressZeroDowntimeKey struct{} @@ -177,6 +178,23 @@ func GetUser(ctx context.Context) string { return k } +func SetAlias(ctx context.Context, a string) context.Context { + if a == "" { + zerolog.Ctx(ctx).Warn().Msg("ignore: trying to set empty alias to ctx") + return ctx + } + if prev := GetAlias(ctx); prev != "" && prev != a { + zerolog.Ctx(ctx).Warn().Msgf("cannot set alias %s, ctx already contains alias %s", a, prev) + return ctx + } + return context.WithValue(ctx, aliasKey{}, a) +} + +func GetAlias(ctx context.Context) string { + k, _ := ctx.Value(aliasKey{}).(string) + return k +} + func GetRoutingPolicy(ctx context.Context) string { p, _ := ctx.Value(routingPolicyKey{}).(string) if p == "" { diff --git a/pkg/objstore/credentials.go b/pkg/objstore/credentials.go index 2d31a99a..8bd7796d 100644 --- a/pkg/objstore/credentials.go +++ b/pkg/objstore/credentials.go @@ -24,6 +24,7 @@ import ( "errors" "fmt" "runtime" + "slices" "strconv" "strings" "sync" @@ -46,8 +47,27 @@ const ( minPasswordLength = 12 ) +// WorkerCredsLookup resolves per-user credentials used by the worker. +type WorkerCredsLookup interface { + GetS3Credentials(storage, user string) (s3.CredentialsV4, error) + GetSwiftCredentials(storage, user string) (swift.Credentials, error) +} + +// ProxyCredsLookup resolves per-alias credentials used by the proxy. +type ProxyCredsLookup interface { + FindS3Credentials(storage, accessKey string) (user, alias string, cred s3.CredentialsV4, err error) + GetS3AliasCredentials(storage, user, alias string) (s3.CredentialsV4, error) + ListS3Aliases(storage, user string) ([]string, error) +} + +// CredsWriter stores dynamic credentials. +type CredsWriter interface { + SetS3Credentials(ctx context.Context, storage, user string, cred s3.CredentialsV4) error + SetS3AliasCredentials(ctx context.Context, storage, user, alias string, cred s3.CredentialsV4) error + SetSwiftCredentials(ctx context.Context, storage, user string, cred swift.Credentials) error +} + type CredsService interface { - FindS3Credentials(storage, accessKey string) (user string, cred s3.CredentialsV4, err error) MainStorage() string Storages() map[string]dom.StorageType GetS3Address(storage string) (s3.StorageAddress, error) @@ -56,11 +76,9 @@ type CredsService interface { HasUser(storage, user string) error ListUsers(storage string) []string - GetSwiftCredentials(storage, user string) (swift.Credentials, error) - GetS3Credentials(storage, user string) (s3.CredentialsV4, error) - - SetSwiftCredentials(ctx context.Context, storage, user string, cred swift.Credentials) error - SetS3Credentials(ctx context.Context, storage, user string, cred s3.CredentialsV4) error + WorkerCredsLookup + ProxyCredsLookup + CredsWriter } type DynamicCredentialsConfig struct { @@ -86,30 +104,43 @@ func (c *DynamicCredentialsConfig) Validate() error { return nil } -func NewCredsSvc(ctx context.Context, conf *Config, rc redis.UniversalClient) (*credsSvc, error) { - if err := conf.DynamicCredentials.Validate(); err != nil { +// New creates a credentials service. Exactly one of worker/proxy must be +// non-nil; that selects the service mode. Role-specific lookups for the +// missing config fail with dom.ErrNotImplemented. +func New(ctx context.Context, rc redis.UniversalClient, dc DynamicCredentialsConfig, + worker *Config, proxy *ProxyConfig) (*credsSvc, error) { + if (worker == nil) == (proxy == nil) { + return nil, fmt.Errorf("%w: exactly one of worker or proxy config must be set", dom.ErrInvalidArg) + } + if err := dc.Validate(); err != nil { return nil, err } instance := &credsSvc{ - types: conf.Types(), - s3AccessKeyIdx: buildS3AccessKeyIdxFromConf(conf), - config: conf, - client: rc, - version: -1, - } - if !conf.DynamicCredentials.Enabled { + worker: worker, + proxy: proxy, + dc: dc, + client: rc, + version: -1, + } + if worker != nil { + instance.types = worker.Types() + } else { + instance.types = proxy.Types() + instance.s3AccessKeyIdx = buildS3AccessKeyIdxFromConf(proxy) + } + if !dc.Enabled { // dynamic credentials disabled, return credentials from config only return instance, nil } - if !conf.DynamicCredentials.DisableEncryption { + if !dc.DisableEncryption { // derive encryption key salt, err := getOrCreateSalt(ctx, rc) if err != nil { return nil, fmt.Errorf("failed to get or create salt: %w", err) } - instance.key = deriveKey(conf.DynamicCredentials.MasterPassword, salt) + instance.key = deriveKey(dc.MasterPassword, salt) } - if conf.DynamicCredentials.DisableEncryption { + if dc.DisableEncryption { zerolog.Ctx(ctx).Warn().Msg("dynamic credentials encryption is disabled, credentials will be stored in Redis in plaintext") } // fetch initial credentials from redis @@ -118,7 +149,7 @@ func NewCredsSvc(ctx context.Context, conf *Config, rc redis.UniversalClient) (* } // start background sync goroutine go func() { - ticker := time.NewTicker(conf.DynamicCredentials.PollInterval) + ticker := time.NewTicker(dc.PollInterval) defer ticker.Stop() for { select { @@ -142,66 +173,134 @@ Implements CredsService interface for dynamic credentials management. If dynamic credentials are disabled, credentials are read from config only. Otherwise, credentials are stored in Redis and cached locally with periodic sync. +The service runs in one of two modes selected at construction: +- worker mode: per-user credentials (GetS3Credentials/GetSwiftCredentials) +- proxy mode: per-alias S3 credentials (FindS3Credentials/GetS3AliasCredentials/ListS3Aliases) +Lookups for the other mode fail with dom.ErrNotImplemented. + Redis HASH structure: - Key: "chorus:dynamic_creds" - Fields: - "version": int64 - version of the credentials, incremented on each update - "salt": string - base64 encoded salt for encryption key derivation - - "::": credentials encoded string value + - "::": worker (per-user) credentials encoded string value + - "S3:::": proxy (per-alias) S3 credentials encoded string value - value: if encryption is enabled: JSON serialized credentials -> encrypted with AES-GCM -> base64 encoded - value: if encryption is disabled: JSON serialized credentials +Alias fields are S3-only and read only by proxy-mode instances; worker-mode +instances skip them. Storage, user, and alias names must not contain ':' - +enforced on the write path so the field format stays unambiguous. + Credentials encryption: - AES-GCM with 256-bit key derived from master password using Argon2id KDF with per-instance salt stored in Redis. Local cache: -- Maps of storage:user to credentials for S3 and Swift, updated on sync if version changes in Redis. +- worker mode: maps of storage:user to credentials for S3 and Swift, updated on sync if version changes in Redis. +- proxy mode: map of storage:user:alias to S3 credentials plus an access-key index seeded from static config. */ type credsSvc struct { client redis.UniversalClient types map[string]dom.StorageType - config *Config + worker *Config + proxy *ProxyConfig swiftCreds map[string]swift.Credentials s3Creds map[string]s3.CredentialsV4 + // proxy mode: dynamic S3 alias credentials keyed "::" + s3AliasCreds map[string]s3.CredentialsV4 // index of S3 credentials by access key for chorus Proxy Auth Middleware s3AccessKeyIdx map[string]s3CredsByAccessKey key []byte + dc DynamicCredentialsConfig version int64 sync.RWMutex } type s3CredsByAccessKey struct { user string + alias string credentials s3.CredentialsV4 } -func (s *credsSvc) FindS3Credentials(storage, accessKey string) (user string, cred s3.CredentialsV4, err error) { - if s.config.DynamicCredentials.Enabled { +func (s *credsSvc) FindS3Credentials(storage, accessKey string) (user, alias string, cred s3.CredentialsV4, err error) { + if s.proxy == nil { + return "", "", s3.CredentialsV4{}, fmt.Errorf("%w: FindS3Credentials requires proxy-mode credentials service", dom.ErrNotImplemented) + } + if s.dc.Enabled { s.RLock() defer s.RUnlock() } key := s3accessKeyIdxKey(storage, accessKey) res, ok := s.s3AccessKeyIdx[key] if !ok { - return "", s3.CredentialsV4{}, dom.ErrNotFound + return "", "", s3.CredentialsV4{}, dom.ErrNotFound } - return res.user, res.credentials, nil + return res.user, res.alias, res.credentials, nil } -func (s *credsSvc) Storages() map[string]dom.StorageType { - return s.types +func (s *credsSvc) GetS3AliasCredentials(storage, user, alias string) (s3.CredentialsV4, error) { + if s.proxy == nil { + return s3.CredentialsV4{}, fmt.Errorf("%w: GetS3AliasCredentials requires proxy-mode credentials service", dom.ErrNotImplemented) + } + // static config takes precedence over dynamic entries + if stor, ok := s.proxy.S3Storages()[storage]; ok { + if cred, ok := stor.Credentials[user][alias]; ok { + return cred, nil + } + } + if !s.dc.Enabled { + return s3.CredentialsV4{}, dom.ErrNotFound + } + s.RLock() + defer s.RUnlock() + cred, ok := s.s3AliasCreds[aliasCacheKey(storage, user, alias)] + if !ok { + return s3.CredentialsV4{}, dom.ErrNotFound + } + return cred, nil } -func (s *credsSvc) StorageType(storage string) (dom.StorageType, error) { - storConf, ok := s.config.Storages[storage] - if !ok { - return "", fmt.Errorf("%w: storage %q not found", dom.ErrNotFound, storage) +func (s *credsSvc) ListS3Aliases(storage, user string) ([]string, error) { + if s.proxy == nil { + return nil, fmt.Errorf("%w: ListS3Aliases requires proxy-mode credentials service", dom.ErrNotImplemented) + } + seen := make(map[string]struct{}) + if stor, ok := s.proxy.S3Storages()[storage]; ok { + for alias := range stor.Credentials[user] { + seen[alias] = struct{}{} + } + } + if s.dc.Enabled { + s.RLock() + prefix := credsCacheKey(storage, user) + ":" + for cacheKey := range s.s3AliasCreds { + if alias, ok := strings.CutPrefix(cacheKey, prefix); ok { + seen[alias] = struct{}{} + } + } + s.RUnlock() + } + res := make([]string, 0, len(seen)) + for alias := range seen { + res = append(res, alias) } - return storConf.CommonConfig.Type, nil + slices.Sort(res) + return res, nil +} + +func (s *credsSvc) Storages() map[string]dom.StorageType { + return s.types } func (s *credsSvc) GetS3Address(storage string) (s3.StorageAddress, error) { - res, ok := s.config.S3Storages()[storage] + if s.worker != nil { + res, ok := s.worker.S3Storages()[storage] + if !ok { + return s3.StorageAddress{}, dom.ErrNotFound + } + return res.StorageAddress, nil + } + res, ok := s.proxy.S3Storages()[storage] if !ok { return s3.StorageAddress{}, dom.ErrNotFound } @@ -209,20 +308,57 @@ func (s *credsSvc) GetS3Address(storage string) (s3.StorageAddress, error) { } func (s *credsSvc) GetSwiftAddress(storage string) (swift.StorageAddress, error) { - res, ok := s.config.SwiftStorages()[storage] + if s.worker != nil { + res, ok := s.worker.SwiftStorages()[storage] + if !ok { + return swift.StorageAddress{}, dom.ErrNotFound + } + return res.StorageAddress, nil + } + res, ok := s.proxy.SwiftStorages()[storage] if !ok { return swift.StorageAddress{}, dom.ErrNotFound } return res.StorageAddress, nil } +// existsInConfig checks static config of whichever mode is configured. +func (s *credsSvc) existsInConfig(storage, user string) error { + if s.worker != nil { + return s.worker.Exists(storage, user) + } + return s.proxy.Exists(storage, user) +} + +// configUserList returns static config users; ok is false for unknown storage. +func (s *credsSvc) configUserList(storage string) (users []string, ok bool) { + if s.worker != nil { + stor, ok := s.worker.Storages[storage] + if !ok { + return nil, false + } + return stor.UserList(), true + } + stor, ok := s.proxy.Storages[storage] + if !ok { + return nil, false + } + return stor.UserList(), true +} + +func (s *credsSvc) sameStorType(fromStorage, toStorage string) error { + if s.worker != nil { + return s.worker.sameStorType(fromStorage, toStorage) + } + return s.proxy.sameStorType(fromStorage, toStorage) +} + func (s *credsSvc) ListUsers(storage string) []string { - storageConf, ok := s.config.Storages[storage] + users, ok := s.configUserList(storage) if !ok { return nil } - users := storageConf.UserList() - if !s.config.DynamicCredentials.Enabled { + if !s.dc.Enabled { // dynamic credentials disabled, return users from config only return users } @@ -230,7 +366,27 @@ func (s *credsSvc) ListUsers(storage string) []string { s.RLock() defer s.RUnlock() - switch storageConf.CommonConfig.Type { + if s.proxy != nil { + // proxy mode: users from dynamic alias entries + seen := make(map[string]struct{}, len(users)) + for _, user := range users { + seen[user] = struct{}{} + } + for cacheKey := range s.s3AliasCreds { + stor, user, _ := aliasCacheKeyToCreds(cacheKey) + if stor != storage { + continue + } + if _, ok := seen[user]; ok { + continue + } + seen[user] = struct{}{} + users = append(users, user) + } + return users + } + + switch s.types[storage] { case dom.S3: for cacheKey := range s.s3Creds { stor, user := cacheKeyToCreds(cacheKey) @@ -247,17 +403,20 @@ func (s *credsSvc) ListUsers(storage string) []string { } default: // should not happen due to prior validation of storage config - panic(fmt.Sprintf("unsupported storage type %q", storageConf.CommonConfig.Type)) + panic(fmt.Sprintf("unsupported storage type %q", s.types[storage])) } return users } func (s *credsSvc) MainStorage() string { - return s.config.Main + if s.worker != nil { + return s.worker.Main + } + return s.proxy.Main } func (s *credsSvc) ValidateReplicationID(id entity.UniversalReplicationID) error { - if err := s.config.sameStorType(id.FromStorage(), id.ToStorage()); err != nil { + if err := s.sameStorType(id.FromStorage(), id.ToStorage()); err != nil { // TODO: allow cross-type replication in the future? return err } @@ -273,8 +432,8 @@ func (s *credsSvc) ValidateReplicationID(id entity.UniversalReplicationID) error } func (s *credsSvc) HasUser(storage string, user string) error { - err := s.config.Exists(storage, user) - if !s.config.DynamicCredentials.Enabled { + err := s.existsInConfig(storage, user) + if !s.dc.Enabled { // dynamic credentials disabled, return result from config return err } @@ -286,6 +445,16 @@ func (s *credsSvc) HasUser(storage string, user string) error { cacheKey := credsCacheKey(storage, user) s.RLock() defer s.RUnlock() + if s.proxy != nil { + // proxy mode: user exists if it has at least one dynamic alias + prefix := cacheKey + ":" + for aliasKey := range s.s3AliasCreds { + if strings.HasPrefix(aliasKey, prefix) { + return nil + } + } + return dom.ErrNotFound + } if _, ok := s.s3Creds[cacheKey]; ok { return nil } @@ -296,7 +465,7 @@ func (s *credsSvc) HasUser(storage string, user string) error { } func (s *credsSvc) sync(ctx context.Context) error { - if !s.config.DynamicCredentials.Enabled { + if !s.dc.Enabled { return nil } // read redis version of creds and update local cache @@ -314,7 +483,11 @@ func (s *credsSvc) sync(ctx context.Context) error { } s3Creds := make(map[string]s3.CredentialsV4) swiftCreds := make(map[string]swift.Credentials) - s3AccessKeyIdx := buildS3AccessKeyIdxFromConf(s.config) + s3AliasCreds := make(map[string]s3.CredentialsV4) + var s3AccessKeyIdx map[string]s3CredsByAccessKey + if s.proxy != nil { + s3AccessKeyIdx = buildS3AccessKeyIdxFromConf(s.proxy) + } for key, value := range allCreds { if key == redisHFieldVersion { // set remote version: @@ -328,35 +501,44 @@ func (s *credsSvc) sync(ctx context.Context) error { // ignore salt field continue } - storType, cacheKey, err := credsFromRedisField(key) + storType, storage, user, alias, err := credsFromRedisField(key) if err != nil { return fmt.Errorf("failed to parse redis field %s: %w", key, err) } + if s.worker != nil && alias != "" { + // proxy alias entry - not used by the worker + continue + } + if s.proxy != nil && (alias == "" || storType != dom.S3) { + // proxy indexes only S3 alias entries; worker per-user entries are skipped + continue + } plaintext, err := s.decrypt([]byte(value)) if err != nil { - return fmt.Errorf("failed to decrypt credentials for %s: %w", cacheKey, err) + return fmt.Errorf("failed to decrypt credentials for %s: %w", key, err) } switch storType { case dom.S3: var cred s3.CredentialsV4 if err := json.Unmarshal(plaintext, &cred); err != nil { - return fmt.Errorf("failed to unmarshal S3 credentials for %s: %w", cacheKey, err) + return fmt.Errorf("failed to unmarshal S3 credentials for %s: %w", key, err) } - s3Creds[cacheKey] = cred - - // update s3AccessKeyIdx - stor, user := cacheKeyToCreds(cacheKey) - s3AccessKeyIdxKey := s3accessKeyIdxKey(stor, cred.AccessKeyID) - s3AccessKeyIdx[s3AccessKeyIdxKey] = s3CredsByAccessKey{ - user: user, - credentials: cred, + if alias != "" { + s3AliasCreds[aliasCacheKey(storage, user, alias)] = cred + s3AccessKeyIdx[s3accessKeyIdxKey(storage, cred.AccessKeyID)] = s3CredsByAccessKey{ + user: user, + alias: alias, + credentials: cred, + } + continue } + s3Creds[credsCacheKey(storage, user)] = cred case dom.Swift: var cred swift.Credentials if err := json.Unmarshal(plaintext, &cred); err != nil { - return fmt.Errorf("failed to unmarshal Swift credentials for %s: %w", cacheKey, err) + return fmt.Errorf("failed to unmarshal Swift credentials for %s: %w", key, err) } - swiftCreds[cacheKey] = cred + swiftCreds[credsCacheKey(storage, user)] = cred default: return fmt.Errorf("%w: unknown storage type %s", dom.ErrInvalidArg, storType) } @@ -365,7 +547,10 @@ func (s *credsSvc) sync(ctx context.Context) error { s.Lock() defer s.Unlock() s.s3Creds = s3Creds - s.s3AccessKeyIdx = s3AccessKeyIdx + s.s3AliasCreds = s3AliasCreds + if s.proxy != nil { + s.s3AccessKeyIdx = s3AccessKeyIdx + } s.swiftCreds = swiftCreds s.version = remoteVer @@ -374,8 +559,11 @@ func (s *credsSvc) sync(ctx context.Context) error { } func (s *credsSvc) GetS3Credentials(storage string, user string) (s3.CredentialsV4, error) { + if s.worker == nil { + return s3.CredentialsV4{}, fmt.Errorf("%w: GetS3Credentials requires worker-mode credentials service", dom.ErrNotImplemented) + } creds, err := s.getS3FromConfig(storage, user) - if !s.config.DynamicCredentials.Enabled { + if !s.dc.Enabled { // dynamic credentials disabled, return result from config return creds, err } @@ -395,7 +583,7 @@ func (s *credsSvc) GetS3Credentials(storage string, user string) (s3.Credentials } func (s *credsSvc) getS3FromConfig(storage string, user string) (s3.CredentialsV4, error) { - stor, ok := s.config.S3Storages()[storage] + stor, ok := s.worker.S3Storages()[storage] if !ok { return s3.CredentialsV4{}, dom.ErrNotFound } @@ -407,8 +595,11 @@ func (s *credsSvc) getS3FromConfig(storage string, user string) (s3.CredentialsV } func (s *credsSvc) GetSwiftCredentials(storage string, user string) (swift.Credentials, error) { + if s.worker == nil { + return swift.Credentials{}, fmt.Errorf("%w: GetSwiftCredentials requires worker-mode credentials service", dom.ErrNotImplemented) + } creds, err := s.getSwiftFromConfig(storage, user) - if !s.config.DynamicCredentials.Enabled { + if !s.dc.Enabled { // dynamic credentials disabled, return result from config return creds, err } @@ -428,7 +619,7 @@ func (s *credsSvc) GetSwiftCredentials(storage string, user string) (swift.Crede } func (s *credsSvc) getSwiftFromConfig(storage string, user string) (swift.Credentials, error) { - stor, ok := s.config.SwiftStorages()[storage] + stor, ok := s.worker.SwiftStorages()[storage] if !ok { return swift.Credentials{}, dom.ErrNotFound } @@ -443,29 +634,44 @@ func (s *credsSvc) SetS3Credentials(ctx context.Context, storage string, user st if err := cred.Validate(); err != nil { return fmt.Errorf("%w: invalid S3 credentials: %w", dom.ErrInvalidArg, err) } - return s.storeCred(ctx, dom.S3, storage, user, cred) + return s.storeCred(ctx, dom.S3, storage, user, "", cred) } -func (s *credsSvc) storeCred(ctx context.Context, storType dom.StorageType, storage, user string, value any) error { - if !s.config.DynamicCredentials.Enabled { - return fmt.Errorf("%w: dynamic credentials are disabled", dom.ErrInvalidArg) +func (s *credsSvc) SetS3AliasCredentials(ctx context.Context, storage, user, alias string, cred s3.CredentialsV4) error { + if err := cred.Validate(); err != nil { + return fmt.Errorf("%w: invalid S3 credentials: %w", dom.ErrInvalidArg, err) } - // cannot overwrite existing user in config - if err := s.config.Exists(storage, user); err == nil { - return fmt.Errorf("%w: cannot overwrite existing user %q in config for storage %q", dom.ErrInvalidArg, user, storage) + if alias == "" { + return fmt.Errorf("%w: alias is required", dom.ErrInvalidArg) } - // check if storage exists in config - switch storType { - case dom.S3: - if _, ok := s.config.S3Storages()[storage]; !ok { - return fmt.Errorf("%w: storage %q not found in config", dom.ErrInvalidArg, storage) + return s.storeCred(ctx, dom.S3, storage, user, alias, cred) +} + +func (s *credsSvc) storeCred(ctx context.Context, storType dom.StorageType, storage, user, alias string, value any) error { + if !s.dc.Enabled { + return fmt.Errorf("%w: dynamic credentials are disabled", dom.ErrInvalidArg) + } + // keep the redis field format unambiguous + for _, name := range []string{storage, user, alias} { + if strings.Contains(name, ":") { + return fmt.Errorf("%w: storage, user, and alias names must not contain ':'", dom.ErrInvalidArg) } - case dom.Swift: - if _, ok := s.config.SwiftStorages()[storage]; !ok { - return fmt.Errorf("%w: storage %q not found in config", dom.ErrInvalidArg, storage) + } + if user == "" { + return fmt.Errorf("%w: user is required", dom.ErrInvalidArg) + } + if alias == "" { + // cannot overwrite existing user in config + if err := s.existsInConfig(storage, user); err == nil { + return fmt.Errorf("%w: cannot overwrite existing user %q in config for storage %q", dom.ErrInvalidArg, user, storage) } - default: - return fmt.Errorf("%w: unknown storage type %q", dom.ErrInvalidArg, storType) + } else if storType != dom.S3 { + // alias entries are S3-only + return fmt.Errorf("%w: alias credentials are supported only for S3 storages", dom.ErrInvalidArg) + } + // check if storage exists in config + if s.types[storage] != storType { + return fmt.Errorf("%w: %s storage %q not found in config", dom.ErrInvalidArg, storType, storage) } // encrypt credentials @@ -473,7 +679,7 @@ func (s *credsSvc) storeCred(ctx context.Context, storType dom.StorageType, stor if err != nil { return fmt.Errorf("failed to encrypt S3 credentials: %w", err) } - key := credsToRedisField(storType, storage, user) + key := credsToRedisField(storType, storage, user, alias) // store in redis and increment version in a transaction tx := s.client.TxPipeline() tx.HSet(ctx, redisHashKey, key, value) @@ -492,7 +698,7 @@ func (s *credsSvc) SetSwiftCredentials(ctx context.Context, storage string, user if err := cred.Validate(); err != nil { return fmt.Errorf("%w: invalid Swift credentials: %w", dom.ErrInvalidArg, err) } - return s.storeCred(ctx, dom.Swift, storage, user, cred) + return s.storeCred(ctx, dom.Swift, storage, user, "", cred) } func getOrCreateSalt(ctx context.Context, rc redis.UniversalClient) ([]byte, error) { @@ -549,6 +755,12 @@ func credsCacheKey(storage, user string) string { return storage + ":" + user } +func aliasCacheKey(storage, user, alias string) string { + // cache key format: "::" + // e.g. "my_storage:alice:laptop" + return storage + ":" + user + ":" + alias +} + func cacheKeyToCreds(cacheKey string) (storage, user string) { parts := strings.SplitN(cacheKey, ":", 2) if len(parts) != 2 { @@ -557,19 +769,35 @@ func cacheKeyToCreds(cacheKey string) (storage, user string) { return parts[0], parts[1] } -func credsToRedisField(storType dom.StorageType, storage, user string) string { - // redis field format: "::" +func aliasCacheKeyToCreds(cacheKey string) (storage, user, alias string) { + parts := strings.SplitN(cacheKey, ":", 3) + if len(parts) != 3 { + panic(fmt.Sprintf("invalid alias cache key format: %s", cacheKey)) + } + return parts[0], parts[1], parts[2] +} + +func credsToRedisField(storType dom.StorageType, storage, user, alias string) string { + // redis field format: "::" for worker entries // e.g. "S3:my_storage:alice" - return fmt.Sprintf("%s:%s", storType, credsCacheKey(storage, user)) + // and "S3:::" for proxy alias entries + // e.g. "S3:my_storage:alice:laptop" + if alias == "" { + return fmt.Sprintf("%s:%s", storType, credsCacheKey(storage, user)) + } + return fmt.Sprintf("%s:%s", storType, aliasCacheKey(storage, user, alias)) } -func credsFromRedisField(field string) (storType dom.StorageType, cacheKey string, err error) { - parts := strings.SplitN(field, ":", 2) - if len(parts) != 2 { - return "", "", fmt.Errorf("%w: invalid redis field format", dom.ErrInvalidArg) +func credsFromRedisField(field string) (storType dom.StorageType, storage, user, alias string, err error) { + parts := strings.Split(field, ":") + switch len(parts) { + case 3: + return dom.StorageType(parts[0]), parts[1], parts[2], "", nil + case 4: + return dom.StorageType(parts[0]), parts[1], parts[2], parts[3], nil + default: + return "", "", "", "", fmt.Errorf("%w: invalid redis field format", dom.ErrInvalidArg) } - storType, cacheKey = dom.StorageType(parts[0]), parts[1] - return storType, cacheKey, nil } func s3accessKeyIdxKey(storage, accessKey string) string { @@ -582,7 +810,7 @@ func (s *credsSvc) encrypt(credsStruct any) (string, error) { if err != nil { return "", fmt.Errorf("failed to marshal credentials to JSON: %w", err) } - if s.config.DynamicCredentials.DisableEncryption { + if s.dc.DisableEncryption { return string(plaintext), nil } // encrypt with AES-GCM @@ -605,7 +833,7 @@ func (s *credsSvc) encrypt(credsStruct any) (string, error) { } func (s *credsSvc) decrypt(ciphertext []byte) ([]byte, error) { - if s.config.DynamicCredentials.DisableEncryption { + if s.dc.DisableEncryption { return ciphertext, nil } // base64 decode @@ -634,14 +862,17 @@ func (s *credsSvc) decrypt(ciphertext []byte) ([]byte, error) { return plaintext, nil } -func buildS3AccessKeyIdxFromConf(conf *Config) map[string]s3CredsByAccessKey { +func buildS3AccessKeyIdxFromConf(conf *ProxyConfig) map[string]s3CredsByAccessKey { res := make(map[string]s3CredsByAccessKey) for storage, storConf := range conf.S3Storages() { - for user, cred := range storConf.Credentials { - key := s3accessKeyIdxKey(storage, cred.AccessKeyID) - res[key] = s3CredsByAccessKey{ - user: user, - credentials: cred, + for user, aliases := range storConf.Credentials { + for alias, cred := range aliases { + key := s3accessKeyIdxKey(storage, cred.AccessKeyID) + res[key] = s3CredsByAccessKey{ + user: user, + alias: alias, + credentials: cred, + } } } } diff --git a/pkg/objstore/credentials_test.go b/pkg/objstore/credentials_test.go index 30aa11a8..406292d4 100644 --- a/pkg/objstore/credentials_test.go +++ b/pkg/objstore/credentials_test.go @@ -35,7 +35,7 @@ func Test_credsSvc_Disabled(t *testing.T) { }, } r.False(conf.DynamicCredentials.Enabled) - s, err := NewCredsSvc(ctx, conf, nil) + s, err := New(ctx, nil, conf.DynamicCredentials, conf, nil) r.NoError(err) // HasUser returns from config @@ -94,24 +94,16 @@ func Test_credsSvc_Disabled(t *testing.T) { r.Error(err, "cannot add creds when dynamic credentials disabled") }) - t.Run("FindS3Credentials", func(t *testing.T) { + t.Run("proxy lookups fail fast in worker mode", func(t *testing.T) { r := require.New(t) creds := validS3.Credentials[validUser] - user, gotCreds, err := s.FindS3Credentials(s3Stor, creds.AccessKeyID) - r.NoError(err) - r.EqualValues(creds, gotCreds) - r.Equal(validUser, user) - - // unknown access key - _, _, err = s.FindS3Credentials(s3Stor, "unknown_access_key") - r.Error(err) - // swift storage - _, _, err = s.FindS3Credentials(swiftStor, creds.AccessKeyID) - r.Error(err) - // secret key match is not supported - _, _, err = s.FindS3Credentials(s3Stor, creds.SecretAccessKey) - r.Error(err) + _, _, _, err := s.FindS3Credentials(s3Stor, creds.AccessKeyID) + r.ErrorIs(err, dom.ErrNotImplemented) + _, err = s.GetS3AliasCredentials(s3Stor, validUser, "alias") + r.ErrorIs(err, dom.ErrNotImplemented) + _, err = s.ListS3Aliases(s3Stor, validUser) + r.ErrorIs(err, dom.ErrNotImplemented) }) t.Run("GetS3Address", func(t *testing.T) { @@ -193,11 +185,11 @@ func Test_credsSvc_Enabled(t *testing.T) { conf.DynamicCredentials.MasterPassword = "superseretlongpassword" // create two instances to emlulate multiple replicas - svcA, err := NewCredsSvc(ctx, conf, c) + svcA, err := New(ctx, c, conf.DynamicCredentials, conf, nil) r.NoError(err) conf.DynamicCredentials.DisableEncryption = !enabled conf.DynamicCredentials.MasterPassword = "superseretlongpassword" - svcB, err := NewCredsSvc(ctx, conf, c) + svcB, err := New(ctx, c, conf.DynamicCredentials, conf, nil) r.NoError(err) // veryfy existing creds from config @@ -324,30 +316,11 @@ func Test_credsSvc_Enabled(t *testing.T) { swiftUsersB := svcB.ListUsers(swiftStor) r.Len(swiftUsersB, 1) r.NotContains(swiftUsersB, newUser) - // test find existing by access key - user, creds, err := svcA.FindS3Credentials(s3Stor, validS3.Credentials[validUser].AccessKeyID) - r.NoError(err) - r.Equal(validUser, user) - r.EqualValues(validS3.Credentials[validUser], creds) - user, creds, err = svcB.FindS3Credentials(s3Stor, validS3.Credentials[validUser].AccessKeyID) - r.NoError(err) - r.Equal(validUser, user) - r.EqualValues(validS3.Credentials[validUser], creds) - // test find new by access key - user, creds, err = svcA.FindS3Credentials(s3Stor, newS3Creds.AccessKeyID) - r.NoError(err) - r.Equal(newUser, user) - r.EqualValues(newS3Creds, creds) - user, creds, err = svcB.FindS3Credentials(s3Stor, newS3Creds.AccessKeyID) - r.NoError(err) - r.Equal(newUser, user) - r.EqualValues(newS3Creds, creds) - // test cannot find new by secret key - _, _, err = svcA.FindS3Credentials(s3Stor, newS3Creds.SecretAccessKey) - r.Error(err) - // test cannot find on Swift storage - _, _, err = svcA.FindS3Credentials(swiftStor, newS3Creds.AccessKeyID) - r.Error(err) + // find by access key is proxy-only and fails fast in worker mode + _, _, _, err = svcA.FindS3Credentials(s3Stor, validS3.Credentials[validUser].AccessKeyID) + r.ErrorIs(err, dom.ErrNotImplemented) + _, _, _, err = svcB.FindS3Credentials(s3Stor, newS3Creds.AccessKeyID) + r.ErrorIs(err, dom.ErrNotImplemented) // update new S3 creds via svcB updatedS3Creds := s3.CredentialsV4{ @@ -461,7 +434,7 @@ func Test_credsSvc_Enabled(t *testing.T) { r.Contains(swiftUsersB, newUser) // verify that creds are stored encrypted in Redis - s3Key := credsToRedisField(dom.S3, s3Stor, newUser) + s3Key := credsToRedisField(dom.S3, s3Stor, newUser, "") rawCred, err := c.HGet(ctx, redisHashKey, s3Key).Result() r.NoError(err) if enabled { @@ -733,7 +706,7 @@ func Test_credsSvc_ValidateReplicationID(t *testing.T) { ctx := t.Context() r.NoError(c.FlushAll(ctx).Err()) // create two instances to emlulate multiple replicas - svcA, err := NewCredsSvc(ctx, conf, c) + svcA, err := New(ctx, c, conf.DynamicCredentials, conf, nil) r.NoError(err) if tt.dynamicS3Users != nil { @@ -778,3 +751,271 @@ func Test_credsSvc_ValidateReplicationID(t *testing.T) { }) } } + +var ( + validAlias = "alias1" + validProxyS3Creds = s3.CredentialsV4{ + AccessKeyID: "proxy_id", + SecretAccessKey: "proxy_key", + } + validProxyS3 = s3.ProxyStorage{ + Credentials: map[string]map[string]s3.CredentialsV4{ + validUser: { + validAlias: validProxyS3Creds, + }, + }, + StorageAddress: s3.StorageAddress{ + Address: "clyso.com", + Provider: "Ceph", + IsSecure: false, + }, + } +) + +func proxyTestConf(dc DynamicCredentialsConfig) *ProxyConfig { + stor := validProxyS3 + return &ProxyConfig{ + Main: "s3storage", + Storages: map[string]GenericStorage[*s3.ProxyStorage, *swift.Storage]{ + "s3storage": { + CommonConfig: CommonConfig{Type: dom.S3}, + S3: &stor, + }, + }, + DynamicCredentials: dc, + } +} + +func Test_credsRedisField_RoundTrip(t *testing.T) { + r := require.New(t) + + // worker entry: 3 segments + field := credsToRedisField(dom.S3, "stor", "alice", "") + r.Equal("S3:stor:alice", field) + storType, storage, user, alias, err := credsFromRedisField(field) + r.NoError(err) + r.Equal(dom.S3, storType) + r.Equal("stor", storage) + r.Equal("alice", user) + r.Equal("", alias) + + // proxy alias entry: 4 segments + field = credsToRedisField(dom.S3, "stor", "alice", "laptop") + r.Equal("S3:stor:alice:laptop", field) + storType, storage, user, alias, err = credsFromRedisField(field) + r.NoError(err) + r.Equal(dom.S3, storType) + r.Equal("stor", storage) + r.Equal("alice", user) + r.Equal("laptop", alias) + + // invalid formats + _, _, _, _, err = credsFromRedisField("S3:stor") + r.Error(err) + _, _, _, _, err = credsFromRedisField("S3:stor:alice:laptop:extra") + r.Error(err) +} + +func Test_credsSvc_ProxyMode_Static(t *testing.T) { + r := require.New(t) + ctx := t.Context() + conf := proxyTestConf(DynamicCredentialsConfig{Enabled: false}) + s, err := New(ctx, nil, conf.DynamicCredentials, nil, conf) + r.NoError(err) + + t.Run("FindS3Credentials", func(t *testing.T) { + r := require.New(t) + user, alias, cred, err := s.FindS3Credentials("s3storage", validProxyS3Creds.AccessKeyID) + r.NoError(err) + r.Equal(validUser, user) + r.Equal(validAlias, alias) + r.EqualValues(validProxyS3Creds, cred) + + // unknown access key + _, _, _, err = s.FindS3Credentials("s3storage", "unknown_access_key") + r.Error(err) + // unknown storage + _, _, _, err = s.FindS3Credentials("unknownstorage", validProxyS3Creds.AccessKeyID) + r.Error(err) + // secret key match is not supported + _, _, _, err = s.FindS3Credentials("s3storage", validProxyS3Creds.SecretAccessKey) + r.Error(err) + }) + + t.Run("GetS3AliasCredentials", func(t *testing.T) { + r := require.New(t) + cred, err := s.GetS3AliasCredentials("s3storage", validUser, validAlias) + r.NoError(err) + r.EqualValues(validProxyS3Creds, cred) + + _, err = s.GetS3AliasCredentials("s3storage", validUser, "unknownalias") + r.ErrorIs(err, dom.ErrNotFound) + _, err = s.GetS3AliasCredentials("s3storage", "unknownuser", validAlias) + r.ErrorIs(err, dom.ErrNotFound) + _, err = s.GetS3AliasCredentials("unknownstorage", validUser, validAlias) + r.ErrorIs(err, dom.ErrNotFound) + }) + + t.Run("ListS3Aliases", func(t *testing.T) { + r := require.New(t) + aliases, err := s.ListS3Aliases("s3storage", validUser) + r.NoError(err) + r.Equal([]string{validAlias}, aliases) + + aliases, err = s.ListS3Aliases("s3storage", "unknownuser") + r.NoError(err) + r.Empty(aliases) + }) + + t.Run("worker lookups fail fast in proxy mode", func(t *testing.T) { + r := require.New(t) + _, err := s.GetS3Credentials("s3storage", validUser) + r.ErrorIs(err, dom.ErrNotImplemented) + _, err = s.GetSwiftCredentials("s3storage", validUser) + r.ErrorIs(err, dom.ErrNotImplemented) + }) + + t.Run("topology", func(t *testing.T) { + r := require.New(t) + r.Equal("s3storage", s.MainStorage()) + r.Equal(map[string]dom.StorageType{"s3storage": dom.S3}, s.Storages()) + addr, err := s.GetS3Address("s3storage") + r.NoError(err) + r.EqualValues(validProxyS3.StorageAddress, addr) + r.NoError(s.HasUser("s3storage", validUser)) + r.Error(s.HasUser("s3storage", "unknownuser")) + users := s.ListUsers("s3storage") + r.Equal([]string{validUser}, users) + }) +} + +func Test_credsSvc_ProxyMode_Dynamic(t *testing.T) { + r := require.New(t) + ctx := t.Context() + c := testutil.SetupRedis(t) + pollInterval := 100 * time.Millisecond + dc := DynamicCredentialsConfig{ + Enabled: true, + DisableEncryption: true, + PollInterval: pollInterval, + } + s3Stor, swiftStor := "s3storage", "swiftstorage" + workerConf := &Config{ + Main: s3Stor, + Storages: map[string]Storage{ + s3Stor: { + CommonConfig: CommonConfig{Type: dom.S3}, + S3: &validS3, + }, + swiftStor: { + CommonConfig: CommonConfig{Type: dom.Swift}, + Swift: &validSwift, + }, + }, + DynamicCredentials: dc, + } + proxyConf := proxyTestConf(dc) + + svcW, err := New(ctx, c, dc, workerConf, nil) + r.NoError(err) + svcP, err := New(ctx, c, dc, nil, proxyConf) + r.NoError(err) + + t.Run("alias entry visible to proxy, ignored by worker", func(t *testing.T) { + r := require.New(t) + aliasCred := s3.CredentialsV4{ + AccessKeyID: "bob_laptop_key", + SecretAccessKey: "bob_laptop_secret", + } + // write via worker instance (as the management API would) + r.NoError(svcW.SetS3AliasCredentials(ctx, s3Stor, "bob", "laptop", aliasCred)) + + // proxy instance picks up the alias entry + r.Eventually(func() bool { + user, alias, cred, err := svcP.FindS3Credentials(s3Stor, aliasCred.AccessKeyID) + return err == nil && user == "bob" && alias == "laptop" && cred == aliasCred + }, pollInterval*3, pollInterval/2) + cred, err := svcP.GetS3AliasCredentials(s3Stor, "bob", "laptop") + r.NoError(err) + r.EqualValues(aliasCred, cred) + aliases, err := svcP.ListS3Aliases(s3Stor, "bob") + r.NoError(err) + r.Equal([]string{"laptop"}, aliases) + r.NoError(svcP.HasUser(s3Stor, "bob")) + r.Contains(svcP.ListUsers(s3Stor), "bob") + + // worker instance ignores the alias entry + r.Error(svcW.HasUser(s3Stor, "bob")) + _, err = svcW.GetS3Credentials(s3Stor, "bob") + r.ErrorIs(err, dom.ErrNotFound) + r.NotContains(svcW.ListUsers(s3Stor), "bob") + }) + + t.Run("worker entry ignored by proxy", func(t *testing.T) { + r := require.New(t) + userCred := s3.CredentialsV4{ + AccessKeyID: "carol_key", + SecretAccessKey: "carol_secret", + } + r.NoError(svcW.SetS3Credentials(ctx, s3Stor, "carol", userCred)) + + // worker sees it + got, err := svcW.GetS3Credentials(s3Stor, "carol") + r.NoError(err) + r.EqualValues(userCred, got) + + // wait until proxy syncs past the write, then confirm it is ignored + time.Sleep(pollInterval * 3) + _, _, _, err = svcP.FindS3Credentials(s3Stor, userCred.AccessKeyID) + r.ErrorIs(err, dom.ErrNotFound) + r.Error(svcP.HasUser(s3Stor, "carol")) + r.NotContains(svcP.ListUsers(s3Stor), "carol") + }) + + t.Run("static config precedence over dynamic alias", func(t *testing.T) { + r := require.New(t) + shadowCred := s3.CredentialsV4{ + AccessKeyID: "shadow_key", + SecretAccessKey: "shadow_secret", + } + // same (user, alias) as static config, different credentials + r.NoError(svcP.SetS3AliasCredentials(ctx, s3Stor, validUser, validAlias, shadowCred)) + // static credential wins + cred, err := svcP.GetS3AliasCredentials(s3Stor, validUser, validAlias) + r.NoError(err) + r.EqualValues(validProxyS3Creds, cred) + }) + + t.Run("write and immediate readback", func(t *testing.T) { + r := require.New(t) + aliasCred := s3.CredentialsV4{ + AccessKeyID: "dave_ci_key", + SecretAccessKey: "dave_ci_secret", + } + r.NoError(svcP.SetS3AliasCredentials(ctx, s3Stor, "dave", "ci", aliasCred)) + // storeCred syncs the writer immediately + cred, err := svcP.GetS3AliasCredentials(s3Stor, "dave", "ci") + r.NoError(err) + r.EqualValues(aliasCred, cred) + }) + + t.Run("write validation", func(t *testing.T) { + r := require.New(t) + cred := s3.CredentialsV4{ + AccessKeyID: "k", + SecretAccessKey: "s", + } + // alias creds are S3-only + r.Error(svcW.SetS3AliasCredentials(ctx, swiftStor, "bob", "laptop", cred)) + // unknown storage + r.Error(svcW.SetS3AliasCredentials(ctx, "unknownstorage", "bob", "laptop", cred)) + // empty alias + r.Error(svcW.SetS3AliasCredentials(ctx, s3Stor, "bob", "", cred)) + // ':' is rejected in names + r.Error(svcW.SetS3AliasCredentials(ctx, s3Stor, "bob", "lap:top", cred)) + r.Error(svcW.SetS3AliasCredentials(ctx, s3Stor, "bo:b", "laptop", cred)) + r.Error(svcW.SetS3Credentials(ctx, s3Stor, "bo:b", cred)) + // invalid credentials + r.Error(svcW.SetS3AliasCredentials(ctx, s3Stor, "bob", "laptop", s3.CredentialsV4{})) + }) +} diff --git a/pkg/objstore/service.go b/pkg/objstore/service.go index 2468ff05..6a4850d4 100644 --- a/pkg/objstore/service.go +++ b/pkg/objstore/service.go @@ -41,6 +41,11 @@ var ( type Config = StoragesConfig[*s3.Storage, *swift.Storage] type Storage = GenericStorage[*s3.Storage, *swift.Storage] +// ProxyConfig is the proxy-side storage config with per-alias S3 credentials. +// The Swift half reuses *swift.Storage purely to satisfy the generic; proxy +// Swift storages carry no credentials and are dropped by the proxy conversion. +type ProxyConfig = StoragesConfig[*s3.ProxyStorage, *swift.Storage] + type NameAndVersion struct { Name string Version string @@ -114,7 +119,12 @@ func WithVersionID(versionID string) func(o *commonObjectOptions) { } type Clients interface { + // AsS3 returns a client with the per-user worker credential. + // Fails with dom.ErrNotImplemented on a proxy-mode registry. AsS3(ctx context.Context, storage, user string) (s3client.Client, error) + // AsS3FromAlias returns a client with the per-alias proxy credential. + // Fails with dom.ErrNotImplemented on a worker-mode registry. + AsS3FromAlias(ctx context.Context, storage, user, alias string) (s3client.Client, error) AsSwift(ctx context.Context, storage, user string) (*gophercloud.ServiceClient, error) commonGetter } @@ -166,9 +176,24 @@ func NewRegistry(ctx context.Context, creds CredsService, metricsSvc metrics.Ser for _, user := range users { switch storageType { case dom.S3: - _, err := res.AsS3(ctx, storageName, user) + aliases, err := creds.ListS3Aliases(storageName, user) + if errors.Is(err, dom.ErrNotImplemented) { + // worker mode: one client per user + _, err := res.AsS3(ctx, storageName, user) + if err != nil { + return nil, fmt.Errorf("failed to init S3 client for storage %q user %q: %w", storageName, user, err) + } + continue + } if err != nil { - return nil, fmt.Errorf("failed to init S3 client for storage %q user %q: %w", storageName, user, err) + return nil, fmt.Errorf("failed to list S3 aliases for storage %q user %q: %w", storageName, user, err) + } + // proxy mode: one client per alias + for _, alias := range aliases { + _, err := res.AsS3FromAlias(ctx, storageName, user, alias) + if err != nil { + return nil, fmt.Errorf("failed to init S3 client for storage %q user %q alias %q: %w", storageName, user, alias, err) + } } case dom.Swift: _, err := res.AsSwift(ctx, storageName, user) @@ -206,14 +231,31 @@ type swiftCacheVal struct { } func (r *clients) AsS3(ctx context.Context, storage, user string) (s3client.Client, error) { - creds, err := r.credsSvc.GetS3Credentials(storage, user) + return r.getOrCreateS3Client(ctx, storage, user, credsCacheKey(storage, user), func() (s3.CredentialsV4, error) { + return r.credsSvc.GetS3Credentials(storage, user) + }) +} + +// AsS3FromAlias mirrors AsS3 for proxy-mode alias credentials. The cache is +// keyed by "::"; metrics labels stay alias-free. +func (r *clients) AsS3FromAlias(ctx context.Context, storage, user, alias string) (s3client.Client, error) { + return r.getOrCreateS3Client(ctx, storage, user, aliasCacheKey(storage, user, alias), func() (s3.CredentialsV4, error) { + return r.credsSvc.GetS3AliasCredentials(storage, user, alias) + }) +} + +// getOrCreateS3Client returns the cached client for cacheKey or creates one +// with the credential from resolveCred. storage and user are passed to the +// client for metrics labels only. +func (r *clients) getOrCreateS3Client(ctx context.Context, storage, user, cacheKey string, + resolveCred func() (s3.CredentialsV4, error)) (s3client.Client, error) { + creds, err := resolveCred() if err != nil { return nil, err } - key := credsCacheKey(storage, user) // obtain read lock to check cache r.RLock() - cacheVal, ok := r.s3Clients[key] + cacheVal, ok := r.s3Clients[cacheKey] r.RUnlock() if ok && cacheVal.cred == creds { // return cached client @@ -229,11 +271,11 @@ func (r *clients) AsS3(ctx context.Context, storage, user string) (s3client.Clie r.Lock() defer r.Unlock() // re-check cache after acquiring write lock - cred, err := r.credsSvc.GetS3Credentials(storage, user) + cred, err := resolveCred() if err != nil { return nil, err } - cacheVal, ok = r.s3Clients[key] + cacheVal, ok = r.s3Clients[cacheKey] if ok && cacheVal.cred == cred { return cacheVal.client, nil } @@ -242,7 +284,7 @@ func (r *clients) AsS3(ctx context.Context, storage, user string) (s3client.Clie if err != nil { return nil, err } - r.s3Clients[key] = s3CacheVal{ + r.s3Clients[cacheKey] = s3CacheVal{ cred: cred, client: client, } diff --git a/pkg/s3/config.go b/pkg/s3/config.go index f5f4f9f5..ede8084f 100644 --- a/pkg/s3/config.go +++ b/pkg/s3/config.go @@ -103,6 +103,10 @@ func (s *Storage) Validate() error { } } + return s.StorageAddress.validate() +} + +func (s *StorageAddress) validate() error { if s.HttpTimeout == 0 { s.HttpTimeout = defaultHttpTimeout } @@ -134,3 +138,54 @@ func (s *Storage) Validate() error { return nil } + +// ProxyStorage is the proxy-side S3 storage config: user -> alias -> credential. +// Each alias is a separate S3 access key belonging to the user. The proxy +// authenticates callers by alias access key and joins credentials across +// storages by alias name when re-signing forwarded requests. +type ProxyStorage struct { + Credentials map[string]map[string]CredentialsV4 `yaml:"credentials"` + StorageAddress `yaml:",inline" mapstructure:",squash"` +} + +func (s *ProxyStorage) HasUser(user string) bool { + _, ok := s.Credentials[user] + return ok +} + +func (s *ProxyStorage) HasUserAlias(user, alias string) bool { + _, ok := s.Credentials[user][alias] + return ok +} + +func (s *ProxyStorage) UserList() []string { + users := make([]string, 0, len(s.Credentials)) + for user := range s.Credentials { + users = append(users, user) + } + return users +} + +func (s *ProxyStorage) Validate() error { + for user, aliases := range s.Credentials { + if strings.Contains(user, ":") { + return fmt.Errorf("%w: user name %q must not contain ':'", dom.ErrInvalidStorageConfig, user) + } + if len(aliases) == 0 { + return fmt.Errorf("%w: user %q has no alias credentials", dom.ErrInvalidStorageConfig, user) + } + for alias, cred := range aliases { + if alias == "" { + return fmt.Errorf("%w: empty alias name for user %q", dom.ErrInvalidStorageConfig, user) + } + if strings.Contains(alias, ":") { + return fmt.Errorf("%w: alias name %q for user %q must not contain ':'", dom.ErrInvalidStorageConfig, alias, user) + } + if err := cred.Validate(); err != nil { + return fmt.Errorf("%w: for user %q alias %q", err, user, alias) + } + } + } + + return s.StorageAddress.validate() +} diff --git a/proto/chorus/chorus.proto b/proto/chorus/chorus.proto index 6f7028bb..026ceba3 100644 --- a/proto/chorus/chorus.proto +++ b/proto/chorus/chorus.proto @@ -45,6 +45,10 @@ message Storage { } message Credential { + // Display name of the credential: + // - in GetStoragesResponse: the user name + // - in GetProxyCredentialsResponse: ":" of the proxy alias + // credential string alias = 1; string access_key = 2; string secret_key = 3; @@ -60,6 +64,10 @@ message SetUserCredentialsRequest { string user = 2; optional S3Credential s3_cred = 3; optional SwiftCredential swift_cred = 4; + // Optional proxy alias name for the credential. S3 only. + // When set, the credential is stored as a proxy alias credential + // used by the proxy to authenticate and re-sign forwarded requests. + optional string alias = 5; } message S3Credential { diff --git a/proto/gen/go/chorus/chorus.pb.go b/proto/gen/go/chorus/chorus.pb.go index ec7a0b2b..7938d968 100644 --- a/proto/gen/go/chorus/chorus.pb.go +++ b/proto/gen/go/chorus/chorus.pb.go @@ -254,10 +254,14 @@ func (x *Storage) GetCredentials() []*Credential { } type Credential struct { - state protoimpl.MessageState `protogen:"open.v1"` - Alias string `protobuf:"bytes,1,opt,name=alias,proto3" json:"alias,omitempty"` - AccessKey string `protobuf:"bytes,2,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` - SecretKey string `protobuf:"bytes,3,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + // Display name of the credential: + // - in GetStoragesResponse: the user name + // - in GetProxyCredentialsResponse: ":" of the proxy alias + // credential + Alias string `protobuf:"bytes,1,opt,name=alias,proto3" json:"alias,omitempty"` + AccessKey string `protobuf:"bytes,2,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` + SecretKey string `protobuf:"bytes,3,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -366,11 +370,15 @@ func (x *GetProxyCredentialsResponse) GetCredentials() []*Credential { } type SetUserCredentialsRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Storage string `protobuf:"bytes,1,opt,name=storage,proto3" json:"storage,omitempty"` - User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` - S3Cred *S3Credential `protobuf:"bytes,3,opt,name=s3_cred,json=s3Cred,proto3,oneof" json:"s3_cred,omitempty"` - SwiftCred *SwiftCredential `protobuf:"bytes,4,opt,name=swift_cred,json=swiftCred,proto3,oneof" json:"swift_cred,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Storage string `protobuf:"bytes,1,opt,name=storage,proto3" json:"storage,omitempty"` + User string `protobuf:"bytes,2,opt,name=user,proto3" json:"user,omitempty"` + S3Cred *S3Credential `protobuf:"bytes,3,opt,name=s3_cred,json=s3Cred,proto3,oneof" json:"s3_cred,omitempty"` + SwiftCred *SwiftCredential `protobuf:"bytes,4,opt,name=swift_cred,json=swiftCred,proto3,oneof" json:"swift_cred,omitempty"` + // Optional proxy alias name for the credential. S3 only. + // When set, the credential is stored as a proxy alias credential + // used by the proxy to authenticate and re-sign forwarded requests. + Alias *string `protobuf:"bytes,5,opt,name=alias,proto3,oneof" json:"alias,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -433,6 +441,13 @@ func (x *SetUserCredentialsRequest) GetSwiftCred() *SwiftCredential { return nil } +func (x *SetUserCredentialsRequest) GetAlias() string { + if x != nil && x.Alias != nil { + return *x.Alias + } + return "" +} + type S3Credential struct { state protoimpl.MessageState `protogen:"open.v1"` AccessKey string `protobuf:"bytes,1,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` @@ -582,16 +597,18 @@ const file_chorus_chorus_proto_rawDesc = "" + "secret_key\x18\x03 \x01(\tR\tsecretKey\"m\n" + "\x1bGetProxyCredentialsResponse\x12\x18\n" + "\aaddress\x18\x01 \x01(\tR\aaddress\x124\n" + - "\vcredentials\x18\x02 \x03(\v2\x12.chorus.CredentialR\vcredentials\"\xd5\x01\n" + + "\vcredentials\x18\x02 \x03(\v2\x12.chorus.CredentialR\vcredentials\"\xfa\x01\n" + "\x19SetUserCredentialsRequest\x12\x18\n" + "\astorage\x18\x01 \x01(\tR\astorage\x12\x12\n" + "\x04user\x18\x02 \x01(\tR\x04user\x122\n" + "\as3_cred\x18\x03 \x01(\v2\x14.chorus.S3CredentialH\x00R\x06s3Cred\x88\x01\x01\x12;\n" + "\n" + - "swift_cred\x18\x04 \x01(\v2\x17.chorus.SwiftCredentialH\x01R\tswiftCred\x88\x01\x01B\n" + + "swift_cred\x18\x04 \x01(\v2\x17.chorus.SwiftCredentialH\x01R\tswiftCred\x88\x01\x01\x12\x19\n" + + "\x05alias\x18\x05 \x01(\tH\x02R\x05alias\x88\x01\x01B\n" + "\n" + "\b_s3_credB\r\n" + - "\v_swift_cred\"L\n" + + "\v_swift_credB\b\n" + + "\x06_alias\"L\n" + "\fS3Credential\x12\x1d\n" + "\n" + "access_key\x18\x01 \x01(\tR\taccessKey\x12\x1d\n" + diff --git a/proto/gen/openapi/chorus/chorus.swagger.json b/proto/gen/openapi/chorus/chorus.swagger.json index 80d408b7..9795a500 100644 --- a/proto/gen/openapi/chorus/chorus.swagger.json +++ b/proto/gen/openapi/chorus/chorus.swagger.json @@ -1140,7 +1140,8 @@ "type": "object", "properties": { "alias": { - "type": "string" + "type": "string", + "title": "Display name of the credential:\n- in GetStoragesResponse: the user name\n- in GetProxyCredentialsResponse: \"\u003cuser\u003e:\u003calias\u003e\" of the proxy alias\n credential" }, "accessKey": { "type": "string" @@ -1759,6 +1760,10 @@ }, "swiftCred": { "$ref": "#/definitions/chorusSwiftCredential" + }, + "alias": { + "type": "string", + "description": "Optional proxy alias name for the credential. S3 only.\nWhen set, the credential is stored as a proxy alias credential\nused by the proxy to authenticate and re-sign forwarded requests." } } }, diff --git a/service/proxy/README.md b/service/proxy/README.md index 04819786..c67cb791 100644 --- a/service/proxy/README.md +++ b/service/proxy/README.md @@ -38,3 +38,31 @@ For S3, Proxy supports S3 signature v4 (v2 can be enabled). See `auth` section i - `auth.custom` - use custom credentials for proxy endpoint For SWIFT, proxy does not perform authentication itself. It checks response status from forwarded requests to Swift storage. + +### Credential aliases + +Unlike the Worker, the Proxy supports multiple S3 credentials per user. Proxy S3 storage +credentials are nested `user -> alias -> credential`: + +```yaml +storage: + storages: + one: + address: s3.example.com + credentials: + user1: # user (routing/replication policy key) + laptop: # alias + accessKeyID: AKIA1 + secretAccessKey: ... + ci: + accessKeyID: AKIA2 + secretAccessKey: ... +``` + +The proxy authenticates callers by alias access key, keeps routing and replication policy +keyed by user only, and re-signs forwarded requests with the target storage's credential of +the same alias name. `auth.custom` uses the same nested form; each custom `(user, alias)` +pair must exist in the main storage credentials. + +Aliases can also be added at runtime via the management API +(`chorctl set-user --alias ...`) - see [docs/dynamic-creds.md](../../docs/dynamic-creds.md). diff --git a/service/proxy/auth/config.go b/service/proxy/auth/config.go index 184a67ff..bfde9745 100644 --- a/service/proxy/auth/config.go +++ b/service/proxy/auth/config.go @@ -21,7 +21,8 @@ import ( ) type Config struct { - AllowV2Signature bool `yaml:"allowV2Signature"` - UseStorage string `yaml:"useStorage"` - Custom map[string]s3.CredentialsV4 `yaml:"custom"` + AllowV2Signature bool `yaml:"allowV2Signature"` + UseStorage string `yaml:"useStorage"` + // Custom credentials for the proxy s3 endpoint: user -> alias -> credential. + Custom map[string]map[string]s3.CredentialsV4 `yaml:"custom"` } diff --git a/service/proxy/auth/middleware.go b/service/proxy/auth/middleware.go index f28ad697..4d64d47c 100644 --- a/service/proxy/auth/middleware.go +++ b/service/proxy/auth/middleware.go @@ -31,12 +31,15 @@ import ( "github.com/clyso/chorus/pkg/util" ) -func Middleware(conf *Config, credsSvc objstore.CredsService, endpointAddress string) *middleware { +func Middleware(conf *Config, credsSvc objstore.ProxyCredsLookup, endpointAddress string) *middleware { custom := map[string]credMeta{} - for user, cred := range conf.Custom { - custom[cred.AccessKeyID] = credMeta{ - cred: cred, - user: user, + for user, aliases := range conf.Custom { + for alias, cred := range aliases { + custom[cred.AccessKeyID] = credMeta{ + cred: cred, + user: user, + alias: alias, + } } } return &middleware{ @@ -49,26 +52,28 @@ func Middleware(conf *Config, credsSvc objstore.CredsService, endpointAddress st } type credMeta struct { - cred s3.CredentialsV4 - user string + cred s3.CredentialsV4 + user string + alias string } type middleware struct { allowV2 bool custom map[string]credMeta storageName string - credsSvc objstore.CredsService + credsSvc objstore.ProxyCredsLookup endpoint string } func (m *middleware) Wrap(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - user, err := m.isReqAuthenticated(r) + cred, err := m.isReqAuthenticated(r) if err != nil { util.WriteError(r.Context(), w, err) return } - ctx := log.WithUser(r.Context(), user) + ctx := log.WithUser(r.Context(), cred.user) + ctx = xctx.SetAlias(ctx, cred.alias) next.ServeHTTP(w, r.WithContext(ctx)) }) @@ -84,12 +89,13 @@ var authDeniedErr = mclient.ErrorResponse{ func (m *middleware) getCred(accessKey string) (credMeta, error) { if m.storageName != "" { // check storage creds - user, cred, err := m.credsSvc.FindS3Credentials(m.storageName, accessKey) + user, alias, cred, err := m.credsSvc.FindS3Credentials(m.storageName, accessKey) if err == nil { // found: return credMeta{ - cred: cred, - user: user, + cred: cred, + user: user, + alias: alias, }, nil } // fallback to custom creds @@ -102,14 +108,14 @@ func (m *middleware) getCred(accessKey string) (credMeta, error) { return res, nil } -func (m *middleware) isReqAuthenticated(r *http.Request) (string, error) { +func (m *middleware) isReqAuthenticated(r *http.Request) (credMeta, error) { if isRequestSignatureV4(r) { sha256sum := getContentSha256Cksum(r) return m.doesSignatureV4Match(sha256sum, r) } else if m.allowV2 && isRequestSignatureV2(r) { return m.doesSignatureV2Match(r) } - return "", mclient.ErrorResponse{ + return credMeta{}, mclient.ErrorResponse{ XMLName: xml.Name{}, Code: "CredentialsNotSupported", Message: "This request does not support given credentials type.", diff --git a/service/proxy/auth/signature_v2.go b/service/proxy/auth/signature_v2.go index bbe44b88..e32431c2 100644 --- a/service/proxy/auth/signature_v2.go +++ b/service/proxy/auth/signature_v2.go @@ -81,14 +81,14 @@ func isRequestSignatureV2(r *http.Request) bool { strings.HasPrefix(r.Header.Get(s3.Authorization), signV2Algorithm) } -func (m *middleware) doesSignatureV2Match(r *http.Request) (string, error) { +func (m *middleware) doesSignatureV2Match(r *http.Request) (credMeta, error) { accessKey, err := getReqAccessKeyV2(r) if err != nil { - return "", err + return credMeta{}, err } credInfo, err := m.getCred(accessKey) if err != nil { - return "", err + return credMeta{}, err } cred := credInfo.cred @@ -102,22 +102,22 @@ func (m *middleware) doesSignatureV2Match(r *http.Request) (string, error) { unescapedQueries, err := unescapeQueries(encodedQuery) if err != nil { - return "", err + return credMeta{}, err } encodedResource, err = getResource(encodedResource, r.Host, m.endpoint) if err != nil { - return "", err + return credMeta{}, err } expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header) v2Auth := r.Header.Get(s3.Authorization) prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKeyID) if !strings.HasPrefix(v2Auth, prefix) { - return "", fmt.Errorf("%w: unsupported sign alhorithm", dom.ErrAuth) + return credMeta{}, fmt.Errorf("%w: unsupported sign alhorithm", dom.ErrAuth) } v2Auth = v2Auth[len(prefix):] if !compareSignatureV2(v2Auth, expectedAuth) { - return "", mclient.ErrorResponse{ + return credMeta{}, mclient.ErrorResponse{ XMLName: xml.Name{}, Code: "SignatureDoesNotMatch", Message: "The request signature that the server calculated does not match the signature that you provided. Check your AWS secret access key and signing method. For more information, see REST Authentication and SOAP Authentication.", @@ -126,7 +126,7 @@ func (m *middleware) doesSignatureV2Match(r *http.Request) (string, error) { StatusCode: http.StatusForbidden, } } - return credInfo.user, nil + return credInfo, nil } func getReqAccessKeyV2(r *http.Request) (string, error) { diff --git a/service/proxy/auth/signature_v2_test.go b/service/proxy/auth/signature_v2_test.go index 5e85d0d8..e9794410 100644 --- a/service/proxy/auth/signature_v2_test.go +++ b/service/proxy/auth/signature_v2_test.go @@ -62,6 +62,7 @@ func TestDoesSignatureV2MatchAcceptsSignedVirtualHostRequest(t *testing.T) { accessKey = "test-access-key" secretKey = "test-secret-key" user = "test-user" + alias = "test-alias" ) req := httptest.NewRequest(http.MethodGet, "/object/nested?acl", nil) @@ -76,7 +77,8 @@ func TestDoesSignatureV2MatchAcceptsSignedVirtualHostRequest(t *testing.T) { AccessKeyID: accessKey, SecretAccessKey: secretKey, }, - user: user, + user: user, + alias: alias, }, }, endpoint: "http://s3.example.com", @@ -84,5 +86,6 @@ func TestDoesSignatureV2MatchAcceptsSignedVirtualHostRequest(t *testing.T) { got, err := m.doesSignatureV2Match(req) require.NoError(t, err) - require.Equal(t, user, got) + require.Equal(t, user, got.user) + require.Equal(t, alias, got.alias) } diff --git a/service/proxy/auth/signature_v4.go b/service/proxy/auth/signature_v4.go index 78191a96..0a7201f7 100644 --- a/service/proxy/auth/signature_v4.go +++ b/service/proxy/auth/signature_v4.go @@ -50,37 +50,37 @@ func compareSignatureV4(sig1, sig2 string) bool { return subtle.ConstantTimeCompare([]byte(sig1), []byte(sig2)) == 1 } -func (m *middleware) doesSignatureV4Match(hashedPayload string, r *http.Request) (string, error) { +func (m *middleware) doesSignatureV4Match(hashedPayload string, r *http.Request) (credMeta, error) { req := *r v4Auth := req.Header.Get(s3.Authorization) signV4Values, err := s3.ParseSignV4(v4Auth) if err != nil { - return "", err + return credMeta{}, err } extractedSignedHeaders, err := s3.ExtractSignedHeaders(signV4Values.SignedHeaders, r) if err != nil { - return "", err + return credMeta{}, err } credInfo, err := m.getCred(signV4Values.Credential.AccessKey) if err != nil { - return "", err + return credMeta{}, err } cred := credInfo.cred var date string if date = req.Header.Get(s3.AmzDate); date == "" { if date = r.Header.Get(s3.Date); date == "" { - return "", fmt.Errorf("%w: invalid signature: %q header is missing", dom.ErrAuth, s3.AmzDate) + return credMeta{}, fmt.Errorf("%w: invalid signature: %q header is missing", dom.ErrAuth, s3.AmzDate) } } t, e := time.Parse(iso8601Format, date) if e != nil { - return "", fmt.Errorf("%w: invalid signature: %q - %q invalid date format", dom.ErrAuth, s3.AmzDate, date) + return credMeta{}, fmt.Errorf("%w: invalid signature: %q - %q invalid date format", dom.ErrAuth, s3.AmzDate, date) } queryStr := req.URL.Query().Encode() canonicalRequest := getCanonicalV4Request(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method) @@ -90,7 +90,7 @@ func (m *middleware) doesSignatureV4Match(hashedPayload string, r *http.Request) newSignature := getV4Signature(signingKey, stringToSign) if !compareSignatureV4(newSignature, signV4Values.Signature) { - return "", mclient.ErrorResponse{ + return credMeta{}, mclient.ErrorResponse{ XMLName: xml.Name{}, Code: "SignatureDoesNotMatch", Message: "The request signature that the server calculated does not match the signature that you provided. Check your AWS secret access key and signing method. For more information, see REST Authentication and SOAP Authentication.", @@ -100,7 +100,7 @@ func (m *middleware) doesSignatureV4Match(hashedPayload string, r *http.Request) } } - return credInfo.user, nil + return credInfo, nil } func getCanonicalV4Request(extractedSignedHeaders http.Header, payload, queryStr, urlPath, method string) string { diff --git a/service/proxy/config.go b/service/proxy/config.go index d63eaedf..141bdeb5 100644 --- a/service/proxy/config.go +++ b/service/proxy/config.go @@ -42,8 +42,8 @@ func defaultConfig() fs.File { return defaultFile } -type Storages = objstore.StoragesConfig[*s3.Storage, *router.SwiftStorage] -type Storage = objstore.GenericStorage[*s3.Storage, *router.SwiftStorage] +type Storages = objstore.StoragesConfig[*s3.ProxyStorage, *router.SwiftStorage] +type Storage = objstore.GenericStorage[*s3.ProxyStorage, *router.SwiftStorage] type Config struct { config.Common `yaml:",inline,omitempty" mapstructure:",squash"` @@ -82,9 +82,15 @@ func ValidateAuth(storage Storages, auth *auth.Config) error { } } if len(auth.Custom) != 0 { - for user := range auth.Custom { - if err := storage.Exists(storage.Main, user); err != nil { - return fmt.Errorf("proxy config: auth custom credentials unknown user %q", user) + main := storage.Storages[storage.Main] + if main.S3 == nil { + return fmt.Errorf("proxy config: auth custom credentials require S3 main storage") + } + for user, aliases := range auth.Custom { + for alias := range aliases { + if !main.S3.HasUserAlias(user, alias) { + return fmt.Errorf("proxy config: auth custom credentials unknown user %q alias %q", user, alias) + } } } } @@ -108,9 +114,9 @@ func GetConfig(src ...config.Opt) (*Config, error) { return &conf, err } -func ProxyToCredsConf(in Storages) (objstore.Config, error) { - res := objstore.Config{ - Storages: map[string]objstore.GenericStorage[*s3.Storage, *swift.Storage]{}, +func ProxyToCredsConf(in Storages) (objstore.ProxyConfig, error) { + res := objstore.ProxyConfig{ + Storages: map[string]objstore.GenericStorage[*s3.ProxyStorage, *swift.Storage]{}, Main: in.Main, DynamicCredentials: in.DynamicCredentials, } @@ -119,14 +125,14 @@ func ProxyToCredsConf(in Storages) (objstore.Config, error) { switch val.Type { case dom.S3: c := *val.S3 - res.Storages[name] = objstore.Storage{ + res.Storages[name] = objstore.GenericStorage[*s3.ProxyStorage, *swift.Storage]{ S3: &c, CommonConfig: val.CommonConfig, } case dom.Swift: // ignore - swift proxy conf does not contains credentials default: - return objstore.Config{}, fmt.Errorf("unsupported storage type %q for storage %q", val.Type, name) + return objstore.ProxyConfig{}, fmt.Errorf("unsupported storage type %q for storage %q", val.Type, name) } } return res, nil diff --git a/service/proxy/config.yaml b/service/proxy/config.yaml index dfd55b4d..2a27aa80 100644 --- a/service/proxy/config.yaml +++ b/service/proxy/config.yaml @@ -7,13 +7,15 @@ cors: auth: allowV2Signature: false useStorage: # use credentials from one of configured storages - custom: # OR use custom credentials for proxy s3 endpoint + custom: # OR use custom credentials for proxy s3 endpoint: user -> alias -> credential # user1: -# accessKeyID: -# secretAccessKey: +# alias1: # alias name for the credential, e.g. laptop or ci +# accessKeyID: +# secretAccessKey: # user2: -# accessKeyID: -# secretAccessKey: +# alias1: +# accessKeyID: +# secretAccessKey: storage: defaultRegion: "us-east-1" # fallback region. Will be used if origin region is not supported by replication destination dynamicCredentials: @@ -24,13 +26,18 @@ storage: storages: # one: # yaml key with some handy storage name # address: -# credentials: +# credentials: # user -> alias -> credential # user1: -# accessKeyID: -# secretAccessKey: +# alias1: # alias name for the credential, e.g. laptop or ci +# accessKeyID: +# secretAccessKey: +# alias2: +# accessKeyID: +# secretAccessKey: # user2: -# accessKeyID: -# secretAccessKey: +# alias1: +# accessKeyID: +# secretAccessKey: # provider: # S3 provider name. Required to use provider-specific APIs, like keeping original versionID. Use 'Other' if your storage is S3 compatible but not listed. # isMain: true # one of the storages in should be main # healthCheckInterval: 10s @@ -41,13 +48,18 @@ storage: # rpm: 60 # two: # yaml key with some handy storage name # address: -# credentials: +# credentials: # user -> alias -> credential. Alias names join credentials across storages # user1: -# accessKeyID: -# secretAccessKey: +# alias1: +# accessKeyID: +# secretAccessKey: +# alias2: +# accessKeyID: +# secretAccessKey: # user2: -# accessKeyID: -# secretAccessKey: +# alias1: +# accessKeyID: +# secretAccessKey: # provider: # S3 provider name. Required to use provider-specific APIs, like keeping original versionID. Use 'Other' if your storage is S3 compatible but not listed. # isMain: false # one of the storages in should be main # healthCheckInterval: 10s diff --git a/service/proxy/config_test.go b/service/proxy/config_test.go index 6571399e..12db7967 100644 --- a/service/proxy/config_test.go +++ b/service/proxy/config_test.go @@ -21,8 +21,13 @@ import ( "testing" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" "github.com/clyso/chorus/pkg/config" + "github.com/clyso/chorus/pkg/dom" + "github.com/clyso/chorus/pkg/objstore" + "github.com/clyso/chorus/pkg/s3" + "github.com/clyso/chorus/service/proxy/auth" ) func TestGetConfigDefaults(t *testing.T) { @@ -41,8 +46,9 @@ func TestHelmEnvOverrides(t *testing.T) { provider: Ceph credentials: user1: - accessKeyID: key - secretAccessKey: secret + alias1: + accessKeyID: key + secretAccessKey: secret address: s3.example.com auth: useStorage: s3` @@ -71,3 +77,154 @@ auth: r.Equal("encryption-key!!", conf.Storage.DynamicCredentials.MasterPassword) }) } + +func validProxyStorages() Storages { + return Storages{ + Main: "main", + Storages: map[string]Storage{ + "main": { + CommonConfig: objstore.CommonConfig{Type: dom.S3}, + S3: &s3.ProxyStorage{ + Credentials: map[string]map[string]s3.CredentialsV4{ + "user1": { + "alias1": {AccessKeyID: "key1", SecretAccessKey: "secret1"}, + "alias2": {AccessKeyID: "key2", SecretAccessKey: "secret2"}, + }, + }, + StorageAddress: s3.StorageAddress{ + Address: "s3.example.com", + Provider: s3.ProviderCeph, + }, + }, + }, + }, + } +} + +func TestStorages_NestedYAMLUnmarshal(t *testing.T) { + r := require.New(t) + const storagesYAML = `main: s3 +storages: + s3: + type: S3 + provider: Ceph + address: s3.example.com + credentials: + user1: + laptop: + accessKeyID: key1 + secretAccessKey: secret1 + ci: + accessKeyID: key2 + secretAccessKey: secret2 + user2: + laptop: + accessKeyID: key3 + secretAccessKey: secret3` + + var conf Storages + r.NoError(yaml.Unmarshal([]byte(storagesYAML), &conf)) + r.NoError(conf.Validate()) + stor := conf.Storages["s3"].S3 + r.NotNil(stor) + r.Len(stor.Credentials, 2) + r.EqualValues("key1", stor.Credentials["user1"]["laptop"].AccessKeyID) + r.EqualValues("secret1", stor.Credentials["user1"]["laptop"].SecretAccessKey) + r.EqualValues("key2", stor.Credentials["user1"]["ci"].AccessKeyID) + r.EqualValues("key3", stor.Credentials["user2"]["laptop"].AccessKeyID) + r.True(stor.HasUser("user1")) + r.True(stor.HasUserAlias("user1", "ci")) + r.False(stor.HasUserAlias("user1", "unknown")) + r.ElementsMatch([]string{"user1", "user2"}, stor.UserList()) +} + +func TestProxyStorage_Validate(t *testing.T) { + validAddr := s3.StorageAddress{ + Address: "s3.example.com", + Provider: s3.ProviderCeph, + } + validCred := s3.CredentialsV4{AccessKeyID: "key", SecretAccessKey: "secret"} + tests := []struct { + name string + creds map[string]map[string]s3.CredentialsV4 + wantErr bool + }{ + { + name: "valid", + creds: map[string]map[string]s3.CredentialsV4{"user1": {"alias1": validCred}}, + }, + { + name: "empty alias name", + creds: map[string]map[string]s3.CredentialsV4{"user1": {"": validCred}}, + wantErr: true, + }, + { + name: "colon in alias name", + creds: map[string]map[string]s3.CredentialsV4{"user1": {"ali:as": validCred}}, + wantErr: true, + }, + { + name: "colon in user name", + creds: map[string]map[string]s3.CredentialsV4{"us:er": {"alias1": validCred}}, + wantErr: true, + }, + { + name: "user without aliases", + creds: map[string]map[string]s3.CredentialsV4{"user1": {}}, + wantErr: true, + }, + { + name: "invalid credential", + creds: map[string]map[string]s3.CredentialsV4{"user1": {"alias1": {AccessKeyID: "key"}}}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := require.New(t) + stor := &s3.ProxyStorage{ + Credentials: tt.creds, + StorageAddress: validAddr, + } + err := stor.Validate() + if tt.wantErr { + r.Error(err) + } else { + r.NoError(err) + } + }) + } +} + +func TestValidateAuth_CustomAlias(t *testing.T) { + r := require.New(t) + storage := validProxyStorages() + + // custom credential matching (user, alias) from main storage + r.NoError(ValidateAuth(storage, &auth.Config{ + Custom: map[string]map[string]s3.CredentialsV4{ + "user1": {"alias1": {AccessKeyID: "custom-key", SecretAccessKey: "custom-secret"}}, + }, + })) + + // unknown user + r.Error(ValidateAuth(storage, &auth.Config{ + Custom: map[string]map[string]s3.CredentialsV4{ + "unknown": {"alias1": {AccessKeyID: "custom-key", SecretAccessKey: "custom-secret"}}, + }, + })) + + // known user, unknown alias + r.Error(ValidateAuth(storage, &auth.Config{ + Custom: map[string]map[string]s3.CredentialsV4{ + "user1": {"unknown": {AccessKeyID: "custom-key", SecretAccessKey: "custom-secret"}}, + }, + })) + + // useStorage must exist + r.Error(ValidateAuth(storage, &auth.Config{UseStorage: "unknown"})) + r.NoError(ValidateAuth(storage, &auth.Config{UseStorage: "main"})) + + // auth credentials must be set + r.Error(ValidateAuth(storage, &auth.Config{})) +} diff --git a/service/proxy/router/router_bucket.go b/service/proxy/router/router_bucket.go index 045a6706..cb7734c7 100644 --- a/service/proxy/router/router_bucket.go +++ b/service/proxy/router/router_bucket.go @@ -34,7 +34,7 @@ func (r *s3Router) createBucket(req *http.Request) (resp *http.Response, task *t bucket := xctx.GetBucket(ctx) storage = xctx.GetRoutingPolicy(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, nil, "", false, err } @@ -65,7 +65,7 @@ func (r *s3Router) deleteBucket(req *http.Request) (resp *http.Response, task *t bucket := xctx.GetBucket(ctx) storage = xctx.GetRoutingPolicy(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, nil, "", false, err } @@ -85,7 +85,7 @@ func (r *s3Router) listBuckets(req *http.Request) (resp *http.Response, storage user := xctx.GetUser(ctx) storage = xctx.GetRoutingPolicy(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } diff --git a/service/proxy/router/router_common.go b/service/proxy/router/router_common.go index f41a3d2a..bb8b982d 100644 --- a/service/proxy/router/router_common.go +++ b/service/proxy/router/router_common.go @@ -41,7 +41,7 @@ func (r *s3Router) commonRead(req *http.Request) (resp *http.Response, storage s ctx = xctx.SetStorage(ctx, storage) req = req.WithContext(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } @@ -56,7 +56,7 @@ func (r *s3Router) commonWrite(req *http.Request) (resp *http.Response, storage ctx = xctx.SetStorage(ctx, storage) req = req.WithContext(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } diff --git a/service/proxy/router/router_multipart.go b/service/proxy/router/router_multipart.go index 266b5505..550eff88 100644 --- a/service/proxy/router/router_multipart.go +++ b/service/proxy/router/router_multipart.go @@ -67,7 +67,7 @@ func (r *s3Router) completeMultipartUpload(req *http.Request) (resp *http.Respon return } - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, nil, "", false, err } @@ -127,7 +127,7 @@ func (r *s3Router) abortMultipartUpload(req *http.Request) (resp *http.Response, return } - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } @@ -151,7 +151,7 @@ func (r *s3Router) listMultipartUploads(req *http.Request) (resp *http.Response, if err != nil { return } - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } @@ -167,7 +167,7 @@ func (r *s3Router) uploadPart(req *http.Request) (resp *http.Response, storage s return } - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, "", false, err } diff --git a/service/proxy/router/router_object.go b/service/proxy/router/router_object.go index 4afaa1c9..90fef5b9 100644 --- a/service/proxy/router/router_object.go +++ b/service/proxy/router/router_object.go @@ -36,7 +36,7 @@ func (r *s3Router) putObject(req *http.Request) (resp *http.Response, taskList [ ctx = xctx.SetStorage(ctx, storage) req = req.WithContext(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, nil, "", false, err } @@ -86,7 +86,7 @@ func (r *s3Router) deleteObjects(req *http.Request) (resp *http.Response, taskLi ctx = xctx.SetStorage(ctx, storage) req = req.WithContext(ctx) - client, err := r.clients.AsS3(ctx, storage, user) + client, err := r.clients.AsS3FromAlias(ctx, storage, user, xctx.GetAlias(ctx)) if err != nil { return nil, nil, "", false, err } diff --git a/service/proxy/server.go b/service/proxy/server.go index d6bdeb8f..b3ee2d92 100644 --- a/service/proxy/server.go +++ b/service/proxy/server.go @@ -123,7 +123,7 @@ func Start(ctx context.Context, app dom.AppInfo, conf *Config) error { if err != nil { return err } - credsSvc, err := objstore.NewCredsSvc(ctx, &credsConf, appRedis) + credsSvc, err := objstore.New(ctx, appRedis, credsConf.DynamicCredentials, nil, &credsConf) if err != nil { return err } @@ -168,7 +168,7 @@ func Start(ctx context.Context, app dom.AppInfo, conf *Config) error { } logger.Info().Msg("proxy created") - credentials := []map[string]s3.CredentialsV4{conf.Auth.Custom} + credentials := []map[string]map[string]s3.CredentialsV4{conf.Auth.Custom} if s3torageConf, ok := conf.Storage.S3Storages()[conf.Auth.UseStorage]; ok { credentials = append(credentials, s3torageConf.Credentials) } @@ -191,15 +191,18 @@ func Start(ctx context.Context, app dom.AppInfo, conf *Config) error { return server.Start(ctx) } -func requestReplyServer(proxyUrl string, credentials ...map[string]s3.CredentialsV4) rpc.ProxyGetCredentials { +func requestReplyServer(proxyUrl string, credentials ...map[string]map[string]s3.CredentialsV4) rpc.ProxyGetCredentials { creds := make([]*pb.Credential, 0) for _, c := range credentials { - for alias, cred := range c { - creds = append(creds, &pb.Credential{ - Alias: alias, - AccessKey: cred.AccessKeyID, - SecretKey: cred.SecretAccessKey, - }) + for user, aliases := range c { + for alias, cred := range aliases { + creds = append(creds, &pb.Credential{ + // display-only connection info: keep the user visible + Alias: user + ":" + alias, + AccessKey: cred.AccessKeyID, + SecretKey: cred.SecretAccessKey, + }) + } } } slices.SortFunc(creds, func(a, b *pb.Credential) int { diff --git a/service/standalone/config.yaml b/service/standalone/config.yaml index 6448f243..0336583a 100644 --- a/service/standalone/config.yaml +++ b/service/standalone/config.yaml @@ -28,11 +28,11 @@ proxy: auth: allowV2Signature: true useStorage: # use credentials from one of configured storages - custom: # use custom credentials for proxy s3 endpoint - # - accessKeyID: - # secretAccessKey: - # - accessKeyID: - # secretAccessKey: + custom: # use custom credentials for proxy s3 endpoint: user -> alias -> credential + # user1: + # alias1: + # accessKeyID: + # secretAccessKey: worker: queueUpdateInterval: 3s swiftRetryInterval: 1m # used for swift task retries caused by swift inconsistent state diff --git a/service/standalone/start.go b/service/standalone/start.go index 27a8cf84..1465968d 100644 --- a/service/standalone/start.go +++ b/service/standalone/start.go @@ -234,7 +234,7 @@ func printCreds(conf *Config, printSecrets bool) string { if !conf.Proxy.Enabled { return "" } - var creds map[string]s3.CredentialsV4 + var creds map[string]map[string]s3.CredentialsV4 if conf.Proxy.Auth.UseStorage != "" { if s3torageConf, ok := conf.Proxy.Storage.S3Storages()[conf.Proxy.Auth.UseStorage]; ok { creds = s3torageConf.Credentials @@ -246,12 +246,14 @@ func printCreds(conf *Config, printSecrets bool) string { return "" } res := make([]string, 0, len(creds)) - for s, v4 := range creds { - secret := "" - if printSecrets { - secret = v4.SecretAccessKey + for user, aliases := range creds { + for alias, v4 := range aliases { + secret := "" + if printSecrets { + secret = v4.SecretAccessKey + } + res = append(res, fmt.Sprintf(" - %s:%s: [%s|%s]", user, alias, v4.AccessKeyID, secret)) } - res = append(res, fmt.Sprintf(" - %s: [%s|%s]", s, v4.AccessKeyID, secret)) } return strings.Join(res, "\n") } diff --git a/service/standalone/test-conf.yaml b/service/standalone/test-conf.yaml index 0185817f..d3896b97 100644 --- a/service/standalone/test-conf.yaml +++ b/service/standalone/test-conf.yaml @@ -7,24 +7,28 @@ proxy: one: # yaml key with some handy storage name type: S3 address: ":9680" # will start fake s3 on port 9680 - credentials: + credentials: # user -> alias -> credential user1: - accessKeyID: testKey1 - secretAccessKey: testSecretKey1 + default: + accessKeyID: testKey1 + secretAccessKey: testSecretKey1 user2: - accessKeyID: testKey2 - secretAccessKey: testSecretKey2 + default: + accessKeyID: testKey2 + secretAccessKey: testSecretKey2 provider: Other two: # yaml key with some handy storage name type: S3 address: ":9681" # will start fake s3 on port 9681 - credentials: + credentials: # user -> alias -> credential user1: - accessKeyID: testKey1 - secretAccessKey: testSecretKey1 + default: + accessKeyID: testKey1 + secretAccessKey: testSecretKey1 user2: - accessKeyID: testKey2 - secretAccessKey: testSecretKey2 + default: + accessKeyID: testKey2 + secretAccessKey: testSecretKey2 provider: Other features: tagging: false # sync object/bucket tags diff --git a/service/worker/server.go b/service/worker/server.go index a3271a0d..28574a61 100644 --- a/service/worker/server.go +++ b/service/worker/server.go @@ -95,7 +95,7 @@ func Start(ctx context.Context, app dom.AppInfo, conf *Config) error { return fmt.Errorf("%w: unable to instrument tracing app redis", err) } - credsSvc, err := objstore.NewCredsSvc(ctx, &conf.Storage, appRedis) + credsSvc, err := objstore.New(ctx, appRedis, conf.Storage.DynamicCredentials, &conf.Storage, nil) if err != nil { return err } diff --git a/test/app/embedded.go b/test/app/embedded.go index b60e7193..7f0ea5b2 100644 --- a/test/app/embedded.go +++ b/test/app/embedded.go @@ -479,12 +479,20 @@ func WorkerS3Config(main string, storages map[string]s3.Storage) objstore.Config func ProxyS3Config(main string, storages map[string]s3.Storage) proxy.Storages { res := proxy.Storages{ Main: main, - Storages: map[string]objstore.GenericStorage[*s3.Storage, *router.SwiftStorage]{}, + Storages: map[string]objstore.GenericStorage[*s3.ProxyStorage, *router.SwiftStorage]{}, } for name, stor := range storages { - s := stor - res.Storages[name] = objstore.GenericStorage[*s3.Storage, *router.SwiftStorage]{ - S3: &s, + // wrap each user credential under alias "default" so existing tests + // keep signing with the same keys and forwarding joins across storages + creds := make(map[string]map[string]s3.CredentialsV4, len(stor.Credentials)) + for user, cred := range stor.Credentials { + creds[user] = map[string]s3.CredentialsV4{"default": cred} + } + res.Storages[name] = objstore.GenericStorage[*s3.ProxyStorage, *router.SwiftStorage]{ + S3: &s3.ProxyStorage{ + Credentials: creds, + StorageAddress: stor.StorageAddress, + }, CommonConfig: objstore.CommonConfig{ Type: dom.S3, }, diff --git a/test/migration/api_test.go b/test/migration/api_test.go index a076e2c8..395ccc71 100644 --- a/test/migration/api_test.go +++ b/test/migration/api_test.go @@ -54,11 +54,11 @@ func Test_api_proxy_creds(t *testing.T) { r.NoError(err) r.Contains(res.Address, "127.0.0.1") r.Len(res.Credentials, 1) - r.EqualValues(res.Credentials[0].Alias, user) + r.EqualValues(res.Credentials[0].Alias, user+":default") r.NotEmpty(res.Credentials[0].AccessKey) r.NotEmpty(res.Credentials[0].SecretKey) - r.EqualValues(res.Credentials[0].AccessKey, proxyConf.Storage.S3Storages()[proxyConf.Auth.UseStorage].Credentials[user].AccessKeyID) - r.EqualValues(res.Credentials[0].SecretKey, proxyConf.Storage.S3Storages()[proxyConf.Auth.UseStorage].Credentials[user].SecretAccessKey) + r.EqualValues(res.Credentials[0].AccessKey, proxyConf.Storage.S3Storages()[proxyConf.Auth.UseStorage].Credentials[user]["default"].AccessKeyID) + r.EqualValues(res.Credentials[0].SecretKey, proxyConf.Storage.S3Storages()[proxyConf.Auth.UseStorage].Credentials[user]["default"].SecretAccessKey) } func Test_api_list_replications(t *testing.T) { diff --git a/test/migration/dynamic_creds_test.go b/test/migration/dynamic_creds_test.go index 6edfbe98..e4b7e264 100644 --- a/test/migration/dynamic_creds_test.go +++ b/test/migration/dynamic_creds_test.go @@ -155,6 +155,18 @@ func Test_Dynamic_creds(t *testing.T) { } } } + // register a proxy alias credential for the new user: the proxy + // authenticates only alias credentials, user-level entries are worker-only + alias := "default" + _, err = e.ChorusClient.SetUserCredentials(tstCtx, &pb.SetUserCredentialsRequest{ + Storage: "main", + User: newUser, + S3Cred: newCred, + Alias: &alias, + }) + r.NoError(err, "success") + time.Sleep(dcInterval * 2) + // create s3 clients with new user creds proxyAddr, err := e.ChorusClient.GetProxyCredentials(tstCtx, &emptypb.Empty{}) r.NoError(err) diff --git a/test/proxy_alias_test.go b/test/proxy_alias_test.go new file mode 100644 index 00000000..549469e1 --- /dev/null +++ b/test/proxy_alias_test.go @@ -0,0 +1,274 @@ +package test + +import ( + "bytes" + "strings" + "testing" + "time" + + mclient "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + + "github.com/clyso/chorus/pkg/objstore" + "github.com/clyso/chorus/pkg/s3" + pb "github.com/clyso/chorus/proto/gen/go/chorus" + "github.com/clyso/chorus/test/app" +) + +// newAliasProxyClient creates minio clients for the proxy signed with the +// given credential without any readiness checks. +func newAliasProxyClient(t *testing.T, proxyAddr string, cred s3.CredentialsV4) (*mclient.Client, *mclient.Core) { + t.Helper() + r := require.New(t) + addr := strings.TrimPrefix(proxyAddr, "http://") + mc, err := mclient.New(addr, &mclient.Options{ + Creds: credentials.NewStaticV4(cred.AccessKeyID, cred.SecretAccessKey, ""), + }) + r.NoError(err) + core, err := mclient.NewCore(addr, &mclient.Options{ + Creds: credentials.NewStaticV4(cred.AccessKeyID, cred.SecretAccessKey, ""), + }) + r.NoError(err) + return mc, core +} + +// TestProxyAlias_CustomAuth verifies that a nested auth.custom credential +// authenticates the caller as (user, alias) and that requests are forwarded +// re-signed with the storage credential of the same alias. Also verifies that +// an unknown access key is rejected. +func TestProxyAlias_CustomAuth(t *testing.T) { + customCred := s3.CredentialsV4{ + AccessKeyID: "CUSTOMALIASACCESSKEY", + SecretAccessKey: "customAliasSecretKey0000000000000000000", + } + // custom credential authenticates as user "test" alias "default": + // the pair must exist in the main storage static config + proxyConf.Auth.Custom = map[string]map[string]s3.CredentialsV4{ + "test": {"default": customCred}, + } + t.Cleanup(func() { proxyConf.Auth.Custom = nil }) + + e := app.SetupEmbedded(t, workerConf, proxyConf) + e.CreateMainFollowerUserReplications(t) + tstCtx := t.Context() + r := require.New(t) + bucket := "proxy-alias-custom" + + customClient, _ := newAliasProxyClient(t, proxyConf.Address, customCred) + err := customClient.MakeBucket(tstCtx, bucket, mclient.MakeBucketOptions{}) + r.NoError(err) + + // bucket is created on main and replicated to followers + r.Eventually(func() bool { + for _, c := range []*mclient.Client{e.MainClient, e.F1Client, e.F2Client} { + ok, err := c.BucketExists(tstCtx, bucket) + if err != nil || !ok { + return false + } + } + return true + }, e.WaitShort, e.RetryShort) + + // object written with the custom alias key replicates to followers + objName := "obj" + payload := bytes.Repeat([]byte("c"), 1024) + _, err = customClient.PutObject(tstCtx, bucket, objName, bytes.NewReader(payload), int64(len(payload)), mclient.PutObjectOptions{}) + r.NoError(err) + r.Eventually(func() bool { + for _, c := range []*mclient.Client{e.MainClient, e.F1Client, e.F2Client} { + if _, err := c.StatObject(tstCtx, bucket, objName, mclient.StatObjectOptions{}); err != nil { + return false + } + } + return true + }, e.WaitShort, e.RetryShort) + + // unknown access key is rejected + bogusClient, _ := newAliasProxyClient(t, proxyConf.Address, s3.CredentialsV4{ + AccessKeyID: "BOGUSACCESSKEY123456", + SecretAccessKey: "bogusSecretKey0000000000000000000000000", + }) + err = bogusClient.MakeBucket(tstCtx, "proxy-alias-bogus", mclient.MakeBucketOptions{}) + r.Error(err) + errResp := mclient.ToErrorResponse(err) + r.EqualValues("InvalidAccessKeyId", errResp.Code) +} + +// TestProxyAlias_DynamicCredentials verifies the dynamic path: a second alias +// added via the management API authenticates at the proxy after the poll +// interval, replication stays keyed by user, and multipart uploads work under +// an alias. +func TestProxyAlias_DynamicCredentials(t *testing.T) { + prevWorkerDC := workerConf.Storage.DynamicCredentials + prevProxyDC := proxyConf.Storage.DynamicCredentials + dc := objstore.DynamicCredentialsConfig{ + Enabled: true, + DisableEncryption: true, + PollInterval: 300 * time.Millisecond, + } + workerConf.Storage.DynamicCredentials = dc + proxyConf.Storage.DynamicCredentials = dc + t.Cleanup(func() { + workerConf.Storage.DynamicCredentials = prevWorkerDC + proxyConf.Storage.DynamicCredentials = prevProxyDC + }) + + e := app.SetupEmbedded(t, workerConf, proxyConf) + e.CreateMainFollowerUserReplications(t) + tstCtx := t.Context() + r := require.New(t) + user := "test" + alias := "laptop" + + // add a second alias for the user on every storage via the management API + aliasCreds := map[string]s3.CredentialsV4{ + "main": {AccessKeyID: "LAPTOPMAINACCESSKEY0", SecretAccessKey: "laptopMainSecretKey00000000000000000000"}, + "f1": {AccessKeyID: "LAPTOPF1ACCESSKEY000", SecretAccessKey: "laptopF1SecretKey0000000000000000000000"}, + "f2": {AccessKeyID: "LAPTOPF2ACCESSKEY000", SecretAccessKey: "laptopF2SecretKey0000000000000000000000"}, + } + for storage, cred := range aliasCreds { + _, err := e.ChorusClient.SetUserCredentials(tstCtx, &pb.SetUserCredentialsRequest{ + Storage: storage, + User: user, + S3Cred: &pb.S3Credential{ + AccessKey: cred.AccessKeyID, + SecretKey: cred.SecretAccessKey, + }, + Alias: &alias, + }) + r.NoError(err) + } + + // alias must not be combined with swift credentials + _, err := e.ChorusClient.SetUserCredentials(tstCtx, &pb.SetUserCredentialsRequest{ + Storage: "main", + User: user, + SwiftCred: &pb.SwiftCredential{ + Username: "u", + Password: "p", + }, + Alias: &alias, + }) + r.Error(err) + + // wait past the poll interval until the proxy accepts the new key + aliasClient, aliasCore := newAliasProxyClient(t, proxyConf.Address, aliasCreds["main"]) + bucket := "proxy-alias-dynamic" + r.Eventually(func() bool { + return aliasClient.MakeBucket(tstCtx, bucket, mclient.MakeBucketOptions{}) == nil + }, e.WaitShort, e.RetryLong) + + // bucket replicates to followers (replication is keyed by user, not alias) + r.Eventually(func() bool { + for _, c := range []*mclient.Client{e.MainClient, e.F1Client, e.F2Client} { + ok, err := c.BucketExists(tstCtx, bucket) + if err != nil || !ok { + return false + } + } + return true + }, e.WaitShort, e.RetryShort) + + // object written with the alias key replicates to followers + objName := "obj" + payload := bytes.Repeat([]byte("d"), 1024) + _, err = aliasClient.PutObject(tstCtx, bucket, objName, bytes.NewReader(payload), int64(len(payload)), mclient.PutObjectOptions{}) + r.NoError(err) + r.Eventually(func() bool { + for _, c := range []*mclient.Client{e.MainClient, e.F1Client, e.F2Client} { + if _, err := c.StatObject(tstCtx, bucket, objName, mclient.StatObjectOptions{}); err != nil { + return false + } + } + return true + }, e.WaitShort, e.RetryShort) + + // multipart upload through the proxy under the alias: + // the multipart namespace is keyed by user and unaffected by alias + mpObjName := "obj-mp" + uploadID, err := aliasCore.NewMultipartUpload(tstCtx, bucket, mpObjName, mclient.PutObjectOptions{DisableContentSha256: true}) + r.NoError(err) + partData := bytes.Repeat([]byte("e"), 1024*1024) + part, err := aliasCore.PutObjectPart(tstCtx, bucket, mpObjName, uploadID, 1, + bytes.NewReader(partData), int64(len(partData)), mclient.PutObjectPartOptions{}) + r.NoError(err) + _, err = aliasCore.CompleteMultipartUpload(tstCtx, bucket, mpObjName, uploadID, + []mclient.CompletePart{{PartNumber: 1, ETag: part.ETag}}, mclient.PutObjectOptions{}) + r.NoError(err) + r.Eventually(func() bool { + for _, c := range []*mclient.Client{e.MainClient, e.F1Client, e.F2Client} { + if _, err := c.StatObject(tstCtx, bucket, mpObjName, mclient.StatObjectOptions{}); err != nil { + return false + } + } + return true + }, e.WaitShort, e.RetryShort) +} + +// TestProxyAlias_CrossStorageForwarding verifies the cross-storage alias join: +// a request authenticated with the main-storage alias key and routed to +// another storage is re-signed with that storage's credential of the same +// alias. +func TestProxyAlias_CrossStorageForwarding(t *testing.T) { + prevWorkerDC := workerConf.Storage.DynamicCredentials + prevProxyDC := proxyConf.Storage.DynamicCredentials + dc := objstore.DynamicCredentialsConfig{ + Enabled: true, + DisableEncryption: true, + PollInterval: 300 * time.Millisecond, + } + workerConf.Storage.DynamicCredentials = dc + proxyConf.Storage.DynamicCredentials = dc + t.Cleanup(func() { + workerConf.Storage.DynamicCredentials = prevWorkerDC + proxyConf.Storage.DynamicCredentials = prevProxyDC + }) + + e := app.SetupEmbedded(t, workerConf, proxyConf) + tstCtx := t.Context() + r := require.New(t) + user := "test" + alias := "laptop" + + aliasCreds := map[string]s3.CredentialsV4{ + "main": {AccessKeyID: "LAPTOPMAINACCESSKEY0", SecretAccessKey: "laptopMainSecretKey00000000000000000000"}, + "f1": {AccessKeyID: "LAPTOPF1ACCESSKEY000", SecretAccessKey: "laptopF1SecretKey0000000000000000000000"}, + } + for storage, cred := range aliasCreds { + _, err := e.ChorusClient.SetUserCredentials(tstCtx, &pb.SetUserCredentialsRequest{ + Storage: storage, + User: user, + S3Cred: &pb.S3Credential{ + AccessKey: cred.AccessKeyID, + SecretKey: cred.SecretAccessKey, + }, + Alias: &alias, + }) + r.NoError(err) + } + + // route the bucket to storage f1 + bucket := "proxy-alias-routed" + _, err := e.PolicyClient.AddRouting(tstCtx, &pb.AddRoutingRequest{ + User: user, + Bucket: &bucket, + ToStorage: "f1", + }) + r.NoError(err) + + // create the bucket signed with the main-storage alias key: the proxy + // authenticates against main and forwards to f1 with the f1 alias cred + aliasClient, _ := newAliasProxyClient(t, proxyConf.Address, aliasCreds["main"]) + r.Eventually(func() bool { + return aliasClient.MakeBucket(tstCtx, bucket, mclient.MakeBucketOptions{}) == nil + }, e.WaitShort, e.RetryLong) + + // bucket exists on f1 and not on main + ok, err := e.F1Client.BucketExists(tstCtx, bucket) + r.NoError(err) + r.True(ok) + ok, err = e.MainClient.BucketExists(tstCtx, bucket) + r.NoError(err) + r.False(ok) +} diff --git a/test/swift/init_test.go b/test/swift/init_test.go index 6827ddc0..6470f912 100644 --- a/test/swift/init_test.go +++ b/test/swift/init_test.go @@ -211,7 +211,7 @@ func TestMain(m *testing.M) { if err := proxyStorages.Validate(); err != nil { panic(err) } - credsSvc, err := objstore.NewCredsSvc(tstCtx, &swiftConf, nil) + credsSvc, err := objstore.New(tstCtx, nil, swiftConf.DynamicCredentials, &swiftConf, nil) if err != nil { panic(err) } diff --git a/tools/bench/go.mod b/tools/bench/go.mod index c68225cd..7071c18f 100644 --- a/tools/bench/go.mod +++ b/tools/bench/go.mod @@ -1,6 +1,6 @@ module github.com/clyso/chorus/tools/bench -go 1.26.4 +go 1.26.5 require ( github.com/boltdb/bolt v1.3.1 diff --git a/tools/chorctl/cmd/set_user.go b/tools/chorctl/cmd/set_user.go index b1e4a79f..650ea56b 100644 --- a/tools/chorctl/cmd/set_user.go +++ b/tools/chorctl/cmd/set_user.go @@ -31,6 +31,7 @@ var ( suStorage string suUser string suType string + suAlias string suAccessKey string suSecretKey string suSwiftUsername string @@ -47,6 +48,9 @@ var setUserCmd = &cobra.Command{ For S3 storage type: chorctl set-user --storage main --user admin --type s3 --access-key AKID --secret-key SECRET +For S3 proxy alias credential: + chorctl set-user --storage main --user admin --type s3 --alias laptop --access-key AKID --secret-key SECRET + For Swift storage type: chorctl set-user --storage main --user admin --type swift --swift-username user --swift-password pass --swift-domain default --swift-tenant tenant`, PreRunE: func(cmd *cobra.Command, args []string) error { @@ -59,6 +63,9 @@ For Swift storage type: } } if suType == "swift" { + if suAlias != "" { + return fmt.Errorf("--alias is supported only for S3 credentials") + } if suSwiftUsername == "" || suSwiftPassword == "" { return fmt.Errorf("--swift-username and --swift-password are required for Swift credentials") } @@ -79,6 +86,9 @@ For Swift storage type: Storage: suStorage, User: suUser, } + if suAlias != "" { + req.Alias = &suAlias + } if suType == "s3" { req.S3Cred = &pb.S3Credential{ @@ -108,6 +118,7 @@ func init() { setUserCmd.Flags().StringVarP(&suStorage, "storage", "s", "", "storage name") setUserCmd.Flags().StringVarP(&suUser, "user", "u", "", "user name") setUserCmd.Flags().StringVarP(&suType, "type", "t", "", "credential type: 's3' or 'swift'") + setUserCmd.Flags().StringVar(&suAlias, "alias", "", "S3 proxy alias name for the credential") setUserCmd.Flags().StringVar(&suAccessKey, "access-key", "", "S3 access key") setUserCmd.Flags().StringVar(&suSecretKey, "secret-key", "", "S3 secret key") setUserCmd.Flags().StringVar(&suSwiftUsername, "swift-username", "", "Swift username") diff --git a/tools/chorctl/go.mod b/tools/chorctl/go.mod index 8ca5f35a..f269930e 100644 --- a/tools/chorctl/go.mod +++ b/tools/chorctl/go.mod @@ -1,6 +1,6 @@ module github.com/clyso/chorus/tools/chorctl -go 1.26.4 +go 1.26.5 require ( github.com/charmbracelet/bubbles v0.16.1