From 9466482f8eaf3b1eb9eff6d2f3e30e7d903f01bf Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 29 Jun 2026 17:08:40 +0300 Subject: [PATCH 1/8] fix(dvcr): upload images as uncompressed layers to remove gzip bottleneck The DVCR importer compressed each image into a single gzip layer via go-containerregistry stream.NewLayer (gzip.BestSpeed, single goroutine). gzip is CPU-bound and the provisioning pod is limited to 750m CPU, so imports of large disk images were capped at ~4 MB/s. Measured on cluster: the same source URL downloads at 133 MB/s while the importer container sits pegged at exactly its 750m CPU limit, so the network and DVCR are not the bottleneck. Disk images barely compress (qcow2 626MB->598Mi, ISO 2.9GB->2.7Gi), so an uncompressed tar layer is roughly the same size with near-zero CPU. Replace stream.NewLayer with a custom single-pass streaming uncompressed layer (application/vnd.docker.image.rootfs.diff.tar). containerd and CDI/containers-image read uncompressed layers transparently. Signed-off-by: Nikita Korolev --- images/dvcr-artifact/pkg/registry/registry.go | 8 +- .../pkg/registry/uncompressed_layer.go | 202 ++++++++++++++++++ .../pkg/registry/uncompressed_layer_test.go | 133 ++++++++++++ 3 files changed, 341 insertions(+), 2 deletions(-) create mode 100644 images/dvcr-artifact/pkg/registry/uncompressed_layer.go create mode 100644 images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index 3f33fa406f..a8830bb7cf 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -36,7 +36,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/stream" "golang.org/x/sync/errgroup" "k8s.io/klog/v2" @@ -339,7 +338,12 @@ func (p DataProcessor) uploadLayersAndImage( return fmt.Errorf("error constructing new repository: %w", err) } - layer := stream.NewLayer(pipeReader) + // Upload the tar stream as an uncompressed layer. gzip compression + // (the default of stream.NewLayer) is single-threaded and CPU-bound, and + // caps the import speed of large disk images in the CPU-limited + // provisioning pod. Disk images barely compress, so skipping gzip removes + // the bottleneck without meaningfully growing the stored layer. + layer := newUncompressedLayer(pipeReader) klog.Infoln("Uploading layer to registry") if err := remote.WriteLayer(repo, layer, remoteOpts...); err != nil { diff --git a/images/dvcr-artifact/pkg/registry/uncompressed_layer.go b/images/dvcr-artifact/pkg/registry/uncompressed_layer.go new file mode 100644 index 0000000000..45a987074e --- /dev/null +++ b/images/dvcr-artifact/pkg/registry/uncompressed_layer.go @@ -0,0 +1,202 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "crypto" + "encoding/hex" + "errors" + "hash" + "io" + "os" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var ( + // errNotComputed is returned when the requested value is not yet computed + // because the stream has not been consumed yet. + errNotComputed = errors.New("value not computed until stream is consumed") + + // errConsumed is returned when the underlying stream has already been + // consumed and closed. + errConsumed = errors.New("stream was already consumed") +) + +// uncompressedLayer is a single-pass streaming v1.Layer that uploads the raw +// (uncompressed) tar stream as an application/vnd.docker.image.rootfs.diff.tar +// layer. +// +// It mirrors go-containerregistry's stream.Layer but skips gzip compression. +// gzip (gzip.BestSpeed, single goroutine) is the CPU bottleneck of the importer +// upload path: the provisioning pod is CPU-limited, so compressing a multi-GB +// disk image caps the whole pipeline at a few MB/s. Disk images barely compress +// anyway, so an uncompressed layer is roughly the same size with near-zero CPU. +// +// For an uncompressed layer the on-disk blob equals the uncompressed content, +// so Digest and DiffID are identical (both the sha256 of the raw tar stream). +type uncompressedLayer struct { + blob io.ReadCloser + consumed bool + + mu sync.Mutex + digest *v1.Hash + size int64 +} + +var _ v1.Layer = (*uncompressedLayer)(nil) + +// newUncompressedLayer creates an uncompressed streaming Layer from rc. +func newUncompressedLayer(rc io.ReadCloser) *uncompressedLayer { + return &uncompressedLayer{blob: rc} +} + +// Digest implements v1.Layer. It returns errNotComputed until the stream is +// consumed, which marks the layer as streaming for remote.WriteLayer. +func (l *uncompressedLayer) Digest() (v1.Hash, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.digest == nil { + return v1.Hash{}, errNotComputed + } + return *l.digest, nil +} + +// DiffID implements v1.Layer. For an uncompressed layer it equals Digest. +func (l *uncompressedLayer) DiffID() (v1.Hash, error) { + return l.Digest() +} + +// Size implements v1.Layer. +func (l *uncompressedLayer) Size() (int64, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.size == 0 { + return 0, errNotComputed + } + return l.size, nil +} + +// MediaType implements v1.Layer. +func (l *uncompressedLayer) MediaType() (types.MediaType, error) { + return types.DockerUncompressedLayer, nil +} + +// Uncompressed implements v1.Layer. +func (l *uncompressedLayer) Uncompressed() (io.ReadCloser, error) { + return l.reader() +} + +// Compressed implements v1.Layer. The layer is not compressed, so this returns +// the raw tar stream unchanged. +func (l *uncompressedLayer) Compressed() (io.ReadCloser, error) { + return l.reader() +} + +func (l *uncompressedLayer) reader() (io.ReadCloser, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.consumed { + return nil, errConsumed + } + return newUncompressedReader(l), nil +} + +// finalize sets the layer to consumed and records the digest and size computed +// while streaming. +func (l *uncompressedLayer) finalize(h hash.Hash, size int64) error { + l.mu.Lock() + defer l.mu.Unlock() + + digest, err := v1.NewHash("sha256:" + hex.EncodeToString(h.Sum(nil))) + if err != nil { + return err + } + + l.digest = &digest + l.size = size + l.consumed = true + return nil +} + +type uncompressedReader struct { + pr io.Reader + closer func() error +} + +func newUncompressedReader(l *uncompressedLayer) *uncompressedReader { + // Collect the digest and size of the raw stream as it is read. + h := crypto.SHA256.New() + count := &countWriter{} + + pr, pw := io.Pipe() + + // Tee the raw blob to the pipe reader (consumed by the uploader), the + // hasher (digest), and the counter (size). + mw := io.MultiWriter(pw, h, count) + + doneDigesting := make(chan struct{}) + + r := &uncompressedReader{ + pr: pr, + closer: func() error { + // NOTE: pw.Close never returns an error. + _ = pw.Close() + + // Close the inner ReadCloser. net/http may have already closed it + // on success, so ignore os.ErrClosed. + if err := l.blob.Close(); err != nil && !errors.Is(err, os.ErrClosed) { + return err + } + + <-doneDigesting + return l.finalize(h, count.n) + }, + } + + go func() { + _, copyErr := io.Copy(mw, l.blob) + if copyErr != nil { + close(doneDigesting) + pw.CloseWithError(copyErr) + return + } + + // Notify closer that digest/size are done being written. + close(doneDigesting) + + // Close the reader to finalize digest/size. This causes pr to return + // EOF so readers of the stream finish. + pw.CloseWithError(r.Close()) + }() + + return r +} + +func (r *uncompressedReader) Read(b []byte) (int, error) { return r.pr.Read(b) } + +func (r *uncompressedReader) Close() error { return r.closer() } + +// countWriter counts bytes written to it. +type countWriter struct{ n int64 } + +func (c *countWriter) Write(p []byte) (int, error) { + c.n += int64(len(p)) + return len(p), nil +} diff --git a/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go b/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go new file mode 100644 index 0000000000..b2868842f0 --- /dev/null +++ b/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +func Test_UncompressedLayer_MediaType(t *testing.T) { + l := newUncompressedLayer(io.NopCloser(bytes.NewReader(nil))) + mt, err := l.MediaType() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if mt != types.DockerUncompressedLayer { + t.Fatalf("media type: got %q, want %q", mt, types.DockerUncompressedLayer) + } +} + +func Test_UncompressedLayer_NotComputedBeforeConsumed(t *testing.T) { + l := newUncompressedLayer(io.NopCloser(bytes.NewReader([]byte("data")))) + + if _, err := l.Digest(); !errors.Is(err, errNotComputed) { + t.Fatalf("Digest before consume: got %v, want errNotComputed", err) + } + if _, err := l.DiffID(); !errors.Is(err, errNotComputed) { + t.Fatalf("DiffID before consume: got %v, want errNotComputed", err) + } + if _, err := l.Size(); !errors.Is(err, errNotComputed) { + t.Fatalf("Size before consume: got %v, want errNotComputed", err) + } +} + +func Test_UncompressedLayer_StreamsRawBytesAndComputesDigest(t *testing.T) { + payload := bytes.Repeat([]byte("virtualization-disk-image"), 4096) + + l := newUncompressedLayer(io.NopCloser(bytes.NewReader(payload))) + + rc, err := l.Compressed() + if err != nil { + t.Fatalf("Compressed: %v", err) + } + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + // The uploaded bytes must equal the input: no compression is applied. + if !bytes.Equal(got, payload) { + t.Fatalf("streamed bytes differ from input: got %d bytes, want %d bytes", len(got), len(payload)) + } + + wantDigest := "sha256:" + hex.EncodeToString(sum256(payload)) + + digest, err := l.Digest() + if err != nil { + t.Fatalf("Digest after consume: %v", err) + } + if digest.String() != wantDigest { + t.Fatalf("digest: got %q, want %q", digest.String(), wantDigest) + } + + // For an uncompressed layer DiffID equals Digest. + diffID, err := l.DiffID() + if err != nil { + t.Fatalf("DiffID after consume: %v", err) + } + if diffID != digest { + t.Fatalf("diffID %q != digest %q", diffID, digest) + } + + size, err := l.Size() + if err != nil { + t.Fatalf("Size after consume: %v", err) + } + if size != int64(len(payload)) { + t.Fatalf("size: got %d, want %d", size, len(payload)) + } +} + +func Test_UncompressedLayer_SecondReadFailsAfterConsumed(t *testing.T) { + l := newUncompressedLayer(io.NopCloser(bytes.NewReader([]byte("payload")))) + + rc, err := l.Compressed() + if err != nil { + t.Fatalf("first Compressed: %v", err) + } + if _, err := io.ReadAll(rc); err != nil { + t.Fatalf("ReadAll: %v", err) + } + if err := rc.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + if _, err := l.Compressed(); !errors.Is(err, errConsumed) { + t.Fatalf("second Compressed: got %v, want errConsumed", err) + } +} + +// satisfy the v1.Layer interface at compile time in the test too. +var _ v1.Layer = (*uncompressedLayer)(nil) + +func sum256(b []byte) []byte { + h := sha256.New() + h.Write(b) + return h.Sum(nil) +} From de003d613a6df42a3e4a363c66b2e89f1f900263 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 29 Jun 2026 17:59:19 +0300 Subject: [PATCH 2/8] fix(dvcr): return stream sentinels from uncompressed layer remote's pusher detects a streaming layer via errors.Is(err, stream.ErrNotComputed) (pusher.go writeLayer/manifest paths) and only then routes the upload through its lazy digest-after-consume path. The uncompressed layer returned its own sentinel errors, so the pusher treated "value not computed until stream is consumed" as fatal and the import failed after retries. Return go-containerregistry's stream.ErrNotComputed (Digest/DiffID/Size) and stream.ErrConsumed (Compressed/Uncompressed) so the layer is handled identically to stream.Layer. The unit test now asserts the exported sentinels, guarding against this regression. Signed-off-by: Nikita Korolev --- .../pkg/registry/uncompressed_layer.go | 24 +++++++------------ .../pkg/registry/uncompressed_layer_test.go | 17 ++++++------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/images/dvcr-artifact/pkg/registry/uncompressed_layer.go b/images/dvcr-artifact/pkg/registry/uncompressed_layer.go index 45a987074e..21f97885c2 100644 --- a/images/dvcr-artifact/pkg/registry/uncompressed_layer.go +++ b/images/dvcr-artifact/pkg/registry/uncompressed_layer.go @@ -26,19 +26,10 @@ import ( "sync" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" ) -var ( - // errNotComputed is returned when the requested value is not yet computed - // because the stream has not been consumed yet. - errNotComputed = errors.New("value not computed until stream is consumed") - - // errConsumed is returned when the underlying stream has already been - // consumed and closed. - errConsumed = errors.New("stream was already consumed") -) - // uncompressedLayer is a single-pass streaming v1.Layer that uploads the raw // (uncompressed) tar stream as an application/vnd.docker.image.rootfs.diff.tar // layer. @@ -67,13 +58,16 @@ func newUncompressedLayer(rc io.ReadCloser) *uncompressedLayer { return &uncompressedLayer{blob: rc} } -// Digest implements v1.Layer. It returns errNotComputed until the stream is -// consumed, which marks the layer as streaming for remote.WriteLayer. +// Digest implements v1.Layer. Until the stream is consumed it returns +// stream.ErrNotComputed: remote's pusher detects a streaming layer by +// errors.Is(err, stream.ErrNotComputed) and only then routes the upload through +// its lazy (chunked, digest-after-consume) path. Returning a different sentinel +// makes the pusher treat the error as fatal. func (l *uncompressedLayer) Digest() (v1.Hash, error) { l.mu.Lock() defer l.mu.Unlock() if l.digest == nil { - return v1.Hash{}, errNotComputed + return v1.Hash{}, stream.ErrNotComputed } return *l.digest, nil } @@ -88,7 +82,7 @@ func (l *uncompressedLayer) Size() (int64, error) { l.mu.Lock() defer l.mu.Unlock() if l.size == 0 { - return 0, errNotComputed + return 0, stream.ErrNotComputed } return l.size, nil } @@ -113,7 +107,7 @@ func (l *uncompressedLayer) reader() (io.ReadCloser, error) { l.mu.Lock() defer l.mu.Unlock() if l.consumed { - return nil, errConsumed + return nil, stream.ErrConsumed } return newUncompressedReader(l), nil } diff --git a/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go b/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go index b2868842f0..f49b86b041 100644 --- a/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go +++ b/images/dvcr-artifact/pkg/registry/uncompressed_layer_test.go @@ -25,6 +25,7 @@ import ( "testing" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" ) @@ -42,14 +43,14 @@ func Test_UncompressedLayer_MediaType(t *testing.T) { func Test_UncompressedLayer_NotComputedBeforeConsumed(t *testing.T) { l := newUncompressedLayer(io.NopCloser(bytes.NewReader([]byte("data")))) - if _, err := l.Digest(); !errors.Is(err, errNotComputed) { - t.Fatalf("Digest before consume: got %v, want errNotComputed", err) + if _, err := l.Digest(); !errors.Is(err, stream.ErrNotComputed) { + t.Fatalf("Digest before consume: got %v, want stream.ErrNotComputed", err) } - if _, err := l.DiffID(); !errors.Is(err, errNotComputed) { - t.Fatalf("DiffID before consume: got %v, want errNotComputed", err) + if _, err := l.DiffID(); !errors.Is(err, stream.ErrNotComputed) { + t.Fatalf("DiffID before consume: got %v, want stream.ErrNotComputed", err) } - if _, err := l.Size(); !errors.Is(err, errNotComputed) { - t.Fatalf("Size before consume: got %v, want errNotComputed", err) + if _, err := l.Size(); !errors.Is(err, stream.ErrNotComputed) { + t.Fatalf("Size before consume: got %v, want stream.ErrNotComputed", err) } } @@ -118,8 +119,8 @@ func Test_UncompressedLayer_SecondReadFailsAfterConsumed(t *testing.T) { t.Fatalf("Close: %v", err) } - if _, err := l.Compressed(); !errors.Is(err, errConsumed) { - t.Fatalf("second Compressed: got %v, want errConsumed", err) + if _, err := l.Compressed(); !errors.Is(err, stream.ErrConsumed) { + t.Fatalf("second Compressed: got %v, want stream.ErrConsumed", err) } } From 9d090ac58c1210d4aec537a2cefba630b04ab037 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 29 Jun 2026 18:32:34 +0300 Subject: [PATCH 3/8] chore(dvcr): add temporary pprof server to importer for profiling TEMPORARY debug hook (to be removed before merge): start net/http/pprof on :6060 in the dvcr-importer so a CPU profile can be captured during an import via kubectl port-forward + go tool pprof. Used to locate the single-threaded ~1-core bottleneck that caps import throughput at ~4.6 MB/s regardless of the provisioning pod CPU limit. Signed-off-by: Nikita Korolev --- images/dvcr-artifact/cmd/dvcr-importer/main.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/images/dvcr-artifact/cmd/dvcr-importer/main.go b/images/dvcr-artifact/cmd/dvcr-importer/main.go index d60fef617c..11f63cb2ba 100644 --- a/images/dvcr-artifact/cmd/dvcr-importer/main.go +++ b/images/dvcr-artifact/cmd/dvcr-importer/main.go @@ -19,7 +19,10 @@ package main import ( "context" "flag" + "net/http" + _ "net/http/pprof" "os" + "time" "github.com/google/go-containerregistry/pkg/logs" "k8s.io/klog/v2" @@ -32,12 +35,27 @@ func init() { flag.Parse() } +// TODO(profiling): temporary CPU-profiling hook, remove before merge. +// Starts net/http/pprof so a CPU profile can be captured during import via +// `kubectl port-forward 6060:6060` + `go tool pprof`. +func startPprof() { + go func() { + klog.Infoln("Starting pprof server on :6060") + srv := &http.Server{Addr: ":6060", ReadHeaderTimeout: 10 * time.Second} + if err := srv.ListenAndServe(); err != nil { + klog.Warningf("pprof server stopped: %v", err) + } + }() +} + func main() { defer klog.Flush() logs.Progress.SetOutput(os.Stdout) logs.Warn.SetOutput(os.Stderr) + startPprof() + klog.Infoln("Starting registry importer") imp := importer.New() From 6b05d33dee81076702712cf60156671661c7efa3 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Mon, 29 Jun 2026 19:14:38 +0300 Subject: [PATCH 4/8] perf(dvcr): force AES-GCM TLS for importer->DVCR upload CPU profiling of the importer showed ~95% of CPU spent in GOST (Kuznyechik/MGM) TLS encryption of the layer upload to the in-cluster DVCR: gost3412128.l alone was 76% flat. The GOST cipher is pure-software (no hardware acceleration) and caps upload throughput at a few MB/s on a single core, which is the real DVCR import bottleneck (gzip was a red herring: incompressible disk images send ~the same bytes through the cipher either way). Pin the destination TLS client to TLS 1.2 with AES-GCM cipher suites so the AES-NI accelerated cipher is negotiated instead of GOST. TLS 1.3 ignores CipherSuites, so the version is capped for the list to apply. Signed-off-by: Nikita Korolev --- images/dvcr-artifact/pkg/registry/registry.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index a8830bb7cf..e835c58455 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -419,7 +419,21 @@ func destNameOptions(destInsecure bool) []name.Option { func destRemoteOptions(ctx context.Context, destUsername, destPassword string, destInsecure bool) []remote.Option { tlsConfig := &tls.Config{ - InsecureSkipVerify: destInsecure, + InsecureSkipVerify: destInsecure, //nolint:gosec // dest is the in-cluster DVCR; verification is controlled by DESTINATION_INSECURE_TLS. + // The DVCR upload dominates importer CPU because the GOST-enabled + // toolchain negotiates a software GOST (Kuznyechik/MGM) TLS cipher, + // which has no hardware acceleration and caps throughput at a few MB/s + // on a single core. Pin TLS 1.2 with AES-GCM so the hardware-accelerated + // (AES-NI) cipher is used instead. TLS 1.3 ignores CipherSuites, so the + // version must be capped for the suite list to take effect. + MinVersion: tls.VersionTLS12, + MaxVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + }, } transport := http.DefaultTransport.(*http.Transport).Clone() From 2f1938cbda1826d23e7db40264abcc71d970cef0 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 30 Jun 2026 12:08:42 +0300 Subject: [PATCH 5/8] perf(dvcr): drop GOST TLS via toolchain API instead of pinning TLS 1.2 The deckhouse GOST Go toolchain unconditionally installs GOST TLS 1.3 cipher suites (crypto/tls init -> GOSTInstall), so the importer always advertises them and the GOST-preferring DVCR selects software GOST, which caps upload at a few MB/s on one core. Replace the previous TLS 1.2 + AES-GCM pin with the toolchain's intended API: SetAllowedTLS13CipherSuites(AES/ChaCha20), keeping TLS 1.3 while removing GOST from the advertised suites. The call uses a toolchain-only API, so it is isolated in cmd/dvcr-importer behind the dvcr_no_gost_tls build tag (added to the importer go build in werf.inc.yaml); standard-Go builds exclude the file. destRemoteOptions is reverted to its original TLS config. Signed-off-by: Nikita Korolev --- .../cmd/dvcr-importer/gost_tls_off.go | 41 +++++++++++++++++++ images/dvcr-artifact/pkg/registry/registry.go | 16 +------- images/dvcr-artifact/werf.inc.yaml | 2 +- 3 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go diff --git a/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go b/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go new file mode 100644 index 0000000000..f103f2b574 --- /dev/null +++ b/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go @@ -0,0 +1,41 @@ +//go:build dvcr_no_gost_tls + +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This file is compiled only when built with the deckhouse GOST-enabled Go +// toolchain (golang-alt/golang-debian) via `-tags=dvcr_no_gost_tls`. That +// toolchain unconditionally installs GOST (Kuznyechik/MGM) TLS 1.3 cipher +// suites, which are pure-software (no hardware acceleration) and cap the +// importer's upload to the in-cluster DVCR at a few MB/s on a single core. +// +// SetAllowedTLS13CipherSuites is a toolchain-specific API (absent from upstream +// Go), so the call is isolated behind a build tag: standard-Go builds (local +// dev, golangci-lint) simply exclude this file and keep upstream behaviour. + +package main + +import "crypto/tls" + +func init() { + // Restrict TLS 1.3 to the AES-NI / ChaCha20 accelerated suites so GOST is + // no longer advertised and a GOST-preferring DVCR cannot select it. + tls.SetAllowedTLS13CipherSuites([]uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }) +} diff --git a/images/dvcr-artifact/pkg/registry/registry.go b/images/dvcr-artifact/pkg/registry/registry.go index e835c58455..a8830bb7cf 100644 --- a/images/dvcr-artifact/pkg/registry/registry.go +++ b/images/dvcr-artifact/pkg/registry/registry.go @@ -419,21 +419,7 @@ func destNameOptions(destInsecure bool) []name.Option { func destRemoteOptions(ctx context.Context, destUsername, destPassword string, destInsecure bool) []remote.Option { tlsConfig := &tls.Config{ - InsecureSkipVerify: destInsecure, //nolint:gosec // dest is the in-cluster DVCR; verification is controlled by DESTINATION_INSECURE_TLS. - // The DVCR upload dominates importer CPU because the GOST-enabled - // toolchain negotiates a software GOST (Kuznyechik/MGM) TLS cipher, - // which has no hardware acceleration and caps throughput at a few MB/s - // on a single core. Pin TLS 1.2 with AES-GCM so the hardware-accelerated - // (AES-NI) cipher is used instead. TLS 1.3 ignores CipherSuites, so the - // version must be capped for the suite list to take effect. - MinVersion: tls.VersionTLS12, - MaxVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - }, + InsecureSkipVerify: destInsecure, } transport := http.DefaultTransport.(*http.Transport).Clone() diff --git a/images/dvcr-artifact/werf.inc.yaml b/images/dvcr-artifact/werf.inc.yaml index c179c240a9..fdadcfe851 100644 --- a/images/dvcr-artifact/werf.inc.yaml +++ b/images/dvcr-artifact/werf.inc.yaml @@ -55,7 +55,7 @@ shell: export GOARCH=amd64 - | {{- $_ := set $ "ProjectName" (list .ImageName "dvcr-importer" | join "/") }} - {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -o /out/dvcr-importer ./cmd/dvcr-importer`) | nindent 6 }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -tags=dvcr_no_gost_tls -ldflags="-s -w" -o /out/dvcr-importer ./cmd/dvcr-importer`) | nindent 6 }} {{- $_ := set $ "ProjectName" (list .ImageName "dvcr-uploader" | join "/") }} {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -o /out/dvcr-uploader ./cmd/dvcr-uploader`) | nindent 6 }} - | From 6c18dae4eeaa6e724b4d2430baf9098c48183e5a Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 30 Jun 2026 13:59:47 +0300 Subject: [PATCH 6/8] perf(dvcr): drop GOST TLS for uploader too via shared gosttls package The dvcr-uploader pushes to DVCR through the same registry.DataProcessor as the importer, so it hits the identical software-GOST TLS bottleneck. Move the TLS 1.3 cipher override out of cmd/dvcr-importer into a shared pkg/gosttls package (toolchain-only SetAllowedTLS13CipherSuites behind the dvcr_no_gost_tls build tag, with a no-op stub when the tag is absent) and blank-import it from both the importer and uploader mains. Add the build tag to the uploader go build in werf.inc.yaml as well. Signed-off-by: Nikita Korolev --- .../dvcr-artifact/cmd/dvcr-importer/main.go | 2 ++ .../dvcr-artifact/cmd/dvcr-uploader/main.go | 2 ++ .../dvcr-artifact/pkg/gosttls/disable_gost.go | 32 +++++++++++++++++++ images/dvcr-artifact/pkg/gosttls/gosttls.go | 30 +++++++++++++++++ images/dvcr-artifact/werf.inc.yaml | 2 +- 5 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 images/dvcr-artifact/pkg/gosttls/disable_gost.go create mode 100644 images/dvcr-artifact/pkg/gosttls/gosttls.go diff --git a/images/dvcr-artifact/cmd/dvcr-importer/main.go b/images/dvcr-artifact/cmd/dvcr-importer/main.go index 11f63cb2ba..29ffccfcfc 100644 --- a/images/dvcr-artifact/cmd/dvcr-importer/main.go +++ b/images/dvcr-artifact/cmd/dvcr-importer/main.go @@ -28,6 +28,8 @@ import ( "k8s.io/klog/v2" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/importer" + // Prefer AES-GCM over GOST for TLS 1.3 when built with -tags=dvcr_no_gost_tls. + _ "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/gosttls" ) func init() { diff --git a/images/dvcr-artifact/cmd/dvcr-uploader/main.go b/images/dvcr-artifact/cmd/dvcr-uploader/main.go index 419aaeeb91..35a29fcc70 100644 --- a/images/dvcr-artifact/cmd/dvcr-uploader/main.go +++ b/images/dvcr-artifact/cmd/dvcr-uploader/main.go @@ -28,6 +28,8 @@ import ( cryptowatch "kubevirt.io/containerized-data-importer/pkg/util/tls-crypto-watch" "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/uploader" + // Prefer AES-GCM over GOST for TLS 1.3 when built with -tags=dvcr_no_gost_tls. + _ "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/gosttls" ) const ( diff --git a/images/dvcr-artifact/pkg/gosttls/disable_gost.go b/images/dvcr-artifact/pkg/gosttls/disable_gost.go new file mode 100644 index 0000000000..91539ae709 --- /dev/null +++ b/images/dvcr-artifact/pkg/gosttls/disable_gost.go @@ -0,0 +1,32 @@ +//go:build dvcr_no_gost_tls + +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gosttls + +import "crypto/tls" + +// SetAllowedTLS13CipherSuites is a deckhouse-toolchain-only API (absent from +// upstream Go), so this call lives behind the dvcr_no_gost_tls build tag: +// builds without the tag exclude this file and keep upstream behaviour. +func init() { + tls.SetAllowedTLS13CipherSuites([]uint16{ + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + }) +} diff --git a/images/dvcr-artifact/pkg/gosttls/gosttls.go b/images/dvcr-artifact/pkg/gosttls/gosttls.go new file mode 100644 index 0000000000..772b651fb9 --- /dev/null +++ b/images/dvcr-artifact/pkg/gosttls/gosttls.go @@ -0,0 +1,30 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package gosttls optionally narrows the process-wide TLS 1.3 cipher suites to +// the hardware-accelerated AES-GCM / ChaCha20 set, removing the GOST +// (Kuznyechik/MGM) suites that the deckhouse GOST Go toolchain installs by +// default. +// +// The deckhouse toolchain advertises software GOST TLS 1.3 suites in every Go +// binary; when DVCR (which prefers GOST) is the peer, the importer/uploader +// upload runs through a pure-software GOST cipher and is capped at a few MB/s on +// a single core. Blank-import this package from a binary that should prefer +// AES, and build it with `-tags=dvcr_no_gost_tls` to activate the override. +// +// Without the build tag this package is a no-op, so standard-Go builds (local +// dev, golangci-lint) compile unchanged. +package gosttls diff --git a/images/dvcr-artifact/werf.inc.yaml b/images/dvcr-artifact/werf.inc.yaml index fdadcfe851..d667722980 100644 --- a/images/dvcr-artifact/werf.inc.yaml +++ b/images/dvcr-artifact/werf.inc.yaml @@ -57,7 +57,7 @@ shell: {{- $_ := set $ "ProjectName" (list .ImageName "dvcr-importer" | join "/") }} {{- include "image-build.build" (set $ "BuildCommand" `go build -tags=dvcr_no_gost_tls -ldflags="-s -w" -o /out/dvcr-importer ./cmd/dvcr-importer`) | nindent 6 }} {{- $_ := set $ "ProjectName" (list .ImageName "dvcr-uploader" | join "/") }} - {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -o /out/dvcr-uploader ./cmd/dvcr-uploader`) | nindent 6 }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -tags=dvcr_no_gost_tls -ldflags="-s -w" -o /out/dvcr-uploader ./cmd/dvcr-uploader`) | nindent 6 }} - | export CGO_ENABLED=0 {{- $_ := set $ "ProjectName" (list .ImageName "dvcr-cleaner" | join "/") }} From 76276aa29d6d28a9e30b20834580d4e9f53be9a2 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 30 Jun 2026 14:28:02 +0300 Subject: [PATCH 7/8] chore(dvcr): remove temporary pprof hook and stale gost_tls_off.go Drop the temporary net/http/pprof server from the importer main (used only to profile the GOST TLS bottleneck), and remove the obsolete cmd/dvcr-importer/gost_tls_off.go whose logic now lives in the shared pkg/gosttls package. Signed-off-by: Nikita Korolev --- .../cmd/dvcr-importer/gost_tls_off.go | 41 ------------------- .../dvcr-artifact/cmd/dvcr-importer/main.go | 18 -------- 2 files changed, 59 deletions(-) delete mode 100644 images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go diff --git a/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go b/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go deleted file mode 100644 index f103f2b574..0000000000 --- a/images/dvcr-artifact/cmd/dvcr-importer/gost_tls_off.go +++ /dev/null @@ -1,41 +0,0 @@ -//go:build dvcr_no_gost_tls - -/* -Copyright 2026 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// This file is compiled only when built with the deckhouse GOST-enabled Go -// toolchain (golang-alt/golang-debian) via `-tags=dvcr_no_gost_tls`. That -// toolchain unconditionally installs GOST (Kuznyechik/MGM) TLS 1.3 cipher -// suites, which are pure-software (no hardware acceleration) and cap the -// importer's upload to the in-cluster DVCR at a few MB/s on a single core. -// -// SetAllowedTLS13CipherSuites is a toolchain-specific API (absent from upstream -// Go), so the call is isolated behind a build tag: standard-Go builds (local -// dev, golangci-lint) simply exclude this file and keep upstream behaviour. - -package main - -import "crypto/tls" - -func init() { - // Restrict TLS 1.3 to the AES-NI / ChaCha20 accelerated suites so GOST is - // no longer advertised and a GOST-preferring DVCR cannot select it. - tls.SetAllowedTLS13CipherSuites([]uint16{ - tls.TLS_AES_128_GCM_SHA256, - tls.TLS_AES_256_GCM_SHA384, - tls.TLS_CHACHA20_POLY1305_SHA256, - }) -} diff --git a/images/dvcr-artifact/cmd/dvcr-importer/main.go b/images/dvcr-artifact/cmd/dvcr-importer/main.go index 29ffccfcfc..1516276de8 100644 --- a/images/dvcr-artifact/cmd/dvcr-importer/main.go +++ b/images/dvcr-artifact/cmd/dvcr-importer/main.go @@ -19,10 +19,7 @@ package main import ( "context" "flag" - "net/http" - _ "net/http/pprof" "os" - "time" "github.com/google/go-containerregistry/pkg/logs" "k8s.io/klog/v2" @@ -37,27 +34,12 @@ func init() { flag.Parse() } -// TODO(profiling): temporary CPU-profiling hook, remove before merge. -// Starts net/http/pprof so a CPU profile can be captured during import via -// `kubectl port-forward 6060:6060` + `go tool pprof`. -func startPprof() { - go func() { - klog.Infoln("Starting pprof server on :6060") - srv := &http.Server{Addr: ":6060", ReadHeaderTimeout: 10 * time.Second} - if err := srv.ListenAndServe(); err != nil { - klog.Warningf("pprof server stopped: %v", err) - } - }() -} - func main() { defer klog.Flush() logs.Progress.SetOutput(os.Stdout) logs.Warn.SetOutput(os.Stderr) - startPprof() - klog.Infoln("Starting registry importer") imp := importer.New() From 8242d57d07c5dfd57ec8e2f40ed25372ce4789a5 Mon Sep 17 00:00:00 2001 From: Nikita Korolev Date: Tue, 30 Jun 2026 15:02:03 +0300 Subject: [PATCH 8/8] style(dvcr): fix gci import order in importer/uploader mains The blank gosttls import must precede the other deckhouse-prefixed imports (gci alphabetical order within the prefix group). Caught by the containerized golangci-lint; the macOS run masked it behind the libnbd/cgo typecheck failure. Signed-off-by: Nikita Korolev --- images/dvcr-artifact/cmd/dvcr-importer/main.go | 2 +- images/dvcr-artifact/cmd/dvcr-uploader/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/images/dvcr-artifact/cmd/dvcr-importer/main.go b/images/dvcr-artifact/cmd/dvcr-importer/main.go index 1516276de8..0eadfb0ffd 100644 --- a/images/dvcr-artifact/cmd/dvcr-importer/main.go +++ b/images/dvcr-artifact/cmd/dvcr-importer/main.go @@ -24,9 +24,9 @@ import ( "github.com/google/go-containerregistry/pkg/logs" "k8s.io/klog/v2" - "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/importer" // Prefer AES-GCM over GOST for TLS 1.3 when built with -tags=dvcr_no_gost_tls. _ "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/gosttls" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/importer" ) func init() { diff --git a/images/dvcr-artifact/cmd/dvcr-uploader/main.go b/images/dvcr-artifact/cmd/dvcr-uploader/main.go index 35a29fcc70..c374317581 100644 --- a/images/dvcr-artifact/cmd/dvcr-uploader/main.go +++ b/images/dvcr-artifact/cmd/dvcr-uploader/main.go @@ -27,9 +27,9 @@ import ( "kubevirt.io/containerized-data-importer/pkg/common" cryptowatch "kubevirt.io/containerized-data-importer/pkg/util/tls-crypto-watch" - "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/uploader" // Prefer AES-GCM over GOST for TLS 1.3 when built with -tags=dvcr_no_gost_tls. _ "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/gosttls" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/uploader" ) const (