diff --git a/go.mod b/go.mod index 831295743a..82db755861 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/anchore/clio v0.0.0-20250715152405-a0fa658e5084 github.com/anchore/stereoscope v0.1.22 github.com/anchore/syft v1.42.3 - github.com/avast/retry-go/v4 v4.7.0 github.com/defenseunicorns/pkg/helpers/v2 v2.0.4 github.com/defenseunicorns/pkg/oci v1.3.0 github.com/derailed/k9s v0.50.18 diff --git a/go.sum b/go.sum index 6a45931a89..c5f0234af8 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,6 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= -github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio= -github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= diff --git a/src/internal/packager/helm/repo.go b/src/internal/packager/helm/repo.go index ec27ad63ee..a44e9bb7b7 100644 --- a/src/internal/packager/helm/repo.go +++ b/src/internal/packager/helm/repo.go @@ -28,8 +28,8 @@ import ( "helm.sh/helm/v4/pkg/getter" "helm.sh/helm/v4/pkg/registry" repov1 "helm.sh/helm/v4/pkg/repo/v1" + "k8s.io/apimachinery/pkg/util/wait" - retry "github.com/avast/retry-go/v4" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" "github.com/zarf-dev/zarf/src/internal/git" @@ -239,31 +239,36 @@ func DownloadPublishedChart(ctx context.Context, chart v1alpha1.ZarfChart, chart }, } - var saved string - err = retry.Do( - func() error { - var downloadErr error - saved, _, downloadErr = chartDownloader.DownloadToCache(chartURL, pull.Version) - return downloadErr - }, - retry.Attempts(uint(config.ZarfDefaultRetries)), - retry.Delay(config.ZarfDefaultRetryDelay), - retry.MaxDelay(config.ZarfDefaultRetryMaxDelay), - retry.DelayType(retry.BackOffDelay), - retry.LastErrorOnly(true), - retry.Context(ctx), - retry.OnRetry(func(n uint, err error) { - if config.ZarfDefaultRetries > 1 && n+1 < uint(config.ZarfDefaultRetries) { - l.Warn("retrying chart download", - "attempt", n+1, - "maxAttempts", config.ZarfDefaultRetries, - "chart", chart.Name, - "error", err, - ) - } - }), + var ( + saved string + lastErr error + attempts int ) + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: config.ZarfDefaultRetryDelay, + Factor: 2.0, + Steps: config.ZarfDefaultRetries, + Cap: config.ZarfDefaultRetryMaxDelay, + }, func(ctx context.Context) (bool, error) { + var downloadErr error + saved, _, downloadErr = chartDownloader.DownloadToCache(chartURL, pull.Version) + if downloadErr == nil { + return true, nil + } + lastErr = downloadErr + attempts++ + l.Warn("retrying chart download", + "attempt", attempts, + "maxAttempts", config.ZarfDefaultRetries, + "chart", chart.Name, + "error", downloadErr, + ) + return false, nil + }) if err != nil { + if lastErr != nil { + return fmt.Errorf("unable to download the helm chart: %w", lastErr) + } return fmt.Errorf("unable to download the helm chart: %w", err) } diff --git a/src/pkg/cluster/cluster.go b/src/pkg/cluster/cluster.go index 38c8ae2666..a7a2c22e68 100644 --- a/src/pkg/cluster/cluster.go +++ b/src/pkg/cluster/cluster.go @@ -12,7 +12,6 @@ import ( "slices" "time" - "github.com/avast/retry-go/v4" "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/internal/healthchecks" "github.com/zarf-dev/zarf/src/pkg/logger" @@ -22,6 +21,7 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" v1ac "k8s.io/client-go/applyconfigurations/core/v1" "k8s.io/client-go/discovery" "k8s.io/client-go/discovery/cached/memory" @@ -76,25 +76,29 @@ func NewWithWait(ctx context.Context) (*Cluster, error) { if err != nil { return nil, err } - err = retry.Do(func() error { + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: time.Second, + Factor: 1.0, + }, func(ctx context.Context) (bool, error) { nodeList, err := c.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - return err + return false, err } if len(nodeList.Items) < 1 { - return fmt.Errorf("cluster does not have any nodes") + return false, fmt.Errorf("cluster does not have any nodes") } + pods, err := c.Clientset.CoreV1().Pods(corev1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { - return err + return false, nil } for _, pod := range pods.Items { if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodRunning { - return nil + return true, nil } } - return fmt.Errorf("no pods are in succeeded or running state") - }, retry.Context(ctx), retry.Attempts(0), retry.DelayType(retry.FixedDelay), retry.Delay(time.Second)) + return false, nil + }) if err != nil { return nil, err } @@ -242,13 +246,16 @@ func (c *Cluster) InitState(ctx context.Context, opts InitStateOptions) (*state. // The default SA is required for pods to start properly. saCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - err = retry.Do(func() error { - _, err := c.Clientset.CoreV1().ServiceAccounts(state.ZarfNamespaceName).Get(saCtx, "default", metav1.GetOptions{}) + err = wait.ExponentialBackoffWithContext(saCtx, wait.Backoff{ + Duration: time.Second, + Factor: 1.0, + }, func(ctx context.Context) (bool, error) { + _, err := c.Clientset.CoreV1().ServiceAccounts(state.ZarfNamespaceName).Get(ctx, "default", metav1.GetOptions{}) if err != nil { - return err + return false, nil } - return nil - }, retry.Context(saCtx), retry.Attempts(0), retry.DelayType(retry.FixedDelay), retry.Delay(time.Second)) + return true, nil + }) if err != nil { return nil, fmt.Errorf("unable get default Zarf service account: %w", err) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index 4d349d5b70..e73c2f22db 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -17,9 +17,9 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "github.com/avast/retry-go/v4" "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/api/v1alpha1" @@ -172,13 +172,17 @@ type podFilter func(pod corev1.Pod) bool // TODO: Test, refactor and/or remove. func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interface, target podLookup, include podFilter) ([]corev1.Pod, error) { l := logger.From(ctx) - readyPods, err := retry.DoWithData(func() ([]corev1.Pod, error) { + var readyPods []corev1.Pod + err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: time.Second, + Factor: 1.0, + }, func(ctx context.Context) (bool, error) { listOpts := metav1.ListOptions{ LabelSelector: target.Selector, } podList, err := clientset.CoreV1().Pods(target.Namespace).List(ctx, listOpts) if err != nil { - return nil, err + return false, nil } l.Debug("found pods matching the target", "count", len(podList.Items), "target", target) // Sort the pods from newest to oldest @@ -186,7 +190,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac return podList.Items[i].CreationTimestamp.After(podList.Items[j].CreationTimestamp.Time) }) - readyPods := []corev1.Pod{} + found := []corev1.Pod{} for _, pod := range podList.Items { l.Debug("testing pod", "name", pod.Name) @@ -204,7 +208,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac isRunning := initContainer.State.Running != nil if initContainer.Name == target.Container && isRunning { // On running match in initContainer break this loop - readyPods = append(readyPods, pod) + found = append(found, pod) break } } @@ -213,7 +217,7 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac for _, container := range pod.Status.ContainerStatuses { isRunning := container.State.Running != nil if container.Name == target.Container && isRunning { - readyPods = append(readyPods, pod) + found = append(found, pod) break } } @@ -222,16 +226,17 @@ func waitForPodsAndContainers(ctx context.Context, clientset kubernetes.Interfac l.Debug(fmt.Sprintf("checking pod for %s status", corev1.PodRunning), "pod", pod.Name, "status", status) // Regular status checking without a container if status == corev1.PodRunning { - readyPods = append(readyPods, pod) + found = append(found, pod) break } } } - if len(readyPods) == 0 { - return nil, fmt.Errorf("no ready pods found") + if len(found) == 0 { + return false, nil } - return readyPods, nil - }, retry.Context(ctx), retry.Attempts(0), retry.DelayType(retry.FixedDelay), retry.Delay(time.Second)) + readyPods = found + return true, nil + }) if err != nil { return nil, err } diff --git a/src/pkg/cluster/injector.go b/src/pkg/cluster/injector.go index 3cc9ba8d95..8ed5253f28 100644 --- a/src/pkg/cluster/injector.go +++ b/src/pkg/cluster/injector.go @@ -6,7 +6,6 @@ package cluster import ( "context" - "errors" "fmt" "os" "path/filepath" @@ -15,7 +14,6 @@ import ( "time" "github.com/Masterminds/semver/v3" - "github.com/avast/retry-go/v4" "github.com/google/go-containerregistry/pkg/crane" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" @@ -375,10 +373,14 @@ func (c *Cluster) GetInjectorDaemonsetImage(ctx context.Context) (string, error) l := logger.From(ctx) var injectorImage string - err := retry.Do(func() error { + err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 5 * time.Second, + Factor: 1.0, + Steps: 15, + }, func(ctx context.Context) (bool, error) { nodes, err := c.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) if err != nil { - return err + return false, nil } // Track images across all nodes @@ -415,12 +417,12 @@ func (c *Cluster) GetInjectorDaemonsetImage(ctx context.Context) (string, error) } } injectorImage = latestPause.name - return nil + return true, nil } // Fallback to smallest image if no valid pause images if len(allImages) == 0 { - return errors.New("no suitable image found on any node") + return false, nil } // Find the smallest image by size @@ -432,11 +434,11 @@ func (c *Cluster) GetInjectorDaemonsetImage(ctx context.Context) (string, error) } if len(smallestImage.Names) == 0 { - return errors.New("selected image has no names") + return false, nil } injectorImage = smallestImage.Names[0] - return nil - }, retry.Attempts(15), retry.Delay(5*time.Second), retry.Context(ctx), retry.DelayType(retry.FixedDelay)) + return true, nil + }) if err != nil { return "", err } @@ -604,7 +606,11 @@ func (c *Cluster) createInjectorNodeportService(ctx context.Context, pkgName str if opts.InjectorNodePort != 0 { portConfiguration.WithNodePort(int32(opts.InjectorNodePort)) } - err := retry.Do(func() error { + err := wait.ExponentialBackoffWithContext(timeoutCtx, wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.0, + Steps: 10, + }, func(ctx context.Context) (bool, error) { svcAc := v1ac.Service("zarf-injector", state.ZarfNamespaceName). WithSpec(v1ac.ServiceSpec(). WithType(corev1.ServiceTypeNodePort). @@ -619,20 +625,19 @@ func (c *Cluster) createInjectorNodeportService(ctx context.Context, pkgName str var err error svc, err = c.Clientset.CoreV1().Services(*svcAc.Namespace).Apply(ctx, svcAc, metav1.ApplyOptions{Force: true, FieldManager: FieldManagerName}) if err != nil { - return err + return false, nil } assignedNodePort := int(svc.Spec.Ports[0].NodePort) if assignedNodePort == int(opts.RegistryNodePort) { l.Info("injector service NodePort conflicts with registry NodePort, recreating service", "conflictingPort", assignedNodePort) - deleteErr := c.Clientset.CoreV1().Services(state.ZarfNamespaceName).Delete(ctx, "zarf-injector", metav1.DeleteOptions{}) - if deleteErr != nil { - return deleteErr + if deleteErr := c.Clientset.CoreV1().Services(state.ZarfNamespaceName).Delete(ctx, "zarf-injector", metav1.DeleteOptions{}); deleteErr != nil { + return false, deleteErr } - return fmt.Errorf("nodePort conflict with registry port %d", opts.RegistryNodePort) + return false, nil } - return nil - }, retry.Attempts(10), retry.Delay(500*time.Millisecond), retry.Context(timeoutCtx)) + return true, nil + }) if err != nil { return nil, fmt.Errorf("failed to create the injector nodeport service: %w", err) } diff --git a/src/pkg/cluster/namespace.go b/src/pkg/cluster/namespace.go index e3f04f185e..4d16ac8fe0 100644 --- a/src/pkg/cluster/namespace.go +++ b/src/pkg/cluster/namespace.go @@ -6,12 +6,11 @@ package cluster import ( "context" - "fmt" "time" - "github.com/avast/retry-go/v4" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/state" @@ -32,16 +31,13 @@ func (c *Cluster) DeleteZarfNamespace(ctx context.Context) error { if err != nil { return err } - err = retry.Do(func() error { + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { _, err := c.Clientset.CoreV1().Namespaces().Get(ctx, state.ZarfNamespaceName, metav1.GetOptions{}) if kerrors.IsNotFound(err) { - return nil + return true, nil } - if err != nil { - return err - } - return fmt.Errorf("namespace still exists") - }, retry.Context(ctx), retry.Attempts(0), retry.DelayType(retry.FixedDelay), retry.Delay(time.Second)) + return false, nil + }) if err != nil { return err } diff --git a/src/pkg/cluster/tunnel.go b/src/pkg/cluster/tunnel.go index d707727b9b..a823294fe1 100644 --- a/src/pkg/cluster/tunnel.go +++ b/src/pkg/cluster/tunnel.go @@ -13,16 +13,17 @@ import ( "strconv" "strings" "sync" + "time" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/portforward" "k8s.io/client-go/transport/spdy" "k8s.io/streaming/pkg/httpstream" - "github.com/avast/retry-go/v4" "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/internal/dns" "github.com/zarf-dev/zarf/src/pkg/logger" @@ -398,13 +399,19 @@ func (tunnel *Tunnel) Wrap(function func() error) error { // Connect will establish a tunnel to the specified target. func (tunnel *Tunnel) Connect(ctx context.Context) ([]string, error) { - urls, err := retry.DoWithData(func() ([]string, error) { - urls, err := tunnel.establish(ctx) + var urls []string + err := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 100 * time.Millisecond, + Factor: 2.0, + Steps: 6, + }, func(ctx context.Context) (bool, error) { + result, err := tunnel.establish(ctx) if err != nil { - return []string{}, err + return false, nil } - return urls, nil - }, retry.Context(ctx), retry.Attempts(6)) + urls = result + return true, nil + }) if err != nil { return []string{}, err } diff --git a/src/pkg/images/pull.go b/src/pkg/images/pull.go index c7e1da8a08..631142a1f0 100644 --- a/src/pkg/images/pull.go +++ b/src/pkg/images/pull.go @@ -16,7 +16,6 @@ import ( "sync" "time" - retry "github.com/avast/retry-go/v4" "github.com/docker/cli/cli/command" "github.com/docker/cli/cli/context/docker" "github.com/docker/cli/cli/flags" @@ -26,22 +25,23 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" clayout "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/moby/moby/client" - "github.com/zarf-dev/zarf/src/config" - "github.com/zarf-dev/zarf/src/pkg/logger" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/util/wait" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/oci" "oras.land/oras-go/v2/registry" + orasRemote "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/credentials" "github.com/defenseunicorns/pkg/helpers/v2" orasCache "github.com/defenseunicorns/pkg/oci/cache" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/internal/dns" + "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/transform" "github.com/zarf-dev/zarf/src/pkg/utils" - orasRemote "oras.land/oras-go/v2/registry/remote" - "oras.land/oras-go/v2/registry/remote/auth" - "oras.land/oras-go/v2/registry/remote/credentials" ) // PullOptions is the configuration for pulling images. @@ -471,34 +471,39 @@ func orasSave(ctx context.Context, imageInfo imagePullInfo, opts PullOptions, ds return fmt.Errorf("failed to create oci formatted directory: %w", err) } pullSrc = orasCache.New(repo, localCache) - var desc ocispec.Descriptor - err = retry.Do( - func() error { - trackedDst := NewTrackedTarget(dst, imageInfo.byteSize, DefaultReport(l, "image pull in progress", imageInfo.registryOverrideRef)) - trackedDst.StartReporting(ctx) - defer trackedDst.StopReporting() - var copyErr error - desc, copyErr = oras.Copy(ctx, pullSrc, imageInfo.registryOverrideRef, trackedDst, imageInfo.ref, copyOpts) - return copyErr - }, - retry.Attempts(uint(config.ZarfDefaultRetries)), - retry.Delay(config.ZarfDefaultRetryDelay), - retry.MaxDelay(config.ZarfDefaultRetryMaxDelay), - retry.DelayType(retry.BackOffDelay), - retry.LastErrorOnly(true), - retry.Context(ctx), - retry.OnRetry(func(n uint, err error) { - if config.ZarfDefaultRetries > 1 && n+1 < uint(config.ZarfDefaultRetries) { - l.Warn("retrying image pull", - "attempt", n+1, - "maxAttempts", config.ZarfDefaultRetries, - "image", imageInfo.registryOverrideRef, - "error", err, - ) - } - }), + var ( + desc ocispec.Descriptor + lastErr error + attempts int ) + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: config.ZarfDefaultRetryDelay, + Factor: 2.0, + Steps: config.ZarfDefaultRetries, + Cap: config.ZarfDefaultRetryMaxDelay, + }, func(ctx context.Context) (bool, error) { + trackedDst := NewTrackedTarget(dst, imageInfo.byteSize, DefaultReport(l, "image pull in progress", imageInfo.registryOverrideRef)) + trackedDst.StartReporting(ctx) + defer trackedDst.StopReporting() + var copyErr error + desc, copyErr = oras.Copy(ctx, pullSrc, imageInfo.registryOverrideRef, trackedDst, imageInfo.ref, copyOpts) + if copyErr == nil { + return true, nil + } + lastErr = copyErr + attempts++ + l.Warn("retrying image pull", + "attempt", attempts, + "maxAttempts", config.ZarfDefaultRetries, + "image", imageInfo.registryOverrideRef, + "error", copyErr, + ) + return false, nil + }) if err != nil { + if lastErr != nil { + return fmt.Errorf("failed to copy: %w", lastErr) + } return fmt.Errorf("failed to copy: %w", err) } desc = addNameAnnotationsToDesc(desc, imageInfo.ref) diff --git a/src/pkg/images/push.go b/src/pkg/images/push.go index 26c27fc1d1..c16eba4f03 100644 --- a/src/pkg/images/push.go +++ b/src/pkg/images/push.go @@ -12,7 +12,6 @@ import ( "strings" "time" - "github.com/avast/retry-go/v4" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content/oci" "oras.land/oras-go/v2/registry" @@ -27,6 +26,7 @@ import ( "github.com/zarf-dev/zarf/src/pkg/pki" "github.com/zarf-dev/zarf/src/pkg/state" "github.com/zarf-dev/zarf/src/pkg/transform" + "k8s.io/apimachinery/pkg/util/wait" ) const defaultRetries = 3 @@ -78,48 +78,63 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri return fmt.Errorf("failed to instantiate oci directory: %w", err) } - err = retry.Do(func() error { + attempt := 0 + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.0, + Steps: cfg.Retries, + }, func(ctx context.Context) (bool, error) { + if attempt > 0 { + if cfg.Retries > 2 && attempt == cfg.Retries-1 { + cfg.ResponseHeaderTimeout = 60 * time.Second // this should really never happen + } + l.Debug("retrying component image(s) push", "responseTimeout", cfg.ResponseHeaderTimeout) + } + attempt++ + // reset concurrency to user-provided value on each component retry ociConcurrency := cfg.OCIConcurrency var registryRef registry.Reference // Include tunnel connection in retry loop in case the port forward breaks, for example, a registry pod could spin down / restart var tunnel *cluster.Tunnel if cfg.Cluster != nil { - var err error + var regErr error var registryURL string - registryURL, tunnel, err = cfg.Cluster.ConnectToZarfRegistryEndpoint(ctx, registryInfo) - if err != nil { - return err + registryURL, tunnel, regErr = cfg.Cluster.ConnectToZarfRegistryEndpoint(ctx, registryInfo) + if regErr != nil { + return false, nil } - registryRef, err = parseRegistryReference(registryURL) - if err != nil { - return fmt.Errorf("failed to get reference from registry from internal registry: %w", err) + registryRef, regErr = parseRegistryReference(registryURL) + if regErr != nil { + return false, nil } if tunnel != nil { defer tunnel.Close() } } else { - registryRef, err = parseRegistryReference(registryInfo.Address) - if err != nil { - return fmt.Errorf("failed to get reference from registry address: %w", err) + var regErr error + registryRef, regErr = parseRegistryReference(registryInfo.Address) + if regErr != nil { + return false, nil } } var transport http.RoundTripper - var certs pki.GeneratedPKI if cfg.Cluster != nil && registryInfo.ShouldUseMTLS() { - certs, err = cfg.Cluster.GetRegistryClientMTLSCert(ctx) - if err != nil { - return err + certs, regErr := cfg.Cluster.GetRegistryClientMTLSCert(ctx) + if regErr != nil { + return false, nil } - transport, err = pki.TransportWithKey(certs) - if err != nil { - return err + var tErr error + transport, tErr = pki.TransportWithKey(certs) + if tErr != nil { + return false, nil } } else { - transport, err = orasTransport(cfg.InsecureSkipTLSVerify, cfg.ResponseHeaderTimeout) - if err != nil { - return err + var tErr error + transport, tErr = orasTransport(cfg.InsecureSkipTLSVerify, cfg.ResponseHeaderTimeout) + if tErr != nil { + return false, nil } } @@ -136,10 +151,10 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri plainHTTP := cfg.PlainHTTP if dns.IsLocalhost(registryRef.Host()) && !cfg.PlainHTTP { - var err error - plainHTTP, err = ShouldUsePlainHTTP(ctx, registryRef.Host(), client) - if err != nil { - return err + var httpErr error + plainHTTP, httpErr = ShouldUsePlainHTTP(ctx, registryRef.Host(), client) + if httpErr != nil { + return false, nil } } @@ -148,8 +163,9 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri PlainHTTP: plainHTTP, Client: client, } - remoteRepo.Reference, err = registry.ParseReference(dstName) - if err != nil { + var regErr error + remoteRepo.Reference, regErr = registry.ParseReference(dstName) + if regErr != nil { return fmt.Errorf("failed to parse ref %s: %w", dstName, err) } defaultPlatform := &ocispec.Platform{ @@ -163,6 +179,7 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri } return copyImage(ctx, src, remoteRepo, srcName, dstName, ociConcurrency, defaultPlatform) } + pushed := []string{} // Delete the images that were already successfully pushed so that they aren't attempted on the next retry defer func() { @@ -174,23 +191,29 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri l.Info("pushing image", "name", img) // If this is not a no checksum image push it for use with the Zarf agent if !cfg.NoChecksum { - offlineNameCRC, err := transform.ImageTransformHost(registryRef.String(), img) - if err != nil { - return err + offlineNameCRC, tErr := transform.ImageTransformHost(registryRef.String(), img) + if tErr != nil { + return false, nil } - - err = retry.Do( - func() error { return pushImage(img, offlineNameCRC) }, - retry.OnRetry(func(_ uint, err error) { + innerAttempt := 0 + var lastPushErr error + if waitErr := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.0, + Steps: 2, + }, func(ctx context.Context) (bool, error) { + if innerAttempt > 0 { ociConcurrency = 1 - l.Debug("retrying image push", "error", err, "concurrency", ociConcurrency) - }), - retry.Context(ctx), - retry.Attempts(2), - retry.Delay(500*time.Millisecond), - ) - if err != nil { - return err + l.Debug("retrying image push", "error", lastPushErr, "concurrency", ociConcurrency) + } + innerAttempt++ + lastPushErr = pushImage(img, offlineNameCRC) + if lastPushErr != nil { + return false, nil + } + return true, nil + }); waitErr != nil { + return false, nil } } @@ -198,32 +221,33 @@ func Push(ctx context.Context, imageList []transform.Image, sourceDirectory stri // (this may result in collisions but this is acceptable for this use case) offlineName, err := transform.ImageTransformHostWithoutChecksum(registryRef.String(), img) if err != nil { - return err + return false, nil } - - err = retry.Do( - func() error { return pushImage(img, offlineName) }, - retry.OnRetry(func(_ uint, err error) { + innerAttempt := 0 + var lastPushErr error + if waitErr := wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.0, + Steps: 2, + }, func(ctx context.Context) (bool, error) { + if innerAttempt > 0 { ociConcurrency = 1 - l.Debug("retrying image push", "error", err, "concurrency", ociConcurrency) - }), - retry.Context(ctx), - retry.Attempts(2), - retry.Delay(500*time.Millisecond), - ) - if err != nil { - return err + l.Debug("retrying image push", "error", lastPushErr, "concurrency", ociConcurrency) + } + innerAttempt++ + lastPushErr = pushImage(img, offlineName) + if lastPushErr != nil { + return false, nil + } + return true, nil + }); waitErr != nil { + return false, nil } pushed = append(pushed, img) } - return nil - }, retry.Context(ctx), retry.Attempts(uint(cfg.Retries)), retry.Delay(500*time.Millisecond), retry.OnRetry(func(attempt uint, _ error) { - if uint(cfg.Retries) > 2 && attempt == uint(cfg.Retries)-2 { - cfg.ResponseHeaderTimeout = 60 * time.Second // this should really never happen - } - l.Debug("retrying component image(s) push", "responseTimeout", cfg.ResponseHeaderTimeout) - })) + return true, nil + }) if err != nil { return err } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 7d66209593..35a03b0678 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -10,7 +10,7 @@ import ( "os" "time" - "github.com/avast/retry-go/v4" + "k8s.io/apimachinery/pkg/util/wait" "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/config" @@ -121,60 +121,64 @@ func pushComponentReposToRegistry(ctx context.Context, component v1alpha1.ZarfCo if err != nil { return err } - err = retry.Do(func() error { + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 2.0, + Steps: retries, + }, func(ctx context.Context) (bool, error) { if !dns.IsServiceURL(gitInfo.Address) { l.Info("pushing repository to server", "repo", repoURL, "server", gitInfo.Address) - err = repository.Push(ctx, gitInfo.Address, gitInfo.PushUsername, gitInfo.PushPassword) - if err != nil { - return err + if pushErr := repository.Push(ctx, gitInfo.Address, gitInfo.PushUsername, gitInfo.PushPassword); pushErr != nil { + return false, nil } - return nil + return true, nil } if c == nil { - return retry.Unrecoverable(errors.New("cannot push to internal Git server when cluster is nil")) + return false, errors.New("cannot push to internal Git server when cluster is nil") } - namespace, name, port, err := dns.ParseServiceURL(gitInfo.Address) - if err != nil { - return retry.Unrecoverable(err) + namespace, name, port, parseErr := dns.ParseServiceURL(gitInfo.Address) + if parseErr != nil { + return false, parseErr } - tunnel, err := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) - if err != nil { - return err + tunnel, tunnelErr := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) + if tunnelErr != nil { + return false, nil } - _, err = tunnel.Connect(ctx) - if err != nil { - return err + _, tunnelErr = tunnel.Connect(ctx) + if tunnelErr != nil { + return false, nil } defer tunnel.Close() // tunnel is create with the default listenAddress - there will only be one endpoint until otherwise supported endpoints := tunnel.HTTPEndpoints() if len(endpoints) == 0 { - return errors.New("no tunnel endpoints found") + return false, nil } - giteaClient, err := gitea.NewClient(endpoints[0], gitInfo.PushUsername, gitInfo.PushPassword) - if err != nil { - return err + repoName, repoErr := transform.GitURLtoRepoName(repoURL) + if repoErr != nil { + return false, fmt.Errorf("unable to add the read only user to the repo %s: %w", repoName, repoErr) + } + giteaClient, giteaErr := gitea.NewClient(endpoints[0], gitInfo.PushUsername, gitInfo.PushPassword) + if giteaErr != nil { + return false, nil } - return tunnel.Wrap(func() error { + if wrapErr := tunnel.Wrap(func() error { l.Info("pushing repository to server", "repo", repoURL, "server", endpoints[0]) - err = repository.Push(ctx, endpoints[0], gitInfo.PushUsername, gitInfo.PushPassword) - if err != nil { + if err := repository.Push(ctx, endpoints[0], gitInfo.PushUsername, gitInfo.PushPassword); err != nil { return err } // Add the read-only user to this repo // TODO: This should not be done here. Or the function name should be changed. - repoName, err := transform.GitURLtoRepoName(repoURL) - if err != nil { - return retry.Unrecoverable(err) - } - err = giteaClient.AddReadOnlyUserToRepository(ctx, repoName, gitInfo.PullUsername) - if err != nil { + if err := giteaClient.AddReadOnlyUserToRepository(ctx, repoName, gitInfo.PullUsername); err != nil { return fmt.Errorf("unable to add the read only user to the repo %s: %w", repoName, err) } return nil - }) - }, retry.Context(ctx), retry.Attempts(uint(retries)), retry.Delay(500*time.Millisecond)) + }); wrapErr != nil { + return false, nil + } + return true, nil + }) if err != nil { return fmt.Errorf("unable to push repo %s to the Git Server: %w", repoURL, err) } diff --git a/src/pkg/utils/network.go b/src/pkg/utils/network.go index 7e514c5048..847fc7f4d6 100644 --- a/src/pkg/utils/network.go +++ b/src/pkg/utils/network.go @@ -17,21 +17,34 @@ import ( "strings" "time" - retry "github.com/avast/retry-go/v4" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/clock" + "github.com/defenseunicorns/pkg/helpers/v2" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" "github.com/zarf-dev/zarf/src/pkg/logger" ) -// retryAfterDuration is returned on a 429 so the custom DelayType can use it -// instead of stacking on top of the normal backoff. +// retryAfterDuration is returned on a 429 so the retry loop can honor +// the server-requested delay before the normal exponential sleep. type retryAfterDuration time.Duration func (d retryAfterDuration) Error() string { return fmt.Sprintf("rate limited (HTTP 429), retry after %s", time.Duration(d)) } +// unrecoverableError wraps errors that must not be retried. +type unrecoverableError struct{ err error } + +func (e unrecoverableError) Error() string { + return e.err.Error() +} + +func (e unrecoverableError) Unwrap() error { + return e.err +} + func parseChecksum(src string) (string, string, error) { atSymbolCount := strings.Count(src, "@") var checksum string @@ -66,40 +79,54 @@ func DownloadToFile(ctx context.Context, src, dst string) (err error) { } l := logger.From(ctx) - err = retry.Do( - func() error { - // Create the file - file, createErr := os.Create(dst) - if createErr != nil { - return retry.Unrecoverable(fmt.Errorf(lang.ErrWritingFile, dst, createErr)) - } - getErr := httpGetFile(ctx, src, file) - closeErr := file.Close() - return errors.Join(getErr, closeErr) - }, - retry.Attempts(uint(config.ZarfDefaultRetries)), - retry.Delay(config.ZarfDefaultRetryDelay), - retry.MaxDelay(config.ZarfDefaultRetryMaxDelay), - retry.DelayType(func(n uint, err error, rc *retry.Config) time.Duration { - var rlErr retryAfterDuration - if errors.As(err, &rlErr) { - return time.Duration(rlErr) - } - return retry.BackOffDelay(n, err, rc) - }), - retry.LastErrorOnly(true), - retry.Context(ctx), - retry.OnRetry(func(n uint, err error) { - if config.ZarfDefaultRetries > 1 && n+1 < uint(config.ZarfDefaultRetries) { - l.Warn("retrying download", - "attempt", n+1, - "maxAttempts", config.ZarfDefaultRetries, - "url", src, - "error", err, - ) - } - }), - ) + + // resetInterval is larger than the total retry window so the backoff never auto-resets + expDelay := wait.Backoff{ + Duration: config.ZarfDefaultRetryDelay, + Factor: 2.0, + Steps: config.ZarfDefaultRetries, + Cap: config.ZarfDefaultRetryMaxDelay, + }.DelayWithReset(clock.RealClock{}, time.Hour) + + // when a 429 Retry-After is seen the condition sets retryAfterOverride so + // the next sleep uses that duration instead of the exponential one + var retryAfterOverride time.Duration + retryDelay := wait.DelayFunc(func() time.Duration { + if retryAfterOverride > 0 { + d := retryAfterOverride + retryAfterOverride = 0 + return d + } + return expDelay() + }) + + attempt := 0 + err = retryDelay.Until(ctx, true, false, func(ctx context.Context) (bool, error) { + file, createErr := os.Create(dst) + if createErr != nil { + return false, fmt.Errorf(lang.ErrWritingFile, dst, createErr) + } + getErr := httpGetFile(ctx, src, file) + closeErr := file.Close() + joinedErr := errors.Join(getErr, closeErr) + if joinedErr == nil { + return true, nil + } + if unrecovErr, ok := errors.AsType[unrecoverableError](joinedErr); ok { + return false, unrecovErr.err + } + if rlErr, ok := errors.AsType[retryAfterDuration](joinedErr); ok { + retryAfterOverride = time.Duration(rlErr) + } + attempt++ + l.Warn("retrying download", + "attempt", attempt, + "maxAttempts", config.ZarfDefaultRetries, + "url", src, + "error", joinedErr, + ) + return false, nil + }) if err != nil { return err } @@ -125,7 +152,7 @@ func httpGetFile(ctx context.Context, url string, destinationFile *os.File) (err req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return retry.Unrecoverable(fmt.Errorf("unable to create request for %s: %w", url, err)) + return &unrecoverableError{fmt.Errorf("unable to create request for %s: %w", url, err)} } resp, err := http.DefaultClient.Do(req) if err != nil { @@ -142,7 +169,7 @@ func httpGetFile(ctx context.Context, url string, destinationFile *os.File) (err if d := parseRetryAfter(resp.Header.Get("Retry-After")); d > 0 { const maxRetryAfter = 60 * time.Second if d > maxRetryAfter { - return retry.Unrecoverable(fmt.Errorf("rate limited (HTTP 429) with Retry-After %s exceeding %s: %s", d, maxRetryAfter, resp.Status)) + return &unrecoverableError{fmt.Errorf("rate limited (HTTP 429) with Retry-After %s exceeding %s: %s", d, maxRetryAfter, resp.Status)} } return retryAfterDuration(d) } @@ -151,7 +178,7 @@ func httpGetFile(ctx context.Context, url string, destinationFile *os.File) (err if resp.StatusCode >= 500 { return fmt.Errorf("server error: %s", resp.Status) } - return retry.Unrecoverable(fmt.Errorf("bad HTTP status: %s", resp.Status)) + return &unrecoverableError{fmt.Errorf("bad HTTP status: %s", resp.Status)} } // Copy response body to file diff --git a/src/pkg/zoci/copier.go b/src/pkg/zoci/copier.go index 761d813e0c..731739cba8 100644 --- a/src/pkg/zoci/copier.go +++ b/src/pkg/zoci/copier.go @@ -8,10 +8,10 @@ import ( "context" "fmt" - "github.com/zarf-dev/zarf/src/pkg/logger" + "k8s.io/apimachinery/pkg/util/wait" "oras.land/oras-go/v2" - retry "github.com/avast/retry-go/v4" + "github.com/zarf-dev/zarf/src/pkg/logger" ) // CopyPackage copies a zarf package from one OCI registry to another using ORAS with retry. @@ -44,44 +44,57 @@ func CopyPackage(ctx context.Context, src *Remote, dst *Remote, opts PublishOpti tag = opts.Tag } - err = retry.Do( - func() error { - l.Info("copying package", - "src", src.Repo().Reference.String(), - "dst", dst.Repo().Reference.String(), - "ref", srcRef, + var ( + lastErr error + attempts int + ) + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: defaultDelayTime, + Factor: 2.0, + Steps: opts.Retries, + Cap: defaultMaxDelayTime, + }, func(ctx context.Context) (bool, error) { + l.Info("copying package", + "src", src.Repo().Reference.String(), + "dst", dst.Repo().Reference.String(), + "ref", srcRef, + ) + defer func() { + if lastErr == nil { + return + } + l.Warn("retrying package copy", + "attempt", attempts, + "maxAttempts", opts.Retries, + "error", lastErr, ) + }() - source := src.Repo() // implements oras.ReadOnlyTarget - destination := dst.Repo() // implements oras.Target + source := src.Repo() // implements oras.ReadOnlyTarget + destination := dst.Repo() // implements oras.Target - // 1) Copy by digest from source → destination - publishedDesc, copyErr := oras.Copy(ctx, source, srcRef, destination, "", copyOpts) - if copyErr != nil { - return copyErr - } + attempts++ - // 2) Update/tag the destination index to the source tag - return dst.OrasRemote.UpdateIndex(ctx, tag, publishedDesc) - }, - retry.Attempts(uint(opts.Retries)), - retry.Delay(defaultDelayTime), - retry.MaxDelay(defaultMaxDelayTime), - retry.DelayType(retry.BackOffDelay), - retry.LastErrorOnly(true), - retry.Context(ctx), - retry.OnRetry(func(n uint, err error) { - // Only log retry if retries are enabled and we're not on the last attempt - if opts.Retries > 1 && n+1 < uint(opts.Retries) { - l.Warn("retrying package copy", - "attempt", n+1, - "maxAttempts", opts.Retries, - "error", err, - ) - } - }), - ) + // 1) Copy by digest from source → destination + publishedDesc, copyErr := oras.Copy(ctx, source, srcRef, destination, "", copyOpts) + if copyErr != nil { + lastErr = copyErr + return false, nil + } + + // 2) Update/tag the destination index to the source tag + if err := dst.OrasRemote.UpdateIndex(ctx, tag, publishedDesc); err != nil { + lastErr = err + return false, nil + } + + lastErr = nil + return true, nil + }) if err != nil { + if lastErr != nil { + return fmt.Errorf("copy failed after retries: %w", lastErr) + } return fmt.Errorf("copy failed after retries: %w", err) } diff --git a/src/pkg/zoci/push.go b/src/pkg/zoci/push.go index da356758d0..2a48cbb3eb 100644 --- a/src/pkg/zoci/push.go +++ b/src/pkg/zoci/push.go @@ -14,17 +14,18 @@ import ( "sort" "time" - "github.com/avast/retry-go/v4" "github.com/defenseunicorns/pkg/oci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "k8s.io/apimachinery/pkg/util/wait" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content/file" + "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/pkg/images" "github.com/zarf-dev/zarf/src/pkg/logger" "github.com/zarf-dev/zarf/src/pkg/packager/layout" "github.com/zarf-dev/zarf/src/pkg/utils" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" ) // OCITimestampFormat is the format used for the OCI timestamp annotation @@ -106,63 +107,74 @@ func (r *Remote) PushPackage(ctx context.Context, pkgLayout *layout.PackageLayou return ocispec.Descriptor{}, fmt.Errorf("invalid annotations: please include value for %q", ocispec.AnnotationTitle) } - var publishedDesc ocispec.Descriptor - err = retry.Do( - func() error { - l.Info("pushing package to registry", "destination", r.Repo().Reference.String(), - "architecture", pkgLayout.Pkg.Build.Architecture, "size", utils.ByteFormat(float64(totalSize), 2)) - - manifestConfigBytes, err := json.Marshal(pkgLayout.Pkg) - if err != nil { - return err - } - manifestConfigDesc, err := r.PushLayer(ctx, manifestConfigBytes, ZarfConfigMediaType) - if err != nil { - return err - } - - root, packErr := r.OrasRemote.PackAndTagManifest(ctx, src, descs, manifestConfigDesc, annotations) - if packErr != nil { - return packErr + var ( + publishedDesc ocispec.Descriptor + lastErr error + attempts int + ) + err = wait.ExponentialBackoffWithContext(ctx, wait.Backoff{ + Duration: defaultDelayTime, + Factor: 2.0, + Steps: opts.Retries, + Cap: defaultMaxDelayTime, + }, func(ctx context.Context) (bool, error) { + l.Info("pushing package to registry", "destination", r.Repo().Reference.String(), + "architecture", pkgLayout.Pkg.Build.Architecture, "size", utils.ByteFormat(float64(totalSize), 2)) + defer func() { + if lastErr == nil { + return } - - // Update the total with manifest + config for better progress (optional) - attemptTotal := totalSize + root.Size + manifestConfigDesc.Size - - trackedRemote := images.NewTrackedTarget( - r.Repo(), - attemptTotal, - images.DefaultReport(r.Log(), "package publish in progress", r.Repo().Reference.String()), + l.Warn("retrying package push", + "attempt", attempts, + "maxAttempts", opts.Retries, + "error", lastErr, ) - trackedRemote.StartReporting(ctx) - defer trackedRemote.StopReporting() + }() - var copyErr error - publishedDesc, copyErr = oras.Copy(ctx, src, root.Digest.String(), trackedRemote, "", copyOpts) - if copyErr != nil { - return copyErr - } + attempts++ - return r.OrasRemote.UpdateIndex(ctx, r.Repo().Reference.Reference, publishedDesc) - }, - retry.Attempts(uint(opts.Retries)), - retry.Delay(defaultDelayTime), - retry.MaxDelay(defaultMaxDelayTime), - retry.DelayType(retry.BackOffDelay), // exponential backoff - retry.LastErrorOnly(true), - retry.Context(ctx), - retry.OnRetry(func(n uint, err error) { - // Only log retry if retries are enabled and this is not the last attempt - if opts.Retries > 1 && n+1 < uint(opts.Retries) { - l.Warn("retrying package push", - "attempt", n+1, - "maxAttempts", opts.Retries, - "error", err, - ) - } - }), - ) + manifestConfigBytes, pushErr := json.Marshal(pkgLayout.Pkg) + if pushErr != nil { + lastErr = pushErr + return false, nil + } + var manifestConfigDesc *ocispec.Descriptor + manifestConfigDesc, pushErr = r.PushLayer(ctx, manifestConfigBytes, ZarfConfigMediaType) + if pushErr != nil { + lastErr = pushErr + return false, nil + } + var root ocispec.Descriptor + root, pushErr = r.OrasRemote.PackAndTagManifest(ctx, src, descs, manifestConfigDesc, annotations) + if pushErr != nil { + lastErr = pushErr + return false, nil + } + // Update the total with manifest + config for better progress (optional) + attemptTotal := totalSize + root.Size + manifestConfigDesc.Size + trackedRemote := images.NewTrackedTarget( + r.Repo(), + attemptTotal, + images.DefaultReport(r.Log(), "package publish in progress", r.Repo().Reference.String()), + ) + trackedRemote.StartReporting(ctx) + defer trackedRemote.StopReporting() + publishedDesc, pushErr = oras.Copy(ctx, src, root.Digest.String(), trackedRemote, "", copyOpts) + if pushErr != nil { + lastErr = err + return false, nil + } + if err := r.OrasRemote.UpdateIndex(ctx, r.Repo().Reference.Reference, publishedDesc); err != nil { + lastErr = err + } + + lastErr = nil + return true, nil + }) if err != nil { + if lastErr != nil { + return ocispec.Descriptor{}, fmt.Errorf("publish failed: %w", lastErr) + } return ocispec.Descriptor{}, fmt.Errorf("publish failed: %w", err) }