diff --git a/src/pkg/packager/load/import.go b/src/pkg/packager/load/import.go index b5a6c37f13..b97f5785fa 100644 --- a/src/pkg/packager/load/import.go +++ b/src/pkg/packager/load/import.go @@ -102,6 +102,7 @@ func resolveImports(ctx context.Context, pkg v1alpha1.ZarfPackage, packagePath, var importedPkg v1alpha1.ZarfPackage var innerSchemas []string + var importRemote *zoci.Remote if component.Import.Path != "" { importPath := filepath.Join(pkgPath.BaseDir, component.Import.Path) for _, sp := range importStack { @@ -143,19 +144,19 @@ func resolveImports(ctx context.Context, pkg v1alpha1.ZarfPackage, packagePath, if err != nil { return v1alpha1.ZarfPackage{}, nil, err } - remote, err := zoci.NewRemote(ctx, component.Import.URL, zoci.PlatformForSkeleton(), + importRemote, err = zoci.NewRemote(ctx, component.Import.URL, zoci.PlatformForSkeleton(), cacheModifier, oci.WithPlainHTTP(plainHTTP), oci.WithInsecureSkipVerify(remoteOptions.InsecureSkipTLSVerify)) if err != nil { return v1alpha1.ZarfPackage{}, nil, err } - _, err = remote.ResolveRoot(ctx) + descriptor, err := importRemote.ResolveRoot(ctx) if err != nil { if strings.Contains(err.Error(), "no matching manifest was found in the manifest list") { return v1alpha1.ZarfPackage{}, nil, fmt.Errorf("package at %s exists but has not been published as a skeleton: %w", component.Import.URL, err) } return v1alpha1.ZarfPackage{}, nil, err } - importedPkg, err = remote.FetchZarfYAML(ctx) + importedPkg, err = importRemote.FetchZarfYAMLFromDescriptor(ctx, descriptor) if err != nil { return v1alpha1.ZarfPackage{}, nil, err } @@ -186,7 +187,7 @@ func resolveImports(ctx context.Context, pkg v1alpha1.ZarfPackage, packagePath, } importedComponent := found[0] - importPath, err := fetchOCISkeleton(ctx, component, pkgPath.BaseDir, cachePath, remoteOptions) + importPath, err := fetchOCISkeleton(ctx, component, pkgPath.BaseDir, cachePath, importRemote) if err != nil { return v1alpha1.ZarfPackage{}, nil, err } @@ -306,10 +307,13 @@ func compatibleComponent(c v1alpha1.ZarfComponent, arch, flavor string) bool { } // TODO (phillebaba): Refactor package structure so that pullOCI can be used instead. -func fetchOCISkeleton(ctx context.Context, component v1alpha1.ZarfComponent, packagePath string, cachePath string, remoteOptions types.RemoteOptions) (string, error) { +func fetchOCISkeleton(ctx context.Context, component v1alpha1.ZarfComponent, packagePath string, cachePath string, remote *zoci.Remote) (string, error) { if component.Import.URL == "" { return component.Import.Path, nil } + if remote == nil { + return "", fmt.Errorf("missing remote for OCI import %s", component.Import.URL) + } name := component.Name if component.Import.Name != "" { @@ -321,25 +325,6 @@ func fetchOCISkeleton(ctx context.Context, component v1alpha1.ZarfComponent, pac return "", err } - // Get the descriptor for the component. - plainHTTP, err := negotiateImportPlainHTTP(ctx, component.Import.URL, remoteOptions) - if err != nil { - return "", err - } - remote, err := zoci.NewRemote(ctx, component.Import.URL, zoci.PlatformForSkeleton(), - oci.WithPlainHTTP(plainHTTP), oci.WithInsecureSkipVerify(remoteOptions.InsecureSkipTLSVerify)) - if err != nil { - return "", err - } - _, err = remote.ResolveRoot(ctx) - if err != nil { - // This error likely won't occur as the root has been resolved before this function is invoked. - // This serves as a secondary mechanism to highlight the potential for the package existing without a published skeleton. - if strings.Contains(err.Error(), "no matching manifest was found in the manifest list") { - return "", fmt.Errorf("package at %s exists but has not been published as a skeleton: %w", component.Import.URL, err) - } - return "", fmt.Errorf("published skeleton package for %s does not exist: %w", component.Import.URL, err) - } manifest, err := remote.FetchRoot(ctx) if err != nil { return "", err diff --git a/src/pkg/packager/pull.go b/src/pkg/packager/pull.go index 02d36e3bfa..d67260a155 100644 --- a/src/pkg/packager/pull.go +++ b/src/pkg/packager/pull.go @@ -139,15 +139,11 @@ func pullOCI(ctx context.Context, opts pullOCIOptions) (*layout.PackageLayout, e if err != nil { return nil, err } - desc, err := remote.ResolveRoot(ctx) + desc, pkg, err := remote.FetchPackageMetadata(ctx) if err != nil { - return nil, fmt.Errorf("could not find package %s with architecture %s: %w", opts.Source, platform.Architecture, err) + return nil, fmt.Errorf("could not fetch metadata for package %s with architecture %s: %w", opts.Source, platform.Architecture, err) } isPartial := false - pkg, err := remote.FetchZarfYAML(ctx) - if err != nil { - return nil, err - } if supportsFiltering(desc.Platform) { pkg.Components, err = opts.Filter.Apply(pkg) if err != nil { diff --git a/src/pkg/zoci/common.go b/src/pkg/zoci/common.go index af95b31662..c2bb7fdb61 100644 --- a/src/pkg/zoci/common.go +++ b/src/pkg/zoci/common.go @@ -6,6 +6,7 @@ package zoci import ( "context" + "fmt" "path/filepath" "time" @@ -59,9 +60,11 @@ type PublishOptions struct { Tag string } -// Remote is a wrapper around the Oras remote repository with zarf specific functions +// Remote is a wrapper around the Oras remote repository with zarf specific functions. type Remote struct { *oci.OrasRemote + root *oci.Manifest + rootDescriptor ocispec.Descriptor } // NewRemote returns an oras remote repository client and context for the given url @@ -76,7 +79,39 @@ func NewRemote(ctx context.Context, url string, platform ocispec.Platform, mods if err != nil { return nil, err } - return &Remote{remote}, nil + return &Remote{OrasRemote: remote}, nil +} + +// FetchRoot fetches and caches the manifest for the remote reference. +func (r *Remote) FetchRoot(ctx context.Context) (*oci.Manifest, error) { + if r.root != nil { + return r.root, nil + } + descriptor, err := r.ResolveRoot(ctx) + if err != nil { + return nil, err + } + return r.FetchRootFromDescriptor(ctx, descriptor) +} + +// FetchRootFromDescriptor fetches and caches the manifest identified by descriptor. +// Subsequent methods on this Remote use the cached manifest, ensuring they all read +// from the same package even if its tag moves in the registry. +func (r *Remote) FetchRootFromDescriptor(ctx context.Context, descriptor ocispec.Descriptor) (*oci.Manifest, error) { + if r.root != nil { + if r.rootDescriptor.Digest != descriptor.Digest { + return nil, fmt.Errorf("remote root is already pinned to %s, cannot use %s", r.rootDescriptor.Digest, descriptor.Digest) + } + return r.root, nil + } + + root, err := r.FetchManifest(ctx, descriptor) + if err != nil { + return nil, err + } + r.root = root + r.rootDescriptor = descriptor + return root, nil } // GetOCICacheModifier takes in a Zarf cachePath and uses it to return an oci.WithCache modifier diff --git a/src/pkg/zoci/fetch.go b/src/pkg/zoci/fetch.go index b7d6f47ca6..906e4a9bad 100644 --- a/src/pkg/zoci/fetch.go +++ b/src/pkg/zoci/fetch.go @@ -15,12 +15,49 @@ import ( "github.com/zarf-dev/zarf/src/pkg/packager/layout" ) +// FetchPackageMetadata resolves the remote reference and fetches its parsed package +// definition from the same platform-selected OCI manifest. It does not download +// unrelated package layers. +func (r *Remote) FetchPackageMetadata(ctx context.Context) (ocispec.Descriptor, v1alpha1.ZarfPackage, error) { + if r.root != nil { + pkg, err := r.FetchZarfYAML(ctx) + if err != nil { + return ocispec.Descriptor{}, v1alpha1.ZarfPackage{}, err + } + return r.rootDescriptor, pkg, nil + } + + descriptor, err := r.ResolveRoot(ctx) + if err != nil { + return ocispec.Descriptor{}, v1alpha1.ZarfPackage{}, err + } + pkg, err := r.FetchZarfYAMLFromDescriptor(ctx, descriptor) + if err != nil { + return ocispec.Descriptor{}, v1alpha1.ZarfPackage{}, err + } + return descriptor, pkg, nil +} + // FetchZarfYAML fetches the zarf.yaml file from the remote repository. func (r *Remote) FetchZarfYAML(ctx context.Context) (v1alpha1.ZarfPackage, error) { manifest, err := r.FetchRoot(ctx) if err != nil { return v1alpha1.ZarfPackage{}, err } + return r.fetchZarfYAMLFromManifest(ctx, manifest) +} + +// FetchZarfYAMLFromDescriptor fetches the zarf.yaml file from the manifest identified by descriptor. +// It pins this Remote to the descriptor so subsequent reads use the same package. +func (r *Remote) FetchZarfYAMLFromDescriptor(ctx context.Context, descriptor ocispec.Descriptor) (v1alpha1.ZarfPackage, error) { + manifest, err := r.FetchRootFromDescriptor(ctx, descriptor) + if err != nil { + return v1alpha1.ZarfPackage{}, err + } + return r.fetchZarfYAMLFromManifest(ctx, manifest) +} + +func (r *Remote) fetchZarfYAMLFromManifest(ctx context.Context, manifest *oci.Manifest) (v1alpha1.ZarfPackage, error) { descriptor := manifest.Locate(layout.ZarfYAML) if oci.IsEmptyDescriptor(descriptor) { return v1alpha1.ZarfPackage{}, fmt.Errorf("unable to find %s in the manifest", layout.ZarfYAML) diff --git a/src/pkg/zoci/pull_test.go b/src/pkg/zoci/pull_test.go index 601f7c03bd..dad021e672 100644 --- a/src/pkg/zoci/pull_test.go +++ b/src/pkg/zoci/pull_test.go @@ -87,6 +87,50 @@ func TestAllLayersRespectsRequestedComponents(t *testing.T) { require.Len(t, allLayersSubset, 3) } +func TestFetchZarfYAMLFromDescriptor_PinsRemoteToResolvedPackage(t *testing.T) { + ctx := testutil.TestContext(t) + first := buildVirtualPackage(ctx, t) + second := buildVirtualPackage(ctx, t) + packageRegistry := testutil.SetupInMemoryRegistryDynamic(ctx, t) + + remote, firstComponents := publishPackage(ctx, t, first.packagePath, packageRegistry) + firstDescriptor, err := remote.ResolveRoot(ctx) + require.NoError(t, err) + + _, secondComponents := publishPackage(ctx, t, second.packagePath, packageRegistry) + secondDescriptor, err := remote.ResolveRoot(ctx) + require.NoError(t, err) + require.NotEqual(t, firstDescriptor.Digest, secondDescriptor.Digest) + require.NotEqual(t, firstComponents, secondComponents) + + metadataRemote, err := zoci.NewRemote(ctx, remote.Repo().Reference.String(), oci.PlatformForArch("amd64"), oci.WithPlainHTTP(true)) + require.NoError(t, err) + metadataDescriptor, metadataPkg, err := metadataRemote.FetchPackageMetadata(ctx) + require.NoError(t, err) + require.Equal(t, secondDescriptor.Digest, metadataDescriptor.Digest) + require.Equal(t, secondComponents, metadataPkg.Components) + + expectedRoot, err := remote.FetchManifest(ctx, firstDescriptor) + require.NoError(t, err) + + pkg, err := remote.FetchZarfYAMLFromDescriptor(ctx, firstDescriptor) + require.NoError(t, err) + require.Equal(t, firstComponents, pkg.Components) + + root, err := remote.FetchRoot(ctx) + require.NoError(t, err) + require.Equal(t, expectedRoot, root) + + layers, err := remote.AssembleLayers(ctx, pkg.Components, zoci.ComponentLayers) + require.NoError(t, err) + for _, layer := range layers { + require.Contains(t, expectedRoot.Layers, layer) + } + + _, err = remote.FetchRootFromDescriptor(ctx, secondDescriptor) + require.ErrorContains(t, err, "already pinned") +} + // writeVirtualPackageDef writes a minimal zarf package definition that references imageRef. func writeVirtualPackageDef(t *testing.T, imageRef string) string { t.Helper() diff --git a/src/pkg/zoci/utils.go b/src/pkg/zoci/utils.go index 3152b5c9b7..1d6c67e594 100644 --- a/src/pkg/zoci/utils.go +++ b/src/pkg/zoci/utils.go @@ -25,6 +25,26 @@ type ReferenceFromMetadataOptions struct { Tag string } +// ReferenceAtDigest returns source's registry and repository pinned to digest. +// Any tag in source is intentionally discarded. +func ReferenceAtDigest(source registry.Reference, digest string) (registry.Reference, error) { + ref := registry.Reference{ + Registry: source.Registry, + Repository: source.Repository, + Reference: digest, + } + if err := ref.ValidateRegistry(); err != nil { + return registry.Reference{}, fmt.Errorf("invalid digest-pinned reference: %w", err) + } + if err := ref.ValidateRepository(); err != nil { + return registry.Reference{}, fmt.Errorf("invalid digest-pinned reference: %w", err) + } + if err := ref.ValidateReferenceAsDigest(); err != nil { + return registry.Reference{}, fmt.Errorf("invalid digest-pinned reference: %w", err) + } + return ref, nil +} + // ReferenceFromMetadataWithOptions returns a reference for the given metadata with optional overrides func ReferenceFromMetadataWithOptions(registryLocation string, pkg v1alpha1.ZarfPackage, opts ReferenceFromMetadataOptions) (registry.Reference, error) { // Explicit requirement for version in order to publish diff --git a/src/pkg/zoci/utils_test.go b/src/pkg/zoci/utils_test.go new file mode 100644 index 0000000000..ed3f483a07 --- /dev/null +++ b/src/pkg/zoci/utils_test.go @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package zoci_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/zarf-dev/zarf/src/pkg/zoci" + "oras.land/oras-go/v2/registry" +) + +func TestReferenceAtDigest(t *testing.T) { + t.Parallel() + + source, err := registry.ParseReference("registry.example/zarf-packages/my-package:latest") + require.NoError(t, err) + + const digest = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + actual, err := zoci.ReferenceAtDigest(source, digest) + require.NoError(t, err) + require.Equal(t, source.Registry, actual.Registry) + require.Equal(t, source.Repository, actual.Repository) + require.Equal(t, digest, actual.Reference) + require.Equal(t, "registry.example/zarf-packages/my-package@"+digest, actual.String()) +} + +func TestReferenceAtDigest_InvalidReference(t *testing.T) { + t.Parallel() + + source, err := registry.ParseReference("registry.example/zarf-packages/my-package:latest") + require.NoError(t, err) + + _, err = zoci.ReferenceAtDigest(source, "not-a-digest") + require.ErrorContains(t, err, "invalid digest-pinned reference") +}