diff --git a/src/ImageBuilder.Tests/BuildCommandTests.cs b/src/ImageBuilder.Tests/BuildCommandTests.cs index 39bc25ae0..7a5c2d2ad 100644 --- a/src/ImageBuilder.Tests/BuildCommandTests.cs +++ b/src/ImageBuilder.Tests/BuildCommandTests.cs @@ -11,18 +11,14 @@ using Azure.ResourceManager.ContainerRegistry.Models; using FluentAssertions; using Microsoft.DotNet.ImageBuilder.Commands; -using Microsoft.DotNet.ImageBuilder.Configuration; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.Models.Manifest; using Microsoft.DotNet.ImageBuilder.Oras; using Microsoft.DotNet.ImageBuilder.Tests.Helpers; using Microsoft.DotNet.ImageBuilder.ViewModel; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; using Moq; using Newtonsoft.Json; using Shouldly; -using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ConfigurationHelper; using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ImageInfoHelper; using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ManifestHelper; using static Microsoft.DotNet.ImageBuilder.Tests.Helpers.ManifestServiceHelper; @@ -143,7 +139,7 @@ public async Task BuildCommand_ImageInfoOutput_Basic() "1.0/aspnet/os", tempFolderContext, $"{runtimeRepo}:{tag}"); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsDockerfileRelativePath)), It.IsAny())) .Returns(dockerfileCommitSha); @@ -346,7 +342,7 @@ public async Task BuildCommand_ImageInfoOutput_DuplicatedPlatform() "1.0/runtime/os", tempFolderContext, baseImageTag); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDockerfileRelativePath)), It.IsAny())) .Returns(dockerfileCommitSha); @@ -522,6 +518,7 @@ public async Task BuildCommand_Publish() TagInfo.GetFullyQualifiedName(repoName, sharedTag) }, It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -538,8 +535,125 @@ public async Task BuildCommand_Publish() } /// - /// Verifies that the manifest's platform architecture settings match the architecture of the base image. + /// Verifies that the OCI source/base image labels and the Dockerfile path label are applied to the built image. + /// + [TestMethod] + public async Task BuildCommand_AppliesOciLabels() + { + const string repoName = "runtime"; + const string tag = "tag"; + const string baseImageRepo = "baserepo"; + string baseImageTag = $"{baseImageRepo}:basetag"; + string baseImageDigest = $"{baseImageRepo}@sha256:baseImageDigestSha"; + const string sourceRepoUrl = "https://github.com/dotnet/test"; + const string commitSha = "c0ff33c0ff33c0ff33c0ff33c0ff33c0ff33c0ff"; + const string dockerfileRepoRootPath = "1.0/runtime/os/Dockerfile"; + + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + Mock dockerServiceMock = CreateDockerServiceMock(); + + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); + gitServiceMock + .Setup(o => o.GetCommitSha(It.IsAny(), true)) + .Returns(commitSha); + + BuildCommand command = CreateBuildCommand( + dockerService: dockerServiceMock.Object, + gitService: gitServiceMock.Object, + copyImageService: Mock.Of(), + manifestServiceFactory: CreateManifestServiceFactoryMock( + localImageDigestResults: [new(baseImageTag, baseImageDigest)]).Object, + imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); + command.Options.SourceRepoUrl = sourceRepoUrl; + + const string runtimeRelativeDir = "1.0/runtime/os"; + Directory.CreateDirectory(Path.Combine(tempFolderContext.Path, runtimeRelativeDir)); + string dockerfileRelativePath = Path.Combine(runtimeRelativeDir, "Dockerfile"); + string dockerfileAbsolutePath = PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, dockerfileRelativePath)); + File.WriteAllText(dockerfileAbsolutePath, $"FROM {baseImageTag}"); + + Platform platform = CreatePlatform(dockerfileRelativePath, new string[] { tag }); + + Manifest manifest = CreateManifest( + CreateRepo(repoName, + CreateImage( + new Platform[] { platform }))); + + File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); + + command.LoadManifest(); + await command.ExecuteAsync(); + + dockerServiceMock.Verify( + o => o.BuildImage( + dockerfileAbsolutePath, + PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeRelativeDir)), + "linux/amd64", + It.IsAny>(), + It.IsAny>(), + It.Is>(labels => + labels.Count == 5 && + labels[OciAnnotations.Source] == sourceRepoUrl && + labels[OciAnnotations.Revision] == commitSha && + labels[OciAnnotations.BaseName] == baseImageTag && + labels[OciAnnotations.BaseDigest] == "sha256:baseImageDigestSha" && + labels[ImageBuilderLabels.Dockerfile] == dockerfileRepoRootPath), + It.IsAny>(), + It.IsAny(), + It.IsAny())); + } + + /// + /// Verifies that all labels are omitted when their source data (source repo, base image) is unavailable. /// + [TestMethod] + public async Task BuildCommand_OmitsOciLabelsWhenDataUnavailable() + { + const string repoName = "runtime"; + const string tag = "tag"; + + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + Mock dockerServiceMock = CreateDockerServiceMock(); + + BuildCommand command = CreateBuildCommand( + dockerService: dockerServiceMock.Object, + copyImageService: Mock.Of(), + manifestServiceFactory: CreateManifestServiceFactoryMock().Object, + imageCacheService: new ImageCacheService(Mock.Of>(), Mock.Of())); + command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json"); + + const string runtimeRelativeDir = "1.0/runtime/os"; + Directory.CreateDirectory(Path.Combine(tempFolderContext.Path, runtimeRelativeDir)); + string dockerfileRelativePath = Path.Combine(runtimeRelativeDir, "Dockerfile"); + string dockerfileAbsolutePath = PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, dockerfileRelativePath)); + File.WriteAllText(dockerfileAbsolutePath, "FROM scratch"); + + Platform platform = CreatePlatform(dockerfileRelativePath, new string[] { tag }); + + Manifest manifest = CreateManifest( + CreateRepo(repoName, + CreateImage( + new Platform[] { platform }))); + + File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest)); + + command.LoadManifest(); + await command.ExecuteAsync(); + + dockerServiceMock.Verify( + o => o.BuildImage( + dockerfileAbsolutePath, + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny>(), + It.Is>(labels => labels.Count == 0), + It.IsAny>(), + It.IsAny(), + It.IsAny())); + } + [TestMethod] public async Task BuildCommand_VerifyOnBaseImageArchMismatch() { @@ -688,6 +802,7 @@ public async Task BuildCommand_BuildArgs() It.IsAny>(), It.Is>( args => args.Count == 3 && args["arg1"] == "val1" && args["arg2"] == "val2b" && args["arg3"] == "val3"), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -742,6 +857,7 @@ public async Task BuildCommand_DockerBuildOptions() It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.Is>(args => args.SequenceEqual(command.Options.DockerBuildOptions)), It.IsAny(), It.IsAny())); @@ -806,6 +922,7 @@ public async Task BuildCommand_NoBaseImage_Build() TagInfo.GetFullyQualifiedName(repoName, sharedTag) }, It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -856,7 +973,7 @@ public async Task BuildCommand_NoBaseImage_Cached() string runtimeDepsLinuxDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/linux", tempFolderContext, "scratch"); - Mock gitServiceMock = new(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -1008,7 +1125,7 @@ public async Task BuildCommand_NoBaseImage_Cached() o.BuildImage( PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), - It.IsAny>(), It.IsAny(), It.IsAny()), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); dockerServiceMock.Verify( o => o.GetImageSize(It.IsAny(), false), @@ -1088,7 +1205,7 @@ public async Task BuildCommand_ImageInfoOutput_CustomDockerfile() File.WriteAllText(Path.Combine(tempFolderContext.Path, dockerfileRelativePath), "FROM repo:tag"); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(dockerfileRelativePath, It.IsAny())) .Returns(dockerfileCommitSha); @@ -1339,7 +1456,7 @@ public async Task BuildCommand_Caching( string runtimeDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime/os", tempFolderContext, $"$REPO:{tag}"); - Mock gitServiceMock = new(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -1549,7 +1666,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn string runtimeDepsWindowsDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/windows", tempFolderContext, windowsBaseImageTag); - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -1779,7 +1896,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn dockerServiceMock.Verify(o => o.BuildImage( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), - It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), + It.IsAny>(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); dockerServiceMock.VerifyNoOtherCalls(); @@ -1863,7 +1980,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn string runtimeDepsDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/os", tempFolderContext, baseImageTag); - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -2087,6 +2204,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_MissingSourceImageInfoEn It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -2151,7 +2269,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_NoExistingImageInfoEntri string runtimeDepsDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/os", tempFolderContext, baseImageTag); - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -2294,6 +2412,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_NoExistingImageInfoEntri It.IsAny(), new string[] { expectedTag }, It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), @@ -2359,7 +2478,7 @@ public async Task BuildCommand_SharedDockerfile() string runtimeDepsDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/os", tempFolderContext, baseImageTag); - Mock gitServiceMock = new(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -2538,6 +2657,7 @@ public async Task BuildCommand_SharedDockerfile() It.IsAny(), new string[] { expectedTag }, It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), @@ -2602,7 +2722,7 @@ public async Task BuildCommand_Caching_TagUpdate() string runtimeDepsLinuxDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/linux", tempFolderContext, baseImageTag); - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -2756,7 +2876,7 @@ public async Task BuildCommand_Caching_TagUpdate() o.BuildImage( PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), - It.IsAny>(), It.IsAny(), It.IsAny()), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); dockerServiceMock.Verify( o => o.GetImageSize(It.IsAny(), false), @@ -2824,7 +2944,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_TagUpdate() string runtimeDepsLinuxDockerfileRelativePath = DockerfileHelper.CreateDockerfile( "1.0/runtime-deps/linux", tempFolderContext, baseImageTag); - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(PathHelper.NormalizePath(Path.Combine(tempFolderContext.Path, runtimeDepsLinuxDockerfileRelativePath)), It.IsAny())) .Returns(currentRuntimeDepsCommitSha); @@ -3015,7 +3135,7 @@ public async Task BuildCommand_Caching_SharedDockerfile_TagUpdate() dockerServiceMock.Verify(o => o.BuildImage( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), - It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), + It.IsAny>(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny()), Times.Never); dockerServiceMock.Verify(o => o.GetCreatedDate(It.IsAny(), false)); @@ -3131,7 +3251,7 @@ public async Task BuildCommand_MirroredImages(bool hasCachedImage, string srcBas "1.0/aspnet/os", tempFolderContext, $"$REPO:{Tag}"); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(It.IsAny(), It.IsAny())) .Returns(dockerfileCommitSha); @@ -3375,6 +3495,7 @@ public async Task BuildCommand_MirroredImages(bool hasCachedImage, string srcBas It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -3476,7 +3597,7 @@ public async Task BuildCommand_MirroredImages_External(string baseImageRegistry, "1.0/samples/os", tempFolderContext, $"{baseImageRegistry}/{RuntimeRepo}:{Tag}"); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new Mock(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(It.IsAny(), It.IsAny())) .Returns(dockerfileCommitSha); @@ -3521,6 +3642,7 @@ public async Task BuildCommand_MirroredImages_External(string baseImageRegistry, It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -3572,7 +3694,7 @@ public async Task BuildCommand_MirroredImages_BaseImageTagOverride() ], []); const string dockerfileCommitSha = "mycommit"; - Mock gitServiceMock = new(); + Mock gitServiceMock = CreateGitServiceMock(tempFolderContext); gitServiceMock .Setup(o => o.GetCommitSha(It.IsAny(), It.IsAny())) .Returns(dockerfileCommitSha); @@ -3663,6 +3785,7 @@ public async Task BuildCommand_MirroredImages_BaseImageTagOverride() It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())); @@ -3735,6 +3858,7 @@ private static Mock CreateDockerServiceMock(string buildOutput = It.IsAny(), It.IsAny>(), It.IsAny>(), + It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny())) @@ -3747,6 +3871,15 @@ private static Mock CreateDockerServiceMock(string buildOutput = return dockerServiceMock; } + private static Mock CreateGitServiceMock(TempFolderContext tempFolderContext) + { + Mock mock = new(); + mock + .Setup(o => o.GetRepoRoot(It.IsAny())) + .Returns(tempFolderContext.Path); + return mock; + } + private static void VerifyImportImage(Mock copyImageServiceMock, BuildCommand command, string[] destTagNames, string srcTagName, string destRegistryName, string srcRegistryName) { diff --git a/src/ImageBuilder/Commands/Build/ImageArtifactDetailsExtensions.cs b/src/ImageBuilder/Commands/Build/ImageArtifactDetailsExtensions.cs new file mode 100644 index 000000000..f347d6fc5 --- /dev/null +++ b/src/ImageBuilder/Commands/Build/ImageArtifactDetailsExtensions.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.DotNet.ImageBuilder.Models.Image; + +namespace Microsoft.DotNet.ImageBuilder.Commands.Build; + +/// +/// Extension methods for image artifact details used during build command processing. +/// +internal static class ImageArtifactDetailsExtensions +{ + /// + /// Enumerates all platform data entries from image-info repo and image groups. + /// + /// The image artifact details to enumerate. + /// All platform data entries contained in the image artifact details. + internal static IEnumerable EnumeratePlatforms(this ImageArtifactDetails imageArtifactDetails) => + imageArtifactDetails.Repos + .Where(repoData => repoData.Images != null) + .SelectMany(repoData => repoData.Images) + .SelectMany(imageData => imageData.Platforms); +} diff --git a/src/ImageBuilder/Commands/BuildCommand.cs b/src/ImageBuilder/Commands/BuildCommand.cs index 8c1dd37cb..5bd40a289 100644 --- a/src/ImageBuilder/Commands/BuildCommand.cs +++ b/src/ImageBuilder/Commands/BuildCommand.cs @@ -7,9 +7,8 @@ using System.IO; using System.Linq; using System.Text.RegularExpressions; -using System.Threading; using System.Threading.Tasks; -using Azure.Core; +using Microsoft.DotNet.ImageBuilder.Commands.Build; using Microsoft.DotNet.ImageBuilder.Models.Image; using Microsoft.DotNet.ImageBuilder.ViewModel; @@ -27,20 +26,9 @@ public class BuildCommand : ManifestCommand private readonly IAzureTokenCredentialProvider _tokenCredentialProvider; private readonly IImageCacheService _imageCacheService; private readonly IImageInfoService _imageInfoService; - private readonly ImageDigestCache _imageDigestCache; - private readonly List _processedTags = new List(); - private readonly HashSet _builtPlatforms = new(); private readonly Lazy _imageNameResolver; private readonly Lazy _storageAccountToken; - /// - /// Maps a source digest from the image info file to the corresponding digest in the copied location for image caching. - /// This is specifically needed to support shared Dockerfile scenarios. - /// - private readonly Dictionary _sourceDigestCopyLocationMapping = new(); - - private ImageArtifactDetails? _imageArtifactDetails; - public BuildCommand( IManifestJsonService manifestJsonService, IDockerService dockerService, @@ -68,7 +56,6 @@ public BuildCommand( ArgumentNullException.ThrowIfNull(manifestServiceFactory); _manifestService = new Lazy(() => manifestServiceFactory.Create(Options.CredentialsOptions)); - _imageDigestCache = new ImageDigestCache(_manifestService); _imageNameResolver = new Lazy(() => new ImageNameResolverForBuild( @@ -98,131 +85,282 @@ public override async Task ExecuteAsync() { Options.BaseImageOverrideOptions.Validate(); - if (Options.ImageInfoOutputPath != null) + bool isImageInfoOutputEnabled = !string.IsNullOrEmpty(Options.ImageInfoOutputPath); + if (isImageInfoOutputEnabled && string.IsNullOrEmpty(Options.SourceRepoUrl)) { - _imageArtifactDetails = new ImageArtifactDetails(); + throw new InvalidOperationException("Source repo URL must be provided when outputting to an image info file."); } - await ExecuteWithDockerCredentialsAsync(PullBaseImagesAsync); - await BuildImagesAsync(); + ImageDigestCache imageDigestCache = new ImageDigestCache(_manifestService); - if (_processedTags.Count > 0 || _imageCacheService.HasAnyCachedPlatforms) - { - // Log in again to refresh token as it may have expired from a long build - await ExecuteWithDockerCredentialsAsync(async () => - { - PushImages(); - await PublishImageInfoAsync(); - }); - } + await ExecuteWithDockerCredentialsAsync(() => PullBaseImagesAsync(imageDigestCache)); - WriteBuildSummary(); - WriteBuiltImagesToOutputVar(); - } + _logger.LogInformation("BUILDING IMAGES"); - private async Task ExecuteWithDockerCredentialsAsync(Func action) - { - await _registryCredentialsProvider.ExecuteWithCredentialsAsync( - isDryRun: Options.IsDryRun, - action: action, - credentialsOptions: Options.CredentialsOptions, - registryName: Manifest.Registry); - } + List builtTags = []; + HashSet builtTagNames = []; + HashSet builtPlatforms = []; - private async Task ExecuteWithDockerCredentialsAsync(Action action) - { - await _registryCredentialsProvider.ExecuteWithCredentialsAsync( - isDryRun: Options.IsDryRun, - action: action, - credentialsOptions: Options.CredentialsOptions, - registryName: Manifest.Registry); - } + // Maps source image-info digests to copied locations so shared Dockerfile cache hits + // can resolve per-repo digests. + Dictionary sourceDigestCopyLocationMapping = []; + Dictionary platformDataByTag = []; + List platformsWithNoPushTags = []; + ImageArtifactDetails? imageArtifactDetails = + isImageInfoOutputEnabled ? new ImageArtifactDetails() : null; - private void WriteBuiltImagesToOutputVar() - { - if (!string.IsNullOrEmpty(Options.OutputVariableName)) + ImageArtifactDetails? srcImageArtifactDetails = null; + if (!string.IsNullOrWhiteSpace(Options.ImageInfoSourcePath)) { - IEnumerable builtDigests = _builtPlatforms - .Select(platform => DockerHelper.GetDigestString(platform.PlatformInfo!.RepoName, DockerHelper.GetDigestSha(platform.Digest))) - .Distinct(); - _logger.LogInformation( - PipelineHelper.FormatOutputVariable( - Options.OutputVariableName, - string.Join(',', builtDigests))); + srcImageArtifactDetails = ImageInfoHelper.LoadFromFile( + Options.ImageInfoSourcePath, + Manifest, + skipManifestValidation: true); } - } - - private async Task PublishImageInfoAsync() - { - if (string.IsNullOrEmpty(Options.ImageInfoOutputPath)) + else if (Manifest.Model.ImageInfo is not null) { - return; - } + string imageInfoContent = await _imageInfoService.PullImageInfoArtifactAsync( + Manifest, + string.IsNullOrWhiteSpace(Options.RegistryOverride) ? Manifest.Model.Registry : Options.RegistryOverride, + Options.RepoPrefix); - if (string.IsNullOrEmpty(Options.SourceRepoUrl)) - { - throw new InvalidOperationException("Source repo URL must be provided when outputting to an image info file."); + srcImageArtifactDetails = ImageInfoHelper.LoadFromContent( + imageInfoContent, + Manifest, + skipManifestValidation: true); } - Dictionary platformDataByTag = new Dictionary(); - foreach (PlatformData platformData in GetProcessedPlatforms()) + foreach (RepoInfo repoInfo in Manifest.FilteredRepos) { - if (platformData.PlatformInfo is not null) + RepoData repoData = CreateRepoData(repoInfo); + RepoData? srcRepoData = srcImageArtifactDetails?.Repos.FirstOrDefault(srcRepo => srcRepo.Repo == repoInfo.Name); + + foreach (ImageInfo image in repoInfo.FilteredImages) { - foreach (TagInfo tag in platformData.PlatformInfo.Tags) + ImageData imageData = CreateImageData(image); + repoData.Images.Add(imageData); + + ImageData? srcImageData = srcRepoData?.Images.FirstOrDefault(srcImage => srcImage.ManifestImage == image); + + foreach (PlatformInfo platform in image.FilteredPlatforms) { - platformDataByTag.Add(tag.FullyQualifiedName, platformData); - } - } - } + // Tag the built images with the shared tags as well as the platform tags. + // Some tests and image FROM instructions depend on these tags. - IEnumerable processedPlatforms = GetProcessedPlatforms(); - List platformsWithNoPushTags = new List(); + List allTagInfos = platform.Tags + .Concat(image.SharedTags) + .ToList(); - foreach (PlatformData platform in processedPlatforms) - { - IEnumerable pushTags = platform.PlatformInfo?.Tags ?? []; + List allTags = allTagInfos + .Select(tag => tag.FullyQualifiedName) + .ToList(); - foreach (TagInfo tag in pushTags) - { - if (Options.IsPushEnabled) - { - await SetPlatformDataDigestAsync(platform, tag.FullyQualifiedName); - SetPlatformDataBaseDigest(platform, platformDataByTag); - await SetPlatformDataLayersAsync(platform, tag.FullyQualifiedName); + List concreteTags = platform.Tags.ToList(); + PlatformData platformData = CreatePlatformData(image, platform); + imageData.Platforms.Add(platformData); + + if (platformData.PlatformInfo is not null) + { + foreach (TagInfo tag in platformData.PlatformInfo.Tags) + { + platformDataByTag.Add(tag.FullyQualifiedName, platformData); + } + } + + bool isCachedImage = false; + bool shouldCheckCache = !Options.NoCache; + if (shouldCheckCache && platform.FinalStageFromImage is not null) + { + string finalStageLocalTag = + _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage); + shouldCheckCache = !builtTagNames.Contains(finalStageLocalTag); + } + + if (shouldCheckCache) + { + ImageCacheResult cacheResult = await _imageCacheService.CheckForCachedImageAsync( + srcImageData, + platformData, + imageDigestCache, + _imageNameResolver.Value, + sourceRepoUrl: Options.SourceRepoUrl, + isLocalBaseImageExpected: true, + isDryRun: Options.IsDryRun); + + if (cacheResult.State.HasFlag(ImageCacheState.Cached)) + { + isCachedImage = true; + + CopyPlatformDataFromCachedPlatform(platformData, cacheResult.Platform!); + platformData.IsUnchanged = cacheResult.State != ImageCacheState.CachedWithMissingTags; + + await OnCacheHitAsync( + repoInfo, + allTagInfos, + pullImage: cacheResult.IsNewCacheHit, + sourceDigest: cacheResult.Platform!.Digest, + imageDigestCache, + sourceDigestCopyLocationMapping); + } + } + + Dictionary pushedDigestByTag = []; + if (!isCachedImage) + { + builtTags.AddRange(allTagInfos); + builtTagNames.UnionWith(allTags); + + Dictionary labels = []; + if (!string.IsNullOrEmpty(Options.SourceRepoUrl)) + { + labels[OciAnnotations.Source] = Options.SourceRepoUrl; + labels[OciAnnotations.Revision] = _gitService.GetCommitSha(platform.DockerfilePath, useFullHash: true); + + // Record which Dockerfile the image was built from, relative to the repo root. + // The repo root must be discovered via Git rather than assuming it's the manifest's directory. + string repoRoot = _gitService.GetRepoRoot(platform.DockerfilePath); + labels[ImageBuilderLabels.Dockerfile] = + PathHelper.NormalizePath(Path.GetRelativePath(repoRoot, platform.DockerfilePath)); + } + + if (platform.FinalStageFromImage is not null) + { + labels[OciAnnotations.BaseName] = + _imageNameResolver.Value.GetFromImagePublicTag(platform.FinalStageFromImage); + + string? baseImageDigest = await imageDigestCache.GetLocalImageDigestAsync( + _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage), Options.IsDryRun); + if (!string.IsNullOrEmpty(baseImageDigest)) + { + labels[OciAnnotations.BaseDigest] = DockerHelper.GetDigestSha(baseImageDigest); + } + } + + BuildImage(platform, allTags, labels); + builtPlatforms.Add(platformData); + + if (Options.IsPushEnabled) + { + IEnumerable tagsForDigest = imageArtifactDetails is null ? [] : concreteTags; + + // Log in again to refresh token as it may have expired from a long build + await ExecuteWithDockerCredentialsAsync( + async () => + { + pushedDigestByTag = await PushTagsAsync(allTagInfos, tagsForDigest, imageDigestCache); + }); + + if (platform.FinalStageFromImage is not null) + { + platformData.BaseImageDigest = + await imageDigestCache.GetLocalImageDigestAsync( + _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage), Options.IsDryRun); + } + } + } + + if (imageArtifactDetails is not null) + { + // Multiple concrete tags for the same platform should all resolve to the same + // digest and created date; validate each tag before preserving the shared values. + foreach (TagInfo tag in concreteTags) + { + if (Options.IsPushEnabled) + { + string? digest = + isCachedImage + ? await imageDigestCache.GetLocalImageDigestAsync(tag.FullyQualifiedName, Options.IsDryRun) + : pushedDigestByTag[tag.FullyQualifiedName]; + + SetPlatformDataDigest(platformData, tag.FullyQualifiedName, digest); + SetPlatformDataBaseDigest(platformData, platformDataByTag); + await SetPlatformDataLayersAsync(platformData, tag.FullyQualifiedName); + } + + SetPlatformDataCreatedDate(platformData, tag.FullyQualifiedName); + } + + if (!concreteTags.Any()) + { + platformsWithNoPushTags.Add(platformData); + } + + if (!string.IsNullOrEmpty(Options.SourceRepoUrl)) + { + platformData.CommitUrl = _gitService.GetDockerfileCommitUrl(platformData.PlatformInfo, Options.SourceRepoUrl); + } + } } + } - SetPlatformDataCreatedDate(platform, tag.FullyQualifiedName); + if (repoData.Images.Any()) + { + imageArtifactDetails?.Repos.Add(repoData); } + } + + if ((builtTags.Count > 0 || _imageCacheService.HasAnyCachedPlatforms) + && !string.IsNullOrEmpty(Options.ImageInfoOutputPath) + && imageArtifactDetails is not null) + { + List allPlatforms = imageArtifactDetails.EnumeratePlatforms().ToList(); - if (!pushTags.Any()) + // Some platforms do not have concrete tags. In such cases, they must be duplicates of a platform in a different + // image which does have a concrete tag. For these platforms that do not have concrete tags, we are unable to + // lookup digest/created info based on their tag. Instead, we find the matching platform which does have that info + // set (as a result of having a concrete tag) and copy its values. + foreach (PlatformData platform in platformsWithNoPushTags) { - platformsWithNoPushTags.Add(platform); + PlatformData matchingBuiltPlatform = allPlatforms.First(builtPlatform => + (builtPlatform.PlatformInfo?.Tags ?? []).Any() && + platform.ImageInfo is not null && + platform.PlatformInfo is not null && + builtPlatform.ImageInfo is not null && + builtPlatform.PlatformInfo is not null && + PlatformInfo.AreMatchingPlatforms(platform.ImageInfo, platform.PlatformInfo, builtPlatform.ImageInfo, builtPlatform.PlatformInfo)); + + platform.Digest = matchingBuiltPlatform.Digest; + platform.Created = matchingBuiltPlatform.Created; } - platform.CommitUrl = _gitService.GetDockerfileCommitUrl(platform.PlatformInfo, Options.SourceRepoUrl); + string imageInfoContent = JsonHelper.SerializeObject(imageArtifactDetails); + File.WriteAllText(Options.ImageInfoOutputPath, imageInfoContent); } - // Some platforms do not have concrete tags. In such cases, they must be duplicates of a platform in a different - // image which does have a concrete tag. For these platforms that do not have concrete tags, we are unable to - // lookup digest/created info based on their tag. Instead, we find the matching platform which does have that info - // set (as a result of having a concrete tag) and copy its values. - foreach (PlatformData platform in platformsWithNoPushTags) + WriteBuildSummary(builtTags); + if (!string.IsNullOrEmpty(Options.OutputVariableName)) { - PlatformData matchingBuiltPlatform = processedPlatforms.First(builtPlatform => - (builtPlatform.PlatformInfo?.Tags ?? []).Any() && - platform.ImageInfo is not null && - platform.PlatformInfo is not null && - builtPlatform.ImageInfo is not null && - builtPlatform.PlatformInfo is not null && - PlatformInfo.AreMatchingPlatforms(platform.ImageInfo, platform.PlatformInfo, builtPlatform.ImageInfo, builtPlatform.PlatformInfo)); - - platform.Digest = matchingBuiltPlatform.Digest; - platform.Created = matchingBuiltPlatform.Created; + WriteBuiltImagesToOutputVar(Options.OutputVariableName, builtPlatforms); } + } + + private async Task ExecuteWithDockerCredentialsAsync(Func action) + { + await _registryCredentialsProvider.ExecuteWithCredentialsAsync( + isDryRun: Options.IsDryRun, + action: action, + credentialsOptions: Options.CredentialsOptions, + registryName: Manifest.Registry); + } + + private async Task ExecuteWithDockerCredentialsAsync(Action action) + { + await _registryCredentialsProvider.ExecuteWithCredentialsAsync( + isDryRun: Options.IsDryRun, + action: action, + credentialsOptions: Options.CredentialsOptions, + registryName: Manifest.Registry); + } - string imageInfoString = JsonHelper.SerializeObject(_imageArtifactDetails); - File.WriteAllText(Options.ImageInfoOutputPath, imageInfoString); + private void WriteBuiltImagesToOutputVar(string outputVariableName, IEnumerable builtPlatforms) + { + IEnumerable builtDigests = builtPlatforms + .Select(platform => DockerHelper.GetDigestString(platform.PlatformInfo!.RepoName, DockerHelper.GetDigestSha(platform.Digest))) + .Distinct(); + _logger.LogInformation( + PipelineHelper.FormatOutputVariable( + outputVariableName, + string.Join(',', builtDigests))); } private void SetPlatformDataCreatedDate(PlatformData platform, string tag) @@ -276,10 +414,9 @@ private async Task SetPlatformDataLayersAsync(PlatformData platform, string tag) } } - private async Task SetPlatformDataDigestAsync(PlatformData platform, string tag) + private void SetPlatformDataDigest(PlatformData platform, string tag, string? digest) { // The digest of an image that is pushed to ACR is guaranteed to be the same when transferred to MCR. - string? digest = await _imageDigestCache.GetLocalImageDigestAsync(tag, Options.IsDryRun); if (digest is not null && platform.PlatformInfo is not null) { digest = DockerHelper.GetDigestString(platform.PlatformInfo.FullRepoModelName, DockerHelper.GetDigestSha(digest)); @@ -303,106 +440,6 @@ private async Task SetPlatformDataDigestAsync(PlatformData platform, string tag) platform.Digest = digest; } - private async Task BuildImagesAsync() - { - _logger.LogInformation("BUILDING IMAGES"); - - ImageArtifactDetails? srcImageArtifactDetails = null; - if (!string.IsNullOrWhiteSpace(Options.ImageInfoSourcePath)) - { - srcImageArtifactDetails = ImageInfoHelper.LoadFromFile( - Options.ImageInfoSourcePath, - Manifest, - skipManifestValidation: true); - } - else if (Manifest.Model.ImageInfo is not null) - { - string imageInfoContent = await _imageInfoService.PullImageInfoArtifactAsync( - Manifest, - string.IsNullOrWhiteSpace(Options.RegistryOverride) ? Manifest.Model.Registry : Options.RegistryOverride, - Options.RepoPrefix); - - srcImageArtifactDetails = ImageInfoHelper.LoadFromContent( - imageInfoContent, - Manifest, - skipManifestValidation: true); - } - - foreach (RepoInfo repoInfo in Manifest.FilteredRepos) - { - RepoData repoData = CreateRepoData(repoInfo); - RepoData? srcRepoData = srcImageArtifactDetails?.Repos.FirstOrDefault(srcRepo => srcRepo.Repo == repoInfo.Name); - - foreach (ImageInfo image in repoInfo.FilteredImages) - { - ImageData imageData = CreateImageData(image); - repoData.Images.Add(imageData); - - ImageData? srcImageData = srcRepoData?.Images.FirstOrDefault(srcImage => srcImage.ManifestImage == image); - - foreach (PlatformInfo platform in image.FilteredPlatforms) - { - // Tag the built images with the shared tags as well as the platform tags. - // Some tests and image FROM instructions depend on these tags. - - IEnumerable allTagInfos = platform.Tags - .Concat(image.SharedTags) - .ToList(); - - IEnumerable allTags = allTagInfos - .Select(tag => tag.FullyQualifiedName) - .ToList(); - - PlatformData platformData = CreatePlatformData(image, platform); - imageData.Platforms.Add(platformData); - - bool isCachedImage = false; - if (!Options.NoCache) - { - ImageCacheResult cacheResult = await _imageCacheService.CheckForCachedImageAsync( - srcImageData, - platformData, - _imageDigestCache, - _imageNameResolver.Value, - sourceRepoUrl: Options.SourceRepoUrl, - isLocalBaseImageExpected: true, - isDryRun: Options.IsDryRun); - - if (cacheResult.State.HasFlag(ImageCacheState.Cached)) - { - isCachedImage = true; - - CopyPlatformDataFromCachedPlatform(platformData, cacheResult.Platform!); - platformData.IsUnchanged = cacheResult.State != ImageCacheState.CachedWithMissingTags; - - await OnCacheHitAsync(repoInfo, allTagInfos, pullImage: cacheResult.IsNewCacheHit, cacheResult.Platform!.Digest); - } - } - - if (!isCachedImage) - { - _processedTags.AddRange(allTagInfos); - - BuildImage(platform, allTags); - _builtPlatforms.Add(platformData); - - if (Options.IsPushEnabled && platform.FinalStageFromImage is not null) - { - platformData.BaseImageDigest = - await _imageDigestCache.GetLocalImageDigestAsync( - _imageNameResolver.Value.GetFromImageLocalTag(platform.FinalStageFromImage), Options.IsDryRun); - } - } - } - } - - if (repoData?.Images.Any() == true) - { - _imageArtifactDetails?.Repos.Add(repoData); - } - } - } - private void CopyPlatformDataFromCachedPlatform(PlatformData dstPlatform, PlatformData srcPlatform) { // When a cache hit occurs for a Dockerfile, we want to transfer some of the metadata about the previously @@ -497,7 +534,7 @@ private void ValidatePlatformIsCompatibleWithBaseImage(PlatformInfo platform) } } - private void BuildImage(PlatformInfo platform, IEnumerable allTags) + private void BuildImage(PlatformInfo platform, IEnumerable allTags, IDictionary labels) { ValidatePlatformIsCompatibleWithBaseImage(platform); @@ -511,6 +548,7 @@ private void BuildImage(PlatformInfo platform, IEnumerable allTags) platform.PlatformLabel, allTags, GetBuildArgs(platform), + labels, GetDockerBuildOptions(), Options.IsRetryEnabled, Options.IsDryRun); @@ -576,7 +614,13 @@ private void BuildImage(PlatformInfo platform, IEnumerable allTags) private IEnumerable GetDockerBuildOptions() => Options.DockerBuildOptions.Where(option => !string.IsNullOrWhiteSpace(option)); - private async Task OnCacheHitAsync(RepoInfo repo, IEnumerable allTags, bool pullImage, string sourceDigest) + private async Task OnCacheHitAsync( + RepoInfo repo, + IEnumerable allTags, + bool pullImage, + string sourceDigest, + ImageDigestCache imageDigestCache, + Dictionary sourceDigestCopyLocationMapping) { _logger.LogInformation(string.Empty); _logger.LogInformation("CACHE HIT"); @@ -602,14 +646,14 @@ await ExecuteWithDockerCredentialsAsync(() => { // Don't need to provide the platform because we're pulling by digest. No need to worry about multi-arch tags. _dockerService.PullImage(copiedSourceDigest, null, Options.IsDryRun); - _sourceDigestCopyLocationMapping[sourceDigest] = copiedSourceDigest; + sourceDigestCopyLocationMapping[sourceDigest] = copiedSourceDigest; }); } // Tag the image as if it were locally built so that subsequent built images can reference it foreach (TagInfo tag in allTags) { - if (!_sourceDigestCopyLocationMapping.TryGetValue(sourceDigest, out string? resolvedSourceDigest)) + if (!sourceDigestCopyLocationMapping.TryGetValue(sourceDigest, out string? resolvedSourceDigest)) { throw new InvalidOperationException("Digest should be mapped by this point"); } @@ -626,7 +670,7 @@ await ExecuteWithDockerCredentialsAsync(() => // Populate the digest cache with the known digest value for the tags assigned to the image. // This is needed in order to prevent a call to the manifest tool to get the digest for these tags // because they haven't yet been pushed to staging by that time. - _imageDigestCache.AddDigest(tag.FullyQualifiedName, newDigest); + imageDigestCache.AddDigest(tag.FullyQualifiedName, newDigest); } } @@ -653,7 +697,7 @@ await _copyImageService.ImportImageAsync( return sourceDigest; } - private async Task PullBaseImagesAsync() + private async Task PullBaseImagesAsync(ImageDigestCache imageDigestCache) { _logger.LogInformation("PULLING LATEST BASE IMAGES"); @@ -712,7 +756,7 @@ await Parallel.ForEachAsync(finalStageExternalFromImages, async (fromImage, canc // the DockerServiceCache for later use. The longer we wait to get the digest after pulling, the // greater chance the tag could be updated resulting in a different digest returned than what was // originally pulled. - await _imageDigestCache.GetLocalImageDigestAsync(fromImage, Options.IsDryRun); + await imageDigestCache.GetLocalImageDigestAsync(fromImage, Options.IsDryRun); }); // Tag the images that were pulled from the mirror as they are referenced in the Dockerfiles @@ -726,23 +770,40 @@ await Parallel.ForEachAsync(finalStageExternalFromImages, async (fromImage, canc }); } - private IEnumerable GetProcessedPlatforms() => _imageArtifactDetails?.Repos - .Where(repoData => repoData.Images != null) - .SelectMany(repoData => repoData.Images) - .SelectMany(imageData => imageData.Platforms) - ?? Enumerable.Empty(); - - private void PushImages() + private async Task> PushTagsAsync( + IEnumerable tagsToPush, + IEnumerable tagsForDigest, + ImageDigestCache imageDigestCache) { - if (Options.IsPushEnabled) + _logger.LogInformation("PUSHING BUILT IMAGES"); + + HashSet digestTagNames = tagsForDigest + .Select(tag => tag.FullyQualifiedName) + .ToHashSet(); + Dictionary pushedDigestByTag = []; + + foreach (TagInfo tag in tagsToPush) { - _logger.LogInformation("PUSHING BUILT IMAGES"); + _dockerService.PushImage(tag.FullyQualifiedName, Options.IsDryRun); - foreach (TagInfo tag in _processedTags) + if (digestTagNames.Contains(tag.FullyQualifiedName)) { - _dockerService.PushImage(tag.FullyQualifiedName, Options.IsDryRun); + string? digest = null; + for (int attempt = 0; attempt <= RetryHelper.MaxRetries && digest is null; attempt++) + { + digest = await imageDigestCache.GetLocalImageDigestAsync(tag.FullyQualifiedName, Options.IsDryRun); + } + + if (digest is null) + { + throw new InvalidOperationException($"Unable to retrieve digest for pushed tag '{tag.FullyQualifiedName}'."); + } + + pushedDigestByTag.Add(tag.FullyQualifiedName, digest); } } + + return pushedDigestByTag; } private bool UpdateDockerfileFromCommands(PlatformInfo platform, out string dockerfilePath) @@ -779,13 +840,13 @@ private bool UpdateDockerfileFromCommands(PlatformInfo platform, out string dock return updateDockerfile; } - private void WriteBuildSummary() + private void WriteBuildSummary(IReadOnlyCollection builtTags) { _logger.LogInformation("IMAGES BUILT"); - if (_processedTags.Any()) + if (builtTags.Any()) { - foreach (TagInfo tag in _processedTags) + foreach (TagInfo tag in builtTags) { _logger.LogInformation(tag.FullyQualifiedName); } @@ -797,5 +858,6 @@ private void WriteBuildSummary() _logger.LogInformation(string.Empty); } + } } diff --git a/src/ImageBuilder/DockerService.cs b/src/ImageBuilder/DockerService.cs index d24dbe9a3..250344449 100644 --- a/src/ImageBuilder/DockerService.cs +++ b/src/ImageBuilder/DockerService.cs @@ -34,6 +34,7 @@ public void CreateManifestList(string manifestListTag, IEnumerable image string platform, IEnumerable tags, IDictionary buildArgs, + IDictionary labels, IEnumerable dockerBuildOptions, bool isRetryEnabled, bool isDryRun) @@ -44,12 +45,16 @@ public void CreateManifestList(string manifestListTag, IEnumerable image .Select(buildArg => $" --build-arg {buildArg.Key}={buildArg.Value}"); string buildArgsString = string.Join(string.Empty, buildArgList); + IEnumerable labelList = labels + .Select(label => $" --label {label.Key}={label.Value}"); + string labelsString = string.Join(string.Empty, labelList); + IEnumerable dockerBuildOptionList = dockerBuildOptions .Where(option => !string.IsNullOrWhiteSpace(option)) .Select(option => $" {option}"); string dockerBuildOptionsString = string.Join(string.Empty, dockerBuildOptionList); - string dockerArgs = $"build --platform {platform} {tagArgs} -f {dockerfilePath}{buildArgsString}{dockerBuildOptionsString} {buildContextPath}"; + string dockerArgs = $"build --platform {platform} {tagArgs} -f {dockerfilePath}{buildArgsString}{labelsString}{dockerBuildOptionsString} {buildContextPath}"; if (isRetryEnabled) { diff --git a/src/ImageBuilder/DockerServiceCache.cs b/src/ImageBuilder/DockerServiceCache.cs index 7878c025b..5ff9f8046 100644 --- a/src/ImageBuilder/DockerServiceCache.cs +++ b/src/ImageBuilder/DockerServiceCache.cs @@ -5,8 +5,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; using Microsoft.DotNet.ImageBuilder.Models.Manifest; namespace Microsoft.DotNet.ImageBuilder @@ -31,9 +29,25 @@ public DockerServiceCache(IDockerService inner) public Architecture Architecture => _inner.Architecture; public string? BuildImage( - string dockerfilePath, string buildContextPath, string platform, IEnumerable tags, - IDictionary buildArgs, IEnumerable dockerBuildOptions, bool isRetryEnabled, bool isDryRun) => - _inner.BuildImage(dockerfilePath, buildContextPath, platform, tags, buildArgs, dockerBuildOptions, isRetryEnabled, isDryRun); + string dockerfilePath, + string buildContextPath, + string platform, + IEnumerable tags, + IDictionary buildArgs, + IDictionary labels, + IEnumerable dockerBuildOptions, + bool isRetryEnabled, + bool isDryRun) => + _inner.BuildImage( + dockerfilePath, + buildContextPath, + platform, + tags, + buildArgs, + labels, + dockerBuildOptions, + isRetryEnabled, + isDryRun); public (Architecture Arch, string? Variant) GetImageArch(string image, bool isDryRun) => _architectureCache.GetOrAdd(image, _ =>_inner.GetImageArch(image, isDryRun)); @@ -49,10 +63,10 @@ public DateTime GetCreatedDate(string image, bool isDryRun) => public long GetImageSize(string image, bool isDryRun) => _imageSizeCache.GetOrAdd(image, _ => _inner.GetImageSize(image, isDryRun)); - + public bool LocalImageExists(string tag, bool isDryRun) => _localImageExistsCache.GetOrAdd(tag, _ => _inner.LocalImageExists(tag, isDryRun)); - + public void PullImage(string image, string? platform, bool isDryRun) { _pulledImages.GetOrAdd(image, _ => diff --git a/src/ImageBuilder/GitHelper.cs b/src/ImageBuilder/GitHelper.cs index aaf9893ae..e22170c88 100644 --- a/src/ImageBuilder/GitHelper.cs +++ b/src/ImageBuilder/GitHelper.cs @@ -24,31 +24,36 @@ public static class GitHelper public static string GetCommitSha(string filePath, bool useFullHash = false) { - // Don't make the assumption that the current working directory is a Git repository - // Find the Git repo that contains the file being checked. - DirectoryInfo directory = new FileInfo(filePath).Directory; - while (!directory.GetDirectories(".git").Any()) - { - directory = directory.Parent; - - if (directory is null) - { - throw new InvalidOperationException($"File '{filePath}' is not contained within a Git repository."); - } - } - - filePath = Path.GetRelativePath(directory.FullName, filePath); + string repoRoot = GetRepoRoot(filePath); + filePath = Path.GetRelativePath(repoRoot, filePath); string format = useFullHash ? "H" : "h"; return ExecuteHelper.Execute( new ProcessStartInfo("git", $"log -1 --format=format:%{format} {filePath}") { - WorkingDirectory = directory.FullName + WorkingDirectory = repoRoot }, false, $"Unable to retrieve the latest commit SHA for {filePath}"); } + // Don't make the assumption that the current working directory is a Git repository. + // Walk up from the given path to find the root of the containing Git repository. + public static string GetRepoRoot(string path) + { + DirectoryInfo directory = Directory.Exists(path) ? new DirectoryInfo(path) : new FileInfo(path).Directory; + + // The repository root is marked by a ".git" entry. It's a directory in a normal + // checkout, but a file (a gitdir pointer) in linked worktrees and submodules. + while (!directory.EnumerateFileSystemInfos(".git").Any()) + { + directory = directory.Parent + ?? throw new InvalidOperationException($"'{path}' is not contained within a Git repository."); + } + + return directory.FullName; + } + public static Uri GetArchiveUrl(IGitHubBranchRef branchRef) => new Uri($"https://github.com/{branchRef.Owner}/{branchRef.Repo}/archive/{branchRef.Branch}.zip"); diff --git a/src/ImageBuilder/GitService.cs b/src/ImageBuilder/GitService.cs index e062a1dd7..1fe2d7704 100644 --- a/src/ImageBuilder/GitService.cs +++ b/src/ImageBuilder/GitService.cs @@ -16,6 +16,9 @@ public string GetCommitSha(string filePath, bool useFullHash = false) return GitHelper.GetCommitSha(filePath, useFullHash); } + /// + public string GetRepoRoot(string path) => GitHelper.GetRepoRoot(path); + public IRepository CloneRepository(string sourceUrl, string workdirPath, CloneOptions options) { _logger.LogInformation($"Cloning repository {sourceUrl} to {workdirPath}"); diff --git a/src/ImageBuilder/IDockerService.cs b/src/ImageBuilder/IDockerService.cs index 8eb379f71..6285adf65 100644 --- a/src/ImageBuilder/IDockerService.cs +++ b/src/ImageBuilder/IDockerService.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; -using System.Threading.Tasks; using Microsoft.DotNet.ImageBuilder.Models.Manifest; namespace Microsoft.DotNet.ImageBuilder @@ -23,12 +22,20 @@ public interface IDockerService void CreateManifestList(string manifestListTag, IEnumerable images, bool isDryRun); + /// + /// Builds a Docker image. + /// + /// + /// Labels to apply to the image. Each entry translates to a --label key=value option on the + /// docker build command. + /// string? BuildImage( string dockerfilePath, string buildContextPath, string platform, IEnumerable tags, IDictionary buildArgs, + IDictionary labels, IEnumerable dockerBuildOptions, bool isRetryEnabled, bool isDryRun); diff --git a/src/ImageBuilder/IGitService.cs b/src/ImageBuilder/IGitService.cs index 0765ddc69..fdde000ef 100644 --- a/src/ImageBuilder/IGitService.cs +++ b/src/ImageBuilder/IGitService.cs @@ -10,6 +10,15 @@ public interface IGitService { string GetCommitSha(string filePath, bool useFullHash = false); + /// + /// Gets the absolute path to the root of the Git repository that contains the given path. + /// + /// + /// An absolute path to a file or directory that resides within a Git repository's working tree. + /// + /// The absolute path to the containing repository's root directory. + string GetRepoRoot(string path); + IRepository CloneRepository(string sourceUrl, string workdirPath, CloneOptions options); void Stage(IRepository repository, string path); diff --git a/src/ImageBuilder/ImageBuilderLabels.cs b/src/ImageBuilder/ImageBuilderLabels.cs new file mode 100644 index 000000000..655fcce13 --- /dev/null +++ b/src/ImageBuilder/ImageBuilderLabels.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.ImageBuilder; + +/// +/// Custom (non-OCI) image label keys applied to built images. +/// +public static class ImageBuilderLabels +{ + /// + /// Path of the Dockerfile the image was built from, relative to the root of the source repository. + /// + public const string Dockerfile = "com.microsoft.imagebuilder.dockerfile"; +} diff --git a/src/ImageBuilder/OciAnnotations.cs b/src/ImageBuilder/OciAnnotations.cs new file mode 100644 index 000000000..f3e2c493f --- /dev/null +++ b/src/ImageBuilder/OciAnnotations.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.ImageBuilder; + +/// +/// Well-known OCI image annotation keys applied to built images as Docker labels. +/// See https://github.com/opencontainers/image-spec/blob/main/annotations.md. +/// +public static class OciAnnotations +{ + /// + /// URL of the source code repository the image was built from. + /// + public const string Source = "org.opencontainers.image.source"; + + /// + /// Source control revision (commit) the image was built from. + /// + public const string Revision = "org.opencontainers.image.revision"; + + /// + /// Image reference of the base image the image was built from. + /// + public const string BaseName = "org.opencontainers.image.base.name"; + + /// + /// Digest of the base image the image was built from. + /// + public const string BaseDigest = "org.opencontainers.image.base.digest"; +}