Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove default dependency on Docker CLI for generating Dockerfiles #1508

Merged
merged 7 commits into from
Dec 2, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -42,6 +42,11 @@ public Command GetCliCommand()

cmd.Handler = CommandHandler.Create<TOptions>(options =>
{
if (!Options.NoVersionLogging)
{
LogDockerVersions();
}

Initialize(options);
return ExecuteAsync();
});
@@ -54,6 +59,13 @@ protected virtual void Initialize(TOptions options)
Options = options;
}

private static void LogDockerVersions()
{
// Capture the Docker version and info in the output.
ExecuteHelper.Execute(fileName: "docker", args: "version", isDryRun: false);
ExecuteHelper.Execute(fileName: "docker", args: "info", isDryRun: false);
}

public abstract Task ExecuteAsync();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections.Generic;
using System.CommandLine;
using static Microsoft.DotNet.ImageBuilder.Commands.CliHelper;

#nullable enable
namespace Microsoft.DotNet.ImageBuilder.Commands;

public class DockerfileFilterOptions
{
public IEnumerable<string> Paths { get; set; } = [];
public IEnumerable<string> ProductVersions { get; set; } = [];
}

public class DockerfileFilterOptionsBuilder
{
public const string PathOptionName = "path";

public IEnumerable<Option> GetCliOptions() =>
[
CreateMultiOption<string>(
alias: PathOptionName,
propertyName: nameof(DockerfileFilterOptions.Paths),
description:
"Directory paths containing the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)"),
CreateMultiOption<string>(
alias: "version",
propertyName: nameof(DockerfileFilterOptions.ProductVersions),
description: "Product versions of the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)")
];

public IEnumerable<Argument> GetCliArguments() => [];
}
Original file line number Diff line number Diff line change
@@ -213,7 +213,7 @@ private static void AddImageBuilderPathsVariable(string[] dockerfilePaths, Build
{
string pathArgs = dockerfilePaths
.Distinct()
.Select(path => $"{CliHelper.FormatAlias(ManifestFilterOptionsBuilder.PathOptionName)} {path}")
.Select(path => $"{CliHelper.FormatAlias(DockerfileFilterOptionsBuilder.PathOptionName)} {path}")
.Aggregate((working, next) => $"{working} {next}");
leg.Variables.Add(("imageBuilderPaths", pathArgs));
}
@@ -231,7 +231,7 @@ private static void AddCommonVariables(
leg.Variables.Add(("architecture", platformGrouping.Key.Architecture.GetDockerName()));

string[] osVersions = subgraph
.Select(platform => $"{CliHelper.FormatAlias(ManifestFilterOptionsBuilder.OsVersionOptionName)} {platform.Model.OsVersion}")
.Select(platform => $"{CliHelper.FormatAlias(PlatformFilterOptionsBuilder.OsVersionOptionName)} {platform.Model.OsVersion}")
.Distinct()
.ToArray();
leg.Variables.Add(("osVersions", string.Join(" ", osVersions)));
Original file line number Diff line number Diff line change
@@ -4,14 +4,13 @@

using System.Collections.Generic;
using System.CommandLine;
using System.Linq;

#nullable enable
namespace Microsoft.DotNet.ImageBuilder.Commands
{
public class GenerateDockerfilesOptions : GenerateArtifactsOptions, IFilterableOptions
public class GenerateDockerfilesOptions : GenerateArtifactsOptions, IDockerfileFilterableOptions
{
public ManifestFilterOptions FilterOptions { get; set; } = new ManifestFilterOptions();
public DockerfileFilterOptions Dockerfile { get; set; } = new DockerfileFilterOptions();

public GenerateDockerfilesOptions() : base()
{
@@ -20,16 +19,18 @@ public GenerateDockerfilesOptions() : base()

public class GenerateDockerfilesOptionsBuilder : GenerateArtifactsOptionsBuilder
{
private readonly ManifestFilterOptionsBuilder _manifestFilterOptionsBuilder =
new ManifestFilterOptionsBuilder();
private readonly DockerfileFilterOptionsBuilder _dockerfileFilterOptionsBuilder = new();

public override IEnumerable<Option> GetCliOptions() =>
base.GetCliOptions()
.Concat(_manifestFilterOptionsBuilder.GetCliOptions());
[
..base.GetCliOptions(),
.._dockerfileFilterOptionsBuilder.GetCliOptions(),
];

public override IEnumerable<Argument> GetCliArguments() =>
base.GetCliArguments()
.Concat(_manifestFilterOptionsBuilder.GetCliArguments());
[
..base.GetCliArguments(),
.._dockerfileFilterOptionsBuilder.GetCliArguments(),
];
}
}
#nullable disable
Original file line number Diff line number Diff line change
@@ -8,4 +8,14 @@ public interface IFilterableOptions
{
ManifestFilterOptions FilterOptions { get; }
}

public interface IPlatformFilterableOptions
{
PlatformFilterOptions Platform { get; }
}

public interface IDockerfileFilterableOptions
{
DockerfileFilterOptions Dockerfile { get; }
}
}
Original file line number Diff line number Diff line change
@@ -2,48 +2,35 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.Collections.Generic;
using System.CommandLine;
using System.Linq;
using Microsoft.DotNet.ImageBuilder.ViewModel;
using static Microsoft.DotNet.ImageBuilder.Commands.CliHelper;

#nullable enable
namespace Microsoft.DotNet.ImageBuilder.Commands
{
public class ManifestFilterOptions
{
public string Architecture { get; set; } = string.Empty;
public string OsType { get; set; } = string.Empty;
public IEnumerable<string> OsVersions { get; set; } = Array.Empty<string>();
public IEnumerable<string> Paths { get; set; } = Array.Empty<string>();
public IEnumerable<string> ProductVersions { get; set; } = Array.Empty<string>();
public PlatformFilterOptions Platform { get; set; } = new();

public DockerfileFilterOptions Dockerfile { get; set; } = new();
}

public class ManifestFilterOptionsBuilder
{
public const string PathOptionName = "path";
public const string OsVersionOptionName = "os-version";
private readonly PlatformFilterOptionsBuilder _platformFilterOptionsBuilder = new();

private readonly DockerfileFilterOptionsBuilder _dockerfileFilterOptionsBuilder = new();

public IEnumerable<Option> GetCliOptions() =>
new Option[]
{
CreateOption("architecture", nameof(ManifestFilterOptions.Architecture),
"Architecture of Dockerfiles to operate on - wildcard chars * and ? supported (default is current OS architecture)",
() => DockerHelper.Architecture.GetDockerName()),
CreateOption("os-type", nameof(ManifestFilterOptions.OsType),
"OS type (linux/windows) of the Dockerfiles to build - wildcard chars * and ? supported (default is the Docker OS)",
() => DockerHelper.OS.GetDockerName()),
CreateMultiOption<string>(OsVersionOptionName, nameof(ManifestFilterOptions.OsVersions),
"OS versions of the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)"),
CreateMultiOption<string>(PathOptionName, nameof(ManifestFilterOptions.Paths),
"Directory paths containing the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)"),
CreateMultiOption<string>("version", nameof(ManifestFilterOptions.ProductVersions),
"Product versions of the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)")
};
[
.._platformFilterOptionsBuilder.GetCliOptions(),
.._dockerfileFilterOptionsBuilder.GetCliOptions(),
];

public IEnumerable<Argument> GetCliArguments() => Enumerable.Empty<Argument>();
public IEnumerable<Argument> GetCliArguments() =>
[
.._platformFilterOptionsBuilder.GetCliArguments(),
.._dockerfileFilterOptionsBuilder.GetCliArguments(),
];
}
}
#nullable disable
Original file line number Diff line number Diff line change
@@ -27,14 +27,35 @@ public virtual ManifestFilter GetManifestFilter()

if (this is IFilterableOptions options)
{
ManifestFilterOptions filterOptions = options.FilterOptions;
filter.IncludeArchitecture = filterOptions.Architecture;
filter.IncludeOsType = filterOptions.OsType;
filter.IncludeOsVersions = filterOptions.OsVersions;
filter.IncludePaths = filterOptions.Paths;
filter.IncludeProductVersions = filterOptions.ProductVersions;
filter = SetPlatformFilters(filter, options.FilterOptions.Platform);
filter = SetDockerfileFilters(filter, options.FilterOptions.Dockerfile);
}

if (this is IPlatformFilterableOptions platformFilterOptions)
{
filter = SetPlatformFilters(filter, platformFilterOptions.Platform);
}

if (this is IDockerfileFilterableOptions dockerfileFilterOptions)
{
filter = SetDockerfileFilters(filter, dockerfileFilterOptions.Dockerfile);
}

return filter;
}

private static ManifestFilter SetDockerfileFilters(ManifestFilter filter, DockerfileFilterOptions options)
{
filter.IncludePaths = options.Paths;
filter.IncludeProductVersions = options.ProductVersions;
return filter;
}

private static ManifestFilter SetPlatformFilters(ManifestFilter filter, PlatformFilterOptions options)
{
filter.IncludeArchitecture = options.Architecture;
filter.IncludeOsType = options.OsType;
filter.IncludeOsVersions = options.OsVersions;
return filter;
}
}
7 changes: 6 additions & 1 deletion src/Microsoft.DotNet.ImageBuilder/src/Commands/Options.cs
Original file line number Diff line number Diff line change
@@ -16,6 +16,7 @@ public class Options : IOptions
{
public bool IsDryRun { get; set; }
public bool IsVerbose { get; set; }
public bool NoVersionLogging { get; set; }
}

public class CliOptionsBuilder
@@ -39,7 +40,11 @@ public virtual IEnumerable<Option> GetCliOptions() =>
CreateOption<bool>(
alias: "verbose",
propertyName: nameof(Options.IsVerbose),
description: "Show details about the tasks run")
description: "Show details about the tasks run"),
CreateOption<bool>(
alias: "no-version-logging",
propertyName: nameof(Options.NoVersionLogging),
description: "Disable automatic logging of Docker version information")
];
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System.Collections.Generic;
using System.CommandLine;
using Microsoft.DotNet.ImageBuilder.ViewModel;
using static Microsoft.DotNet.ImageBuilder.Commands.CliHelper;

#nullable enable
namespace Microsoft.DotNet.ImageBuilder.Commands;

public class PlatformFilterOptions
{
public string Architecture { get; set; } = string.Empty;
public string OsType { get; set; } = string.Empty;
public IEnumerable<string> OsVersions { get; set; } = [];

}

public class PlatformFilterOptionsBuilder
{
public const string OsVersionOptionName = "os-version";

public IEnumerable<Option> GetCliOptions() =>
[
CreateOption(
alias: "architecture",
propertyName: nameof(PlatformFilterOptions.Architecture),
description: "Architecture of Dockerfiles to operate on - wildcard chars * and ? supported (default is current OS architecture)",
defaultValue: () => DockerHelper.Architecture.GetDockerName()),
CreateOption(
alias: "os-type",
propertyName: nameof(PlatformFilterOptions.OsType),
description: "OS type (linux/windows) of the Dockerfiles to build - wildcard chars * and ? supported (default is the Docker OS)",
defaultValue: () => DockerHelper.OS.GetDockerName()),
CreateMultiOption<string>(
alias: OsVersionOptionName,
propertyName: nameof(PlatformFilterOptions.OsVersions),
description: "OS versions of the Dockerfiles to build - wildcard chars * and ? supported (default is to build all)"),
];

public IEnumerable<Argument> GetCliArguments() => [];
}
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ private IEnumerable<SubscriptionImagePaths> GetAllSubscriptionImagePaths()
{
// This data comes from the GetStaleImagesCommand and represents a mapping of a subscription to the Dockerfile paths
// of the images that need to be built. A given subscription may have images that are spread across Linux/Windows, AMD64/ARM
// which means that the paths collected for that subscription were spread across multiple jobs. Each of the items in the
// which means that the paths collected for that subscription were spread across multiple jobs. Each of the items in the
// Enumerable here represents the data collected by one job. We need to consolidate the paths for a given subscription since
// they could be spread across multiple items in the Enumerable.
return Options.AllSubscriptionImagePaths
@@ -92,7 +92,7 @@ private async Task QueueBuildForStaleImages(Subscription subscription, IEnumerab
}

string formattedPathsToRebuild = pathsToRebuild
.Select(path => $"{CliHelper.FormatAlias(ManifestFilterOptionsBuilder.PathOptionName)} '{path}'")
.Select(path => $"{CliHelper.FormatAlias(DockerfileFilterOptionsBuilder.PathOptionName)} '{path}'")
.Aggregate((p1, p2) => $"{p1} {p2}");

string parameters = "{\"" + subscription.PipelineTrigger.PathVariable + "\": \"" + formattedPathsToRebuild + "\"}";
9 changes: 0 additions & 9 deletions src/Microsoft.DotNet.ImageBuilder/src/ImageBuilder.cs
Original file line number Diff line number Diff line change
@@ -53,15 +53,6 @@ public static int Main(string[] args)
Parser parser = new CommandLineBuilder(rootCliCommand)
.UseDefaults()
.UseMiddleware(context =>
{
if (context.ParseResult.CommandResult.Command != rootCliCommand)
{
// Capture the Docker version and info in the output.
ExecuteHelper.Execute(fileName: "docker", args: "version", isDryRun: false);
ExecuteHelper.Execute(fileName: "docker", args: "info", isDryRun: false);
}
})
.UseMiddleware(context =>
{
context.BindingContext.AddModelBinder(new ModelBinder<AzdoOptions>());
context.BindingContext.AddModelBinder(new ModelBinder<GitOptions>());
Original file line number Diff line number Diff line change
@@ -47,7 +47,7 @@ public static class SubscriptionHelper
{
// If the command is filtered with an OS type that does not match the OsType filter of the subscription,
// then there are no images that need to be inspected.
string osTypeRegexPattern = ManifestFilter.GetFilterRegexPattern(filterOptions.OsType);
string osTypeRegexPattern = ManifestFilter.GetFilterRegexPattern(filterOptions.Platform.OsType);
if (!string.IsNullOrEmpty(subscription.OsType) &&
!Regex.IsMatch(subscription.OsType, osTypeRegexPattern, RegexOptions.IgnoreCase))
{
Original file line number Diff line number Diff line change
@@ -41,7 +41,7 @@ public async Task GenerateBuildMatrixCommand_PlatformVersionedOs(string filterPa
command.Options.ProductVersionComponents = 3;
if (filterPaths != null)
{
command.Options.FilterOptions.Paths = filterPaths.Replace("--path ", "").Split(" ");
command.Options.FilterOptions.Dockerfile.Paths = filterPaths.Replace("--path ", "").Split(" ");
}

const string runtimeDepsRelativeDir = "2.1.1/runtime-deps/os";
@@ -108,7 +108,7 @@ public async Task GenerateBuildMatrixCommand_PlatformDependencyGraph(string filt
command.Options.MatrixType = MatrixType.PlatformDependencyGraph;
if (filterPaths != null)
{
command.Options.FilterOptions.Paths = filterPaths.Replace("--path ", "").Split(" ");
command.Options.FilterOptions.Dockerfile.Paths = filterPaths.Replace("--path ", "").Split(" ");
}

const string runtimeDepsRelativeDir = "1.0/runtimedeps/os/amd64";
@@ -258,7 +258,7 @@ public async Task FilterOutCachedImages(
command.Options.MatrixType = MatrixType.PlatformDependencyGraph;
command.Options.ImageInfoPath = Path.Combine(tempFolderContext.Path, "imageinfo.json");
command.Options.TrimCachedImages = true;
command.Options.FilterOptions.Paths = inputPathFilters.Split(" ", StringSplitOptions.RemoveEmptyEntries);
command.Options.FilterOptions.Dockerfile.Paths = inputPathFilters.Split(" ", StringSplitOptions.RemoveEmptyEntries);

File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));

Original file line number Diff line number Diff line change
@@ -1768,7 +1768,7 @@ private GetStaleImagesCommand CreateCommand()
this.ManifestServiceFactoryMock.Object, this.loggerServiceMock.Object, this.octokitClientFactory, this.gitService);
command.Options.SubscriptionOptions.SubscriptionsPath = this.subscriptionsPath;
command.Options.VariableName = VariableName;
command.Options.FilterOptions.OsType = this.osType;
command.Options.FilterOptions.Platform.OsType = this.osType;
command.Options.GitOptions.Email = "test";
command.Options.GitOptions.Username = "test";
command.Options.GitOptions.AuthToken = "test";
Original file line number Diff line number Diff line change
@@ -694,7 +694,7 @@ private static bool FilterBuildToParameters(string buildParametersJson, string p
IList<string> paths = pathString
.Split(" ", StringSplitOptions.RemoveEmptyEntries)
.Select(p => p.Trim().Trim('\''))
.Except(new string[] { CliHelper.FormatAlias(ManifestFilterOptionsBuilder.PathOptionName) })
.Except([ CliHelper.FormatAlias(DockerfileFilterOptionsBuilder.PathOptionName) ])
.ToList();
return CompareLists(expectedPaths, paths);
}