From 692d989486488a617ae816d6f4bca3df91a9b51f Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 14 Jan 2025 14:02:47 -0700 Subject: [PATCH 01/72] validate shape of `packages.config` and consolidate error handling --- .../EntryPointTests.Analyze.cs | 8 +- .../EntryPointTests.Discover.cs | 52 ++++++- .../EntryPointTests.Update.cs | 8 ++ .../Commands/AnalyzeCommand.cs | 10 +- .../Commands/DiscoverCommand.cs | 15 +- .../NuGetUpdater.Cli/Commands/RunCommand.cs | 8 +- .../Commands/UpdateCommand.cs | 22 ++- .../Analyze/AnalyzeWorkerTestBase.cs | 19 ++- .../Analyze/AnalyzeWorkerTests.cs | 24 ++-- .../Discover/DiscoveryWorkerTestBase.cs | 32 +++-- .../Discover/DiscoveryWorkerTests.cs | 28 ++-- .../Discover/ExpectedDiscoveryResults.cs | 2 +- .../Run/RunWorkerTests.cs | 9 +- .../Run/SerializationTests.cs | 10 +- .../NuGetUpdater.Core.Test/TestBase.cs | 24 ++++ .../Update/ExpectedUpdateOperationResult.cs | 2 +- .../Update/UpdateWorkerTestBase.cs | 26 ++-- .../Update/UpdateWorkerTests.DirsProj.cs | 2 +- .../Update/UpdateWorkerTests.Mixed.cs | 22 +-- .../UpdateWorkerTests.PackageReference.cs | 6 +- .../UpdateWorkerTests.PackagesConfig.cs | 26 ++-- .../Analyze/AnalyzeWorker.cs | 22 +-- .../NuGetUpdater.Core/Clone/CloneWorker.cs | 3 +- .../Discover/DiscoveryWorker.cs | 27 +--- .../Discover/ProjectDiscoveryResult.cs | 6 +- .../NuGetUpdater.Core/ErrorType.cs | 12 -- .../NuGetUpdater.Core/ExperimentsManager.cs | 31 ++--- .../Files/PackagesConfigBuildFile.cs | 6 +- .../NuGetUpdater.Core/MissingFileException.cs | 3 +- .../NuGetUpdater.Core/NativeResult.cs | 6 +- .../Run/ApiModel/DependencyFileNotFound.cs | 10 +- .../ApiModel/DependencyFileNotParseable.cs | 15 ++ .../Run/ApiModel/JobErrorBase.cs | 24 ++++ .../NuGetUpdater.Core/Run/RunWorker.cs | 27 +--- .../UnparseableFileException.cs | 12 ++ .../Updater/UpdaterWorker.cs | 36 +---- nuget/lib/dependabot/nuget/native_helpers.rb | 57 +++++--- .../dependabot/nuget/file_fetcher_spec.rb | 8 +- .../spec/dependabot/nuget/file_parser_spec.rb | 2 + .../dependabot/nuget/file_updater_spec.rb | 2 + .../dependabot/nuget/native_helpers_spec.rb | 130 ++++++++++++++---- .../dependabot/nuget/update_checker_spec.rb | 8 +- 42 files changed, 510 insertions(+), 292 deletions(-) delete mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ErrorType.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotParseable.cs create mode 100644 nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/UnparseableFileException.cs diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Analyze.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Analyze.cs index ef0799a4e4..0f2429f006 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Analyze.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Analyze.cs @@ -26,6 +26,8 @@ public async Task FindsUpdatedPackageAndReturnsTheCorrectData() await RunAsync(path => [ "analyze", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -146,6 +148,8 @@ public async Task DotNetToolsJsonCanBeAnalyzed() await RunAsync(path => [ "analyze", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -235,6 +239,8 @@ public async Task GlobalJsonCanBeAnalyzed() await RunAsync(path => [ "analyze", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -345,7 +351,7 @@ private static async Task RunAsync( { if (args[i] == "--job-path") { - var experimentsResult = await ExperimentsManager.FromJobFileAsync(args[i + 1]); + var experimentsResult = await ExperimentsManager.FromJobFileAsync("TEST-JOB-ID", args[i + 1]); experimentsManager = experimentsResult.ExperimentsManager; } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs index 2622c56e85..be65bdc4f8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Discover.cs @@ -1,5 +1,6 @@ using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using NuGetUpdater.Core; using NuGetUpdater.Core.Discover; @@ -25,6 +26,8 @@ public async Task PathWithSpaces(bool useDirectDiscovery) await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -83,6 +86,8 @@ public async Task WithSolution(bool useDirectDiscovery) await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -178,6 +183,8 @@ public async Task WithProject(bool useDirectDiscovery) await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -251,6 +258,8 @@ public async Task WithDirectory(bool useDirectDiscovery) await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -321,6 +330,8 @@ public async Task WithDuplicateDependenciesOfDifferentTypes() await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -398,6 +409,8 @@ public async Task JobFileParseErrorIsReported_InvalidJson() await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", jobFilePath, "--repo-root", @@ -412,8 +425,7 @@ await RunAsync(path => { Path = "/", Projects = [], - ErrorType = ErrorType.Unknown, - ErrorDetailsPattern = "JsonException", + ErrorRegex = "Error deserializing job file contents", } ); } @@ -445,6 +457,8 @@ await File.WriteAllTextAsync(jobFilePath, """ await RunAsync(path => [ "discover", + "--job-id", + "TEST-JOB-ID", "--job-path", jobFilePath, "--repo-root", @@ -459,8 +473,7 @@ await RunAsync(path => { Path = "/", Projects = [], - ErrorType = ErrorType.BadRequirement, - ErrorDetailsPattern = "not a valid requirement", + Error = new Core.Run.ApiModel.BadRequirement("not a valid requirement"), } ); } @@ -497,7 +510,7 @@ private static async Task RunAsync( switch (args[i]) { case "--job-path": - var experimentsResult = await ExperimentsManager.FromJobFileAsync(args[i + 1]); + var experimentsResult = await ExperimentsManager.FromJobFileAsync("TEST-JOB-ID", args[i + 1]); experimentsManager = experimentsResult.ExperimentsManager; break; case "--output": @@ -520,11 +533,38 @@ private static async Task RunAsync( resultPath ??= Path.Join(path, DiscoveryWorker.DiscoveryResultFileName); var resultJson = await File.ReadAllTextAsync(resultPath); - var resultObject = JsonSerializer.Deserialize(resultJson, DiscoveryWorker.SerializerOptions); + var serializerOptions = new JsonSerializerOptions() + { + Converters = { new TestJobErrorBaseConverter() } + }; + foreach (var converter in DiscoveryWorker.SerializerOptions.Converters) + { + serializerOptions.Converters.Add(converter); + } + var resultObject = JsonSerializer.Deserialize(resultJson, serializerOptions); return resultObject!; }); ValidateWorkspaceResult(expectedResult, actualResult, experimentsManager); } + + private class TestJobErrorBaseConverter : JsonConverter + { + public override Core.Run.ApiModel.JobErrorBase? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dict = JsonSerializer.Deserialize>(ref reader, options)!; + return dict["error-type"].GetString() switch + { + "illformed_requirement" => new Core.Run.ApiModel.BadRequirement(dict["error-details"].GetProperty("message").GetString()!), + "unknown_error" => new Core.Run.ApiModel.UnknownError(new Exception("Error deserializing job file contents"), "TEST-JOB-ID"), + _ => throw new NotImplementedException($"Unknown error type: {dict["error-type"]}"), + }; + } + + public override void Write(Utf8JsonWriter writer, Core.Run.ApiModel.JobErrorBase value, JsonSerializerOptions options) + { + throw new NotImplementedException(); + } + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs index 6981b2d369..a0d9b64fab 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli.Test/EntryPointTests.Update.cs @@ -19,6 +19,8 @@ public async Task WithSolution() await Run(path => [ "update", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -122,6 +124,8 @@ public async Task WithProject() await Run(path => [ "update", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -202,6 +206,8 @@ public async Task WithDirsProjAndDirectoryBuildPropsThatIsOutOfDirectoryButStill await Run(path => [ "update", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(path, "job.json"), "--repo-root", @@ -361,6 +367,8 @@ await File.WriteAllTextAsync(projectPath, """ IEnumerable executableArgs = [ executableName, "update", + "--job-id", + "TEST-JOB-ID", "--job-path", Path.Combine(tempDir.DirectoryPath, "job.json"), "--repo-root", diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/AnalyzeCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/AnalyzeCommand.cs index 0133b62b87..7c3d6363a6 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/AnalyzeCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/AnalyzeCommand.cs @@ -7,6 +7,7 @@ namespace NuGetUpdater.Cli.Commands; internal static class AnalyzeCommand { + internal static readonly Option JobIdOption = new("--job-id") { IsRequired = true }; internal static readonly Option JobPathOption = new("--job-path") { IsRequired = true }; internal static readonly Option RepoRootOption = new("--repo-root") { IsRequired = true }; internal static readonly Option DependencyFilePathOption = new("--dependency-file-path") { IsRequired = true }; @@ -17,6 +18,7 @@ internal static Command GetCommand(Action setExitCode) { Command command = new("analyze", "Determines how to update a dependency based on the workspace discovery information.") { + JobIdOption, JobPathOption, RepoRootOption, DependencyFilePathOption, @@ -26,13 +28,13 @@ internal static Command GetCommand(Action setExitCode) command.TreatUnmatchedTokensAsErrors = true; - command.SetHandler(async (jobPath, repoRoot, discoveryPath, dependencyPath, analysisDirectory) => + command.SetHandler(async (jobId, jobPath, repoRoot, discoveryPath, dependencyPath, analysisDirectory) => { var logger = new ConsoleLogger(); - var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobPath.FullName); - var worker = new AnalyzeWorker(experimentsManager, logger); + var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobId, jobPath.FullName); + var worker = new AnalyzeWorker(jobId, experimentsManager, logger); await worker.RunAsync(repoRoot.FullName, discoveryPath.FullName, dependencyPath.FullName, analysisDirectory.FullName); - }, JobPathOption, RepoRootOption, DiscoveryFilePathOption, DependencyFilePathOption, AnalysisFolderOption); + }, JobIdOption, JobPathOption, RepoRootOption, DiscoveryFilePathOption, DependencyFilePathOption, AnalysisFolderOption); return command; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/DiscoverCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/DiscoverCommand.cs index d3d62b1a1e..d346d90659 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/DiscoverCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/DiscoverCommand.cs @@ -7,6 +7,7 @@ namespace NuGetUpdater.Cli.Commands; internal static class DiscoverCommand { + internal static readonly Option JobIdOption = new("--job-id") { IsRequired = true }; internal static readonly Option JobPathOption = new("--job-path") { IsRequired = true }; internal static readonly Option RepoRootOption = new("--repo-root") { IsRequired = true }; internal static readonly Option WorkspaceOption = new("--workspace") { IsRequired = true }; @@ -16,6 +17,7 @@ internal static Command GetCommand(Action setExitCode) { Command command = new("discover", "Generates a report of the workspace dependencies and where they are located.") { + JobIdOption, JobPathOption, RepoRootOption, WorkspaceOption, @@ -24,27 +26,26 @@ internal static Command GetCommand(Action setExitCode) command.TreatUnmatchedTokensAsErrors = true; - command.SetHandler(async (jobPath, repoRoot, workspace, outputPath) => + command.SetHandler(async (jobId, jobPath, repoRoot, workspace, outputPath) => { - var (experimentsManager, errorResult) = await ExperimentsManager.FromJobFileAsync(jobPath.FullName); - if (errorResult is not null) + var (experimentsManager, error) = await ExperimentsManager.FromJobFileAsync(jobId, jobPath.FullName); + if (error is not null) { // to make testing easier, this should be a `WorkspaceDiscoveryResult` object var discoveryErrorResult = new WorkspaceDiscoveryResult { + Error = error, Path = workspace, Projects = [], - ErrorType = errorResult.ErrorType, - ErrorDetails = errorResult.ErrorDetails, }; await DiscoveryWorker.WriteResultsAsync(repoRoot.FullName, outputPath.FullName, discoveryErrorResult); return; } var logger = new ConsoleLogger(); - var worker = new DiscoveryWorker(experimentsManager, logger); + var worker = new DiscoveryWorker(jobId, experimentsManager, logger); await worker.RunAsync(repoRoot.FullName, workspace, outputPath.FullName); - }, JobPathOption, RepoRootOption, WorkspaceOption, OutputOption); + }, JobIdOption, JobPathOption, RepoRootOption, WorkspaceOption, OutputOption); return command; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs index 439c2b07d6..fe6a064d9a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/RunCommand.cs @@ -33,11 +33,11 @@ internal static Command GetCommand(Action setExitCode) command.SetHandler(async (jobPath, repoContentsPath, apiUrl, jobId, outputPath, baseCommitSha) => { var apiHandler = new HttpApiHandler(apiUrl.ToString(), jobId); - var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobPath.FullName); + var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobId, jobPath.FullName); var logger = new ConsoleLogger(); - var discoverWorker = new DiscoveryWorker(experimentsManager, logger); - var analyzeWorker = new AnalyzeWorker(experimentsManager, logger); - var updateWorker = new UpdaterWorker(experimentsManager, logger); + var discoverWorker = new DiscoveryWorker(jobId, experimentsManager, logger); + var analyzeWorker = new AnalyzeWorker(jobId, experimentsManager, logger); + var updateWorker = new UpdaterWorker(jobId, experimentsManager, logger); var worker = new RunWorker(jobId, apiHandler, discoverWorker, analyzeWorker, updateWorker, logger); await worker.RunAsync(jobPath, repoContentsPath, baseCommitSha, outputPath); }, JobPathOption, RepoContentsPathOption, ApiUrlOption, JobIdOption, OutputPathOption, BaseCommitShaOption); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/UpdateCommand.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/UpdateCommand.cs index bd04c652cc..91b920be99 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/UpdateCommand.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Cli/Commands/UpdateCommand.cs @@ -6,6 +6,7 @@ namespace NuGetUpdater.Cli.Commands; internal static class UpdateCommand { + internal static readonly Option JobIdOption = new("--job-id") { IsRequired = true }; internal static readonly Option JobPathOption = new("--job-path") { IsRequired = true }; internal static readonly Option RepoRootOption = new("--repo-root", () => new DirectoryInfo(Environment.CurrentDirectory)) { IsRequired = false }; internal static readonly Option SolutionOrProjectFileOption = new("--solution-or-project") { IsRequired = true }; @@ -19,6 +20,7 @@ internal static Command GetCommand(Action setExitCode) { Command command = new("update", "Applies the changes from an analysis report to update a dependency.") { + JobIdOption, JobPathOption, RepoRootOption, SolutionOrProjectFileOption, @@ -30,15 +32,25 @@ internal static Command GetCommand(Action setExitCode) }; command.TreatUnmatchedTokensAsErrors = true; - - command.SetHandler(async (jobPath, repoRoot, solutionOrProjectFile, dependencyName, newVersion, previousVersion, isTransitive, resultOutputPath) => + command.SetHandler(async (context) => { - var (experimentsManager, _errorResult) = await ExperimentsManager.FromJobFileAsync(jobPath.FullName); + // since we have more than 8 arguments, we have to pull them out manually + var jobId = context.ParseResult.GetValueForOption(JobIdOption)!; + var jobPath = context.ParseResult.GetValueForOption(JobPathOption)!; + var repoRoot = context.ParseResult.GetValueForOption(RepoRootOption)!; + var solutionOrProjectFile = context.ParseResult.GetValueForOption(SolutionOrProjectFileOption)!; + var dependencyName = context.ParseResult.GetValueForOption(DependencyNameOption)!; + var newVersion = context.ParseResult.GetValueForOption(NewVersionOption)!; + var previousVersion = context.ParseResult.GetValueForOption(PreviousVersionOption)!; + var isTransitive = context.ParseResult.GetValueForOption(IsTransitiveOption); + var resultOutputPath = context.ParseResult.GetValueForOption(ResultOutputPathOption); + + var (experimentsManager, _error) = await ExperimentsManager.FromJobFileAsync(jobId, jobPath.FullName); var logger = new ConsoleLogger(); - var worker = new UpdaterWorker(experimentsManager, logger); + var worker = new UpdaterWorker(jobId, experimentsManager, logger); await worker.RunAsync(repoRoot.FullName, solutionOrProjectFile.FullName, dependencyName, previousVersion, newVersion, isTransitive, resultOutputPath); setExitCode(0); - }, JobPathOption, RepoRootOption, SolutionOrProjectFileOption, DependencyNameOption, NewVersionOption, PreviousVersionOption, IsTransitiveOption, ResultOutputPathOption); + }); return command; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTestBase.cs index ce60bf9eeb..104ac9abf1 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTestBase.cs @@ -12,7 +12,7 @@ namespace NuGetUpdater.Core.Test.Analyze; using TestFile = (string Path, string Content); -public class AnalyzeWorkerTestBase +public class AnalyzeWorkerTestBase : TestBase { protected static async Task TestAnalyzeAsync( WorkspaceDiscoveryResult discovery, @@ -39,7 +39,7 @@ protected static async Task TestAnalyzeAsync( var discoveryPath = Path.GetFullPath(DiscoveryWorker.DiscoveryResultFileName, directoryPath); var dependencyPath = Path.GetFullPath(relativeDependencyPath, directoryPath); - var worker = new AnalyzeWorker(experimentsManager, new TestLogger()); + var worker = new AnalyzeWorker("TEST-JOB-ID", experimentsManager, new TestLogger()); var result = await worker.RunWithErrorHandlingAsync(directoryPath, discoveryPath, dependencyPath); return result; }); @@ -55,8 +55,7 @@ protected static void ValidateAnalysisResult(ExpectedAnalysisResult expectedResu Assert.Equal(expectedResult.VersionComesFromMultiDependencyProperty, actualResult.VersionComesFromMultiDependencyProperty); ValidateDependencies(expectedResult.UpdatedDependencies, actualResult.UpdatedDependencies); Assert.Equal(expectedResult.ExpectedUpdatedDependenciesCount ?? expectedResult.UpdatedDependencies.Length, actualResult.UpdatedDependencies.Length); - Assert.Equal(expectedResult.ErrorType, actualResult.ErrorType); - Assert.Equal(expectedResult.ErrorDetails, actualResult.ErrorDetails); + ValidateResult(expectedResult, actualResult); return; @@ -81,6 +80,18 @@ void ValidateDependencies(ImmutableArray expectedDependencies, Immut } } + protected static void ValidateResult(ExpectedAnalysisResult? expectedResult, AnalysisResult actualResult) + { + if (expectedResult?.Error is not null) + { + ValidateError(expectedResult.Error, actualResult.Error); + } + else + { + Assert.Null(actualResult.Error); + } + } + protected static async Task RunAnalyzerAsync(string dependencyName, TestFile[] files, Func> action) { // write initial files diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs index 4e93815025..0b546c97c3 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs @@ -4,6 +4,7 @@ using NuGet; using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Run.ApiModel; using Xunit; @@ -1015,8 +1016,7 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() using var temporaryDirectory = await TemporaryDirectory.CreateWithContentsAsync([]); await AnalyzeWorker.WriteResultsAsync(temporaryDirectory.DirectoryPath, "Some.Dependency", new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "", + Error = new PrivateSourceAuthenticationFailure([""]), UpdatedVersion = "", UpdatedDependencies = [], }, new TestLogger()); @@ -1025,16 +1025,23 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() // raw result file should look like this: // { // ... - // "ErrorType": "AuthenticationFailure", + // "Error": { + // "error-type": "private_source_authentication_failure", + // "error-details": { + // "source": "()" + // } + // } // "ErrorDetails": "", // ... // } var jsonDocument = JsonDocument.Parse(discoveryContents); - var errorType = jsonDocument.RootElement.GetProperty("ErrorType"); - var errorDetails = jsonDocument.RootElement.GetProperty("ErrorDetails"); + var error = jsonDocument.RootElement.GetProperty("Error"); + var errorType = error.GetProperty("error-type"); + var errorDetails = error.GetProperty("error-details"); + var errorSource = errorDetails.GetProperty("source"); - Assert.Equal("AuthenticationFailure", errorType.GetString()); - Assert.Equal("", errorDetails.GetString()); + Assert.Equal("private_source_authentication_failure", errorType.GetString()); + Assert.Equal("()", errorSource.GetString()); } [Fact] @@ -1110,8 +1117,7 @@ await TestAnalyzeAsync( }, expectedResult: new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = $"({http.BaseUrl.TrimEnd('/')}/index.json)", + Error = new PrivateSourceAuthenticationFailure([$"{http.BaseUrl.TrimEnd('/')}/index.json"]), UpdatedVersion = string.Empty, CanUpdate = false, UpdatedDependencies = [], diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs index 4da255d04f..f9bf1b0b09 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTestBase.cs @@ -5,7 +5,9 @@ using NuGetUpdater.Core.Discover; using NuGetUpdater.Core.Test.Update; +using NuGetUpdater.Core.Test.Updater; using NuGetUpdater.Core.Test.Utilities; +using NuGetUpdater.Core.Updater; using NuGetUpdater.Core.Utilities; using Xunit; @@ -28,7 +30,7 @@ protected static async Task TestDiscoveryAsync( { await UpdateWorkerTestBase.MockNuGetPackagesInDirectory(packages, directoryPath); - var worker = new DiscoveryWorker(experimentsManager, new TestLogger()); + var worker = new DiscoveryWorker("TEST-JOB-ID", experimentsManager, new TestLogger()); var result = await worker.RunWithErrorHandlingAsync(directoryPath, workspacePath); return result; }); @@ -44,17 +46,7 @@ protected static void ValidateWorkspaceResult(ExpectedWorkspaceDiscoveryResult e ValidateResultWithDependencies(expectedResult.DotNetToolsJson, actualResult.DotNetToolsJson); ValidateProjectResults(expectedResult.Projects, actualResult.Projects, experimentsManager); Assert.Equal(expectedResult.ExpectedProjectCount ?? expectedResult.Projects.Length, actualResult.Projects.Length); - Assert.Equal(expectedResult.ErrorType, actualResult.ErrorType); - if (expectedResult.ErrorDetailsPattern is not null) - { - var errorDetails = actualResult.ErrorDetails?.ToString(); - Assert.NotNull(errorDetails); - Assert.Matches(expectedResult.ErrorDetailsPattern, errorDetails); - } - else - { - Assert.Equal(expectedResult.ErrorDetails, actualResult.ErrorDetails); - } + ValidateDiscoveryOperationResult(expectedResult, actualResult); return; @@ -76,6 +68,22 @@ static void ValidateResultWithDependencies(ExpectedDependencyDiscoveryResult? ex } } + protected static void ValidateDiscoveryOperationResult(ExpectedWorkspaceDiscoveryResult? expectedResult, WorkspaceDiscoveryResult actualResult) + { + if (expectedResult?.Error is not null) + { + ValidateError(expectedResult.Error, actualResult.Error); + } + else if (expectedResult?.ErrorRegex is not null) + { + ValidateErrorRegex(expectedResult.ErrorRegex, actualResult.Error); + } + else + { + Assert.Null(actualResult.Error); + } + } + internal static void ValidateProjectResults(ImmutableArray expectedProjects, ImmutableArray actualProjects, ExperimentsManager experimentsManager) { if (expectedProjects.IsDefaultOrEmpty) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.cs index 066595385c..9f12965692 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/DiscoveryWorkerTests.cs @@ -3,6 +3,7 @@ using System.Text.Json; using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Run.ApiModel; using Xunit; @@ -1130,8 +1131,7 @@ await TestDiscoveryAsync( { Path = "", Projects = [], - ErrorType = ErrorType.DependencyFileNotParseable, - ErrorDetails = "project2.csproj", + Error = new DependencyFileNotParseable("project2.csproj"), }); } @@ -1142,8 +1142,7 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() var discoveryResultPath = Path.Combine(temporaryDirectory.DirectoryPath, DiscoveryWorker.DiscoveryResultFileName); await DiscoveryWorker.WriteResultsAsync(temporaryDirectory.DirectoryPath, discoveryResultPath, new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "", + Error = new PrivateSourceAuthenticationFailure([""]), Path = "/", Projects = [], }); @@ -1152,16 +1151,22 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() // raw result file should look like this: // { // ... - // "ErrorType": "AuthenticationFailure", - // "ErrorDetails": "", + // "Error": { + // "error-type": "private_source_authentication_failure", + // "error-detail": { + // "source": "()" + // } + // } // ... // } var jsonDocument = JsonDocument.Parse(discoveryContents); - var errorType = jsonDocument.RootElement.GetProperty("ErrorType"); - var errorDetails = jsonDocument.RootElement.GetProperty("ErrorDetails"); + var error = jsonDocument.RootElement.GetProperty("Error"); + var errorType = error.GetProperty("error-type"); + var errorDetail = error.GetProperty("error-details"); + var errorSource = errorDetail.GetProperty("source"); - Assert.Equal("AuthenticationFailure", errorType.GetString()); - Assert.Equal("", errorDetails.GetString()); + Assert.Equal("private_source_authentication_failure", errorType.GetString()); + Assert.Equal("()", errorSource.GetString()); } [Theory] @@ -1236,8 +1241,7 @@ await TestDiscoveryAsync( ], expectedResult: new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = $"({http.BaseUrl.TrimEnd('/')}/index.json)", + Error = new PrivateSourceAuthenticationFailure([$"{http.BaseUrl.TrimEnd('/')}/index.json"]), Path = "", Projects = [], } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs index d9d434aad6..3d6da437bc 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Discover/ExpectedDiscoveryResults.cs @@ -12,7 +12,7 @@ public record ExpectedWorkspaceDiscoveryResult : NativeResult public int? ExpectedProjectCount { get; init; } public ExpectedDependencyDiscoveryResult? GlobalJson { get; init; } public ExpectedDependencyDiscoveryResult? DotNetToolsJson { get; init; } - public string? ErrorDetailsPattern { get; init; } = null; + public string? ErrorRegex { get; init; } = null; } public record ExpectedSdkProjectDiscoveryResult : ExpectedDependencyDiscoveryResult diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs index c58a29fcd7..cef94b63b6 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/RunWorkerTests.cs @@ -1729,11 +1729,12 @@ private static async Task RunAsync(Job job, TestFile[] files, IDiscoveryWorker? experimentsManager ??= new ExperimentsManager(); var testApiHandler = new TestApiHandler(); var logger = new TestLogger(); - discoveryWorker ??= new DiscoveryWorker(experimentsManager, logger); - analyzeWorker ??= new AnalyzeWorker(experimentsManager, logger); - updaterWorker ??= new UpdaterWorker(experimentsManager, logger); + var jobId = "TEST-JOB-ID"; + discoveryWorker ??= new DiscoveryWorker(jobId, experimentsManager, logger); + analyzeWorker ??= new AnalyzeWorker(jobId, experimentsManager, logger); + updaterWorker ??= new UpdaterWorker(jobId, experimentsManager, logger); - var worker = new RunWorker("TEST-JOB-ID", testApiHandler, discoveryWorker, analyzeWorker, updaterWorker, logger); + var worker = new RunWorker(jobId, testApiHandler, discoveryWorker, analyzeWorker, updaterWorker, logger); var repoContentsPathDirectoryInfo = new DirectoryInfo(tempDirectory.DirectoryPath); var actualResult = await worker.RunAsync(job, repoContentsPathDirectoryInfo, "TEST-COMMIT-SHA"); var actualApiMessages = testApiHandler.ReceivedMessages.ToArray(); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs index 68264c1a2f..f9541cd6b7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/SerializationTests.cs @@ -523,12 +523,20 @@ public void DeserializeCommitOptions() yield return [ - new DependencyFileNotFound("some message", "/some/file"), + new DependencyFileNotFound("/some/file", "some message"), """ {"data":{"error-type":"dependency_file_not_found","error-details":{"message":"some message","file-path":"/some/file"}}} """ ]; + yield return + [ + new DependencyFileNotParseable("/some/file", "some message"), + """ + {"data":{"error-type":"dependency_file_not_parseable","error-details":{"message":"some message","file-path":"/some/file"}}} + """ + ]; + yield return [ new JobRepoNotFound("some message"), diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/TestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/TestBase.cs index f427981ec2..711c6943c3 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/TestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/TestBase.cs @@ -1,3 +1,10 @@ +using System.Text.Json; + +using NuGetUpdater.Core.Run.ApiModel; +using NuGetUpdater.Core.Run; + +using Xunit; + namespace NuGetUpdater.Core.Test { public abstract class TestBase @@ -6,5 +13,22 @@ protected TestBase() { MSBuildHelper.RegisterMSBuild(Environment.CurrentDirectory, Environment.CurrentDirectory); } + + protected static void ValidateError(JobErrorBase expected, JobErrorBase? actual) + { + var expectedErrorString = JsonSerializer.Serialize(expected, RunWorker.SerializerOptions); + var actualErrorString = actual is null + ? null + : JsonSerializer.Serialize(actual, RunWorker.SerializerOptions); + Assert.Equal(expectedErrorString, actualErrorString); + } + + protected static void ValidateErrorRegex(string expectedErrorRegex, JobErrorBase? actual) + { + var actualErrorString = actual is null + ? null + : JsonSerializer.Serialize(actual, RunWorker.SerializerOptions); + Assert.Matches(expectedErrorRegex, actualErrorString); + } } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/ExpectedUpdateOperationResult.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/ExpectedUpdateOperationResult.cs index 6f4b9d4089..dba5d4436a 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/ExpectedUpdateOperationResult.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/ExpectedUpdateOperationResult.cs @@ -4,5 +4,5 @@ namespace NuGetUpdater.Core.Test.Updater; public record ExpectedUpdateOperationResult : UpdateOperationResult { - public string? ErrorDetailsRegex { get; init; } = null; + public string? ErrorRegex { get; init; } = null; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs index 2b6d15ac18..e23cbf8bef 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTestBase.cs @@ -152,17 +152,10 @@ protected static async Task TestUpdateForProject( // run update experimentsManager ??= new ExperimentsManager(); - var worker = new UpdaterWorker(experimentsManager, new TestLogger()); + var worker = new UpdaterWorker("TEST-JOB-ID", experimentsManager, new TestLogger()); var projectPath = placeFilesInSrc ? $"src/{projectFilePath}" : projectFilePath; var actualResult = await worker.RunWithErrorHandlingAsync(temporaryDirectory, projectPath, dependencyName, oldVersion, newVersion, isTransitive); - if (expectedResult is { }) - { - ValidateUpdateOperationResult(expectedResult, actualResult!); - } - else if ((actualResult.ErrorType ?? ErrorType.None) != ErrorType.None) - { - throw new Exception($"Result indicates failure: ErrorType={actualResult.ErrorType}, ErrorDetails={actualResult.ErrorDetails}"); - } + ValidateUpdateOperationResult(expectedResult, actualResult!); if (additionalChecks is not null) { @@ -185,16 +178,19 @@ protected static async Task TestUpdateForProject( AssertContainsFiles(expectedResultFiles, actualResult); } - protected static void ValidateUpdateOperationResult(ExpectedUpdateOperationResult expectedResult, UpdateOperationResult actualResult) + protected static void ValidateUpdateOperationResult(ExpectedUpdateOperationResult? expectedResult, UpdateOperationResult actualResult) { - Assert.Equal(expectedResult.ErrorType, actualResult.ErrorType); - if (expectedResult.ErrorDetailsRegex is not null && actualResult.ErrorDetails is string errorDetails) + if (expectedResult?.Error is not null) + { + ValidateError(expectedResult.Error, actualResult.Error); + } + else if (expectedResult?.ErrorRegex is not null) { - Assert.Matches(expectedResult.ErrorDetailsRegex, errorDetails); + ValidateErrorRegex(expectedResult.ErrorRegex, actualResult.Error); } else { - Assert.Equivalent(expectedResult.ErrorDetails, actualResult.ErrorDetails); + Assert.Null(actualResult.Error); } } @@ -274,7 +270,7 @@ protected static async Task TestUpdateForSolution( experimentsManager ??= new ExperimentsManager(); var slnPath = Path.Combine(temporaryDirectory, slnName); - var worker = new UpdaterWorker(experimentsManager, new TestLogger()); + var worker = new UpdaterWorker("TEST-JOB-ID", experimentsManager, new TestLogger()); await worker.RunAsync(temporaryDirectory, slnPath, dependencyName, oldVersion, newVersion, isTransitive); }); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.DirsProj.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.DirsProj.cs index 82cd79478c..4c1bf4d7b8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.DirsProj.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.DirsProj.cs @@ -366,7 +366,7 @@ static async Task TestUpdateForDirsProj( experimentsManager ??= new ExperimentsManager(); var projectPath = Path.Combine(temporaryDirectory, projectFileName); - var worker = new UpdaterWorker(experimentsManager, new TestLogger()); + var worker = new UpdaterWorker("TEST-JOB-ID", experimentsManager, new TestLogger()); await worker.RunAsync(temporaryDirectory, projectPath, dependencyName, oldVersion, newVersion, isTransitive); }); diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.Mixed.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.Mixed.cs index 5f3dc797f1..b7145c355e 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.Mixed.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.Mixed.cs @@ -1,5 +1,6 @@ using System.Text.Json; +using NuGetUpdater.Core.Run.ApiModel; using NuGetUpdater.Core.Updater; using Xunit; @@ -16,8 +17,7 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() using var temporaryDirectory = await TemporaryDirectory.CreateWithContentsAsync([]); var result = new UpdateOperationResult() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "", + Error = new PrivateSourceAuthenticationFailure([""]), }; var resultFilePath = Path.Combine(temporaryDirectory.DirectoryPath, "update-result.json"); await UpdaterWorker.WriteResultFile(result, resultFilePath, new TestLogger()); @@ -26,16 +26,22 @@ public async Task ResultFileHasCorrectShapeForAuthenticationFailure() // raw result file should look like this: // { // ... - // "ErrorType": "AuthenticationFailure", - // "ErrorDetails": "", + // "Error": { + // "error-type": "private_source_authentication_failure", + // "error-details": { + // "source": "" + // } + // } // ... // } var jsonDocument = JsonDocument.Parse(resultContent); - var errorType = jsonDocument.RootElement.GetProperty("ErrorType"); - var errorDetails = jsonDocument.RootElement.GetProperty("ErrorDetails"); + var error = jsonDocument.RootElement.GetProperty("Error"); + var errorType = error.GetProperty("error-type"); + var errorDetails = error.GetProperty("error-details"); + var source = errorDetails.GetProperty("source"); - Assert.Equal("AuthenticationFailure", errorType.GetString()); - Assert.Equal("", errorDetails.GetString()); + Assert.Equal("private_source_authentication_failure", errorType.GetString()); + Assert.Equal("()", source.GetString()); } [Fact] diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs index f4b17749d3..5269ae61fb 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackageReference.cs @@ -2,6 +2,7 @@ using System.Text; using System.Text.Json; +using NuGetUpdater.Core.Run.ApiModel; using NuGetUpdater.Core.Updater; using Xunit; @@ -571,7 +572,7 @@ await File.WriteAllTextAsync(Path.Combine(srcDirectory, "NuGet.Config"), $""" // // do the update // - UpdaterWorker worker = new(new ExperimentsManager(), new TestLogger()); + UpdaterWorker worker = new("TEST-JOB-ID", new ExperimentsManager(), new TestLogger()); await worker.RunAsync(tempDirectory.DirectoryPath, projectPath, "Some.Package", "1.0.0", "1.1.0", isTransitive: false); // @@ -3482,8 +3483,7 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", """, expectedResult: new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = $"({http.BaseUrl.TrimEnd('/')}/index.json)", + Error = new PrivateSourceAuthenticationFailure([$"{http.BaseUrl.TrimEnd('/')}/index.json"]), } ); } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackagesConfig.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackagesConfig.cs index fa04a35238..a220963014 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackagesConfig.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Update/UpdateWorkerTests.PackagesConfig.cs @@ -4,6 +4,7 @@ using NuGet; +using NuGetUpdater.Core.Run.ApiModel; using NuGetUpdater.Core.Test.Updater; using NuGetUpdater.Core.Updater; @@ -2282,13 +2283,13 @@ public async Task MissingTargetsAreReported() await MockNuGetPackagesInDirectory(packages, Path.Combine(temporaryDirectory.DirectoryPath, "packages")); var resultOutputPath = Path.Combine(temporaryDirectory.DirectoryPath, "result.json"); - var worker = new UpdaterWorker(new ExperimentsManager(), new TestLogger()); + var worker = new UpdaterWorker("TEST-JOB-ID", new ExperimentsManager(), new TestLogger()); await worker.RunAsync(temporaryDirectory.DirectoryPath, "project.csproj", "Some.Package", "1.0.0", "1.1.0", isTransitive: false, resultOutputPath: resultOutputPath); var resultContents = await File.ReadAllTextAsync(resultOutputPath); - var result = JsonSerializer.Deserialize(resultContents, UpdaterWorker.SerializerOptions)!; - Assert.Equal(ErrorType.MissingFile, result.ErrorType); - Assert.Equal(Path.Combine(temporaryDirectory.DirectoryPath, "this.file.does.not.exist.targets"), result.ErrorDetails!.ToString()); + var rawResult = JsonDocument.Parse(resultContents); + Assert.Equal("dependency_file_not_found", rawResult.RootElement.GetProperty("Error").GetProperty("error-type").GetString()); + Assert.Equal(Path.Combine(temporaryDirectory.DirectoryPath, "this.file.does.not.exist.targets").NormalizePathToUnix(), rawResult.RootElement.GetProperty("Error").GetProperty("error-details").GetProperty("file-path").GetString()); } [Fact] @@ -2338,13 +2339,13 @@ public async Task MissingVisualStudioComponentTargetsAreReportedAsMissingFiles() await MockNuGetPackagesInDirectory(packages, Path.Combine(temporaryDirectory.DirectoryPath, "packages")); var resultOutputPath = Path.Combine(temporaryDirectory.DirectoryPath, "result.json"); - var worker = new UpdaterWorker(new ExperimentsManager(), new TestLogger()); + var worker = new UpdaterWorker("TEST-JOB-ID", new ExperimentsManager(), new TestLogger()); await worker.RunAsync(temporaryDirectory.DirectoryPath, "project.csproj", "Some.Package", "1.0.0", "1.1.0", isTransitive: false, resultOutputPath: resultOutputPath); var resultContents = await File.ReadAllTextAsync(resultOutputPath); - var result = JsonSerializer.Deserialize(resultContents, UpdaterWorker.SerializerOptions)!; - Assert.Equal(ErrorType.MissingFile, result.ErrorType); - Assert.Equal("$(MSBuildExtensionsPath32)/Microsoft/VisualStudio/v$(VisualStudioVersion)/Some.Visual.Studio.Component.props", result.ErrorDetails!.ToString().NormalizePathToUnix()); + var rawResult = JsonDocument.Parse(resultContents); + Assert.Equal("dependency_file_not_found", rawResult.RootElement.GetProperty("Error").GetProperty("error-type").GetString()); + Assert.Equal("$(MSBuildExtensionsPath32)/Microsoft/VisualStudio/v$(VisualStudioVersion)/Some.Visual.Studio.Component.props", rawResult.RootElement.GetProperty("Error").GetProperty("error-details").GetProperty("file-path").GetString()); } [Theory] @@ -2424,8 +2425,7 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", """, expectedResult: new() { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = $"({http.BaseUrl.TrimEnd('/')}/index.json)", + Error = new PrivateSourceAuthenticationFailure([$"{http.BaseUrl.TrimEnd('/')}/index.json"]), } ); } @@ -2555,8 +2555,7 @@ await TestUpdateForProject("Some.Package", "1.0.0", "1.1.0", """, expectedResult: new() { - ErrorType = ErrorType.Unknown, - ErrorDetailsRegex = "Response status code does not indicate success", + ErrorRegex = "Response status code does not indicate success", } ); } @@ -2633,8 +2632,7 @@ await TestUpdateForProject("Some.Package", "1.0.1", "1.0.2", """, expectedResult: new() { - ErrorType = ErrorType.UpdateNotPossible, - ErrorDetails = new[] { "Unrelated.Package.1.0.0" }, + Error = new UpdateNotPossible(["Unrelated.Package.1.0.0"]), } ); } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs index eb0b8fcd86..c4c9163dd8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/AnalyzeWorker.cs @@ -7,6 +7,7 @@ using NuGet.Versioning; using NuGetUpdater.Core.Discover; +using NuGetUpdater.Core.Run.ApiModel; namespace NuGetUpdater.Core.Analyze; @@ -16,6 +17,7 @@ public partial class AnalyzeWorker : IAnalyzeWorker { public const string AnalysisDirectoryName = "./.dependabot/analysis"; + private readonly string _jobId; private readonly ExperimentsManager _experimentsManager; private readonly ILogger _logger; @@ -25,8 +27,9 @@ public partial class AnalyzeWorker : IAnalyzeWorker Converters = { new JsonStringEnumConverter(), new RequirementArrayConverter() }, }; - public AnalyzeWorker(ExperimentsManager experimentsManager, ILogger logger) + public AnalyzeWorker(string jobId, ExperimentsManager experimentsManager, ILogger logger) { + _jobId = jobId; _experimentsManager = experimentsManager; _logger = logger; } @@ -48,26 +51,11 @@ internal async Task RunWithErrorHandlingAsync(string repoRoot, s { analysisResult = await RunAsync(repoRoot, discovery, dependencyInfo); } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) - { - var localPath = PathHelper.JoinPath(repoRoot, discovery.Path); - using var nugetContext = new NuGetContext(localPath); - analysisResult = new AnalysisResult - { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "(" + string.Join("|", nugetContext.PackageSources.Select(s => s.Source)) + ")", - UpdatedVersion = string.Empty, - CanUpdate = false, - UpdatedDependencies = [], - }; - } catch (Exception ex) { analysisResult = new AnalysisResult { - ErrorType = ErrorType.Unknown, - ErrorDetails = ex.ToString(), + Error = JobErrorBase.ErrorFromException(ex, _jobId, PathHelper.JoinPath(repoRoot, discovery.Path)), UpdatedVersion = string.Empty, CanUpdate = false, UpdatedDependencies = [], diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Clone/CloneWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Clone/CloneWorker.cs index 2e4d379195..f9dfc6b6e8 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Clone/CloneWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Clone/CloneWorker.cs @@ -70,11 +70,12 @@ public async Task RunAsync(Job job, string repoContentsPath) catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) { + // this is a _very_ specific case we want to handle before the common error handling kicks in error = new JobRepoNotFound(ex.Message); } catch (Exception ex) { - error = new UnknownError(ex, _jobId); + error = JobErrorBase.ErrorFromException(ex, _jobId, repoContentsPath); } if (error is not null) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs index 3253517c9c..951179f724 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/DiscoveryWorker.cs @@ -1,5 +1,4 @@ using System.Collections.Immutable; -using System.Net; using System.Text.Json; using System.Text.Json.Serialization; @@ -10,7 +9,7 @@ using NuGet.Frameworks; -using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Run.ApiModel; using NuGetUpdater.Core.Utilities; namespace NuGetUpdater.Core.Discover; @@ -19,6 +18,7 @@ public partial class DiscoveryWorker : IDiscoveryWorker { public const string DiscoveryResultFileName = "./.dependabot/discovery.json"; + private readonly string _jobId; private readonly ExperimentsManager _experimentsManager; private readonly ILogger _logger; private readonly HashSet _processedProjectPaths = new(StringComparer.Ordinal); private readonly HashSet _restoredMSBuildSdks = new(StringComparer.OrdinalIgnoreCase); @@ -29,8 +29,9 @@ public partial class DiscoveryWorker : IDiscoveryWorker Converters = { new JsonStringEnumConverter() }, }; - public DiscoveryWorker(ExperimentsManager experimentsManager, ILogger logger) + public DiscoveryWorker(string jobId, ExperimentsManager experimentsManager, ILogger logger) { + _jobId = jobId; _experimentsManager = experimentsManager; _logger = logger; } @@ -48,23 +49,11 @@ internal async Task RunWithErrorHandlingAsync(string r { result = await RunAsync(repoRootPath, workspacePath); } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) - { - result = new WorkspaceDiscoveryResult - { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "(" + string.Join("|", NuGetContext.GetPackageSourceUrls(PathHelper.JoinPath(repoRootPath, workspacePath))) + ")", - Path = workspacePath, - Projects = [], - }; - } catch (Exception ex) { result = new WorkspaceDiscoveryResult { - ErrorType = ErrorType.Unknown, - ErrorDetails = ex.ToString(), + Error = JobErrorBase.ErrorFromException(ex, _jobId, PathHelper.JoinPath(repoRootPath, workspacePath)), Path = workspacePath, Projects = [], }; @@ -123,8 +112,7 @@ public async Task RunAsync(string repoRootPath, string DotNetToolsJson = null, GlobalJson = null, Projects = projectResults.Where(p => p.IsSuccess).OrderBy(p => p.FilePath).ToImmutableArray(), - ErrorType = failedProjectResult.ErrorType, - ErrorDetails = failedProjectResult.FilePath, + Error = failedProjectResult.Error, IsSuccess = false, }; @@ -192,8 +180,7 @@ private async Task> RunForDirectoryAsync( ImportedFiles = ImmutableArray.Empty, AdditionalFiles = ImmutableArray.Empty, IsSuccess = false, - ErrorType = ErrorType.DependencyFileNotParseable, - ErrorDetails = "Failed to parse project file found at " + invalidProjectFile, + Error = new DependencyFileNotParseable(invalidProjectFile), }]; } if (projects.IsEmpty) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs index 2781239e9f..1011f9b122 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Discover/ProjectDiscoveryResult.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; -using System.Text.Json.Serialization; + +using NuGetUpdater.Core.Run.ApiModel; namespace NuGetUpdater.Core.Discover; @@ -8,8 +9,7 @@ public record ProjectDiscoveryResult : IDiscoveryResultWithDependencies public required string FilePath { get; init; } public required ImmutableArray Dependencies { get; init; } public bool IsSuccess { get; init; } = true; - public string? ErrorDetails { get; init; } - public ErrorType? ErrorType { get; init; } + public JobErrorBase? Error { get; init; } = null; public ImmutableArray Properties { get; init; } = []; public ImmutableArray TargetFrameworks { get; init; } = []; public ImmutableArray ReferencedProjectPaths { get; init; } = []; diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ErrorType.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ErrorType.cs deleted file mode 100644 index 23dab4bb8d..0000000000 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ErrorType.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace NuGetUpdater.Core; - -public enum ErrorType -{ - None, - AuthenticationFailure, - BadRequirement, - MissingFile, - UpdateNotPossible, - DependencyFileNotParseable, - Unknown, -} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs index 96a00680b7..e517b4dbe7 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/ExperimentsManager.cs @@ -1,6 +1,7 @@ using System.Text.Json; using NuGetUpdater.Core.Run; +using NuGetUpdater.Core.Run.ApiModel; namespace NuGetUpdater.Core; @@ -30,42 +31,28 @@ public static ExperimentsManager GetExperimentsManager(Dictionary FromJobFileAsync(string jobFilePath) + public static async Task<(ExperimentsManager ExperimentsManager, JobErrorBase? Error)> FromJobFileAsync(string jobId, string jobFilePath) { var experimentsManager = new ExperimentsManager(); - NativeResult? errorResult = null; + JobErrorBase? error = null; + var jobFileContent = string.Empty; try { - var jobFileContent = await File.ReadAllTextAsync(jobFilePath); + jobFileContent = await File.ReadAllTextAsync(jobFilePath); var jobWrapper = RunWorker.Deserialize(jobFileContent); experimentsManager = GetExperimentsManager(jobWrapper.Job.Experiments); } - catch (BadRequirementException ex) - { - errorResult = new NativeResult - { - ErrorType = ErrorType.BadRequirement, - ErrorDetails = ex.Message, - }; - } catch (JsonException ex) { - errorResult = new NativeResult - { - ErrorType = ErrorType.Unknown, - ErrorDetails = $"Error deserializing job file: {ex}: {File.ReadAllText(jobFilePath)}", - }; + // this is a very specific case where we want to log the JSON contents for easier debugging + error = JobErrorBase.ErrorFromException(new NotSupportedException($"Error deserializing job file contents: {jobFileContent}", ex), jobId, Environment.CurrentDirectory); // TODO } catch (Exception ex) { - errorResult = new NativeResult - { - ErrorType = ErrorType.Unknown, - ErrorDetails = ex.ToString(), - }; + error = JobErrorBase.ErrorFromException(ex, jobId, Environment.CurrentDirectory); // TODO } - return (experimentsManager, errorResult); + return (experimentsManager, error); } private static bool IsEnabled(Dictionary? experiments, string experimentName) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Files/PackagesConfigBuildFile.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Files/PackagesConfigBuildFile.cs index dadca2d58b..1432687b39 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Files/PackagesConfigBuildFile.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Files/PackagesConfigBuildFile.cs @@ -13,12 +13,16 @@ public static PackagesConfigBuildFile Parse(string basePath, string path, string public PackagesConfigBuildFile(string basePath, string path, XmlDocumentSyntax contents) : base(basePath, path, contents) { + var invalidPackageElements = Packages.Where(p => p.GetAttribute("id") is null || p.GetAttribute("version") is null).ToList(); + if (invalidPackageElements.Any()) + { + throw new UnparseableFileException("`package` element missing `id` or `version` attributes", path); + } } public IEnumerable Packages => Contents.RootSyntax.GetElements("package", StringComparison.OrdinalIgnoreCase); public IEnumerable GetDependencies() => Packages - .Where(p => p.GetAttribute("id") is not null && p.GetAttribute("version") is not null) .Select(p => new Dependency( p.GetAttributeValue("id", StringComparison.OrdinalIgnoreCase), p.GetAttributeValue("version", StringComparison.OrdinalIgnoreCase), diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/MissingFileException.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/MissingFileException.cs index 6d92ebed60..fad1ede35f 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/MissingFileException.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/MissingFileException.cs @@ -4,7 +4,8 @@ internal class MissingFileException : Exception { public string FilePath { get; } - public MissingFileException(string filePath) + public MissingFileException(string filePath, string? message = null) + : base(message) { FilePath = filePath; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NativeResult.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NativeResult.cs index 1ebc767ca0..be87915be2 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NativeResult.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/NativeResult.cs @@ -1,8 +1,8 @@ +using NuGetUpdater.Core.Run.ApiModel; + namespace NuGetUpdater.Core; public record NativeResult { - // TODO: nullable not required, `ErrorType.None` is the default anyway - public ErrorType? ErrorType { get; init; } - public object? ErrorDetails { get; init; } + public JobErrorBase? Error { get; init; } = null; } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs index 91d385da47..da3d722002 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotFound.cs @@ -2,10 +2,14 @@ namespace NuGetUpdater.Core.Run.ApiModel; public record DependencyFileNotFound : JobErrorBase { - public DependencyFileNotFound(string message, string filePath) + public DependencyFileNotFound(string filePath, string? message = null) : base("dependency_file_not_found") { - Details["message"] = message; - Details["file-path"] = filePath; + if (message is not null) + { + Details["message"] = message; + } + + Details["file-path"] = filePath.NormalizePathToUnix(); } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotParseable.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotParseable.cs new file mode 100644 index 0000000000..0efc03b0b1 --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/DependencyFileNotParseable.cs @@ -0,0 +1,15 @@ +namespace NuGetUpdater.Core.Run.ApiModel; + +public record DependencyFileNotParseable : JobErrorBase +{ + public DependencyFileNotParseable(string filePath, string? message = null) + : base("dependency_file_not_parseable") + { + if (message is not null) + { + Details["message"] = message; + } + + Details["file-path"] = filePath.NormalizePathToUnix(); + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs index 3b38b43d3c..1522f2d42d 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/JobErrorBase.cs @@ -1,5 +1,10 @@ +using System.Net; using System.Text.Json.Serialization; +using Microsoft.Build.Exceptions; + +using NuGetUpdater.Core.Analyze; + namespace NuGetUpdater.Core.Run.ApiModel; public abstract record JobErrorBase @@ -14,4 +19,23 @@ public JobErrorBase(string type) [JsonPropertyName("error-details")] public Dictionary Details { get; init; } = new(); + + public static JobErrorBase ErrorFromException(Exception ex, string jobId, string currentDirectory) + { + return ex switch + { + BadRequirementException badRequirement => new BadRequirement(badRequirement.Message), + HttpRequestException httpRequest => httpRequest.StatusCode switch + { + HttpStatusCode.Unauthorized or + HttpStatusCode.Forbidden => new PrivateSourceAuthenticationFailure(NuGetContext.GetPackageSourceUrls(currentDirectory)), + _ => new UnknownError(ex, jobId), + }, + InvalidProjectFileException invalidProjectFile => new DependencyFileNotParseable(invalidProjectFile.ProjectFile), + MissingFileException missingFile => new DependencyFileNotFound(missingFile.FilePath, missingFile.Message), + UnparseableFileException unparseableFile => new DependencyFileNotParseable(unparseableFile.FilePath, unparseableFile.Message), + UpdateNotPossibleException updateNotPossible => new UpdateNotPossible(updateNotPossible.Dependencies), + _ => new UnknownError(ex, jobId), + }; + } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs index 43c82ce7e3..7bce84a7e6 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs @@ -53,7 +53,7 @@ public Task RunAsync(Job job, DirectoryInfo repoContentsPath, string private async Task RunWithErrorHandlingAsync(Job job, DirectoryInfo repoContentsPath, string baseCommitSha) { JobErrorBase? error = null; - string[] lastUsedPackageSourceUrls = []; // used for error reporting below + var currentDirectory = repoContentsPath.FullName; // used for error reporting below var runResult = new RunResult() { Base64DependencyFiles = [], @@ -69,7 +69,7 @@ private async Task RunWithErrorHandlingAsync(Job job, DirectoryInfo r foreach (var directory in job.GetAllDirectories()) { var localPath = PathHelper.JoinPath(repoContentsPath.FullName, directory); - lastUsedPackageSourceUrls = NuGetContext.GetPackageSourceUrls(localPath); + currentDirectory = localPath; var result = await RunForDirectory(job, repoContentsPath, directory, baseCommitSha, experimentsManager); foreach (var dependencyFile in result.Base64DependencyFiles) { @@ -84,26 +84,9 @@ private async Task RunWithErrorHandlingAsync(Job job, DirectoryInfo r BaseCommitSha = baseCommitSha, }; } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) - { - error = new PrivateSourceAuthenticationFailure(lastUsedPackageSourceUrls); - } - catch (BadRequirementException ex) - { - error = new BadRequirement(ex.Message); - } - catch (MissingFileException ex) - { - error = new DependencyFileNotFound("file not found", ex.FilePath); - } - catch (UpdateNotPossibleException ex) - { - error = new UpdateNotPossible(ex.Dependencies); - } catch (Exception ex) { - error = new UnknownError(ex, _jobId); + error = JobErrorBase.ErrorFromException(ex, _jobId, currentDirectory); } if (error is not null) @@ -123,6 +106,8 @@ private async Task RunForDirectory(Job job, DirectoryInfo repoContent _logger.Info("Discovery JSON content:"); _logger.Info(JsonSerializer.Serialize(discoveryResult, DiscoveryWorker.SerializerOptions)); + // TODO: report errors + // report dependencies var discoveredUpdatedDependencies = GetUpdatedDependencyListFromDiscovery(discoveryResult, repoContentsPath.FullName); await _apiHandler.UpdateDependencyList(discoveredUpdatedDependencies); @@ -221,7 +206,7 @@ async Task TrackOriginalContentsAsync(string directory, string fileName) var dependencyFilePath = Path.Join(discoveryResult.Path, project.FilePath).FullyNormalizedRootedPath(); var updateResult = await _updaterWorker.RunAsync(repoContentsPath.FullName, dependencyFilePath, dependency.Name, dependency.Version!, analysisResult.UpdatedVersion, isTransitive: false); // TODO: need to report if anything was actually updated - if (updateResult.ErrorType is null || updateResult.ErrorType == ErrorType.None) + if (updateResult.Error is null) { if (dependencyLocation != dependencyFilePath) { diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/UnparseableFileException.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/UnparseableFileException.cs new file mode 100644 index 0000000000..823cd321af --- /dev/null +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/UnparseableFileException.cs @@ -0,0 +1,12 @@ +namespace NuGetUpdater.Core; + +internal class UnparseableFileException : Exception +{ + public string FilePath { get; } + + public UnparseableFileException(string message, string filePath) + : base(message) + { + FilePath = filePath; + } +} diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs index f6aaae756c..efb2fc31cf 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Updater/UpdaterWorker.cs @@ -3,6 +3,7 @@ using System.Text.Json.Serialization; using NuGetUpdater.Core.Analyze; +using NuGetUpdater.Core.Run.ApiModel; using NuGetUpdater.Core.Updater; using NuGetUpdater.Core.Utilities; @@ -10,6 +11,7 @@ namespace NuGetUpdater.Core; public class UpdaterWorker : IUpdaterWorker { + private readonly string _jobId; private readonly ExperimentsManager _experimentsManager; private readonly ILogger _logger; private readonly HashSet _processedProjectPaths = new(StringComparer.OrdinalIgnoreCase); @@ -20,8 +22,9 @@ public class UpdaterWorker : IUpdaterWorker Converters = { new JsonStringEnumConverter() }, }; - public UpdaterWorker(ExperimentsManager experimentsManager, ILogger logger) + public UpdaterWorker(string jobId, ExperimentsManager experimentsManager, ILogger logger) { + _jobId = jobId; _experimentsManager = experimentsManager; _logger = logger; } @@ -43,42 +46,15 @@ internal async Task RunWithErrorHandlingAsync(string repo { result = await RunAsync(repoRootPath, workspacePath, dependencyName, previousDependencyVersion, newDependencyVersion, isTransitive); } - catch (HttpRequestException ex) - when (ex.StatusCode == HttpStatusCode.Unauthorized || ex.StatusCode == HttpStatusCode.Forbidden) + catch (Exception ex) { if (!Path.IsPathRooted(workspacePath) || !File.Exists(workspacePath)) { workspacePath = Path.GetFullPath(Path.Join(repoRootPath, workspacePath)); } - - result = new() - { - ErrorType = ErrorType.AuthenticationFailure, - ErrorDetails = "(" + string.Join("|", NuGetContext.GetPackageSourceUrls(workspacePath)) + ")", - }; - } - catch (MissingFileException ex) - { - result = new() - { - ErrorType = ErrorType.MissingFile, - ErrorDetails = ex.FilePath, - }; - } - catch (UpdateNotPossibleException ex) - { - result = new() - { - ErrorType = ErrorType.UpdateNotPossible, - ErrorDetails = ex.Dependencies, - }; - } - catch (Exception ex) - { result = new() { - ErrorType = ErrorType.Unknown, - ErrorDetails = ex.ToString(), + Error = JobErrorBase.ErrorFromException(ex, _jobId, workspacePath), }; } diff --git a/nuget/lib/dependabot/nuget/native_helpers.rb b/nuget/lib/dependabot/nuget/native_helpers.rb index 2de499c279..963b85d572 100644 --- a/nuget/lib/dependabot/nuget/native_helpers.rb +++ b/nuget/lib/dependabot/nuget/native_helpers.rb @@ -11,6 +11,11 @@ module Nuget module NativeHelpers extend T::Sig + sig { returns(String) } + def self.job_id + ENV.fetch("DEPENDABOT_JOB_ID") + end + sig { returns(String) } def self.native_helpers_root helpers_root = ENV.fetch("DEPENDABOT_NATIVE_HELPERS_PATH", nil) @@ -66,6 +71,8 @@ def self.get_nuget_discover_tool_command(job_path:, repo_root:, workspace_path:, command_parts = [ exe_path, "discover", + "--job-id", + job_id, "--job-path", job_path, "--repo-root", @@ -81,6 +88,8 @@ def self.get_nuget_discover_tool_command(job_path:, repo_root:, workspace_path:, fingerprint = [ exe_path, "discover", + "--job-id", + "", "--job-path", "", "--repo-root", @@ -127,6 +136,8 @@ def self.get_nuget_analyze_tool_command(job_path:, repo_root:, discovery_file_pa command_parts = [ exe_path, "analyze", + "--job-id", + job_id, "--job-path", job_path, "--repo-root", @@ -144,6 +155,8 @@ def self.get_nuget_analyze_tool_command(job_path:, repo_root:, discovery_file_pa fingerprint = [ exe_path, "analyze", + "--job-id", + "", "--job-path", "", "--discovery-file-path", @@ -190,6 +203,8 @@ def self.get_nuget_updater_tool_command(job_path:, repo_root:, proj_path:, depen command_parts = [ exe_path, "update", + "--job-id", + job_id, "--job-path", job_path, "--repo-root", @@ -212,6 +227,8 @@ def self.get_nuget_updater_tool_command(job_path:, repo_root:, proj_path:, depen fingerprint = [ exe_path, "update", + "--job-id", + "", "--job-path", "", "--repo-root", @@ -290,29 +307,37 @@ def self.install_dotnet_sdks puts output end + # rubocop:disable Metrics/AbcSize sig { params(json: T::Hash[String, T.untyped]).void } def self.ensure_no_errors(json) - error_type = T.let(json.fetch("ErrorType", nil), T.nilable(String)) - error_details = json.fetch("ErrorDetails", nil) + error = T.let(json.fetch("Error", nil), T.nilable(T::Hash[String, T.untyped])) + return if error.nil? + + error_type = T.let(error.fetch("error-type"), String) + error_details = T.let(error.fetch("error-details", {}), T::Hash[String, T.untyped]) + case error_type - when "None", nil - # no issue - when "DependencyFileNotParseable" - raise DependencyFileNotParseable, T.must(T.let(error_details, T.nilable(String))) - when "AuthenticationFailure" - raise PrivateSourceAuthenticationFailure, T.let(error_details, T.nilable(String)) - when "BadRequirement" - raise BadRequirementError, T.let(error_details, T.nilable(String)) - when "MissingFile" - raise DependencyFileNotFound, T.let(error_details, T.nilable(String)) - when "UpdateNotPossible" - raise UpdateNotPossible, T.let(error_details, T::Array[String]) - when "Unknown" - raise DependabotError, T.let(error_details, String) + when "dependency_file_not_found" + file_path = T.let(error_details.fetch("file-path"), String) + message = T.let(error_details.fetch("message", nil), T.nilable(String)) + raise DependencyFileNotFound.new(file_path, message) + when "dependency_file_not_parseable" + file_path = T.let(error_details.fetch("file-path"), String) + message = T.let(error_details.fetch("message", nil), T.nilable(String)) + raise DependencyFileNotParseable.new(file_path, message) + when "illformed_requirement" + raise BadRequirementError, T.let(error_details.fetch("message"), String) + when "private_source_authentication_failure" + raise PrivateSourceAuthenticationFailure, T.let(error_details.fetch("source"), String) + when "update_not_possible" + raise UpdateNotPossible, T.let(error_details.fetch("dependencies"), T::Array[String]) + when "unknown_error" + raise DependabotError, error_details.to_json else raise "Unexpected error type from native tool: #{error_type}: #{error_details}" end end + # rubocop:enable Metrics/AbcSize end end end diff --git a/nuget/spec/dependabot/nuget/file_fetcher_spec.rb b/nuget/spec/dependabot/nuget/file_fetcher_spec.rb index b2eb1be9a1..e56c04e589 100644 --- a/nuget/spec/dependabot/nuget/file_fetcher_spec.rb +++ b/nuget/spec/dependabot/nuget/file_fetcher_spec.rb @@ -260,8 +260,12 @@ def run_fetch_test(files_on_disk:, discovery_content_hash:, &_block) Projects: [], GlobalJson: nil, DotNetToolsJson: nil, - ErrorType: "AuthenticationFailure", - ErrorDetails: "the-error-details" + Error: { + "error-type": "private_source_authentication_failure", + "error-details": { + source: "some-package-source" + } + } } ) do expect { fetched_file_paths }.to raise_error(Dependabot::PrivateSourceAuthenticationFailure) diff --git a/nuget/spec/dependabot/nuget/file_parser_spec.rb b/nuget/spec/dependabot/nuget/file_parser_spec.rb index 7b9fecdcd0..6bb943f574 100644 --- a/nuget/spec/dependabot/nuget/file_parser_spec.rb +++ b/nuget/spec/dependabot/nuget/file_parser_spec.rb @@ -69,6 +69,7 @@ def clean_common_files def run_parser_test(&_block) ENV["DEPENDABOT_NUGET_CACHE_DISABLED"] = "true" + ENV["DEPENDABOT_JOB_ID"] = "TEST-JOB-ID" clean_common_files Dependabot::Nuget::DiscoveryJsonReader.testonly_clear_caches @@ -89,6 +90,7 @@ def run_parser_test(&_block) ensure Dependabot::Nuget::DiscoveryJsonReader.testonly_clear_caches ENV.delete("DEPENDABOT_NUGET_CACHE_DISABLED") + ENV.delete("DEPENDABOT_JOB_ID") clean_common_files end diff --git a/nuget/spec/dependabot/nuget/file_updater_spec.rb b/nuget/spec/dependabot/nuget/file_updater_spec.rb index becb6418ce..0cd3983541 100644 --- a/nuget/spec/dependabot/nuget/file_updater_spec.rb +++ b/nuget/spec/dependabot/nuget/file_updater_spec.rb @@ -92,6 +92,7 @@ def clean_common_files def run_update_test(&_block) # caching is explicitly required for these tests ENV["DEPENDABOT_NUGET_CACHE_DISABLED"] = "false" + ENV["DEPENDABOT_JOB_ID"] = "TEST-JOB-ID" Dependabot::Nuget::DiscoveryJsonReader.testonly_clear_caches clean_common_files @@ -123,6 +124,7 @@ def run_update_test(&_block) ensure Dependabot::Nuget::DiscoveryJsonReader.testonly_clear_caches ENV["DEPENDABOT_NUGET_CACHE_DISABLED"] = "true" + ENV.delete("DEPENDABOT_JOB_ID") clean_common_files end diff --git a/nuget/spec/dependabot/nuget/native_helpers_spec.rb b/nuget/spec/dependabot/nuget/native_helpers_spec.rb index 122aa8a656..c93b8d02ef 100644 --- a/nuget/spec/dependabot/nuget/native_helpers_spec.rb +++ b/nuget/spec/dependabot/nuget/native_helpers_spec.rb @@ -36,8 +36,17 @@ let(:is_transitive) { false } let(:result_output_path) { "/path/to/result.json" } + before do + ENV["DEPENDABOT_JOB_ID"] = "TEST-JOB-ID" + end + + after do + ENV.delete("DEPENDABOT_JOB_ID") + end + it "returns a properly formatted command with spaces on the path" do - expect(command).to eq("/path/to/NuGetUpdater.Cli update --job-path /path/to/job.json --repo-root /path/to/repo " \ + expect(command).to eq("/path/to/NuGetUpdater.Cli update --job-id TEST-JOB-ID --job-path /path/to/job.json " \ + "--repo-root /path/to/repo " \ '--solution-or-project /path/to/repo/src/some\ project/some_project.csproj ' \ "--dependency Some.Package --new-version 1.2.3 --previous-version 1.2.2 " \ "--result-output-path /path/to/result.json") @@ -92,8 +101,12 @@ .to receive(:run_shell_command) .and_wrap_original do |_original_method, *_args, &_block| result = { - ErrorType: "AuthenticationFailure", - ErrorDetails: "the-error-details" + Error: { + "error-type": "private_source_authentication_failure", + "error-details": { + source: "some-url" + } + } } File.write(described_class.update_result_file_path, result.to_json) end @@ -118,8 +131,13 @@ .to receive(:run_shell_command) .and_wrap_original do |_original_method, *_args, &_block| result = { - ErrorType: "MissingFile", - ErrorDetails: "the-error-details" + Error: { + "error-type": "dependency_file_not_found", + "error-details": { + message: "some message", + "file-path": "/some/file" + } + } } File.write(described_class.update_result_file_path, result.to_json) end @@ -145,7 +163,7 @@ # defaults to no error return nil rescue StandardError => e - return e.to_s + return e end context "when nothing is reported" do @@ -154,61 +172,121 @@ it { is_expected.to be_nil } end - context "when the error is expclicitly none" do + context "when the error is expclicitly null" do let(:json) do { - ErrorType: "None" + Error: nil }.to_json end it { is_expected.to be_nil } end - context "when an authentication failure is encountered" do + context "when a dependency file was not found" do + let(:json) do + { + Error: { + "error-type": "dependency_file_not_found", + "error-details": { + message: "some message", + "file-path": "/some/file" + } + } + }.to_json + end + + it { is_expected.to be_a Dependabot::DependencyFileNotFound } + end + + context "when a file is not parseable" do let(:json) do { - ErrorType: "AuthenticationFailure", - ErrorDetails: "(some-source)" + Error: { + "error-type": "dependency_file_not_parseable", + "error-details": { + message: "some message", + "file-path": "/some/file" + } + } }.to_json end - it { is_expected.to include(": (some-source)") } + it { is_expected.to be_a Dependabot::DependencyFileNotParseable } end - context "when a file is missing" do + context "when a requirement cannot be parsed" do let(:json) do { - ErrorType: "MissingFile", - ErrorDetails: "some.file" + Error: { + "error-type": "illformed_requirement", + "error-details": { + message: "some message" + } + } }.to_json end - it { is_expected.to include("some.file not found") } + it { is_expected.to be_a Dependabot::BadRequirementError } + end + + context "when an authenticated feed was rejected" do + let(:json) do + { + Error: { + "error-type": "private_source_authentication_failure", + "error-details": { + source: "some-url" + } + } + }.to_json + end + + it { is_expected.to be_a Dependabot::PrivateSourceAuthenticationFailure } end context "when an update is not possible" do let(:json) do { - ErrorType: "UpdateNotPossible", - ErrorDetails: [ - "dependency 1", - "dependency 2" - ] + Error: { + "error-type": "update_not_possible", + "error-details": { + dependencies: %w(dep1 dep2) + } + } }.to_json end - it { is_expected.to include("dependency 1, dependency 2") } + it { is_expected.to be_a Dependabot::UpdateNotPossible } end - context "when any other error is returned" do + context "when an unknown error is reported" do let(:json) do { - ErrorType: "Unknown", - ErrorDetails: "some error details" + Error: { + "error-type": "unknown_error", + "error-details": { + message: "some message" + } + } + }.to_json + end + + it { is_expected.to be_a Dependabot::DependabotError } + end + + context "when any other type of error is returned" do + let(:json) do + { + Error: { + "error-type": "some_error_type_that_is_not_handled", + "error-details": { + message: "some message" + } + } }.to_json end - it { is_expected.to include("some error details") } + it { is_expected.to be_a StandardError } end end end diff --git a/nuget/spec/dependabot/nuget/update_checker_spec.rb b/nuget/spec/dependabot/nuget/update_checker_spec.rb index 8904a31ff4..c4b9caa6e7 100644 --- a/nuget/spec/dependabot/nuget/update_checker_spec.rb +++ b/nuget/spec/dependabot/nuget/update_checker_spec.rb @@ -287,8 +287,12 @@ def intercept_native_tools(discovery_content_hash:, dependency_name:, analysis_c CanUpdate: false, VersionComesFromMultiDependencyProperty: false, UpdatedDependencies: [], - ErrorType: "AuthenticationFailure", - ErrorDetails: "the-error-details" + Error: { + "error-type": "private_source_authentication_failure", + "error-details": { + source: "some-package-source" + } + } } ) end From 43e0298e0694e13a9ea83e88d21157b45d29a9bf Mon Sep 17 00:00:00 2001 From: Alfred Mazimbe Date: Thu, 16 Jan 2025 09:54:15 +0000 Subject: [PATCH 02/72] Handle DependencyFileNotParseable error on Updater.run (#11288) * Handle DependencyFileNotParseable error on Updater.run * Stop handling standard error * Fix failing CI checks * Count error as stdout * Remove rescued exception from stdout test --- common/lib/dependabot/errors.rb | 10 ++++++ .../tests/testdata/vu-group-semver-error.txt | 1 - .../lib/dependabot/update_files_command.rb | 30 ++++++++++++++--- .../dependabot/update_files_command_spec.rb | 32 +++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/common/lib/dependabot/errors.rb b/common/lib/dependabot/errors.rb index 3c604199e7..2171a9ea62 100644 --- a/common/lib/dependabot/errors.rb +++ b/common/lib/dependabot/errors.rb @@ -166,6 +166,7 @@ def self.parser_error_details(error) # rubocop:disable Lint/RedundantCopDisableDirective # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/AbcSize sig { params(error: StandardError).returns(T.nilable(T::Hash[Symbol, T.untyped])) } def self.updater_error_details(error) case error @@ -179,6 +180,14 @@ def self.updater_error_details(error) "error-type": "dependency_file_not_evaluatable", "error-detail": { message: error.message } } + when Dependabot::DependencyFileNotParseable + { + "error-type": "dependency_file_not_parseable", + "error-detail": { + message: error.message, + "file-path": error.file_path + } + } when Dependabot::GitDependenciesNotReachable { "error-type": "git_dependencies_not_reachable", @@ -294,6 +303,7 @@ def self.updater_error_details(error) # rubocop:enable Metrics/MethodLength # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Lint/RedundantCopDisableDirective + # rubocop:enable Metrics/AbcSize class DependabotError < StandardError extend T::Sig diff --git a/silent/tests/testdata/vu-group-semver-error.txt b/silent/tests/testdata/vu-group-semver-error.txt index e99ac0e32c..8d24fcb248 100644 --- a/silent/tests/testdata/vu-group-semver-error.txt +++ b/silent/tests/testdata/vu-group-semver-error.txt @@ -2,7 +2,6 @@ ! dependabot update -f input.yml --local . --updater-image ghcr.io/dependabot/dependabot-updater-silent # If this appears in the logs twice that means it tried to create a PR again -stderr -count=1 DependencyFileNotParseable stdout -count=2 create_pull_request pr-created expected-group.json pr-created expected-individual.json diff --git a/updater/lib/dependabot/update_files_command.rb b/updater/lib/dependabot/update_files_command.rb index d04b4abc8d..ddcdb1f1ef 100644 --- a/updater/lib/dependabot/update_files_command.rb +++ b/updater/lib/dependabot/update_files_command.rb @@ -37,11 +37,16 @@ def perform_job # # As above, we can remove the responsibility for handling fatal/job halting # errors from Dependabot::Updater entirely. - Dependabot::Updater.new( - service: service, - job: job, - dependency_snapshot: dependency_snapshot - ).run + begin + Dependabot::Updater.new( + service: service, + job: job, + dependency_snapshot: dependency_snapshot + ).run + rescue Dependabot::DependencyFileNotParseable => e + handle_dependency_file_not_parseable_error(e) + return service.mark_job_as_processed(Environment.job_definition["base_commit_sha"]) + end # Wait for all PRs to be created service.wait_for_calls_to_finish @@ -129,5 +134,20 @@ def handle_parser_error(error) ) end # rubocop:enable Metrics/AbcSize, Layout/LineLength, Metrics/MethodLength + + def handle_dependency_file_not_parseable_error(error) + error_details = Dependabot.updater_error_details(error) + + service.record_update_job_error( + error_type: T.must(error_details).fetch(:"error-type"), + error_details: T.must(error_details)[:"error-detail"] + ) + return unless Experiments.enabled?(:record_update_job_unknown_error) + + service.record_update_job_unknown_error( + error_type: T.must(error_details).fetch(:"error-type"), + error_details: T.must(error_details)[:"error-detail"] + ) + end end end diff --git a/updater/spec/dependabot/update_files_command_spec.rb b/updater/spec/dependabot/update_files_command_spec.rb index db17767e0f..5e41194382 100644 --- a/updater/spec/dependabot/update_files_command_spec.rb +++ b/updater/spec/dependabot/update_files_command_spec.rb @@ -296,6 +296,38 @@ end end + context "with a Dependabot::DependencyFileNotParseable error" do + let(:error) { Dependabot::DependencyFileNotParseable.new("path/to/file", "a") } + + let(:snapshot) do + instance_double(Dependabot::DependencySnapshot, + base_commit_sha: "1c6331732c41e4557a16dacb82534f1d1c831848") + end + + let(:updater) do + instance_double(Dependabot::Updater, + service: service, + job: job, + dependency_snapshot: snapshot) + end + + before do + allow(Dependabot::DependencySnapshot).to receive(:create_from_job_definition).and_return(snapshot) + allow(Dependabot::Updater).to receive(:new).and_return(updater) + allow(updater).to receive(:run).and_raise(error) + end + + it "only records a job error" do + expect(service).not_to receive(:capture_exception) + expect(service).to receive(:record_update_job_error).with( + error_type: "dependency_file_not_parseable", + error_details: { "file-path": "path/to/file", message: "a" } + ) + + perform_job + end + end + context "with a Dependabot::DependencyFileNotFound error" do let(:error) { Dependabot::DependencyFileNotFound.new("path/to/file") } From 1b47713d345cc248f5a647a41f427510492d51b7 Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Thu, 16 Jan 2025 19:43:35 +0900 Subject: [PATCH 03/72] Bump Hex Erlang to 25 (#11308) Currently we are using 24 from apt, and there are notes that we should not use ESL for this version of Ubuntu, but times have changed and ESL seems to have caught up. I think ESL is the best way forward outside of using the RabbitMQ erlang solutions (which I'm fine with doing if desired, I'm just more familiar with ESL). This also bumps Elixir by a patch version. Co-authored-by: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> --- hex/Dockerfile | 24 +++++++++------------ hex/spec/dependabot/hex/file_parser_spec.rb | 2 +- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/hex/Dockerfile b/hex/Dockerfile index 2b9060e88f..454bd91e16 100644 --- a/hex/Dockerfile +++ b/hex/Dockerfile @@ -2,25 +2,21 @@ FROM ghcr.io/dependabot/dependabot-updater-core RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ - gnupg2 + gnupg2 sudo wget + +ARG ERLANG_MAJOR_VERSION=25 + +RUN echo "deb http://binaries2.erlang-solutions.com/ubuntu/ jammy-esl-erlang-25 contrib" >> /etc/apt/sources.list +RUN wget https://binaries2.erlang-solutions.com/GPG-KEY-pmanager.asc \ + && sudo apt-key add GPG-KEY-pmanager.asc -# Install Erlang -# Note: Currently we install from Ubuntu PPA to unblock upgrading to Ubuntu 22.04, but we'd be happy to accept a PR -# switching to a more up to date PPA. See also: -# * https://github.com/esl/packages/issues/15 -# * https://github.com/dependabot/dependabot-core/pull/7865 -# * https://erlangforums.com/t/erlang-solutions-apt-package-for-otp-25/1552/1 -# * https://erlangforums.com/t/the-eef-is-looking-for-volunteers-to-take-over-esls-build-packages/2238/1 -ARG ERLANG_MAJOR_VERSION=24 -ARG ERLANG_VERSION=1:${ERLANG_MAJOR_VERSION}.2.1+dfsg-1ubuntu0.1 RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - erlang=${ERLANG_VERSION} + && apt-get install -y --no-install-recommends esl-erlang # Install Elixir # https://github.com/elixir-lang/elixir/releases -ARG ELIXIR_VERSION=v1.14.4 -ARG ELIXIR_CHECKSUM=b5705275d62ce3dae172d25d7ea567fffdabb45458abeb29c0189923b907367c +ARG ELIXIR_VERSION=v1.14.5 +ARG ELIXIR_CHECKSUM=f3b35d9fa61da7e93c9979cb8a08f64a9ce7074aeda66fae994f2a4ea2e4f82e RUN curl -sSLfO https://github.com/elixir-lang/elixir/releases/download/${ELIXIR_VERSION}/elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ && echo "$ELIXIR_CHECKSUM elixir-otp-${ERLANG_MAJOR_VERSION}.zip" | sha256sum -c - \ && unzip -d /usr/local/elixir -x elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ diff --git a/hex/spec/dependabot/hex/file_parser_spec.rb b/hex/spec/dependabot/hex/file_parser_spec.rb index bc5fca2d5e..fd47982dd7 100644 --- a/hex/spec/dependabot/hex/file_parser_spec.rb +++ b/hex/spec/dependabot/hex/file_parser_spec.rb @@ -476,7 +476,7 @@ it "returns the correct language" do expect(language.name).to eq "elixir" expect(language.requirement).to be_nil - expect(language.version.to_s).to eq "1.14.4" + expect(language.version.to_s).to eq "1.14.5" end end end From e0c8d704e367a3362cf44e301d1de46bbe7d5bc4 Mon Sep 17 00:00:00 2001 From: "dependabot-core-action-automation[bot]" <98560086+dependabot-core-action-automation[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:16:42 +0000 Subject: [PATCH 04/72] v0.293.0 (#11258) Release notes: https://github.com/dependabot/dependabot-core/releases/tag/v0.293.0 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- Gemfile.lock | 84 ++++++++++++++++++++-------------------- common/lib/dependabot.rb | 2 +- updater/Gemfile.lock | 84 ++++++++++++++++++++-------------------- 3 files changed, 85 insertions(+), 85 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 3e717b33e3..8879a68d6a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,20 +1,20 @@ PATH remote: bundler specs: - dependabot-bundler (0.292.0) - dependabot-common (= 0.292.0) + dependabot-bundler (0.293.0) + dependabot-common (= 0.293.0) parallel (~> 1.24) PATH remote: cargo specs: - dependabot-cargo (0.292.0) - dependabot-common (= 0.292.0) + dependabot-cargo (0.293.0) + dependabot-common (= 0.293.0) PATH remote: common specs: - dependabot-common (0.292.0) + dependabot-common (0.293.0) aws-sdk-codecommit (~> 1.28) aws-sdk-ecr (~> 1.5) bundler (>= 1.16, < 3.0.0) @@ -38,113 +38,113 @@ PATH PATH remote: composer specs: - dependabot-composer (0.292.0) - dependabot-common (= 0.292.0) + dependabot-composer (0.293.0) + dependabot-common (= 0.293.0) PATH remote: devcontainers specs: - dependabot-devcontainers (0.292.0) - dependabot-common (= 0.292.0) + dependabot-devcontainers (0.293.0) + dependabot-common (= 0.293.0) PATH remote: docker specs: - dependabot-docker (0.292.0) - dependabot-common (= 0.292.0) + dependabot-docker (0.293.0) + dependabot-common (= 0.293.0) PATH remote: dotnet_sdk specs: - dependabot-dotnet_sdk (0.292.0) - dependabot-common (= 0.292.0) + dependabot-dotnet_sdk (0.293.0) + dependabot-common (= 0.293.0) PATH remote: elm specs: - dependabot-elm (0.292.0) - dependabot-common (= 0.292.0) + dependabot-elm (0.293.0) + dependabot-common (= 0.293.0) PATH remote: git_submodules specs: - dependabot-git_submodules (0.292.0) - dependabot-common (= 0.292.0) + dependabot-git_submodules (0.293.0) + dependabot-common (= 0.293.0) parseconfig (~> 1.0, < 1.1.0) PATH remote: github_actions specs: - dependabot-github_actions (0.292.0) - dependabot-common (= 0.292.0) + dependabot-github_actions (0.293.0) + dependabot-common (= 0.293.0) PATH remote: go_modules specs: - dependabot-go_modules (0.292.0) - dependabot-common (= 0.292.0) + dependabot-go_modules (0.293.0) + dependabot-common (= 0.293.0) PATH remote: gradle specs: - dependabot-gradle (0.292.0) - dependabot-common (= 0.292.0) - dependabot-maven (= 0.292.0) + dependabot-gradle (0.293.0) + dependabot-common (= 0.293.0) + dependabot-maven (= 0.293.0) PATH remote: hex specs: - dependabot-hex (0.292.0) - dependabot-common (= 0.292.0) + dependabot-hex (0.293.0) + dependabot-common (= 0.293.0) PATH remote: maven specs: - dependabot-maven (0.292.0) - dependabot-common (= 0.292.0) + dependabot-maven (0.293.0) + dependabot-common (= 0.293.0) PATH remote: npm_and_yarn specs: - dependabot-npm_and_yarn (0.292.0) - dependabot-common (= 0.292.0) + dependabot-npm_and_yarn (0.293.0) + dependabot-common (= 0.293.0) PATH remote: nuget specs: - dependabot-nuget (0.292.0) - dependabot-common (= 0.292.0) + dependabot-nuget (0.293.0) + dependabot-common (= 0.293.0) rubyzip (>= 2.3.2, < 3.0) PATH remote: pub specs: - dependabot-pub (0.292.0) - dependabot-common (= 0.292.0) + dependabot-pub (0.293.0) + dependabot-common (= 0.293.0) PATH remote: python specs: - dependabot-python (0.292.0) - dependabot-common (= 0.292.0) + dependabot-python (0.293.0) + dependabot-common (= 0.293.0) PATH remote: silent specs: - dependabot-silent (0.292.0) - dependabot-common (= 0.292.0) + dependabot-silent (0.293.0) + dependabot-common (= 0.293.0) PATH remote: swift specs: - dependabot-swift (0.292.0) - dependabot-common (= 0.292.0) + dependabot-swift (0.293.0) + dependabot-common (= 0.293.0) PATH remote: terraform specs: - dependabot-terraform (0.292.0) - dependabot-common (= 0.292.0) + dependabot-terraform (0.293.0) + dependabot-common (= 0.293.0) GEM remote: https://rubygems.org/ diff --git a/common/lib/dependabot.rb b/common/lib/dependabot.rb index 1195a7ca24..af32c00ad3 100644 --- a/common/lib/dependabot.rb +++ b/common/lib/dependabot.rb @@ -2,5 +2,5 @@ # frozen_string_literal: true module Dependabot - VERSION = "0.292.0" + VERSION = "0.293.0" end diff --git a/updater/Gemfile.lock b/updater/Gemfile.lock index f3096655cd..c5575a91d7 100644 --- a/updater/Gemfile.lock +++ b/updater/Gemfile.lock @@ -1,20 +1,20 @@ PATH remote: ../bundler specs: - dependabot-bundler (0.292.0) - dependabot-common (= 0.292.0) + dependabot-bundler (0.293.0) + dependabot-common (= 0.293.0) parallel (~> 1.24) PATH remote: ../cargo specs: - dependabot-cargo (0.292.0) - dependabot-common (= 0.292.0) + dependabot-cargo (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../common specs: - dependabot-common (0.292.0) + dependabot-common (0.293.0) aws-sdk-codecommit (~> 1.28) aws-sdk-ecr (~> 1.5) bundler (>= 1.16, < 3.0.0) @@ -38,113 +38,113 @@ PATH PATH remote: ../composer specs: - dependabot-composer (0.292.0) - dependabot-common (= 0.292.0) + dependabot-composer (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../devcontainers specs: - dependabot-devcontainers (0.292.0) - dependabot-common (= 0.292.0) + dependabot-devcontainers (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../docker specs: - dependabot-docker (0.292.0) - dependabot-common (= 0.292.0) + dependabot-docker (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../dotnet_sdk specs: - dependabot-dotnet_sdk (0.292.0) - dependabot-common (= 0.292.0) + dependabot-dotnet_sdk (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../elm specs: - dependabot-elm (0.292.0) - dependabot-common (= 0.292.0) + dependabot-elm (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../git_submodules specs: - dependabot-git_submodules (0.292.0) - dependabot-common (= 0.292.0) + dependabot-git_submodules (0.293.0) + dependabot-common (= 0.293.0) parseconfig (~> 1.0, < 1.1.0) PATH remote: ../github_actions specs: - dependabot-github_actions (0.292.0) - dependabot-common (= 0.292.0) + dependabot-github_actions (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../go_modules specs: - dependabot-go_modules (0.292.0) - dependabot-common (= 0.292.0) + dependabot-go_modules (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../gradle specs: - dependabot-gradle (0.292.0) - dependabot-common (= 0.292.0) - dependabot-maven (= 0.292.0) + dependabot-gradle (0.293.0) + dependabot-common (= 0.293.0) + dependabot-maven (= 0.293.0) PATH remote: ../hex specs: - dependabot-hex (0.292.0) - dependabot-common (= 0.292.0) + dependabot-hex (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../maven specs: - dependabot-maven (0.292.0) - dependabot-common (= 0.292.0) + dependabot-maven (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../npm_and_yarn specs: - dependabot-npm_and_yarn (0.292.0) - dependabot-common (= 0.292.0) + dependabot-npm_and_yarn (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../nuget specs: - dependabot-nuget (0.292.0) - dependabot-common (= 0.292.0) + dependabot-nuget (0.293.0) + dependabot-common (= 0.293.0) rubyzip (>= 2.3.2, < 3.0) PATH remote: ../pub specs: - dependabot-pub (0.292.0) - dependabot-common (= 0.292.0) + dependabot-pub (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../python specs: - dependabot-python (0.292.0) - dependabot-common (= 0.292.0) + dependabot-python (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../silent specs: - dependabot-silent (0.292.0) - dependabot-common (= 0.292.0) + dependabot-silent (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../swift specs: - dependabot-swift (0.292.0) - dependabot-common (= 0.292.0) + dependabot-swift (0.293.0) + dependabot-common (= 0.293.0) PATH remote: ../terraform specs: - dependabot-terraform (0.292.0) - dependabot-common (= 0.292.0) + dependabot-terraform (0.293.0) + dependabot-common (= 0.293.0) GEM remote: https://rubygems.org/ From cd8ff57cf4a528798f0f82021c3c7857b1e81b07 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Thu, 16 Jan 2025 13:38:12 +0000 Subject: [PATCH 05/72] Sorbet: add strict typing for GemfileUpdater and GemspecDependencyNameFinder --- .../bundler/file_updater/gemfile_updater.rb | 38 ++++++++++++++----- .../gemspec_dependency_name_finder.rb | 11 +++++- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/bundler/lib/dependabot/bundler/file_updater/gemfile_updater.rb b/bundler/lib/dependabot/bundler/file_updater/gemfile_updater.rb index b8eeb8965e..c889b64722 100644 --- a/bundler/lib/dependabot/bundler/file_updater/gemfile_updater.rb +++ b/bundler/lib/dependabot/bundler/file_updater/gemfile_updater.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/bundler/file_updater" @@ -7,19 +7,23 @@ module Dependabot module Bundler class FileUpdater class GemfileUpdater + extend T::Sig + GEMFILE_FILENAMES = %w(Gemfile gems.rb).freeze require_relative "git_pin_replacer" require_relative "git_source_remover" require_relative "requirement_replacer" + sig { params(dependencies: T::Array[Dependabot::Dependency], gemfile: Dependabot::DependencyFile).void } def initialize(dependencies:, gemfile:) @dependencies = dependencies @gemfile = gemfile end + sig { returns(String) } def updated_gemfile_content - content = gemfile.content + content = T.must(gemfile.content) dependencies.each do |dependency| content = replace_gemfile_version_requirement( @@ -38,21 +42,27 @@ def updated_gemfile_content private + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependencies + + sig { returns(Dependabot::DependencyFile) } attr_reader :gemfile + sig do + params(dependency: Dependabot::Dependency, file: Dependabot::DependencyFile, content: String).returns(String) + end def replace_gemfile_version_requirement(dependency, file, content) return content unless requirement_changed?(file, dependency) updated_requirement = dependency.requirements .find { |r| r[:file] == file.name } - .fetch(:requirement) + &.fetch(:requirement) previous_requirement = dependency.previous_requirements - .find { |r| r[:file] == file.name } - .fetch(:requirement) + &.find { |r| r[:file] == file.name } + &.fetch(:requirement) RequirementReplacer.new( dependency: dependency, @@ -62,17 +72,19 @@ def replace_gemfile_version_requirement(dependency, file, content) ).rewrite(content) end + sig { params(file: Dependabot::DependencyFile, dependency: Dependabot::Dependency).returns(T::Boolean) } def requirement_changed?(file, dependency) changed_requirements = - dependency.requirements - dependency.previous_requirements + dependency.requirements - T.must(dependency.previous_requirements) changed_requirements.any? { |f| f[:file] == file.name } end + sig { params(dependency: Dependabot::Dependency).returns(T::Boolean) } def remove_git_source?(dependency) old_gemfile_req = dependency.previous_requirements - .find { |f| GEMFILE_FILENAMES.include?(f[:file]) } + &.find { |f| GEMFILE_FILENAMES.include?(f[:file]) } return false unless old_gemfile_req&.dig(:source, :type) == "git" @@ -80,9 +92,10 @@ def remove_git_source?(dependency) dependency.requirements .find { |f| GEMFILE_FILENAMES.include?(f[:file]) } - new_gemfile_req[:source].nil? + T.must(new_gemfile_req)[:source].nil? end + sig { params(dependency: Dependabot::Dependency, file: Dependabot::DependencyFile).returns(T::Boolean) } def update_git_pin?(dependency, file) new_gemfile_req = dependency.requirements @@ -91,18 +104,23 @@ def update_git_pin?(dependency, file) # If the new requirement is a git dependency with a ref then there's # no harm in doing an update - new_gemfile_req.dig(:source, :ref) + !T.must(new_gemfile_req).dig(:source, :ref).nil? end + sig { params(dependency: Dependabot::Dependency, content: String).returns(String) } def remove_gemfile_git_source(dependency, content) GitSourceRemover.new(dependency: dependency).rewrite(content) end + sig do + params(dependency: Dependabot::Dependency, file: Dependabot::DependencyFile, content: String).returns(String) + end def update_gemfile_git_pin(dependency, file, content) new_pin = dependency.requirements .find { |f| f[:file] == file.name } - .fetch(:source).fetch(:ref) + &.fetch(:source) + &.fetch(:ref) GitPinReplacer .new(dependency: dependency, new_pin: new_pin) diff --git a/bundler/lib/dependabot/bundler/file_updater/gemspec_dependency_name_finder.rb b/bundler/lib/dependabot/bundler/file_updater/gemspec_dependency_name_finder.rb index 587586a268..6fd44a04d2 100644 --- a/bundler/lib/dependabot/bundler/file_updater/gemspec_dependency_name_finder.rb +++ b/bundler/lib/dependabot/bundler/file_updater/gemspec_dependency_name_finder.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "parser/current" @@ -8,13 +8,20 @@ module Dependabot module Bundler class FileUpdater class GemspecDependencyNameFinder + extend T::Sig + + ChildNode = T.type_alias { T.nilable(T.any(Parser::AST::Node, Symbol, String)) } + + sig { returns(String) } attr_reader :gemspec_content + sig { params(gemspec_content: String).void } def initialize(gemspec_content:) @gemspec_content = gemspec_content end # rubocop:disable Security/Eval + sig { returns(T.nilable(String)) } def dependency_name ast = Parser::CurrentRuby.parse(gemspec_content) dependency_name_node = find_dependency_name_node(ast) @@ -30,6 +37,7 @@ def dependency_name private + sig { params(node: ChildNode).returns(T.nilable(Parser::AST::Node)) } def find_dependency_name_node(node) return unless node.is_a?(Parser::AST::Node) return node if declares_dependency_name?(node) @@ -40,6 +48,7 @@ def find_dependency_name_node(node) end end + sig { params(node: ChildNode).returns(T::Boolean) } def declares_dependency_name?(node) return false unless node.is_a?(Parser::AST::Node) From 66caeefc0ae0207797d0eb71e3e8776556d907c9 Mon Sep 17 00:00:00 2001 From: "S.Sandhu" <167903774+sachin-sandhu@users.noreply.github.com> Date: Thu, 16 Jan 2025 11:50:44 -0500 Subject: [PATCH 06/72] Fixes [1.3k weekly errors] [JS] Adds error handlers for PNPM exceptions (#11316) Adds error handlers for pnpm exceptions --- .../file_updater/pnpm_lockfile_updater.rb | 74 +- .../npm_and_yarn/pnpm_error_handler_spec.rb | 74 ++ .../git_dependency_local_file/package.json | 15 + .../pnpm/git_dependency_local_file/pnpm.lock | 639 ++++++++++++++++++ 4 files changed, 796 insertions(+), 6 deletions(-) create mode 100644 npm_and_yarn/spec/dependabot/npm_and_yarn/pnpm_error_handler_spec.rb create mode 100644 npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/package.json create mode 100644 npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/pnpm.lock diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb index bba17e2593..a7d1ffc725 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb @@ -18,6 +18,10 @@ def initialize(dependencies:, dependency_files:, repo_contents_path:, credential @dependency_files = dependency_files @repo_contents_path = repo_contents_path @credentials = credentials + @error_handler = PnpmErrorHandler.new( + dependencies: dependencies, + dependency_files: dependency_files + ) end def updated_pnpm_lock_content(pnpm_lock) @@ -36,6 +40,7 @@ def updated_pnpm_lock_content(pnpm_lock) attr_reader :dependency_files attr_reader :repo_contents_path attr_reader :credentials + attr_reader :error_handler IRRESOLVABLE_PACKAGE = "ERR_PNPM_NO_MATCHING_VERSION" INVALID_REQUIREMENT = "ERR_PNPM_SPEC_NOT_SUPPORTED_BY_ANY_RESOLVER" @@ -46,12 +51,12 @@ def updated_pnpm_lock_content(pnpm_lock) UNAUTHORIZED_PACKAGE = /ERR_PNPM_FETCH_401[ [^:print:]]+GET (?.*): Unauthorized - 401/ # ERR_PNPM_FETCH ERROR CODES - ERR_PNPM_FETCH_401 = /ERR_PNPM_FETCH_401.*GET (?.*): - 401/ - ERR_PNPM_FETCH_403 = /ERR_PNPM_FETCH_403.*GET (?.*): - 403/ - ERR_PNPM_FETCH_404 = /ERR_PNPM_FETCH_404.*GET (?.*): - 404/ - ERR_PNPM_FETCH_500 = /ERR_PNPM_FETCH_500.*GET (?.*): - 500/ - ERR_PNPM_FETCH_502 = /ERR_PNPM_FETCH_502.*GET (?.*): - 502/ - ERR_PNPM_FETCH_503 = /ERR_PNPM_FETCH_503.*GET (?.*): - 503/ + ERR_PNPM_FETCH_401 = /ERR_PNPM_FETCH_401.*GET (?.*):/ + ERR_PNPM_FETCH_403 = /ERR_PNPM_FETCH_403.*GET (?.*):/ + ERR_PNPM_FETCH_404 = /ERR_PNPM_FETCH_404.*GET (?.*):/ + ERR_PNPM_FETCH_500 = /ERR_PNPM_FETCH_500.*GET (?.*):/ + ERR_PNPM_FETCH_502 = /ERR_PNPM_FETCH_502.*GET (?.*):/ + ERR_PNPM_FETCH_503 = /ERR_PNPM_FETCH_503.*GET (?.*):/ # ERR_PNPM_UNSUPPORTED_ENGINE ERR_PNPM_UNSUPPORTED_ENGINE = /ERR_PNPM_UNSUPPORTED_ENGINE/ @@ -251,6 +256,8 @@ def handle_pnpm_lock_updater_error(error, pnpm_lock) pnpm_lock) end + error_handler.handle_pnpm_error(error) + raise end # rubocop:enable Metrics/AbcSize @@ -360,5 +367,60 @@ def sanitize_message(message) end end end + + class PnpmErrorHandler + extend T::Sig + + # remote connection closed + ECONNRESET_ERROR = /ECONNRESET/ + + # socket hang up error code + SOCKET_HANG_UP = /socket hang up/ + + # ERR_PNPM_CATALOG_ENTRY_NOT_FOUND_FOR_SPEC error + ERR_PNPM_CATALOG_ENTRY_NOT_FOUND_FOR_SPEC = /ERR_PNPM_CATALOG_ENTRY_NOT_FOUND_FOR_SPEC/ + + # duplicate package error code + DUPLICATE_PACKAGE = /Found duplicates/ + + ERR_PNPM_NO_VERSIONS = /ERR_PNPM_NO_VERSIONS/ + + # Initializes the YarnErrorHandler with dependencies and dependency files + sig do + params( + dependencies: T::Array[Dependabot::Dependency], + dependency_files: T::Array[Dependabot::DependencyFile] + ).void + end + def initialize(dependencies:, dependency_files:) + @dependencies = dependencies + @dependency_files = dependency_files + end + + private + + sig { returns(T::Array[Dependabot::Dependency]) } + attr_reader :dependencies + + sig { returns(T::Array[Dependabot::DependencyFile]) } + attr_reader :dependency_files + + public + + # Handles errors with specific to yarn error codes + sig { params(error: SharedHelpers::HelperSubprocessFailed).void } + def handle_pnpm_error(error) + if error.message.match?(DUPLICATE_PACKAGE) || error.message.match?(ERR_PNPM_NO_VERSIONS) || + error.message.match?(ERR_PNPM_CATALOG_ENTRY_NOT_FOUND_FOR_SPEC) + + raise DependencyFileNotResolvable, "Error resolving dependency" + end + + ## Clean error message from ANSI escape codes + return unless error.message.match?(ECONNRESET_ERROR) || error.message.match?(SOCKET_HANG_UP) + + raise InconsistentRegistryResponse, "Inconsistent registry response while resolving dependency" + end + end end end diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/pnpm_error_handler_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/pnpm_error_handler_spec.rb new file mode 100644 index 0000000000..9ad11efa09 --- /dev/null +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/pnpm_error_handler_spec.rb @@ -0,0 +1,74 @@ +# typed: false +# frozen_string_literal: true + +require "spec_helper" +require "dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater" +require "dependabot/npm_and_yarn/dependency_files_filterer" +require "dependabot/dependency" +require "dependabot/dependency_file" +require "dependabot/shared_helpers" +require "dependabot/errors" + +RSpec.describe Dependabot::NpmAndYarn::PnpmErrorHandler do + subject(:error_handler) { described_class.new(dependencies: dependencies, dependency_files: dependency_files) } + + let(:dependencies) { [dependency] } + let(:error) { instance_double(Dependabot::SharedHelpers::HelperSubprocessFailed, message: error_message) } + + let(:dependency) do + Dependabot::Dependency.new( + name: dependency_name, + version: version, + requirements: [], + previous_requirements: [], + package_manager: "npm_and_yarn" + ) + end + let(:dependency_files) { project_dependency_files("pnpm/git_dependency_local_file") } + + let(:credentials) do + [Dependabot::Credential.new({ + "type" => "git_source", + "host" => "github.com" + })] + end + + let(:dependency_name) { "@segment/analytics.js-integration-facebook-pixel" } + let(:version) { "github:segmentio/analytics.js-integrations#2.4.1" } + let(:yarn_lock) do + dependency_files.find { |f| f.name == "pnpm.lock" } + end + + describe "#initialize" do + it "initializes with dependencies and dependency files" do + expect(error_handler.send(:dependencies)).to eq(dependencies) + expect(error_handler.send(:dependency_files)).to eq(dependency_files) + end + end + + describe "#handle_error" do + context "when the error message contains Inconsistent Registry Response" do + let(:error_message) do + "ECONNRESET  request to https://artifactory.schaeffler.com/as.zip failed, reason: socket hang up" + end + + it "raises a InconsistentRegistryResponse error with the correct message" do + expect do + error_handler.handle_pnpm_error(error) + end.to raise_error(Dependabot::InconsistentRegistryResponse) + end + end + + context "when the error message contains package error" do + let(:error_message) do + "ERR_PNPM_NO_VERSIONS  No versions available for prosemirror-gapcursor. The package may be unpublished." + end + + it "raises a DependencyFileNotResolvable error with the correct message" do + expect do + error_handler.handle_pnpm_error(error) + end.to raise_error(Dependabot::DependencyFileNotResolvable) + end + end + end +end diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/package.json b/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/package.json new file mode 100644 index 0000000000..9c9882a037 --- /dev/null +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/package.json @@ -0,0 +1,15 @@ +{ + "name": "test", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@segment/analytics.js-integration-facebook-pixel": "github:segmentio/analytics.js-integrations#2.4.1" + } +} diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/pnpm.lock b/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/pnpm.lock new file mode 100644 index 0000000000..e1e6422a00 --- /dev/null +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/git_dependency_local_file/pnpm.lock @@ -0,0 +1,639 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@ndhoule/after@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/after/-/after-1.0.0.tgz#e6d86d121448247ac742ff3a61c63fae83ee1191" + integrity sha1-5thtEhRIJHrHQv86YcY/roPuEZE= + dependencies: + "@ndhoule/arity" "^2.0.0" + +"@ndhoule/arity@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/arity/-/arity-2.0.0.tgz#26bfa0b9755ced9aea819d4e6e7a93db27a5b658" + integrity sha1-Jr+guXVc7ZrqgZ1ObnqT2yeltlg= + +"@ndhoule/clone@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/clone/-/clone-1.0.0.tgz#0f68394a95008cf360370e101924564a70927afc" + integrity sha1-D2g5SpUAjPNgNw4QGSRWSnCSevw= + dependencies: + component-type "^1.2.1" + +"@ndhoule/defaults@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/defaults/-/defaults-2.0.1.tgz#704aae3c601a4e4a1a10f0876a2d3253bc7d4d9b" + integrity sha1-cEquPGAaTkoaEPCHai0yU7x9TZs= + dependencies: + "@ndhoule/drop" "^2.0.0" + "@ndhoule/rest" "^2.0.0" + +"@ndhoule/drop@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/drop/-/drop-2.0.0.tgz#bcab1f3041555eaf84ce84e16475ff42ee949c8c" + integrity sha1-vKsfMEFVXq+EzoThZHX/Qu6UnIw= + +"@ndhoule/each@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/each/-/each-2.0.1.tgz#bbed372a603e0713a3193c706a73ddebc5b426a9" + integrity sha1-u+03KmA+BxOjGTxwanPd68W0Jqk= + dependencies: + "@ndhoule/keys" "^2.0.0" + +"@ndhoule/every@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/every/-/every-2.0.1.tgz#3907d8b6c430493dbb619c18071ce9055f8a106d" + integrity sha1-OQfYtsQwST27YZwYBxzpBV+KEG0= + dependencies: + "@ndhoule/each" "^2.0.1" + +"@ndhoule/extend@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/extend/-/extend-2.0.0.tgz#8c9aa5c9b2f0a012104ffe214cd9746572b9aeb6" + integrity sha1-jJqlybLwoBIQT/4hTNl0ZXK5rrY= + +"@ndhoule/foldl@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/foldl/-/foldl-2.0.1.tgz#788acedfa2cfd12ecb0b84d2beaf650d97be84f2" + integrity sha1-eIrO36LP0S7LC4TSvq9lDZe+hPI= + dependencies: + "@ndhoule/each" "^2.0.1" + +"@ndhoule/includes@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/includes/-/includes-2.0.1.tgz#051ff5eb042c8fa17e7158f0a8a70172e1affaa5" + integrity sha1-BR/16wQsj6F+cVjwqKcBcuGv+qU= + dependencies: + "@ndhoule/each" "^2.0.1" + +"@ndhoule/keys@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/keys/-/keys-2.0.0.tgz#3d64ae677c65a261747bf3a457c62eb292a4e0ce" + integrity sha1-PWSuZ3xlomF0e/OkV8YuspKk4M4= + +"@ndhoule/map@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@ndhoule/map/-/map-2.0.1.tgz#f5ca0a47424ea67f46e2a6d499b9e9bc886aefa8" + integrity sha1-9coKR0JOpn9G4qbUmbnpvIhq76g= + dependencies: + "@ndhoule/each" "^2.0.1" + +"@ndhoule/rest@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@ndhoule/rest/-/rest-2.0.0.tgz#0346b02a964a513ed2ba24d164f01d34f2107a0f" + integrity sha1-A0awKpZKUT7SuiTRZPAdNPIQeg8= + +"@segment/analytics.js-integration-facebook-pixel@segmentio/analytics.js-integrations#2.4.1": + version "2.4.1" + resolved "https://codeload.github.com/segmentio/analytics.js-integrations/tar.gz/3b1bb80b302c2e552685dc8a029797ec832ea7c9" + dependencies: + "@ndhoule/each" "^2.0.1" + "@ndhoule/foldl" "^2.0.1" + "@segment/analytics.js-integration" "^3.1.0" + dateformat "^1.0.12" + is "^3.2.1" + reject "0.0.1" + segmentio-facade "^3.1.0" + to-camel-case "^1.0.0" + +"@segment/analytics.js-integration@^3.1.0": + version "3.3.0" + resolved "https://registry.yarnpkg.com/@segment/analytics.js-integration/-/analytics.js-integration-3.3.0.tgz#a44eac133e0ba1a26dc7387adc59c84b0ab15178" + integrity sha512-xDwQ6xQKYFMIaZd+I9SYIZNAWUk806mU18JeGpo1wj84bqKjwi8WaFYTk+keuh+B8GtIOTkH16S/orzvzmBDxw== + dependencies: + "@ndhoule/after" "^1.0.0" + "@ndhoule/clone" "^1.0.0" + "@ndhoule/defaults" "^2.0.1" + "@ndhoule/each" "^2.0.1" + "@ndhoule/every" "^2.0.1" + "@ndhoule/extend" "^2.0.0" + "@ndhoule/foldl" "^2.0.1" + "@ndhoule/includes" "^2.0.1" + "@segment/fmt" "^1.0.0" + "@segment/load-script" "^1.0.1" + analytics-events "^2.0.2" + component-bind "^1.0.0" + component-emitter "^1.2.0" + debug "^2.2.0" + domify "^1.4.0" + extend "3.0.1" + is "^3.1.0" + load-iframe "^1.0.0" + next-tick "^0.2.2" + slug-component "^1.1.0" + to-no-case "^0.1.3" + +"@segment/fmt@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@segment/fmt/-/fmt-1.0.0.tgz#7b2db58a12d8420b0a004889af048ab22765557f" + integrity sha1-ey21ihLYQgsKAEiJrwSKsidlVX8= + +"@segment/isodate-traverse@^1.0.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@segment/isodate-traverse/-/isodate-traverse-1.1.0.tgz#4fb36f30d94c2c5a3c8a75d25c1504822a1bb9ab" + integrity sha1-T7NvMNlMLFo8inXSXBUEgiobuas= + dependencies: + "@segment/isodate" "^1.0.0" + component-each "^0.2.6" + component-type "^1.2.1" + +"@segment/isodate@1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@segment/isodate/-/isodate-1.0.2.tgz#71fef69e816b77593a80b4cfbf2d22443222eafe" + integrity sha1-cf72noFrd1k6gLTPvy0iRDIi6v4= + +"@segment/isodate@^1.0.0": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@segment/isodate/-/isodate-1.0.3.tgz#f44e8202d5edd277ce822785239474b2c9411d4a" + integrity sha512-BtanDuvJqnACFkeeYje7pWULVv8RgZaqKHWwGFnL/g/TH/CcZjkIVTfGDp/MAxmilYHUkrX70SqwnYSTNEaN7A== + +"@segment/load-script@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@segment/load-script/-/load-script-1.0.1.tgz#adb7a2def2c99ac248cc8e2c154fb4bb03094399" + integrity sha1-rbei3vLJmsJIzI4sFU+0uwMJQ5k= + dependencies: + component-type "^1.2.0" + next-tick "^0.2.2" + script-onload "^1.0.2" + +analytics-events@^2.0.2: + version "2.2.0" + resolved "https://registry.yarnpkg.com/analytics-events/-/analytics-events-2.2.0.tgz#f00f55946940a6357809582f6fded6ab80034b84" + integrity sha1-8A9VlGlApjV4CVgvb97Wq4ADS4Q= + dependencies: + "@ndhoule/foldl" "^2.0.1" + "@ndhoule/map" "^2.0.1" + +array-find-index@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1" + integrity sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E= + +camelcase-keys@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7" + integrity sha1-MIvur/3ygRkFHvodkyITyRuPkuc= + dependencies: + camelcase "^2.0.0" + map-obj "^1.0.0" + +camelcase@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f" + integrity sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8= + +component-bind@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/component-bind/-/component-bind-1.0.0.tgz#00c608ab7dcd93897c0009651b1d3a8e1e73bbd1" + integrity sha1-AMYIq33Nk4l8AAllGx06jh5zu9E= + +component-each@^0.2.6: + version "0.2.6" + resolved "https://registry.yarnpkg.com/component-each/-/component-each-0.2.6.tgz#991faf31ef4fcafbad04237124d381b3394941d5" + integrity sha1-mR+vMe9PyvutBCNxJNOBszlJQdU= + dependencies: + component-type "1.0.0" + to-function "2.0.6" + +component-emitter@^1.2.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.3.0.tgz#16e4070fba8ae29b679f2215853ee181ab2eabc0" + integrity sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg== + +component-props@*: + version "1.1.1" + resolved "https://registry.yarnpkg.com/component-props/-/component-props-1.1.1.tgz#f9b7df9b9927b6e6d97c9bd272aa867670f34944" + integrity sha1-+bffm5kntubZfJvScqqGdnDzSUQ= + +component-type@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.0.0.tgz#1ed8812e32dd65099d433570757f111ea3d3d871" + integrity sha1-HtiBLjLdZQmdQzVwdX8RHqPT2HE= + +component-type@^1.2.0, component-type@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9" + integrity sha1-ikeQFwAjjk/DIml3EjAibyS0Fak= + +currently-unhandled@^0.4.1: + version "0.4.1" + resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" + integrity sha1-mI3zP+qxke95mmE2nddsF635V+o= + dependencies: + array-find-index "^1.0.1" + +dateformat@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-1.0.12.tgz#9f124b67594c937ff706932e4a642cca8dbbfee9" + integrity sha1-nxJLZ1lMk3/3BpMuSmQsyo27/uk= + dependencies: + get-stdin "^4.0.1" + meow "^3.3.0" + +debug@^2.2.0: + version "2.6.9" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + +decamelize@^1.1.2: + version "1.2.0" + resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" + integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA= + +domify@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/domify/-/domify-1.4.0.tgz#11483617f764f8695975b4bdc79b14f0803b629b" + integrity sha1-EUg2F/dk+GlZdbS9x5sU8IA7Yps= + +error-ex@^1.2.0: + version "1.3.2" + resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" + integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== + dependencies: + is-arrayish "^0.2.1" + +extend@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" + integrity sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ= + +find-up@^1.0.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f" + integrity sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8= + dependencies: + path-exists "^2.0.0" + pinkie-promise "^2.0.0" + +get-stdin@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" + integrity sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4= + +graceful-fs@^4.1.2: + version "4.1.15" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00" + integrity sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA== + +hosted-git-info@^2.1.4: + version "2.7.1" + resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047" + integrity sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w== + +indent-string@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-2.1.0.tgz#8e2d48348742121b4a8218b7a137e9a52049dc80" + integrity sha1-ji1INIdCEhtKghi3oTfppSBJ3IA= + dependencies: + repeating "^2.0.0" + +inherits@^2.0.1: + version "2.0.3" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" + integrity sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4= + +is-arrayish@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" + integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= + +is-email@0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/is-email/-/is-email-0.1.0.tgz#e7eb1aefe8d4a3183980a7172b851272ebd04a95" + integrity sha1-5+sa7+jUoxg5gKcXK4UScuvQSpU= + +is-finite@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-finite/-/is-finite-1.0.2.tgz#cc6677695602be550ef11e8b4aa6305342b6d0aa" + integrity sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko= + dependencies: + number-is-nan "^1.0.0" + +is-utf8@^0.2.0: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" + integrity sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI= + +is@^3.0.1, is@^3.1.0, is@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" + integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== + +load-iframe@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/load-iframe/-/load-iframe-1.0.0.tgz#b2c111f1a2b982897463232f5286191343bbdbd8" + integrity sha1-ssER8aK5gol0YyMvUoYZE0O729g= + dependencies: + is "^3.0.1" + next-tick "^0.2.2" + script-onload "^1.0.2" + +load-json-file@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" + integrity sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA= + dependencies: + graceful-fs "^4.1.2" + parse-json "^2.2.0" + pify "^2.0.0" + pinkie-promise "^2.0.0" + strip-bom "^2.0.0" + +loud-rejection@^1.0.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f" + integrity sha1-W0b4AUft7leIcPCG0Eghz5mOVR8= + dependencies: + currently-unhandled "^0.4.1" + signal-exit "^3.0.0" + +map-obj@^1.0.0, map-obj@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d" + integrity sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0= + +meow@^3.3.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb" + integrity sha1-cstmi0JSKCkKu/qFaJJYcwioAfs= + dependencies: + camelcase-keys "^2.0.0" + decamelize "^1.1.2" + loud-rejection "^1.0.0" + map-obj "^1.0.1" + minimist "^1.1.3" + normalize-package-data "^2.3.4" + object-assign "^4.0.1" + read-pkg-up "^1.0.1" + redent "^1.0.0" + trim-newlines "^1.0.0" + +minimist@^1.1.3: + version "1.2.0" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" + integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= + +ms@2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" + integrity sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g= + +new-date@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/new-date/-/new-date-1.0.1.tgz#0044981b1739161e653099c44e6e9e4ff72e02d2" + integrity sha1-AESYGxc5Fh5lMJnETm6eT/cuAtI= + dependencies: + "@segment/isodate" "1.0.2" + is "^3.1.0" + +next-tick@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-0.2.2.tgz#75da4a927ee5887e39065880065b7336413b310d" + integrity sha1-ddpKkn7liH45BliABltzNkE7MQ0= + +normalize-package-data@^2.3.2, normalize-package-data@^2.3.4: + version "2.5.0" + resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.5.0.tgz#e66db1838b200c1dfc233225d12cb36520e234a8" + integrity sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA== + dependencies: + hosted-git-info "^2.1.4" + resolve "^1.10.0" + semver "2 || 3 || 4 || 5" + validate-npm-package-license "^3.0.1" + +number-is-nan@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" + integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0= + +obj-case@0.x: + version "0.2.0" + resolved "https://registry.yarnpkg.com/obj-case/-/obj-case-0.2.0.tgz#841c0b26784fc329968396fd871f830255c1ff2d" + integrity sha1-hBwLJnhPwymWg5b9hx+DAlXB/y0= + +object-assign@^4.0.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + +parse-json@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" + integrity sha1-9ID0BDTvgHQfhGkJn43qGPVaTck= + dependencies: + error-ex "^1.2.0" + +path-exists@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b" + integrity sha1-D+tsZPD8UY2adU3V77YscCJ2H0s= + dependencies: + pinkie-promise "^2.0.0" + +path-parse@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" + integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== + +path-type@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" + integrity sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE= + dependencies: + graceful-fs "^4.1.2" + pify "^2.0.0" + pinkie-promise "^2.0.0" + +pify@^2.0.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" + integrity sha1-7RQaasBDqEnqWISY59yosVMw6Qw= + +pinkie-promise@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" + integrity sha1-ITXW36ejWMBprJsXh3YogihFD/o= + dependencies: + pinkie "^2.0.0" + +pinkie@^2.0.0: + version "2.0.4" + resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" + integrity sha1-clVrgM+g1IqXToDnckjoDtT3+HA= + +read-pkg-up@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02" + integrity sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI= + dependencies: + find-up "^1.0.0" + read-pkg "^1.0.0" + +read-pkg@^1.0.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" + integrity sha1-9f+qXs0pyzHAR0vKfXVra7KePyg= + dependencies: + load-json-file "^1.0.0" + normalize-package-data "^2.3.2" + path-type "^1.0.0" + +redent@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde" + integrity sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94= + dependencies: + indent-string "^2.1.0" + strip-indent "^1.0.1" + +reject@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/reject/-/reject-0.0.1.tgz#9fa05ad126ef2ad22ef79cfe088eb96a1a116eaf" + integrity sha1-n6Ba0SbvKtIu95z+CI65ahoRbq8= + dependencies: + type-component "0.0.1" + +repeating@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda" + integrity sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo= + dependencies: + is-finite "^1.0.0" + +resolve@^1.10.0: + version "1.10.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.10.1.tgz#664842ac960795bbe758221cdccda61fb64b5f18" + integrity sha512-KuIe4mf++td/eFb6wkaPbMDnP6kObCaEtIDuHOUED6MNUo4K670KZUHuuvYPZDxNF0WVLw49n06M2m2dXphEzA== + dependencies: + path-parse "^1.0.6" + +script-onload@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/script-onload/-/script-onload-1.0.2.tgz#6bdca122875487192ccaf4e6884fcfdd0c7fde32" + integrity sha1-a9yhIodUhxksyvTmiE/P3Qx/3jI= + +segmentio-facade@^3.1.0: + version "3.2.3" + resolved "https://registry.yarnpkg.com/segmentio-facade/-/segmentio-facade-3.2.3.tgz#7ddc6971801a482475b6a768184696a847e9197d" + integrity sha1-fdxpcYAaSCR1tqdoGEaWqEfpGX0= + dependencies: + "@ndhoule/clone" "^1.0.0" + "@segment/isodate-traverse" "^1.0.0" + inherits "^2.0.1" + is-email "0.1.0" + new-date "^1.0.0" + obj-case "0.x" + trim "0.0.1" + type-component "0.0.1" + +"semver@2 || 3 || 4 || 5": + version "5.7.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.0.tgz#790a7cf6fea5459bac96110b29b60412dc8ff96b" + integrity sha512-Ya52jSX2u7QKghxeoFGpLwCtGlt7j0oY9DYb5apt9nPlJ42ID+ulTXESnt/qAQcoSERyZ5sl3LDIOw0nAn/5DA== + +signal-exit@^3.0.0: + version "3.0.2" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" + integrity sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0= + +slug-component@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/slug-component/-/slug-component-1.1.0.tgz#224290a04591bf9ac08b9c622d3a14f43e3a0df7" + integrity sha1-IkKQoEWRv5rAi5xiLToU9D46Dfc= + +spdx-correct@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.1.0.tgz#fb83e504445268f154b074e218c87c003cd31df4" + integrity sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q== + dependencies: + spdx-expression-parse "^3.0.0" + spdx-license-ids "^3.0.0" + +spdx-exceptions@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz#2ea450aee74f2a89bfb94519c07fcd6f41322977" + integrity sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA== + +spdx-expression-parse@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz#99e119b7a5da00e05491c9fa338b7904823b41d0" + integrity sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg== + dependencies: + spdx-exceptions "^2.1.0" + spdx-license-ids "^3.0.0" + +spdx-license-ids@^3.0.0: + version "3.0.4" + resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.4.tgz#75ecd1a88de8c184ef015eafb51b5b48bfd11bb1" + integrity sha512-7j8LYJLeY/Yb6ACbQ7F76qy5jHkp0U6jgBfJsk97bwWlVUnUWsAgpyaCvo17h0/RQGnQ036tVDomiwoI4pDkQA== + +strip-bom@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" + integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4= + dependencies: + is-utf8 "^0.2.0" + +strip-indent@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-1.0.1.tgz#0c7962a6adefa7bbd4ac366460a638552ae1a0a2" + integrity sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI= + dependencies: + get-stdin "^4.0.1" + +to-camel-case@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/to-camel-case/-/to-camel-case-1.0.0.tgz#1a56054b2f9d696298ce66a60897322b6f423e46" + integrity sha1-GlYFSy+daWKYzmamCJcyK29CPkY= + dependencies: + to-space-case "^1.0.0" + +to-function@2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/to-function/-/to-function-2.0.6.tgz#7d56cd9c3b92fa8dbd7b22e83d51924de740ebc5" + integrity sha1-fVbNnDuS+o29eyLoPVGSTedA68U= + dependencies: + component-props "*" + +to-no-case@^0.1.3: + version "0.1.3" + resolved "https://registry.yarnpkg.com/to-no-case/-/to-no-case-0.1.3.tgz#f761b1ea1931680967b79886a3303106d966e061" + integrity sha1-92Gx6hkxaAlnt5iGozAxBtlm4GE= + +to-no-case@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/to-no-case/-/to-no-case-1.0.2.tgz#c722907164ef6b178132c8e69930212d1b4aa16a" + integrity sha1-xyKQcWTvaxeBMsjmmTAhLRtKoWo= + +to-space-case@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/to-space-case/-/to-space-case-1.0.0.tgz#b052daafb1b2b29dc770cea0163e5ec0ebc9fc17" + integrity sha1-sFLar7Gysp3HcM6gFj5ewOvJ/Bc= + dependencies: + to-no-case "^1.0.0" + +trim-newlines@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613" + integrity sha1-WIeWa7WCpFA6QetST301ARgVphM= + +trim@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" + integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= + +type-component@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/type-component/-/type-component-0.0.1.tgz#952a6c81c21efd24d13d811d0c8498cb860e1956" + integrity sha1-lSpsgcIe/STRPYEdDISYy4YOGVY= + +validate-npm-package-license@^3.0.1: + version "3.0.4" + resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" + integrity sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew== + dependencies: + spdx-correct "^3.0.0" + spdx-expression-parse "^3.0.0" From 8aad5cac94675139909e923d4225cbf90bfb400a Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 12 Jan 2025 17:06:42 +0000 Subject: [PATCH 07/72] Adding support for path based tag structure in github_actions module --- .../dependabot/github_actions/file_parser.rb | 5 +- .../lib/dependabot/github_actions/version.rb | 9 +- .../github_actions/file_parser_spec.rb | 280 ++++++++++++++++++ .../github_actions/file_updater_spec.rb | 80 +++++ .../github_actions/update_checker_spec.rb | 128 ++++++++ .../dependabot/github_actions/version_spec.rb | 48 +++ .../upload_packs/github-monorepo-path-based | Bin 0 -> 899 bytes .../workflow_monorepo_path_based_semver.yml | 13 + ...orepo_path_based_semver_and_direct_ref.yml | 22 ++ ...low_monorepo_path_based_without_semver.yml | 11 + ...th_based_without_semver_and_direct_ref.yml | 18 ++ 11 files changed, 610 insertions(+), 4 deletions(-) create mode 100644 github_actions/spec/fixtures/git/upload_packs/github-monorepo-path-based create mode 100644 github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver.yml create mode 100644 github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver_and_direct_ref.yml create mode 100644 github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver.yml create mode 100644 github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver_and_direct_ref.yml diff --git a/github_actions/lib/dependabot/github_actions/file_parser.rb b/github_actions/lib/dependabot/github_actions/file_parser.rb index ba80eb1e62..aac4f4ca27 100644 --- a/github_actions/lib/dependabot/github_actions/file_parser.rb +++ b/github_actions/lib/dependabot/github_actions/file_parser.rb @@ -111,9 +111,10 @@ def build_github_dependency(file, string) sig { params(file: Dependabot::DependencyFile, string: String, hostname: String).returns(Dependabot::Dependency) } def github_dependency(file, string, hostname) details = T.must(string.match(GITHUB_REPO_REFERENCE)).named_captures - name = "#{details.fetch(OWNER_KEY)}/#{details.fetch(REPO_KEY)}" + repo_name = "#{details.fetch(OWNER_KEY)}/#{details.fetch(REPO_KEY)}" ref = details.fetch(REF_KEY) version = version_class.new(ref).to_s if version_class.correct?(ref) + name = version_class.path_based?(ref) ? string : repo_name Dependency.new( name: name, version: version, @@ -122,7 +123,7 @@ def github_dependency(file, string, hostname) groups: [], source: { type: "git", - url: "https://#{hostname}/#{name}".downcase, + url: "https://#{hostname}/#{repo_name}".downcase, ref: ref, branch: nil }, diff --git a/github_actions/lib/dependabot/github_actions/version.rb b/github_actions/lib/dependabot/github_actions/version.rb index edcc777f6c..9f0c06593e 100644 --- a/github_actions/lib/dependabot/github_actions/version.rb +++ b/github_actions/lib/dependabot/github_actions/version.rb @@ -24,9 +24,14 @@ def self.new(version) sig { params(version: VersionParameter).returns(VersionParameter) } def self.remove_leading_v(version) - return version unless version.to_s.match?(/\Av([0-9])/) + return version unless version.to_s.match?(/\A(?:.*\/)?v?([0-9])/) - version.to_s.delete_prefix("v") + version.to_s.sub(/\A(?:.*\/)?v?/, "") + end + + sig { params(version: VersionParameter).returns(T::Boolean) } + def self.path_based?(version) + version.to_s.match?(/\A.+\/v?([0-9])/) end sig { override.params(version: VersionParameter).returns(T::Boolean) } diff --git a/github_actions/spec/dependabot/github_actions/file_parser_spec.rb b/github_actions/spec/dependabot/github_actions/file_parser_spec.rb index 4756049feb..8a2f1df983 100644 --- a/github_actions/spec/dependabot/github_actions/file_parser_spec.rb +++ b/github_actions/spec/dependabot/github_actions/file_parser_spec.rb @@ -560,6 +560,286 @@ def mock_service_pack_request(nwo) end end end + + context "with path based semver tag pinned to workflow action" do + let(:workflow_file_fixture_name) { "workflow_monorepo_path_based_semver.yml" } + + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + end + + it "has dependencies" do + expect(dependencies.count).to be(2) + end + + describe "the path based first dependency" do + subject(:dependency) { dependencies.first } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("gopidesupavan/monorepo-actions/first/init@init/v1.0.0") + expect(dependency.version).to eq("1.0.0") + expect(dependency.requirements).to eq(expected_requirements) + end + end + + describe "the path based last dependency" do + subject(:dependency) { dependencies.last } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("gopidesupavan/monorepo-actions/first/run@run/v2.0.0") + expect(dependency.version).to eq("2.0.0") + expect(dependency.requirements).to eq(expected_requirements) + end + end + end + + context "with path based without semver tag pinned to workflow action" do + let(:workflow_file_fixture_name) { "workflow_monorepo_path_based_without_semver.yml" } + + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + end + + it "has dependencies" do + expect(dependencies.count).to be(1) + end + + describe "the path based first dependency" do + subject(:dependency) { dependencies.first } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("gopidesupavan/monorepo-actions/second/exec@exec/1.0.0") + expect(dependency.version).to eq("1.0.0") + expect(dependency.requirements).to eq(expected_requirements) + end + end + end + + context "with mix of path based semver tag pinned to workflow action and direct ref" do + let(:workflow_file_fixture_name) { "workflow_monorepo_path_based_semver_and_direct_ref.yml" } + + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + mock_service_pack_request("actions/checkout") + end + + it "has dependencies" do + expect(dependencies.count).to be(3) + end + + describe "the path based first dependency" do + subject(:dependency) { dependencies.first } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("gopidesupavan/monorepo-actions/first/init@init/v1.0.0") + expect(dependency.version).to eq("1.0.0") + expect(dependency.requirements).to eq(expected_requirements) + end + end + + describe "the path based last dependency" do + subject(:dependency) { dependencies.last } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/actions/checkout", + ref: "v1", + branch: nil + }, + metadata: { declaration_string: "actions/checkout@v1" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("actions/checkout") + expect(dependency.version).to eq("1") + expect(dependency.requirements).to eq(expected_requirements) + end + end + end + + context "with mix of path based without semver tag pinned to workflow action and direct ref" do + let(:workflow_file_fixture_name) { "workflow_monorepo_path_based_without_semver_and_direct_ref.yml" } + + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + mock_service_pack_request("actions/checkout") + end + + it "has dependencies" do + expect(dependencies.count).to be(2) + end + + describe "the path based first dependency" do + subject(:dependency) { dependencies.first } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/1.0.0" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("gopidesupavan/monorepo-actions/first/init@init/1.0.0") + expect(dependency.version).to eq("1.0.0") + expect(dependency.requirements).to eq(expected_requirements) + end + end + + describe "the path based last dependency" do + subject(:dependency) { dependencies.last } + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/actions/checkout", + ref: "v1", + branch: nil + }, + metadata: { declaration_string: "actions/checkout@v1" } + }] + end + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("actions/checkout") + expect(dependency.version).to eq("1") + expect(dependency.requirements).to eq(expected_requirements) + end + end + end end describe "#ecosystem" do diff --git a/github_actions/spec/dependabot/github_actions/file_updater_spec.rb b/github_actions/spec/dependabot/github_actions/file_updater_spec.rb index 1b6ff1addb..614aa6d45d 100644 --- a/github_actions/spec/dependabot/github_actions/file_updater_spec.rb +++ b/github_actions/spec/dependabot/github_actions/file_updater_spec.rb @@ -524,6 +524,86 @@ end end end + + context "with a path based tag with semver" do + let(:workflow_file_body) do + fixture("workflow_files", "workflow_monorepo_path_based_semver.yml") + end + + let(:dependency) do + Dependabot::Dependency.new( + name: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0", + version: "5273d0df9c603edc4284ac8402cf650b4f1f6686", + previous_version: nil, + requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v3.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } + }], + previous_requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } + }], + package_manager: "github_actions" + ) + end + its(:content) { is_expected.to include "gopidesupavan/monorepo-actions/first/run@run/v3.0.0\n" } + end + + context "with a path based tag with without semver" do + let(:workflow_file_body) do + fixture("workflow_files", "workflow_monorepo_path_based_without_semver.yml") + end + + let(:dependency) do + Dependabot::Dependency.new( + name: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0", + version: "5273d0df9c603edc4284ac8402cf650b4f1f6686", + previous_version: nil, + requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }], + previous_requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }], + package_manager: "github_actions" + ) + end + its(:content) { is_expected.to include "gopidesupavan/monorepo-actions/second/exec@exec/2.0.0\n" } + end end end end diff --git a/github_actions/spec/dependabot/github_actions/update_checker_spec.rb b/github_actions/spec/dependabot/github_actions/update_checker_spec.rb index b69eb50a86..11711294c8 100644 --- a/github_actions/spec/dependabot/github_actions/update_checker_spec.rb +++ b/github_actions/spec/dependabot/github_actions/update_checker_spec.rb @@ -1101,5 +1101,133 @@ expect(updated_requirements).to eq(expected_requirements) end end + + context "when a dependency has a path based tag reference with semver" do + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + end + let(:upload_pack_fixture) { "github-monorepo-path-based" } + let(:dependency) do + Dependabot::Dependency.new( + name: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0", + version: "1.0.0", + requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } + }], + package_manager: "github_actions" + ) + end + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v3.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } + }] + end + + it { is_expected.to eq(expected_requirements) } + + # context "when the latest version is being ignored" do + # let(:ignored_versions) { [">= 1.1.0"] } + # let(:expected_requirements) do + # [{ + # requirement: nil, + # groups: [], + # file: ".github/workflows/workflow.yml", + # source: { + # type: "git", + # url: "https://github.com/actions/setup-node", + # ref: "v1.0.4", + # branch: nil + # }, + # metadata: { declaration_string: "actions/setup-node@master" } + # }] + # end + # + # it { is_expected.to eq(expected_requirements) } + # end + end + + context "when a dependency has a path based tag reference without semver" do + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) + end + let(:upload_pack_fixture) { "github-monorepo-path-based" } + let(:dependency) do + Dependabot::Dependency.new( + name: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0", + version: "1.0.0", + requirements: [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }], + package_manager: "github_actions" + ) + end + + let(:expected_requirements) do + [{ + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }] + end + + it { is_expected.to eq(expected_requirements) } + end end end diff --git a/github_actions/spec/dependabot/github_actions/version_spec.rb b/github_actions/spec/dependabot/github_actions/version_spec.rb index fc61fcfb09..98cd4a853b 100644 --- a/github_actions/spec/dependabot/github_actions/version_spec.rb +++ b/github_actions/spec/dependabot/github_actions/version_spec.rb @@ -7,6 +7,8 @@ RSpec.describe Dependabot::GithubActions::Version do semver_version = "v1.2.3" semver_without_v = "1.2.3" + path_based_sem_version = "dummy/v1.2.3" + path_based_sem_without_v = "dummy/1.2.3" describe "#correct?" do it "rejects nil" do @@ -20,6 +22,14 @@ it "accepts semver without v" do expect(described_class.correct?(semver_without_v)).to be(true) end + + it "accepts path based sem version" do + expect(described_class.correct?(path_based_sem_version)).to be(true) + end + + it "accepts path based sem version without v" do + expect(described_class.correct?(path_based_sem_without_v)).to be(true) + end end describe "#initialize" do @@ -38,5 +48,43 @@ version_without_v = described_class.new(semver_without_v) expect(version).to eq(version_without_v) end + + it "accepts path based sem version" do + version = described_class.new(path_based_sem_version) + expect(version.to_s).to eq(semver_without_v) + end + + it "accepts path based sem version without v" do + version = described_class.new(path_based_sem_without_v) + expect(version.to_s).to eq(semver_without_v) + end + + it "normalizes path based semver v" do + version = described_class.new(path_based_sem_version) + version_without_v = described_class.new(path_based_sem_without_v) + expect(version).to eq(version_without_v) + end + end + + describe "#path_based" do + it "rejects nil" do + expect(described_class.path_based?(nil)).to be(false) + end + + it "accepts when tag structure like path based with semver" do + expect(described_class.path_based?(path_based_sem_version)).to be(true) + end + + it "accepts when tag structure like path based without semver" do + expect(described_class.path_based?(path_based_sem_without_v)).to be(true) + end + + it "reject when tag structure not like path based with semver" do + expect(described_class.path_based?(semver_version)).to be(false) + end + + it "reject when tag structure not like path based without semver" do + expect(described_class.path_based?(semver_without_v)).to be(false) + end end end diff --git a/github_actions/spec/fixtures/git/upload_packs/github-monorepo-path-based b/github_actions/spec/fixtures/git/upload_packs/github-monorepo-path-based new file mode 100644 index 0000000000000000000000000000000000000000..4f00c0fe92a359ecfd4b6f0a51fd4fa134de0ebe GIT binary patch literal 899 zcmb7=-EJEp5QTf4r$FlU#=s0LRPsfoR_a?s0}M9FnzeSZ)4qMNmBeX!(;^`aKl7b4 zM@q$?#mdXMMBeU78&1b*rZgNWKHbU7QU;95&Hx5V4MYOTCiBS`rU|YPwZ}#V0Z6bx1rl66>C(X?_-o`N(xxOO5<`sB?QQ z7oMnE&8R6-*5sp8n1fTnD~FCD7>*vSLCoYN0E%mKPi5WkPmUXP zFYo1*@PG0*f_@=Dwpw|elEEO6b)s&{Xdji739~1ew6}ST@XkZ2J&?1y7y#>X|Kh|s(iXa7Vi}n^dxNY3GMPL&Wz_o_u7sP105)4<6D%a@<;d)OP|*czymT;H Q@3KXmK6-!`4^rO!2IoB!(*OVf literal 0 HcmV?d00001 diff --git a/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver.yml b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver.yml new file mode 100644 index 0000000000..e7a729e084 --- /dev/null +++ b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver.yml @@ -0,0 +1,13 @@ +on: [push] + +name: Integration +jobs: + chore: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - uses: gopidesupavan/monorepo-actions/first/init@init/v1.0.0 + + - name: run action + uses: gopidesupavan/monorepo-actions/first/run@run/v2.0.0 diff --git a/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver_and_direct_ref.yml b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver_and_direct_ref.yml new file mode 100644 index 0000000000..ca0df7e4f3 --- /dev/null +++ b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_semver_and_direct_ref.yml @@ -0,0 +1,22 @@ +on: [push] + +name: Integration +jobs: + chore: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - uses: gopidesupavan/monorepo-actions/first/init@init/v1.0.0 + + - name: run action + uses: gopidesupavan/monorepo-actions/first/run@run/v2.0.0 + + - uses: actions/checkout@master + + direct: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 diff --git a/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver.yml b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver.yml new file mode 100644 index 0000000000..d1c62aac10 --- /dev/null +++ b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver.yml @@ -0,0 +1,11 @@ +on: [push] + +name: Integration +jobs: + chore: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - name: run exec + uses: gopidesupavan/monorepo-actions/second/exec@exec/1.0.0 diff --git a/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver_and_direct_ref.yml b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver_and_direct_ref.yml new file mode 100644 index 0000000000..34d771e185 --- /dev/null +++ b/github_actions/spec/fixtures/workflow_files/workflow_monorepo_path_based_without_semver_and_direct_ref.yml @@ -0,0 +1,18 @@ +on: [push] + +name: Integration +jobs: + chore: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - uses: gopidesupavan/monorepo-actions/first/init@init/1.0.0 + + direct: + name: Testing chores + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + From c6ba49d6584d56284e21228012cb0ae423ae18c9 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 12 Jan 2025 18:54:02 +0000 Subject: [PATCH 08/72] Fix linter offenses --- .../lib/dependabot/github_actions/version.rb | 6 +- .../github_actions/update_checker_spec.rb | 148 ++++++++---------- 2 files changed, 67 insertions(+), 87 deletions(-) diff --git a/github_actions/lib/dependabot/github_actions/version.rb b/github_actions/lib/dependabot/github_actions/version.rb index 9f0c06593e..9c25a27fa1 100644 --- a/github_actions/lib/dependabot/github_actions/version.rb +++ b/github_actions/lib/dependabot/github_actions/version.rb @@ -24,14 +24,14 @@ def self.new(version) sig { params(version: VersionParameter).returns(VersionParameter) } def self.remove_leading_v(version) - return version unless version.to_s.match?(/\A(?:.*\/)?v?([0-9])/) + return version unless version.to_s.match?(%r{\A(?:.*/)?v?([0-9])}) - version.to_s.sub(/\A(?:.*\/)?v?/, "") + version.to_s.sub(%r{\A(?:.*/)?v?}, "") end sig { params(version: VersionParameter).returns(T::Boolean) } def self.path_based?(version) - version.to_s.match?(/\A.+\/v?([0-9])/) + version.to_s.match?(%r{\A.+/v?([0-9])}) end sig { override.params(version: VersionParameter).returns(T::Boolean) } diff --git a/github_actions/spec/dependabot/github_actions/update_checker_spec.rb b/github_actions/spec/dependabot/github_actions/update_checker_spec.rb index 11711294c8..b57e5fd0ab 100644 --- a/github_actions/spec/dependabot/github_actions/update_checker_spec.rb +++ b/github_actions/spec/dependabot/github_actions/update_checker_spec.rb @@ -1107,80 +1107,41 @@ "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ "?service=git-upload-pack" end - before do - stub_request(:get, service_pack_url) - .to_return( - status: 200, - body: fixture("git", "upload_packs", "github-monorepo-path-based"), - headers: { - "content-type" => "application/x-git-upload-pack-advertisement" - } - ) - end let(:upload_pack_fixture) { "github-monorepo-path-based" } let(:dependency) do Dependabot::Dependency.new( name: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0", version: "1.0.0", requirements: [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "run/v1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } - }], + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } + }], package_manager: "github_actions" ) end - let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "run/v3.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v3.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v1.0.0" } + }] end - it { is_expected.to eq(expected_requirements) } - - # context "when the latest version is being ignored" do - # let(:ignored_versions) { [">= 1.1.0"] } - # let(:expected_requirements) do - # [{ - # requirement: nil, - # groups: [], - # file: ".github/workflows/workflow.yml", - # source: { - # type: "git", - # url: "https://github.com/actions/setup-node", - # ref: "v1.0.4", - # branch: nil - # }, - # metadata: { declaration_string: "actions/setup-node@master" } - # }] - # end - # - # it { is_expected.to eq(expected_requirements) } - # end - end - - context "when a dependency has a path based tag reference without semver" do - let(:service_pack_url) do - "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ - "?service=git-upload-pack" - end before do stub_request(:get, service_pack_url) .to_return( @@ -1191,40 +1152,59 @@ } ) end + + it { is_expected.to eq(expected_requirements) } + end + + context "when a dependency has a path based tag reference without semver" do + let(:service_pack_url) do + "https://github.com/gopidesupavan/monorepo-actions.git/info/refs" \ + "?service=git-upload-pack" + end let(:upload_pack_fixture) { "github-monorepo-path-based" } let(:dependency) do Dependabot::Dependency.new( name: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0", version: "1.0.0", requirements: [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "exec/1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } - }], + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }], package_manager: "github_actions" ) end - let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "exec/2.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }] + end + + before do + stub_request(:get, service_pack_url) + .to_return( + status: 200, + body: fixture("git", "upload_packs", "github-monorepo-path-based"), + headers: { + "content-type" => "application/x-git-upload-pack-advertisement" + } + ) end it { is_expected.to eq(expected_requirements) } From 5463710ccd91c9077bf5ffcbf2fde0cd25d7b23f Mon Sep 17 00:00:00 2001 From: Pavan Kumar Date: Sun, 12 Jan 2025 18:58:36 +0000 Subject: [PATCH 09/72] Fix linter offenses --- .../github_actions/file_parser_spec.rb | 154 +++++++++--------- .../github_actions/file_updater_spec.rb | 46 +++--- 2 files changed, 101 insertions(+), 99 deletions(-) diff --git a/github_actions/spec/dependabot/github_actions/file_parser_spec.rb b/github_actions/spec/dependabot/github_actions/file_parser_spec.rb index 8a2f1df983..6f24d41268 100644 --- a/github_actions/spec/dependabot/github_actions/file_parser_spec.rb +++ b/github_actions/spec/dependabot/github_actions/file_parser_spec.rb @@ -589,17 +589,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "init/v1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } + }] end it "has the right details" do @@ -615,17 +615,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "run/v2.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } + }] end it "has the right details" do @@ -665,17 +665,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "exec/1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }] end it "has the right details" do @@ -716,17 +716,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "init/v1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/v1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/v1.0.0" } + }] end it "has the right details" do @@ -742,17 +742,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/actions/checkout", - ref: "v1", - branch: nil - }, - metadata: { declaration_string: "actions/checkout@v1" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/actions/checkout", + ref: "v1", + branch: nil + }, + metadata: { declaration_string: "actions/checkout@v1" } + }] end it "has the right details" do @@ -793,17 +793,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "init/1.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/1.0.0" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "init/1.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/init@init/1.0.0" } + }] end it "has the right details" do @@ -819,17 +819,17 @@ def mock_service_pack_request(nwo) let(:expected_requirements) do [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/actions/checkout", - ref: "v1", - branch: nil - }, - metadata: { declaration_string: "actions/checkout@v1" } - }] + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/actions/checkout", + ref: "v1", + branch: nil + }, + metadata: { declaration_string: "actions/checkout@v1" } + }] end it "has the right details" do diff --git a/github_actions/spec/dependabot/github_actions/file_updater_spec.rb b/github_actions/spec/dependabot/github_actions/file_updater_spec.rb index 614aa6d45d..965c5d7374 100644 --- a/github_actions/spec/dependabot/github_actions/file_updater_spec.rb +++ b/github_actions/spec/dependabot/github_actions/file_updater_spec.rb @@ -536,17 +536,17 @@ version: "5273d0df9c603edc4284ac8402cf650b4f1f6686", previous_version: nil, requirements: [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "run/v3.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } - }], + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "run/v3.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/first/run@run/v2.0.0" } + }], previous_requirements: [{ requirement: nil, groups: [], @@ -562,6 +562,7 @@ package_manager: "github_actions" ) end + its(:content) { is_expected.to include "gopidesupavan/monorepo-actions/first/run@run/v3.0.0\n" } end @@ -576,17 +577,17 @@ version: "5273d0df9c603edc4284ac8402cf650b4f1f6686", previous_version: nil, requirements: [{ - requirement: nil, - groups: [], - file: ".github/workflows/workflow.yml", - source: { - type: "git", - url: "https://github.com/gopidesupavan/monorepo-actions", - ref: "exec/2.0.0", - branch: nil - }, - metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } - }], + requirement: nil, + groups: [], + file: ".github/workflows/workflow.yml", + source: { + type: "git", + url: "https://github.com/gopidesupavan/monorepo-actions", + ref: "exec/2.0.0", + branch: nil + }, + metadata: { declaration_string: "gopidesupavan/monorepo-actions/second/exec@exec/1.0.0" } + }], previous_requirements: [{ requirement: nil, groups: [], @@ -602,6 +603,7 @@ package_manager: "github_actions" ) end + its(:content) { is_expected.to include "gopidesupavan/monorepo-actions/second/exec@exec/2.0.0\n" } end end From 960ff4b1bbdd75ca4a62fb6069a076df2f10364e Mon Sep 17 00:00:00 2001 From: Daniil Fajnberg <60156134+daniil-berg@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:54:27 +0100 Subject: [PATCH 10/72] Fix TOML parsing in Python ecosystem (fixes #10523) (#10540) * Fix TOML parsing in Python ecosystem (fixes #10523) Use `tomli` instead of outdated `toml` package * Adds test cases --------- Co-authored-by: sachin-sandhu --- python/helpers/lib/parser.py | 9 +++-- python/helpers/requirements.txt | 4 +- .../pyproject_files_parser_spec.rb | 39 +++++++++++++++++++ .../pyproject_files/pyproject_1_0_0.toml | 22 +++++++++++ .../pyproject_1_0_0_nodeps.toml | 19 +++++++++ .../pyproject_1_0_0_optional_deps.toml | 29 ++++++++++++++ 6 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index c2cfc8ff40..17c5e6f80b 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -14,9 +14,9 @@ ) from packaging.requirements import InvalidRequirement, Requirement -# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop -# support for Python 3.10. -import toml +# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we +# drop support for Python 3.10. +import tomli # Inspired by pips internal check: # https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35 @@ -24,7 +24,8 @@ def parse_pep621_dependencies(pyproject_path): - project_toml = toml.load(pyproject_path) + with open(pyproject_path, "rb") as file: + project_toml = tomli.load(file) def parse_toml_section_pep621_dependencies(pyproject_path, dependencies): requirement_packages = [] diff --git a/python/helpers/requirements.txt b/python/helpers/requirements.txt index f5998ed5e9..f16e275938 100644 --- a/python/helpers/requirements.txt +++ b/python/helpers/requirements.txt @@ -7,8 +7,8 @@ hashin==1.0.3; python_version >= '3.9' pipenv==2024.0.2 plette==2.1.0 poetry==1.8.5 -# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10. -toml==0.10.2 +# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10. +tomli==2.0.1 # Some dependencies will only install if Cython is present Cython==3.0.10 diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index cd042f22ee..e2737b4141 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -376,5 +376,44 @@ its(:length) { is_expected.to be > 0 } end + + describe "parse standard python files" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0.toml" } + + # fixture has 1 build system requires and plus 1 dependencies exists + + its(:length) { is_expected.to eq(1) } + + context "with a string declaration" do + subject(:dependency) { dependencies.first } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("pydantic") + expect(dependency.version).to eq("2.7.0") + end + end + + context "without dependencies" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0_nodeps.toml" } + + # fixture has 1 build system requires and no dependencies or + # optional dependencies exists + + its(:length) { is_expected.to eq(0) } + end + + context "with optional dependencies only" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0_optional_deps.toml" } + + its(:length) { is_expected.to be > 0 } + end + end end end diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml new file mode 100644 index 0000000000..998a8616ec --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] +dependencies = [ + '''pydantic==2.7.0''', +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml new file mode 100644 index 0000000000..ee091069eb --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml new file mode 100644 index 0000000000..7bad5cb673 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] +dependencies = [ + '''pydantic==2.7.0''', +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] +[project.optional-dependencies] +socks = [ 'PySocks >= 1.5.6, != 1.5.7, < 2' ] +tests = [ + 'ddt >= 1.2.2, < 2', + 'pytest < 6', + 'mock >= 1.0.1, < 4; python_version < "3.4"', +] From 6a4f8be5c6a0e6cd2e804a342077266d6eeb06eb Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:42:10 +0000 Subject: [PATCH 11/72] Removing corepack usage from else clause (#11299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Removing corepack usage from else clause * Fixing an rspec failure. * Removing the .npmrc file changes, as mark already fixed it. * Removing unnecessay comments. * Reverting last change. --------- Co-authored-by: “Thavachelvam <“thavaahariharangit@git.com”> --- npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb index 1f028de1f8..7cd5ac3956 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb @@ -324,8 +324,8 @@ def self.run_npm_command(command, fingerprint: command) package_manager_run_command(NpmPackageManager::NAME, command, fingerprint: fingerprint) else Dependabot::SharedHelpers.run_shell_command( - "corepack npm #{command}", - fingerprint: "corepack npm #{fingerprint}" + "npm #{command}", + fingerprint: "npm #{fingerprint}" ) end end From 028a428e8d16894746030cf3e83f9d89e414774f Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Fri, 17 Jan 2025 04:10:26 +0900 Subject: [PATCH 12/72] Bump Hex Elixir version to 1.15.8 (#11319) Bumps Elixir to 1.15.8 within Hex. There's a few notable changes due to the upgrade. 1. When Mix tries to process remote_convergers (Hex.RemoteConverger), Hex.Mix and other nested hex modules weren't loaded. 2. The term -> binary -> term message passing from run.exs to the other scripts now has Base.encode64/decode64 calls in order to make sure the binaries can be processed by IO.write 3. :hex and :ssh are added as extra_applications 4. The deps_warning mixfile needed a new kind of warning for the associated test to still pass. The old mixfile raises an error due to the missing parens on `deps` now. Co-authored-by: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> --- hex/Dockerfile | 4 +-- hex/helpers/lib/check_update.exs | 39 ++++++++++++--------- hex/helpers/lib/do_update.exs | 14 ++++---- hex/helpers/lib/parse_deps.exs | 12 ++++--- hex/helpers/lib/run.exs | 2 ++ hex/helpers/mix.exs | 2 +- hex/spec/dependabot/hex/file_parser_spec.rb | 2 +- hex/spec/fixtures/mixfiles/deps_warning | 6 +++- 8 files changed, 50 insertions(+), 31 deletions(-) diff --git a/hex/Dockerfile b/hex/Dockerfile index 454bd91e16..8a5bcf38a0 100644 --- a/hex/Dockerfile +++ b/hex/Dockerfile @@ -15,8 +15,8 @@ RUN apt-get update \ # Install Elixir # https://github.com/elixir-lang/elixir/releases -ARG ELIXIR_VERSION=v1.14.5 -ARG ELIXIR_CHECKSUM=f3b35d9fa61da7e93c9979cb8a08f64a9ce7074aeda66fae994f2a4ea2e4f82e +ARG ELIXIR_VERSION=v1.15.8 +ARG ELIXIR_CHECKSUM=62d33c51417191e027c9b6f0c46e11daeb236a7dda6f0746ec4dd53263531092 RUN curl -sSLfO https://github.com/elixir-lang/elixir/releases/download/${ELIXIR_VERSION}/elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ && echo "$ELIXIR_CHECKSUM elixir-otp-${ERLANG_MAJOR_VERSION}.zip" | sha256sum -c - \ && unzip -d /usr/local/elixir -x elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ diff --git a/hex/helpers/lib/check_update.exs b/hex/helpers/lib/check_update.exs index 229e968861..873ef099f8 100644 --- a/hex/helpers/lib/check_update.exs +++ b/hex/helpers/lib/check_update.exs @@ -1,5 +1,8 @@ defmodule UpdateChecker do def run(dependency_name) do + # This is necessary because we can't specify :extra_applications to have :hex in other mixfiles. + Mix.ensure_application!(:hex) + # Update the lockfile in a session that we can time out task = Task.async(fn -> do_resolution(dependency_name) end) @@ -45,24 +48,28 @@ end [dependency_name] = System.argv() -case UpdateChecker.run(dependency_name) do - {:ok, version} -> - version = :erlang.term_to_binary({:ok, version}) - IO.write(:stdio, version) +result = + case UpdateChecker.run(dependency_name) do + {:ok, version} -> + {:ok, version} + + {:error, %Version.InvalidRequirementError{} = error} -> + {:error, "Invalid requirement: #{error.requirement}"} - {:error, %Version.InvalidRequirementError{} = error} -> - result = :erlang.term_to_binary({:error, "Invalid requirement: #{error.requirement}"}) - IO.write(:stdio, result) + {:error, %Mix.Error{} = error} -> + {:error, "Dependency resolution failed: #{error.message}"} - {:error, %Mix.Error{} = error} -> - result = :erlang.term_to_binary({:error, "Dependency resolution failed: #{error.message}"}) - IO.write(:stdio, result) + {:error, :dependency_resolution_timed_out} -> + # We do nothing here because Hex is already printing out a message in stdout + nil - {:error, :dependency_resolution_timed_out} -> - # We do nothing here because Hex is already printing out a message in stdout - nil + {:error, error} -> + {:error, "Unknown error in check_update: #{inspect(error)}"} + end - {:error, error} -> - result = :erlang.term_to_binary({:error, "Unknown error in check_update: #{inspect(error)}"}) - IO.write(:stdio, result) +if not is_nil(result) do + result + |> :erlang.term_to_binary() + |> Base.encode64() + |> IO.write() end diff --git a/hex/helpers/lib/do_update.exs b/hex/helpers/lib/do_update.exs index 8eb862a7a4..35c4b9fcb0 100644 --- a/hex/helpers/lib/do_update.exs +++ b/hex/helpers/lib/do_update.exs @@ -1,3 +1,6 @@ +# This is necessary because we can't specify :extra_applications to have :hex in other mixfiles. +Mix.ensure_application!(:hex) + dependency = System.argv() |> List.first() @@ -23,9 +26,8 @@ System.cmd( ] ) -lockfile_content = - "mix.lock" - |> File.read() - |> :erlang.term_to_binary() - -IO.write(:stdio, lockfile_content) +"mix.lock" +|> File.read() +|> :erlang.term_to_binary() +|> Base.encode64() +|> IO.write() diff --git a/hex/helpers/lib/parse_deps.exs b/hex/helpers/lib/parse_deps.exs index 8e2f2458f5..d212c7c99a 100644 --- a/hex/helpers/lib/parse_deps.exs +++ b/hex/helpers/lib/parse_deps.exs @@ -1,5 +1,8 @@ defmodule Parser do def run do + # This is necessary because we can't specify :extra_applications to have :hex in other mixfiles. + Mix.ensure_application!(:hex) + Mix.Dep.load_on_environment([]) |> Enum.flat_map(&parse_dep/1) |> Enum.map(&build_dependency(&1.opts[:lock], &1)) @@ -82,7 +85,7 @@ defmodule Parser do |> empty_str_to_nil() end - defp maybe_regex_to_str(s), do: if Regex.regex?(s), do: Regex.source(s), else: s + defp maybe_regex_to_str(s), do: if(Regex.regex?(s), do: Regex.source(s), else: s) defp empty_str_to_nil(""), do: nil defp empty_str_to_nil(s), do: s @@ -99,6 +102,7 @@ defmodule Parser do end end -dependencies = :erlang.term_to_binary({:ok, Parser.run()}) - -IO.write(:stdio, dependencies) +{:ok, Parser.run()} +|> :erlang.term_to_binary() +|> Base.encode64() +|> IO.write() diff --git a/hex/helpers/lib/run.exs b/hex/helpers/lib/run.exs index 21e55d7670..71b0c2d91c 100644 --- a/hex/helpers/lib/run.exs +++ b/hex/helpers/lib/run.exs @@ -5,6 +5,7 @@ defmodule DependencyHelper do |> run() |> case do {output, 0} -> + output = Base.decode64!(output) if output =~ "No authenticated organization found" do {:error, output} else @@ -12,6 +13,7 @@ defmodule DependencyHelper do end {error, 1} -> + Base.decode64!(error) {:error, error} end |> handle_result() diff --git a/hex/helpers/mix.exs b/hex/helpers/mix.exs index 30f28a211b..d758c9b66c 100644 --- a/hex/helpers/mix.exs +++ b/hex/helpers/mix.exs @@ -12,7 +12,7 @@ defmodule DependabotCore.Mixfile do end def application do - [extra_applications: [:logger]] + [extra_applications: [:hex, :logger, :ssh]] end defp deps() do diff --git a/hex/spec/dependabot/hex/file_parser_spec.rb b/hex/spec/dependabot/hex/file_parser_spec.rb index fd47982dd7..539b575455 100644 --- a/hex/spec/dependabot/hex/file_parser_spec.rb +++ b/hex/spec/dependabot/hex/file_parser_spec.rb @@ -476,7 +476,7 @@ it "returns the correct language" do expect(language.name).to eq "elixir" expect(language.requirement).to be_nil - expect(language.version.to_s).to eq "1.14.5" + expect(language.version.to_s).to eq "1.15.8" end end end diff --git a/hex/spec/fixtures/mixfiles/deps_warning b/hex/spec/fixtures/mixfiles/deps_warning index f46492765c..c22526b76e 100644 --- a/hex/spec/fixtures/mixfiles/deps_warning +++ b/hex/spec/fixtures/mixfiles/deps_warning @@ -7,7 +7,7 @@ defmodule DependabotTest.Mixfile do version: "0.1.0", elixir: "~> 1.5", start_permanent: Mix.env == :prod, - deps: deps + deps: deps() ] end @@ -15,6 +15,10 @@ defmodule DependabotTest.Mixfile do [extra_applications: [:logger]] end + defp test do + nil + end + defp deps do [ {:plug, "~> 1.3.0"}, From 2fd554a38f2fc9106d372fddbbad24e54d35f100 Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Thu, 16 Jan 2025 19:41:28 +0000 Subject: [PATCH 13/72] Revert "Removing corepack usage from else clause (#11299)" (#11322) This reverts commit 6a4f8be5c6a0e6cd2e804a342077266d6eeb06eb. --- npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb index 7cd5ac3956..1f028de1f8 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb @@ -324,8 +324,8 @@ def self.run_npm_command(command, fingerprint: command) package_manager_run_command(NpmPackageManager::NAME, command, fingerprint: fingerprint) else Dependabot::SharedHelpers.run_shell_command( - "npm #{command}", - fingerprint: "npm #{fingerprint}" + "corepack npm #{command}", + fingerprint: "corepack npm #{fingerprint}" ) end end From 6754717df03dec87b263ba15ef584617ea15d704 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Wed, 15 Jan 2025 17:19:42 -0700 Subject: [PATCH 14/72] do null check before attempting to install sdk --- nuget/updater/common.ps1 | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nuget/updater/common.ps1 b/nuget/updater/common.ps1 index 657400d9d3..fab27e6b14 100755 --- a/nuget/updater/common.ps1 +++ b/nuget/updater/common.ps1 @@ -57,8 +57,15 @@ function Install-Sdks([string]$jobFilePath, [string]$repoContentsPath, [string]$ foreach ($globalJsonRelativePath in $globalJsonRelativePaths) { $globalJsonPath = "$rootDir/$globalJsonRelativePath" $globalJson = Get-Content $globalJsonPath | ConvertFrom-Json + if ("sdk" -notin $globalJson.PSobject.Properties.Name) { + continue + } + if ("version" -notin $globalJson.sdk.PSobject.Properties.Name) { + continue + } + $sdkVersion = $globalJson.sdk.version - if (-Not ($sdkVersion -in $installedSdks)) { + if (($Null -ne $sdkVersion) -And (-Not ($sdkVersion -in $installedSdks))) { $installedSdks += $sdkVersion Write-Host "Installing SDK $sdkVersion as specified in $globalJsonRelativePath" & $dotnetInstallScriptPath --version $sdkVersion --install-dir $dotnetInstallDir From 5b1320050fad07462c0a2ee6fcab2bf545b50159 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 20:46:12 +0000 Subject: [PATCH 15/72] Added sorbet typecheck under python module for setup_file_parser.rb , setup_file_sanitizer.rb, language_version_manager.rb,pipenv_runner.rb,index_finder.rb --- .../dependabot/python/file_parser/setup_file_parser.rb | 8 +++++++- .../python/file_updater/setup_file_sanitizer.rb | 3 +++ python/lib/dependabot/python/language_version_manager.rb | 6 ++++++ python/lib/dependabot/python/pipenv_runner.rb | 8 ++++++++ .../lib/dependabot/python/update_checker/index_finder.rb | 6 ++++++ 5 files changed, 30 insertions(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index a5336e20c2..4413311026 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -8,11 +8,13 @@ require "dependabot/python/file_parser" require "dependabot/python/native_helpers" require "dependabot/python/name_normaliser" +require "sorbet-runtime" module Dependabot module Python class FileParser class SetupFileParser + extend T::Sig INSTALL_REQUIRES_REGEX = /install_requires\s*=\s*\[/m SETUP_REQUIRES_REGEX = /setup_requires\s*=\s*\[/m TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m @@ -98,6 +100,7 @@ def parsed_sanitized_setup_file [] end + sig { params(requirements: T.untyped).void } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -140,19 +143,21 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - + sig { params(regex: Regexp).returns(T.untyped) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end + sig { params(regex: Regexp).returns(T.untyped) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) "{#{mch.post_match[0..closing_bracket_index(mch.post_match, '{')]}" end + sig { params(string: String, bracket: String).returns(T.untyped) } def closing_bracket_index(string, bracket) closes_required = 1 @@ -165,6 +170,7 @@ def closing_bracket_index(string, bracket) 0 end + sig { params(name: String, extras: T::Array[String]).returns(T.untyped) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index de74f69fc9..75d1d1918c 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -3,6 +3,7 @@ require "dependabot/python/file_updater" require "dependabot/python/file_parser/setup_file_parser" +require "sorbet-runtime" module Dependabot module Python @@ -10,6 +11,7 @@ class FileUpdater # Take a setup.py, parses it (carefully!) and then create a new, clean # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer + extend T::Sig def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg @@ -86,6 +88,7 @@ def parsed_setup_file ).dependency_set end + sig{ returns(String) } def package_name content = setup_file.content match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 2bd4eb1e26..970c41731e 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -3,10 +3,12 @@ require "dependabot/logger" require "dependabot/python/version" +require "sorbet-runtime" module Dependabot module Python class LanguageVersionManager + extend T::Sig # This list must match the versions specified at the top of `python/Dockerfile` PRE_INSTALLED_PYTHON_VERSIONS = %w( 3.13.1 @@ -21,6 +23,7 @@ def initialize(python_requirement_parser:) @python_requirement_parser = python_requirement_parser end + sig { returns(T.nilable(String)) } def install_required_python # The leading space is important in the version check return if SharedHelpers.run_shell_command("pyenv versions").include?(" #{python_major_minor}.") @@ -46,6 +49,7 @@ def python_version @python_version ||= python_version_from_supported_versions end + sig { returns(String) } def python_requirement_string if user_specified_python_version if user_specified_python_version.start_with?(/\d/) @@ -59,6 +63,7 @@ def python_requirement_string end end + sig { returns(String) } def python_version_from_supported_versions requirement_string = python_requirement_string @@ -88,6 +93,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end + sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index f59f0ec2bd..f242491a2a 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -4,16 +4,20 @@ require "dependabot/shared_helpers" require "dependabot/python/file_parser" require "json" +require "sorbet-runtime" module Dependabot module Python class PipenvRunner + extend T::Sig + def initialize(dependency:, lockfile:, language_version_manager:) @dependency = dependency @lockfile = lockfile @language_version_manager = language_version_manager end + sig { params(constraint: String).void } def run_upgrade(constraint) constraint = "" if constraint == "*" command = "pyenv exec pipenv upgrade --verbose #{dependency_name}#{constraint}" @@ -22,6 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end + sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -30,6 +35,7 @@ def run_upgrade_and_fetch_version(constraint) fetch_version_from_parsed_lockfile(updated_lockfile) end + sig { params(command: String, fingerprint: T.nilable(String)).void } def run(command, fingerprint: nil) run_command( "pyenv local #{language_version_manager.python_major_minor}", @@ -45,6 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager + sig { params(updated_lockfile: []).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -52,6 +59,7 @@ def fetch_version_from_parsed_lockfile(updated_lockfile) &.gsub(/^==/, "") end + sig { params(command: String, fingerprint: T.nilable(String)).void } def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 5f3e6443dd..3a109cb021 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -9,6 +9,7 @@ module Dependabot module Python class UpdateChecker class IndexFinder + extend T::Sig PYPI_BASE_URL = "https://pypi.org/simple/" ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/ @@ -55,6 +56,7 @@ def main_index_url clean_check_and_remove_environment_variables(url) end + sig { returns({ main: NilClass, extra: [] }) } def requirement_file_index_urls urls = { main: nil, extra: [] } @@ -74,6 +76,7 @@ def requirement_file_index_urls urls end + sig { returns({ main: NilClass, extra: [] }) } def pip_conf_index_urls urls = { main: nil, extra: [] } @@ -90,6 +93,7 @@ def pip_conf_index_urls urls end + sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pipfile_index_urls urls = { main: nil, extra: [] } @@ -109,6 +113,7 @@ def pipfile_index_urls urls end + sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pyproject_index_urls urls = { main: nil, extra: [] } @@ -139,6 +144,7 @@ def pyproject_index_urls urls end + sig { returns({ main: NilClass, extra: [] }) } def config_variable_index_urls urls = { main: nil, extra: [] } From 86a31e6d1cbada7c1f147eec948621640efc66a2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 21:01:59 +0000 Subject: [PATCH 16/72] Fixed issue in index_finder --- python/lib/dependabot/python/update_checker/index_finder.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 3a109cb021..2efc34dc06 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -93,7 +93,6 @@ def pip_conf_index_urls urls end - sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pipfile_index_urls urls = { main: nil, extra: [] } @@ -113,7 +112,6 @@ def pipfile_index_urls urls end - sig { returns(T.nilable({ main: NilClass, extra: [] })) } def pyproject_index_urls urls = { main: nil, extra: [] } From 8df8696b26a19727a4fdfe7b899d788c85d2c9ac Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 21:07:13 +0000 Subject: [PATCH 17/72] Fixed issue in setupfilesanitizer --- .../lib/dependabot/python/file_updater/setup_file_sanitizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 75d1d1918c..d89dc16020 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -88,7 +88,7 @@ def parsed_setup_file ).dependency_set end - sig{ returns(String) } + sig { returns(String) } def package_name content = setup_file.content match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) From 2057befbb9f84699445e785212304550269bcffa Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 23:57:22 +0000 Subject: [PATCH 18/72] Included TypeCheck for one more file --- .../lib/dependabot/python/file_parser/setup_file_parser.rb | 7 +++---- .../dependabot/python/file_updater/pyproject_preparer.rb | 5 +++++ .../lib/dependabot/python/update_checker/index_finder.rb | 3 --- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 4413311026..1b841af615 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -100,7 +100,6 @@ def parsed_sanitized_setup_file [] end - sig { params(requirements: T.untyped).void } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -150,14 +149,14 @@ def get_regexed_req_array(regex) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - sig { params(regex: Regexp).returns(T.untyped) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) "{#{mch.post_match[0..closing_bracket_index(mch.post_match, '{')]}" end - sig { params(string: String, bracket: String).returns(T.untyped) } + sig { params(string: String, bracket: String).returns(Integer) } def closing_bracket_index(string, bracket) closes_required = 1 @@ -170,7 +169,7 @@ def closing_bracket_index(string, bracket) 0 end - sig { params(name: String, extras: T::Array[String]).returns(T.untyped) } + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 17254e734b..2e12c190a6 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -14,6 +14,7 @@ module Dependabot module Python class FileUpdater class PyprojectPreparer + extend T::Sig def initialize(pyproject_content:, lockfile: nil) @pyproject_content = pyproject_content @lockfile = lockfile @@ -21,6 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. + sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } @@ -37,6 +39,7 @@ def add_auth_env_vars(credentials) end end + sig { params(requirement: String).void } def update_python_requirement(requirement) pyproject_object = TomlRB.parse(@pyproject_content) if (python_specification = pyproject_object.dig("tool", "poetry", "dependencies", "python")) @@ -48,6 +51,7 @@ def update_python_requirement(requirement) TomlRB.dump(pyproject_object) end + sig { returns(String) } def sanitize # {{ name }} syntax not allowed pyproject_content @@ -111,6 +115,7 @@ def locked_details(dep_name) .find { |d| d["name"] == normalise(dep_name) } end + sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 2efc34dc06..3f84e1f1f0 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -9,7 +9,6 @@ module Dependabot module Python class UpdateChecker class IndexFinder - extend T::Sig PYPI_BASE_URL = "https://pypi.org/simple/" ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/ @@ -56,7 +55,6 @@ def main_index_url clean_check_and_remove_environment_variables(url) end - sig { returns({ main: NilClass, extra: [] }) } def requirement_file_index_urls urls = { main: nil, extra: [] } @@ -76,7 +74,6 @@ def requirement_file_index_urls urls end - sig { returns({ main: NilClass, extra: [] }) } def pip_conf_index_urls urls = { main: nil, extra: [] } From a43fc93b09f91f4bc17321a511f8c1e885bf7f6a Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 9 Jan 2025 23:59:23 +0000 Subject: [PATCH 19/72] Remvoed from index finder file --- python/lib/dependabot/python/update_checker/index_finder.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/python/lib/dependabot/python/update_checker/index_finder.rb b/python/lib/dependabot/python/update_checker/index_finder.rb index 3f84e1f1f0..5f3e6443dd 100644 --- a/python/lib/dependabot/python/update_checker/index_finder.rb +++ b/python/lib/dependabot/python/update_checker/index_finder.rb @@ -139,7 +139,6 @@ def pyproject_index_urls urls end - sig { returns({ main: NilClass, extra: [] }) } def config_variable_index_urls urls = { main: nil, extra: [] } From 271aeb92cb6807cd82ad3426a67d8a629f764773 Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 00:22:24 +0000 Subject: [PATCH 20/72] Removed from pyproject_preparer --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 1 - python/lib/dependabot/python/pipenv_runner.rb | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 2e12c190a6..5b52ad6236 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -39,7 +39,6 @@ def add_auth_env_vars(credentials) end end - sig { params(requirement: String).void } def update_python_requirement(requirement) pyproject_object = TomlRB.parse(@pyproject_content) if (python_specification = pyproject_object.dig("tool", "poetry", "dependencies", "python")) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index f242491a2a..2689afaf51 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -51,7 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - sig { params(updated_lockfile: []).void } + sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} From ecaa4d2ec32257bc3622e7b9a11b53267f68ac26 Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 00:52:25 +0000 Subject: [PATCH 21/72] Commented since return type is not correct --- .../dependabot/python/file_parser/setup_file_parser.rb | 5 +++-- .../dependabot/python/file_updater/pyproject_preparer.rb | 2 +- python/lib/dependabot/python/language_version_manager.rb | 2 +- python/lib/dependabot/python/pipenv_runner.rb | 8 ++++---- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 1b841af615..403f0ae87a 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -142,14 +142,15 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - sig { params(regex: Regexp).returns(T.untyped) } + + # sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - sig { params(regex: Regexp).returns(T.nilable(String)) } + # sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 5b52ad6236..c43a1c71e2 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - sig { params(credentials: T.nilable(Hash)).void } + # sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 970c41731e..e6a38df299 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -93,7 +93,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end - sig { params(requirements: T.untyped).returns(T.nilable(String)) } + # sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index 2689afaf51..a7031faa0c 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,7 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).void } + # sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -35,7 +35,7 @@ def run_upgrade_and_fetch_version(constraint) fetch_version_from_parsed_lockfile(updated_lockfile) end - sig { params(command: String, fingerprint: T.nilable(String)).void } + sig { params(command: String, fingerprint: T.nilable(String)).returns(String) } def run(command, fingerprint: nil) run_command( "pyenv local #{language_version_manager.python_major_minor}", @@ -51,7 +51,7 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - sig { params(updated_lockfile: Array).void } + # sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -59,7 +59,7 @@ def fetch_version_from_parsed_lockfile(updated_lockfile) &.gsub(/^==/, "") end - sig { params(command: String, fingerprint: T.nilable(String)).void } + sig { params(command: String, fingerprint: T.nilable(String)).returns(String) } def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end From 4c981affa7120beefd325750bc38c8118007bd8c Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 01:09:19 +0000 Subject: [PATCH 22/72] Added TypeCheck back to setup_file_parser --- python/lib/dependabot/python/file_parser/setup_file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 403f0ae87a..2e634bdb6c 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -143,14 +143,14 @@ def write_sanitized_setup_file File.write("setup.py", tmp) end - # sig { params(regex: Regexp).returns(T.nilable(String)) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_array(regex) return unless (mch = setup_file.content.match(regex)) "[#{mch.post_match[0..closing_bracket_index(mch.post_match, '[')]}" end - # sig { params(regex: Regexp).returns(T.nilable(String)) } + sig { params(regex: Regexp).returns(T.nilable(String)) } def get_regexed_req_dict(regex) return unless (mch = setup_file.content.match(regex)) From a383d86a4bfae6f282cf6409a5db9b1e11c0c07d Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 01:38:08 +0000 Subject: [PATCH 23/72] Added TypeCheck pyproject_preparer.rb --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 2 +- python/lib/dependabot/python/language_version_manager.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index c43a1c71e2..5b52ad6236 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,7 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - # sig { params(credentials: T.nilable(Hash)).void } + sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index e6a38df299..3ec07661ea 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -93,7 +93,6 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end - # sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) From cfc50448a11a46873a31fdc3436d6a5a2dab200f Mon Sep 17 00:00:00 2001 From: Randhir Date: Fri, 10 Jan 2025 02:28:33 +0000 Subject: [PATCH 24/72] Removed commented code --- python/lib/dependabot/python/file_updater/pyproject_preparer.rb | 1 - python/lib/dependabot/python/pipenv_runner.rb | 2 -- 2 files changed, 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index 5b52ad6236..a3f0a09afc 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -22,7 +22,6 @@ def initialize(pyproject_content:, lockfile: nil) # For hosted Dependabot token will be nil since the credentials aren't present. # This is for those running Dependabot themselves and for dry-run. - sig { params(credentials: T.nilable(Hash)).void } def add_auth_env_vars(credentials) TomlRB.parse(@pyproject_content).dig("tool", "poetry", "source")&.each do |source| cred = credentials&.find { |c| c["index-url"] == source["url"] } diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index a7031faa0c..83502aa812 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,7 +26,6 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - # sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -51,7 +50,6 @@ def run(command, fingerprint: nil) attr_reader :lockfile attr_reader :language_version_manager - # sig { params(updated_lockfile: Array).void } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} From a8d8ad597d8015332eaf241a0cbc30316b9f1241 Mon Sep 17 00:00:00 2001 From: Randhir Date: Mon, 13 Jan 2025 22:25:30 +0000 Subject: [PATCH 25/72] Added TypeCheck as strict for file_parser --- python/lib/dependabot/python/file_parser.rb | 90 ++++++++++++------- python/lib/dependabot/python/pipenv_runner.rb | 1 + 2 files changed, 57 insertions(+), 34 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 0de97a942d..62c97c88d1 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/dependency" @@ -18,12 +18,13 @@ module Dependabot module Python class FileParser < Dependabot::FileParsers::Base + extend T::Sig require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/pyproject_files_parser" require_relative "file_parser/setup_file_parser" require_relative "file_parser/python_requirement_parser" - DEPENDENCY_GROUP_KEYS = [ + DEPENDENCY_GROUP_KEYS = T.let([ { pipfile: "packages", lockfile: "default" @@ -32,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze + ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -43,6 +44,7 @@ class FileParser < Dependabot::FileParsers::Base # in any way if any metric collection exception start happening UNDETECTED_PACKAGE_MANAGER_VERSION = "0.0" + sig { override.returns(T::Array[Dependabot::Dependency]) } def parse # TODO: setup.py from external dependencies is evaluated. Provide guards before removing this. raise Dependabot::UnexpectedExternalCode if @reject_external_code @@ -57,7 +59,7 @@ def parse dependency_set.dependencies end - sig { returns(Ecosystem) } + sig { override.returns(Ecosystem) } def ecosystem @ecosystem ||= T.let( Ecosystem.new( @@ -71,18 +73,16 @@ def ecosystem private + sig { returns(Dependabot::Python::LanguageVersionManager) } def language_version_manager - @language_version_manager ||= - LanguageVersionManager.new( - python_requirement_parser: python_requirement_parser - ) + @language_version_manager ||= T.let(LanguageVersionManager.new(python_requirement_parser: + python_requirement_parser), T.nilable(LanguageVersionManager)) end + sig { returns(Dependabot::Python::FileParser::PythonRequirementParser) } def python_requirement_parser - @python_requirement_parser ||= - FileParser::PythonRequirementParser.new( - dependency_files: dependency_files - ) + @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: dependency_files), + T.nilable(FileParser::PythonRequirementParser)) end sig { returns(Ecosystem::VersionManager) } @@ -91,7 +91,7 @@ def package_manager Dependabot.logger.info("Detected package manager : #{detected_package_manager.name}") end - @package_manager ||= detected_package_manager + @package_manager ||= T.let(detected_package_manager, T.nilable(Dependabot::Ecosystem::VersionManager)) end sig { returns(Ecosystem::VersionManager) } @@ -188,7 +188,7 @@ def package_manager_version(package_manager) end # setup python local setup on file parser stage - sig { void } + sig { returns(T.nilable(String)) } def setup_python_environment language_version_manager.install_required_python @@ -231,24 +231,24 @@ def language ) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end + sig { returns(T.nilable(PipfileFilesParser)) } def pipenv_dependencies - @pipenv_dependencies ||= - PipfileFilesParser - .new(dependency_files: dependency_files) - .dependency_set + @pipenv_dependencies ||= T.let(PipfileFilesParser.new(dependency_files: + dependency_files).dependency_set, T.nilable(PipfileFilesParser)) end + sig { returns(T.nilable(PyprojectFilesParser))} def pyproject_file_dependencies - @pyproject_file_dependencies ||= - PyprojectFilesParser - .new(dependency_files: dependency_files) - .dependency_set + @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: + dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) end + sig { returns(DependencySet)} def requirement_dependencies dependencies = DependencySet.new parsed_requirement_files.each do |dep| @@ -286,13 +286,15 @@ def requirement_dependencies dependencies end + sig { params(name: String, version: String).returns(T::Boolean) } def old_pyyaml?(name, version) - major_version = version&.split(".")&.first + major_version = version.split(".").first return false unless major_version name == "pyyaml" && major_version < "6" end + sig { params(filename: String).returns(T::Array[String]) } def group_from_filename(filename) if filename.include?("dev") then ["dev-dependencies"] else @@ -300,6 +302,7 @@ def group_from_filename(filename) end end + sig { params(dep: T.untyped).returns(T::Boolean) } def blocking_marker?(dep) return false if dep["markers"] == "None" @@ -316,6 +319,7 @@ def blocking_marker?(dep) end end + sig { params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) } def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -356,13 +360,15 @@ def evaluate_condition(condition, python_version) end end + sig { void } def setup_file_dependencies - @setup_file_dependencies ||= + @setup_file_dependencies ||= T.let( SetupFileParser .new(dependency_files: dependency_files) - .dependency_set + .dependency_set, T.untyped) end + sig { returns(T.untyped) } def parsed_requirement_files SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files @@ -383,6 +389,7 @@ def parsed_requirement_files raise Dependabot::DependencyFileNotEvaluatable, e.message end + sig { params(requirements: T.untyped).returns(T.untyped) } def check_requirements(requirements) requirements.each do |dep| next unless dep["requirement"] @@ -393,18 +400,22 @@ def check_requirements(requirements) end end + sig { returns(T::Boolean) } def pipcompile_in_file requirement_files.any? { |f| f.name.end_with?(PipCompilePackageManager::MANIFEST_FILENAME) } end + sig { returns(T::Boolean) } def pipenv_files dependency_files.any? { |f| f.name == PipenvPackageManager::LOCKFILE_FILENAME } end + sig { returns(T.nilable(TrueClass)) } def poetry_files true if get_original_file(PoetryPackageManager::LOCKFILE_NAME) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def write_temporary_dependency_files dependency_files .reject { |f| f.name == ".python-version" } @@ -415,6 +426,7 @@ def write_temporary_dependency_files end end + sig { params(file: T.untyped).returns(T.untyped) } def remove_imports(file) return file.content if file.path.end_with?(".tar.gz", ".whl", ".zip") @@ -424,10 +436,12 @@ def remove_imports(file) .join end + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras = []) NameNormaliser.normalise_including_extras(name, extras) end + sig { override.void } def check_required_files filenames = dependency_files.map(&:name) return if filenames.any? { |name| name.end_with?(".txt", ".in") } @@ -439,37 +453,45 @@ def check_required_files raise "Missing required files!" end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pipfile - @pipfile ||= get_original_file("Pipfile") + @pipfile ||= T.let(get_original_file("Pipfile"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pipfile_lock - @pipfile_lock ||= get_original_file("Pipfile.lock") + @pipfile_lock ||= T.let(get_original_file("Pipfile.lock"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pyproject - @pyproject ||= get_original_file("pyproject.toml") + @pyproject ||= T.let(get_original_file("pyproject.toml"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def poetry_lock - @poetry_lock ||= get_original_file("poetry.lock") + @poetry_lock ||= T.let(get_original_file("poetry.lock"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile))} def setup_file - @setup_file ||= get_original_file("setup.py") + @setup_file ||= T.let(get_original_file("setup.py"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T.nilable(Dependabot::DependencyFile)) } def setup_cfg_file - @setup_cfg_file ||= get_original_file("setup.cfg") + @setup_cfg_file ||= T.let(get_original_file("setup.cfg"), T.nilable(Dependabot::DependencyFile)) end + sig { returns(T::Array[Dependabot::DependencyFile]) } def pip_compile_files - @pip_compile_files ||= - dependency_files.select { |f| f.name.end_with?(".in") } + @pip_compile_files ||= T.let(dependency_files.select { |f| f.name.end_with?(".in") }, T.untyped) end + sig { returns(Dependabot::Python::PipCompileFileMatcher) } def pip_compile_file_matcher - @pip_compile_file_matcher ||= PipCompileFileMatcher.new(pip_compile_files) + @pip_compile_file_matcher ||= T.let(PipCompileFileMatcher.new(pip_compile_files), + T.nilable(Dependabot::Python::PipCompileFileMatcher)) end end end diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index 83502aa812..d65ab0ff9f 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -26,6 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end + sig { params(constraint: String).void } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) From 604024fa38b8d72d897dd926f6ea6db962c77190 Mon Sep 17 00:00:00 2001 From: Randhir Date: Mon, 13 Jan 2025 22:29:41 +0000 Subject: [PATCH 26/72] Added TypeCheck as strict for file_parser --- python/lib/dependabot/python/file_parser.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 62c97c88d1..46b6c6e0c8 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -242,13 +242,13 @@ def pipenv_dependencies dependency_files).dependency_set, T.nilable(PipfileFilesParser)) end - sig { returns(T.nilable(PyprojectFilesParser))} + sig { returns(T.nilable(PyprojectFilesParser)) } def pyproject_file_dependencies @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) end - sig { returns(DependencySet)} + sig { returns(DependencySet) } def requirement_dependencies dependencies = DependencySet.new parsed_requirement_files.each do |dep| @@ -319,7 +319,9 @@ def blocking_marker?(dep) end end - sig { params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) } + sig do + params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + end def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -341,6 +343,10 @@ def marker_satisfied?(marker, python_version) result end + sig do + params(condition: T.untyped, + python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + end def evaluate_condition(condition, python_version) operator, version = condition.match(/([<>=!]=?)\s*"?([\d.]+)"?/)&.captures @@ -365,7 +371,8 @@ def setup_file_dependencies @setup_file_dependencies ||= T.let( SetupFileParser .new(dependency_files: dependency_files) - .dependency_set, T.untyped) + .dependency_set, T.untyped + ) end sig { returns(T.untyped) } @@ -473,7 +480,7 @@ def poetry_lock @poetry_lock ||= T.let(get_original_file("poetry.lock"), T.nilable(Dependabot::DependencyFile)) end - sig { returns(T.nilable(Dependabot::DependencyFile))} + sig { returns(T.nilable(Dependabot::DependencyFile)) } def setup_file @setup_file ||= T.let(get_original_file("setup.py"), T.nilable(Dependabot::DependencyFile)) end @@ -483,7 +490,7 @@ def setup_cfg_file @setup_cfg_file ||= T.let(get_original_file("setup.cfg"), T.nilable(Dependabot::DependencyFile)) end - sig { returns(T::Array[Dependabot::DependencyFile]) } + sig { returns(T::Array[Dependabot::Python::Requirement]) } def pip_compile_files @pip_compile_files ||= T.let(dependency_files.select { |f| f.name.end_with?(".in") }, T.untyped) end From 711a96f0154edc40b287d68f7bb39b3db19355c2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 14:48:01 +0000 Subject: [PATCH 27/72] Removed nilable to fix the issue --- python/lib/dependabot/python/file_parser.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 46b6c6e0c8..28679db25c 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) + ].freeze, T::Array[T::Hash[String, String]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -320,7 +320,7 @@ def blocking_marker?(dep) end sig do - params(marker: T.untyped, python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + params(marker: T.untyped, python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean) end def marker_satisfied?(marker, python_version) conditions = marker.split(/\s+(and|or)\s+/) @@ -345,7 +345,7 @@ def marker_satisfied?(marker, python_version) sig do params(condition: T.untyped, - python_version: T.nilable(T.any(String, Integer, Gem::Version))).returns(T::Boolean) + python_version: T.any(String, Integer, Gem::Version)).returns(T::Boolean) end def evaluate_condition(condition, python_version) operator, version = condition.match(/([<>=!]=?)\s*"?([\d.]+)"?/)&.captures From 44d27ccf781db205f201ff1409291c04a2e80a05 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:00:06 +0000 Subject: [PATCH 28/72] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 28679db25c..f24f052e32 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[String, String]]) + ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError From e0e0646f3e75fc1482a346e9253fdf8111ed0f14 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:05:26 +0000 Subject: [PATCH 29/72] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index f24f052e32..80d07207fd 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -17,7 +17,7 @@ module Dependabot module Python - class FileParser < Dependabot::FileParsers::Base + class FileParser < Dependabot::FileParsers::Base # rubocop:disable Metrics/ClassLength extend T::Sig require_relative "file_parser/pipfile_files_parser" require_relative "file_parser/pyproject_files_parser" From d9320d2eaec95f2ca017072b53a503005245400f Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 15:08:27 +0000 Subject: [PATCH 30/72] Moved back to untyped --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 80d07207fd..37724bac88 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -81,8 +81,8 @@ def language_version_manager sig { returns(Dependabot::Python::FileParser::PythonRequirementParser) } def python_requirement_parser - @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: dependency_files), - T.nilable(FileParser::PythonRequirementParser)) + @python_requirement_parser ||= T.let(FileParser::PythonRequirementParser.new(dependency_files: + dependency_files), T.nilable(FileParser::PythonRequirementParser)) end sig { returns(Ecosystem::VersionManager) } From 85d266a81c8f93fed4f3cb4b492125b711fc8ac8 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 19:21:03 +0000 Subject: [PATCH 31/72] corrected return type --- python/lib/dependabot/python/file_parser.rb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 37724bac88..4c150f9481 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,14 +198,13 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).void } + sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found - return true if version.match?(/^\d+(?:\.\d+)*$/) - Dependabot.logger.warn( "Detected #{package_manager} with malformed version #{version}" ) + true if version.match?(/^\d+(?:\.\d+)*$/) end sig { returns(String) } @@ -366,13 +365,10 @@ def evaluate_condition(condition, python_version) end end - sig { void } + sig { returns(T.nilable(SetupFileParser)) } def setup_file_dependencies - @setup_file_dependencies ||= T.let( - SetupFileParser - .new(dependency_files: dependency_files) - .dependency_set, T.untyped - ) + @setup_file_dependencies ||= T.let(SetupFileParser.new(dependency_files: dependency_files) + .dependency_set, T.nilable(SetupFileParser)) end sig { returns(T.untyped) } From 2955a6427746b58b7e8d18eb4b7ebfce6ed7621c Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 19:46:40 +0000 Subject: [PATCH 32/72] Corrected returntype to pass end to end error --- python/lib/dependabot/python/file_parser.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 4c150f9481..811e05bc70 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,13 +198,14 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } + sig { params(package_manager: String, version: String).void } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found + return true if version.match?(/^\d+(?:\.\d+)*$/) + Dependabot.logger.warn( "Detected #{package_manager} with malformed version #{version}" ) - true if version.match?(/^\d+(?:\.\d+)*$/) end sig { returns(String) } @@ -285,9 +286,9 @@ def requirement_dependencies dependencies end - sig { params(name: String, version: String).returns(T::Boolean) } + sig { params(name: T.nilable(String), version: T.nilable(String)).returns(T::Boolean) } def old_pyyaml?(name, version) - major_version = version.split(".").first + major_version = version&.split(".")&.first return false unless major_version name == "pyyaml" && major_version < "6" @@ -444,7 +445,7 @@ def normalised_name(name, extras = []) NameNormaliser.normalise_including_extras(name, extras) end - sig { override.void } + sig { override.returns(T.untyped) } def check_required_files filenames = dependency_files.map(&:name) return if filenames.any? { |name| name.end_with?(".txt", ".in") } From 4f3843941b758d03814445ef3104741eba2bff6b Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:05:04 +0000 Subject: [PATCH 33/72] Changed requirementfile method return type --- python/lib/dependabot/python/file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 811e05bc70..3356c5a76f 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -231,7 +231,7 @@ def language ) end - sig { returns(T::Array[Dependabot::DependencyFile]) } + sig { returns(T.untyped) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end From b847188e81d16754a71e784757b31833ea00df27 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:19:12 +0000 Subject: [PATCH 34/72] Changed return type to dependabotset --- python/lib/dependabot/python/file_parser.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 3356c5a76f..2a8f478ac0 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -231,21 +231,21 @@ def language ) end - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } def requirement_files dependency_files.select { |f| f.name.end_with?(".txt", ".in") } end - sig { returns(T.nilable(PipfileFilesParser)) } + sig { returns(DependencySet) } def pipenv_dependencies @pipenv_dependencies ||= T.let(PipfileFilesParser.new(dependency_files: - dependency_files).dependency_set, T.nilable(PipfileFilesParser)) + dependency_files).dependency_set, T.nilable(DependencySet)) end - sig { returns(T.nilable(PyprojectFilesParser)) } + sig { returns(DependencySet) } def pyproject_file_dependencies @pyproject_file_dependencies ||= T.let(PyprojectFilesParser.new(dependency_files: - dependency_files).dependency_set, T.nilable(PyprojectFilesParser)) + dependency_files).dependency_set, T.nilable(DependencySet)) end sig { returns(DependencySet) } From e03d781fcc97b1e4636ef99643c8134626f310a2 Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 20:34:44 +0000 Subject: [PATCH 35/72] Changed return type to dependencySet --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 2a8f478ac0..8685eac848 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -366,10 +366,10 @@ def evaluate_condition(condition, python_version) end end - sig { returns(T.nilable(SetupFileParser)) } + sig { returns(DependencySet) } def setup_file_dependencies @setup_file_dependencies ||= T.let(SetupFileParser.new(dependency_files: dependency_files) - .dependency_set, T.nilable(SetupFileParser)) + .dependency_set, T.nilable(DependencySet)) end sig { returns(T.untyped) } From 12d3f3dbaf05659b55271fba31e18a58de86ff5f Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 21:07:59 +0000 Subject: [PATCH 36/72] Corrected log_if_version_malformed --- python/lib/dependabot/python/file_parser.rb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 8685eac848..9d0940298b 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -198,14 +198,15 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).void } + sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found - return true if version.match?(/^\d+(?:\.\d+)*$/) - - Dependabot.logger.warn( - "Detected #{package_manager} with malformed version #{version}" - ) + if version.match?(/^\d+(?:\.\d+)*$/) + true + else + Dependabot.logger.warn("Detected #{package_manager} with malformed version #{version}") + false + end end sig { returns(String) } From ee979f992dd390545e914ee8107b972c0674a77d Mon Sep 17 00:00:00 2001 From: Randhir Date: Tue, 14 Jan 2025 22:08:33 +0000 Subject: [PATCH 37/72] Changed return type in pipenv_runner --- python/lib/dependabot/python/pipenv_runner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index d65ab0ff9f..bb7fc6857d 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -17,7 +17,7 @@ def initialize(dependency:, lockfile:, language_version_manager:) @language_version_manager = language_version_manager end - sig { params(constraint: String).void } + sig { params(constraint: String).returns(String) } def run_upgrade(constraint) constraint = "" if constraint == "*" command = "pyenv exec pipenv upgrade --verbose #{dependency_name}#{constraint}" @@ -26,7 +26,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).void } + sig { params(constraint: String).returns(T.untyped) } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) From 90772e10882b96f33074d851b3738b48cf86ffee Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 00:01:03 +0000 Subject: [PATCH 38/72] Corrected return type --- python/lib/dependabot/python/file_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser.rb b/python/lib/dependabot/python/file_parser.rb index 9d0940298b..9ab46bb3fa 100644 --- a/python/lib/dependabot/python/file_parser.rb +++ b/python/lib/dependabot/python/file_parser.rb @@ -33,7 +33,7 @@ class FileParser < Dependabot::FileParsers::Base # rubocop:disable Metrics/Class pipfile: "dev-packages", lockfile: "develop" } - ].freeze, T::Array[T::Hash[T.untyped, T.untyped]]) + ].freeze, T::Array[T::Hash[Symbol, String]]) REQUIREMENT_FILE_EVALUATION_ERRORS = %w( InstallationError RequirementsFileParseError InvalidMarker InvalidRequirement ValueError RecursionError @@ -198,7 +198,7 @@ def setup_python_environment nil end - sig { params(package_manager: String, version: String).returns(T.nilable(T::Boolean)) } + sig { params(package_manager: String, version: String).returns(T::Boolean) } def log_if_version_malformed(package_manager, version) # logs warning if malformed version is found if version.match?(/^\d+(?:\.\d+)*$/) From db2df60ead470b6ae8c60fcc3389fe9f408db14e Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 19:22:46 +0000 Subject: [PATCH 39/72] Added TypeCheck as strict for pipenv_runner --- python/lib/dependabot/python/pipenv_runner.rb | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index bb7fc6857d..a526c72a54 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/shared_helpers" @@ -11,6 +11,14 @@ module Python class PipenvRunner extend T::Sig + sig do + params( + dependency: Dependabot::Dependency, + lockfile: DependencyFile, + language_version_manager: LanguageVersionManager + ) + .void + end def initialize(dependency:, lockfile:, language_version_manager:) @dependency = dependency @lockfile = lockfile @@ -26,7 +34,7 @@ def run_upgrade(constraint) run(command, fingerprint: "pyenv exec pipenv upgrade --verbose ") end - sig { params(constraint: String).returns(T.untyped) } + sig { params(constraint: String).returns(String) } def run_upgrade_and_fetch_version(constraint) run_upgrade(constraint) @@ -47,10 +55,14 @@ def run(command, fingerprint: nil) private + sig { returns(Dependabot::Dependency) } attr_reader :dependency + sig { returns(DependencyFile) } attr_reader :lockfile + sig { returns(LanguageVersionManager) } attr_reader :language_version_manager + sig { params(updated_lockfile: T.untyped).returns(T.untyped) } def fetch_version_from_parsed_lockfile(updated_lockfile) deps = updated_lockfile[lockfile_section] || {} @@ -63,21 +75,24 @@ def run_command(command, fingerprint: nil) SharedHelpers.run_shell_command(command, env: pipenv_env_variables, fingerprint: fingerprint) end + sig { returns(String) } def lockfile_section if dependency.requirements.any? - dependency.requirements.first[:groups].first + T.must(dependency.requirements.first)[:groups].first else Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys| section = keys.fetch(:lockfile) - return section if JSON.parse(lockfile.content)[section].keys.any?(dependency_name) + return section if JSON.parse(T.must(lockfile.content))[section].keys.any?(dependency_name) end end end + sig { returns(String) } def dependency_name dependency.metadata[:original_name] || dependency.name end + sig { returns(T::Hash[String, String]) } def pipenv_env_variables { "PIPENV_YES" => "true", # Install new Python ver if needed From ed5a286db234e3cea8a89e05ec874c0c8a18c316 Mon Sep 17 00:00:00 2001 From: Randhir Date: Wed, 15 Jan 2025 19:42:53 +0000 Subject: [PATCH 40/72] Changed param DependcyFile Nilable --- python/lib/dependabot/python/pipenv_runner.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/pipenv_runner.rb b/python/lib/dependabot/python/pipenv_runner.rb index a526c72a54..de76e17631 100644 --- a/python/lib/dependabot/python/pipenv_runner.rb +++ b/python/lib/dependabot/python/pipenv_runner.rb @@ -14,7 +14,7 @@ class PipenvRunner sig do params( dependency: Dependabot::Dependency, - lockfile: DependencyFile, + lockfile: T.nilable(Dependabot::DependencyFile), language_version_manager: LanguageVersionManager ) .void @@ -57,7 +57,7 @@ def run(command, fingerprint: nil) sig { returns(Dependabot::Dependency) } attr_reader :dependency - sig { returns(DependencyFile) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } attr_reader :lockfile sig { returns(LanguageVersionManager) } attr_reader :language_version_manager @@ -82,7 +82,7 @@ def lockfile_section else Python::FileParser::DEPENDENCY_GROUP_KEYS.each do |keys| section = keys.fetch(:lockfile) - return section if JSON.parse(T.must(lockfile.content))[section].keys.any?(dependency_name) + return section if JSON.parse(T.must(T.must(lockfile).content))[section].keys.any?(dependency_name) end end end From 4aab29fc611e32a17635c9e3d6d9dbc5782bde2a Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 17:08:22 +0000 Subject: [PATCH 41/72] Changed setup file typecheck as strict --- .../python/file_parser/setup_file_parser.rb | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 2e634bdb6c..e14661241f 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/dependency" @@ -20,12 +20,14 @@ class SetupFileParser TESTS_REQUIRE_REGEX = /tests_require\s*=\s*\[/m EXTRAS_REQUIRE_REGEX = /extras_require\s*=\s*\{/m - CLOSING_BRACKET = { "[" => "]", "{" => "}" }.freeze + CLOSING_BRACKET = T.let({ "[" => "]", "{" => "}" }.freeze, T.any(T.untyped, T.untyped)) + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def dependency_set dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -56,8 +58,10 @@ def dependency_set private + sig { returns(T.untyped) } attr_reader :dependency_files + sig { returns(T.untyped) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files @@ -79,6 +83,7 @@ def parsed_setup_file parsed_sanitized_setup_file end + sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } def parsed_sanitized_setup_file SharedHelpers.in_a_temporary_directory do write_sanitized_setup_file @@ -100,16 +105,18 @@ def parsed_sanitized_setup_file [] end + sig { params(requirements: T.untyped).returns(T.nilable(Python::Requirement)) } def check_requirements(requirements) - requirements.each do |dep| + requirements&.each do |dep| next unless dep["requirement"] - Python::Requirement.new(dep["requirement"].split(",")) + T.let(Python::Requirement.new(dep["requirement"].split(",")), Python::Requirement) rescue Gem::Requirement::BadRequirementError => e raise Dependabot::DependencyFileNotEvaluatable, e.message end end + sig { void } def write_temporary_dependency_files dependency_files .reject { |f| f.name == ".python-version" } @@ -125,6 +132,7 @@ def write_temporary_dependency_files # This sanitization is far from perfect (it will fail if any of the # entries are dynamic), but it is an alternative approach to the one # used in parser.py which sometimes succeeds when that has failed. + sig { void } def write_sanitized_setup_file install_requires = get_regexed_req_array(INSTALL_REQUIRES_REGEX) setup_requires = get_regexed_req_array(SETUP_REQUIRES_REGEX) @@ -175,6 +183,7 @@ def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end + sig { returns(T.untyped) } def setup_file dependency_files.find { |f| f.name == "setup.py" } end From 7cdf83b93376a5f873fc9c3d0fe0664e6cd4c844 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 19:15:46 +0000 Subject: [PATCH 42/72] Added files with typecheck strict --- .../file_parser/pyproject_files_parser.rb | 51 +++++++++++++++---- .../python/file_parser/setup_file_parser.rb | 4 +- .../file_updater/setup_file_sanitizer.rb | 33 ++++++++---- .../python/language_version_manager.rb | 15 ++++-- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 744d3457dc..65d4b43704 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "toml-rb" @@ -14,15 +14,18 @@ module Dependabot module Python class FileParser class PyprojectFilesParser + extend T::Sig POETRY_DEPENDENCY_TYPES = %w(dependencies dev-dependencies).freeze # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def dependency_set dependency_set = Dependabot::FileParsers::Base::DependencySet.new @@ -34,8 +37,10 @@ def dependency_set private + sig { returns(T.untyped) } attr_reader :dependency_files + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def pyproject_dependencies if using_poetry? missing_keys = missing_poetry_keys @@ -54,10 +59,12 @@ def pyproject_dependencies end end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def poetry_dependencies - @poetry_dependencies ||= parse_poetry_dependencies + @poetry_dependencies ||= T.let(parse_poetry_dependencies, T.untyped) end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def parse_poetry_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -73,6 +80,7 @@ def parse_poetry_dependencies dependencies end + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def pep621_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -107,6 +115,7 @@ def pep621_dependencies dependencies end + sig { params(type: T.untyped, deps_hash: T.untyped).returns(Dependabot::FileParsers::Base::DependencySet) } def parse_poetry_dependency_group(type, deps_hash) dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -126,11 +135,13 @@ def parse_poetry_dependency_group(type, deps_hash) dependencies end + sig { params(name: String, extras: T::Array[String]).returns(String) } def normalised_name(name, extras) NameNormaliser.normalise_including_extras(name, extras) end # @param req can be an Array, Hash or String that represents the constraints for a dependency + sig { params(req: T.untyped, type: T.untyped).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } def parse_requirements_from(req, type) [req].flatten.compact.filter_map do |requirement| next if requirement.is_a?(Hash) && UNSUPPORTED_DEPENDENCY_TYPES.intersect?(requirement.keys) @@ -155,26 +166,31 @@ def parse_requirements_from(req, type) end end + sig { returns(T.nilable(T::Boolean)) } def using_poetry? !poetry_root.nil? end + sig { returns(T::Array[String]) } def missing_poetry_keys package_mode = poetry_root.fetch("package-mode", true) required_keys = package_mode ? %w(name version description authors) : [] required_keys.reject { |key| poetry_root.key?(key) } end + sig { returns(T.untyped) } def using_pep621? !parsed_pyproject.dig("project", "dependencies").nil? || !parsed_pyproject.dig("project", "optional-dependencies").nil? || !parsed_pyproject.dig("build-system", "requires").nil? end + sig { returns(T.untyped) } def poetry_root parsed_pyproject.dig("tool", "poetry") end + sig { returns(T.untyped) } def using_pdm? using_pep621? && pdm_lock end @@ -182,6 +198,7 @@ def using_pdm? # Create a DependencySet where each element has no requirement. Any # requirements will be added when combining the DependencySet with # other DependencySets. + sig { returns(Dependabot::FileParsers::Base::DependencySet) } def lockfile_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -206,10 +223,13 @@ def lockfile_dependencies dependencies end + sig { returns(T::Array[T.nilable(String)]) } def production_dependency_names - @production_dependency_names ||= parse_production_dependency_names + @production_dependency_names ||= T.let(parse_production_dependency_names, + T.nilable(T::Array[T.nilable(String)])) end + sig { returns(T::Array[T.nilable(String)]) } def parse_production_dependency_names SharedHelpers.in_a_temporary_directory do File.write(pyproject.name, pyproject.content) @@ -232,6 +252,7 @@ def parse_production_dependency_names end end + sig { params(dep_name: T.untyped).returns(T.untyped) } def version_from_lockfile(dep_name) return unless parsed_lockfile @@ -240,6 +261,7 @@ def version_from_lockfile(dep_name) &.fetch("version", nil) end + sig { params(req: T.untyped).returns(T::Array[Dependabot::Python::Requirement]) } def check_requirements(req) requirement = req.is_a?(String) ? req : req["version"] Python::Requirement.requirements_array(requirement) @@ -247,31 +269,36 @@ def check_requirements(req) raise Dependabot::DependencyFileNotEvaluatable, e.message end + sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end + sig { returns(T.untyped) } def parsed_pyproject - @parsed_pyproject ||= TomlRB.parse(pyproject.content) + @parsed_pyproject ||= T.let(TomlRB.parse(pyproject.content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, pyproject.path end + sig { returns(T.untyped) } def parsed_poetry_lock - @parsed_poetry_lock ||= TomlRB.parse(poetry_lock.content) + @parsed_poetry_lock ||= T.let(TomlRB.parse(poetry_lock.content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError raise Dependabot::DependencyFileNotParseable, poetry_lock.path end + sig { returns(T.untyped) } def pyproject - @pyproject ||= - dependency_files.find { |f| f.name == "pyproject.toml" } + @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end + sig { returns(T.untyped) } def lockfile poetry_lock end + sig { returns(T.untyped) } def parsed_pep621_dependencies SharedHelpers.in_a_temporary_directory do write_temporary_pyproject @@ -284,24 +311,26 @@ def parsed_pep621_dependencies end end + sig { returns(Integer) } def write_temporary_pyproject path = pyproject.name FileUtils.mkdir_p(Pathname.new(path).dirname) File.write(path, pyproject.content) end + sig { returns(T.untyped) } def parsed_lockfile parsed_poetry_lock if poetry_lock end + sig { returns(T.untyped) } def poetry_lock - @poetry_lock ||= - dependency_files.find { |f| f.name == "poetry.lock" } + @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, T.untyped) end + sig { returns(T.untyped) } def pdm_lock - @pdm_lock ||= - dependency_files.find { |f| f.name == "pdm.lock" } + @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, T.untyped) end end end diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index e14661241f..3c27a8d456 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -105,12 +105,12 @@ def parsed_sanitized_setup_file [] end - sig { params(requirements: T.untyped).returns(T.nilable(Python::Requirement)) } + sig { params(requirements: T.untyped).returns(T.untyped) } def check_requirements(requirements) requirements&.each do |dep| next unless dep["requirement"] - T.let(Python::Requirement.new(dep["requirement"].split(",")), Python::Requirement) + Python::Requirement.new(dep["requirement"].split(",")) rescue Gem::Requirement::BadRequirementError => e raise Dependabot::DependencyFileNotEvaluatable, e.message end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index d89dc16020..2018c14b6f 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/python/file_updater" @@ -12,11 +12,14 @@ class FileUpdater # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer extend T::Sig + + sig { params(setup_file: DependencyFile, setup_cfg: T.untyped).void } def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg end + sig { returns(String) } def sanitized_content # The part of the setup.py that Pipenv cares about appears to be the # install_requires. A name and version are required by don't end up @@ -33,14 +36,19 @@ def sanitized_content private + sig { returns(DependencyFile) } attr_reader :setup_file + sig { returns(String) } attr_reader :setup_cfg + sig { returns(T::Boolean) } def include_pbr? setup_requires_array.any? { |d| d.start_with?("pbr") } end + sig { returns(T.untyped) } def install_requires_array + @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -50,7 +58,9 @@ def install_requires_array end end + sig { returns(T::Array[String]) } def setup_requires_array + @setup_requires_array = T.let(T.untyped, T.untyped) @setup_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -60,7 +70,9 @@ def setup_requires_array end end + sig { returns(T::Hash[T.untyped, T.untyped]) } def extras_require_hash + @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} @@ -78,20 +90,21 @@ def extras_require_hash end end + sig { returns(T.untyped) } def parsed_setup_file - @parsed_setup_file ||= - Python::FileParser::SetupFileParser.new( - dependency_files: [ - setup_file&.dup&.tap { |f| f.name = "setup.py" }, - setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } - ].compact - ).dependency_set + @parsed_setup_file ||= T.let(Python::FileParser::SetupFileParser.new( + dependency_files: [ + setup_file.dup.tap { |f| f.name = "setup.py" }, + setup_cfg.dup.tap { |f| f.name = "setup.cfg" } + ].compact + ) + .dependency_set, T.untyped) end - sig { returns(String) } + sig { returns(T.nilable(String)) } def package_name content = setup_file.content - match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) + match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) match ? match[:package_name] : "default_package_name" end end diff --git a/python/lib/dependabot/python/language_version_manager.rb b/python/lib/dependabot/python/language_version_manager.rb index 3ec07661ea..d921b268ba 100644 --- a/python/lib/dependabot/python/language_version_manager.rb +++ b/python/lib/dependabot/python/language_version_manager.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/logger" @@ -19,6 +19,7 @@ class LanguageVersionManager 3.8.20 ).freeze + sig { params(python_requirement_parser: T.untyped).void } def initialize(python_requirement_parser:) @python_requirement_parser = python_requirement_parser end @@ -33,20 +34,23 @@ def install_required_python ) end + sig { returns(String) } def installed_version # Use `pyenv exec` to query the active Python version output, _status = SharedHelpers.run_shell_command("pyenv exec python --version") version = output.strip.split.last # Extract the version number (e.g., "3.13.1") - version + T.must(version) end + sig { returns(T.untyped) } def python_major_minor - @python_major_minor ||= T.must(Python::Version.new(python_version).segments[0..1]).join(".") + @python_major_minor ||= T.let(T.must(Python::Version.new(python_version).segments[0..1]).join("."), T.untyped) end + sig { returns(String) } def python_version - @python_version ||= python_version_from_supported_versions + @python_version ||= T.let(python_version_from_supported_versions, T.nilable(String)) end sig { returns(String) } @@ -81,10 +85,12 @@ def python_version_from_supported_versions raise ToolVersionNotSupported.new("Python", python_requirement_string, supported_versions) end + sig { returns(T.untyped) } def user_specified_python_version @python_requirement_parser.user_specified_requirements.first end + sig { returns(T.nilable(String)) } def python_version_matching_imputed_requirements compiled_file_python_requirement_markers = @python_requirement_parser.imputed_requirements.map do |r| @@ -93,6 +99,7 @@ def python_version_matching_imputed_requirements python_version_matching(compiled_file_python_requirement_markers) end + sig { params(requirements: T.untyped).returns(T.nilable(String)) } def python_version_matching(requirements) PRE_INSTALLED_PYTHON_VERSIONS.find do |version_string| version = Python::Version.new(version_string) From 2a539e621d8d93a235770c7216d1ad6242c9e8fa Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:17:34 +0000 Subject: [PATCH 43/72] Removed typecheck form pyproject_preparer file --- .../lib/dependabot/python/file_updater/pyproject_preparer.rb | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb index a3f0a09afc..17254e734b 100644 --- a/python/lib/dependabot/python/file_updater/pyproject_preparer.rb +++ b/python/lib/dependabot/python/file_updater/pyproject_preparer.rb @@ -14,7 +14,6 @@ module Dependabot module Python class FileUpdater class PyprojectPreparer - extend T::Sig def initialize(pyproject_content:, lockfile: nil) @pyproject_content = pyproject_content @lockfile = lockfile @@ -49,7 +48,6 @@ def update_python_requirement(requirement) TomlRB.dump(pyproject_object) end - sig { returns(String) } def sanitize # {{ name }} syntax not allowed pyproject_content @@ -113,7 +111,6 @@ def locked_details(dep_name) .find { |d| d["name"] == normalise(dep_name) } end - sig { params(name: String).returns(String) } def normalise(name) NameNormaliser.normalise(name) end From d8e59c392cf6f5f7bf757acae78ebffc9304d684 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:38:04 +0000 Subject: [PATCH 44/72] To fix removed hash --- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 2018c14b6f..8993f379fc 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -68,11 +68,11 @@ def setup_requires_array dep.name + dep.requirements.first[:requirement].to_s end - end + ends - sig { returns(T::Hash[T.untyped, T.untyped]) } + sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(T.untyped, T.untyped) + @extras_require_hash = T.let(Hash, T.untyped) @extras_require_hash ||= begin hash = {} From 136d3c11878ff3f74ed14949a49e9f901e35d536 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 20:41:53 +0000 Subject: [PATCH 45/72] Added removed typo error --- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 8993f379fc..d8aa7f3b32 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -68,11 +68,11 @@ def setup_requires_array dep.name + dep.requirements.first[:requirement].to_s end - ends + end sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(Hash, T.untyped) + @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} From 2dbfed7d8255edffad25ea4696753e5ef40fe7b4 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:07:20 +0000 Subject: [PATCH 46/72] Removed typecheck form setup_file_sanitixer --- .../file_updater/setup_file_sanitizer.rb | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index d8aa7f3b32..8f0abad7d8 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -1,4 +1,4 @@ -# typed: strict +# typed: true # frozen_string_literal: true require "dependabot/python/file_updater" @@ -13,13 +13,11 @@ class FileUpdater class SetupFileSanitizer extend T::Sig - sig { params(setup_file: DependencyFile, setup_cfg: T.untyped).void } def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg end - sig { returns(String) } def sanitized_content # The part of the setup.py that Pipenv cares about appears to be the # install_requires. A name and version are required by don't end up @@ -36,17 +34,13 @@ def sanitized_content private - sig { returns(DependencyFile) } attr_reader :setup_file - sig { returns(String) } attr_reader :setup_cfg - sig { returns(T::Boolean) } def include_pbr? setup_requires_array.any? { |d| d.start_with?("pbr") } end - sig { returns(T.untyped) } def install_requires_array @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= @@ -58,9 +52,7 @@ def install_requires_array end end - sig { returns(T::Array[String]) } def setup_requires_array - @setup_requires_array = T.let(T.untyped, T.untyped) @setup_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -70,9 +62,7 @@ def setup_requires_array end end - sig { returns(T.untyped) } def extras_require_hash - @extras_require_hash = T.let(T.untyped, T.untyped) @extras_require_hash ||= begin hash = {} @@ -90,18 +80,15 @@ def extras_require_hash end end - sig { returns(T.untyped) } def parsed_setup_file - @parsed_setup_file ||= T.let(Python::FileParser::SetupFileParser.new( + @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( dependency_files: [ setup_file.dup.tap { |f| f.name = "setup.py" }, setup_cfg.dup.tap { |f| f.name = "setup.cfg" } ].compact - ) - .dependency_set, T.untyped) + ).dependency_set end - sig { returns(T.nilable(String)) } def package_name content = setup_file.content match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) From 57952cf9b18f389c0fa21ef4830268e3d6a7553b Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:26:09 +0000 Subject: [PATCH 47/72] Reverted back setup_file_sanitizer --- .../dependabot/python/file_parser/pyproject_files_parser.rb | 6 +++--- .../dependabot/python/file_updater/setup_file_sanitizer.rb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 65d4b43704..acd0fd7b3e 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::Dependency]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } @@ -288,7 +288,7 @@ def parsed_poetry_lock raise Dependabot::DependencyFileNotParseable, poetry_lock.path end - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::Dependency]) } def pyproject @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 8f0abad7d8..24e43749a6 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -83,8 +83,8 @@ def extras_require_hash def parsed_setup_file @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( dependency_files: [ - setup_file.dup.tap { |f| f.name = "setup.py" }, - setup_cfg.dup.tap { |f| f.name = "setup.cfg" } + setup_file&.dup&.tap { |f| f.name = "setup.py" }, + setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } ].compact ).dependency_set end From cf4a42c1d603a93a4f90868a697a99baefab7c7c Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:29:01 +0000 Subject: [PATCH 48/72] Reverted back setup_file_sanitizer --- .../lib/dependabot/python/file_parser/pyproject_files_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index acd0fd7b3e..c80635ae67 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -288,7 +288,7 @@ def parsed_poetry_lock raise Dependabot::DependencyFileNotParseable, poetry_lock.path end - sig { returns(T::Array[Dependabot::Dependency]) } + sig { returns(T.untyped) } def pyproject @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) end From 27b91c4c5a1e694e63363015f362ba580c5f8502 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:32:21 +0000 Subject: [PATCH 49/72] Reverted back setup_file_sanitizer --- .../dependabot/python/file_parser/pyproject_files_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index c80635ae67..65d4b43704 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T::Array[Dependabot::Dependency]).void } + sig { params(dependency_files: T.untyped).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T::Array[Dependabot::Dependency]) } + sig { returns(T.untyped) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } From 5842b20f10e1fb9378cb515e4cf92f574d00bd99 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 21:48:07 +0000 Subject: [PATCH 50/72] Pulled back --- .../file_updater/setup_file_sanitizer.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb index 24e43749a6..de74f69fc9 100644 --- a/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb +++ b/python/lib/dependabot/python/file_updater/setup_file_sanitizer.rb @@ -3,7 +3,6 @@ require "dependabot/python/file_updater" require "dependabot/python/file_parser/setup_file_parser" -require "sorbet-runtime" module Dependabot module Python @@ -11,8 +10,6 @@ class FileUpdater # Take a setup.py, parses it (carefully!) and then create a new, clean # setup.py using only the information which will appear in the lockfile. class SetupFileSanitizer - extend T::Sig - def initialize(setup_file:, setup_cfg:) @setup_file = setup_file @setup_cfg = setup_cfg @@ -42,7 +39,6 @@ def include_pbr? end def install_requires_array - @install_requires_array = T.let(T.untyped, T.untyped) @install_requires_array ||= parsed_setup_file.dependencies.filter_map do |dep| next unless dep.requirements.first[:groups] @@ -81,17 +77,18 @@ def extras_require_hash end def parsed_setup_file - @parsed_setup_file ||= Python::FileParser::SetupFileParser.new( - dependency_files: [ - setup_file&.dup&.tap { |f| f.name = "setup.py" }, - setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } - ].compact - ).dependency_set + @parsed_setup_file ||= + Python::FileParser::SetupFileParser.new( + dependency_files: [ + setup_file&.dup&.tap { |f| f.name = "setup.py" }, + setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" } + ].compact + ).dependency_set end def package_name content = setup_file.content - match = T.must(content).match(/name\s*=\s*['"](?[^'"]+)['"]/) + match = content.match(/name\s*=\s*['"](?[^'"]+)['"]/) match ? match[:package_name] : "default_package_name" end end From 2aa399217ca6c19f865a2452a932b9ea013d15c7 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 22:42:15 +0000 Subject: [PATCH 51/72] Reduced untyped --- .../lib/dependabot/python/file_parser/setup_file_parser.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index 3c27a8d456..f9bcca8639 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -22,7 +22,7 @@ class SetupFileParser CLOSING_BRACKET = T.let({ "[" => "]", "{" => "}" }.freeze, T.any(T.untyped, T.untyped)) - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -58,10 +58,10 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files - sig { returns(T.untyped) } + sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files From e2b06be16c6c3b7ad6db484427f19d4a39ec0b92 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 22:44:29 +0000 Subject: [PATCH 52/72] Reduced untyped --- python/lib/dependabot/python/file_parser/setup_file_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/dependabot/python/file_parser/setup_file_parser.rb b/python/lib/dependabot/python/file_parser/setup_file_parser.rb index f9bcca8639..27cef6e9cc 100644 --- a/python/lib/dependabot/python/file_parser/setup_file_parser.rb +++ b/python/lib/dependabot/python/file_parser/setup_file_parser.rb @@ -61,7 +61,7 @@ def dependency_set sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files - sig { returns(T.nilable(T.any(T::Hash[String, T.untyped], String, T::Array[T::Hash[String, T.untyped]]))) } + sig { returns(T.untyped) } def parsed_setup_file SharedHelpers.in_a_temporary_directory do write_temporary_dependency_files From d6806d1783bc24d67e133fdadb18d089935a6b7e Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 23:23:20 +0000 Subject: [PATCH 53/72] Reduced untyped in pyproject_files_parser --- .../file_parser/pyproject_files_parser.rb | 57 +++++++++++-------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index 65d4b43704..bfb0524332 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -20,7 +20,7 @@ class PyprojectFilesParser # https://python-poetry.org/docs/dependency-specification/ UNSUPPORTED_DEPENDENCY_TYPES = %w(git path url).freeze - sig { params(dependency_files: T.untyped).void } + sig { params(dependency_files: T::Array[Dependabot::DependencyFile]).void } def initialize(dependency_files:) @dependency_files = dependency_files end @@ -37,7 +37,7 @@ def dependency_set private - sig { returns(T.untyped) } + sig { returns(T::Array[Dependabot::DependencyFile]) } attr_reader :dependency_files sig { returns(Dependabot::FileParsers::Base::DependencySet) } @@ -47,8 +47,8 @@ def pyproject_dependencies if missing_keys.any? raise DependencyFileNotParseable.new( - pyproject.path, - "#{pyproject.path} is missing the following sections:\n" \ + T.must(pyproject).path, + "#{T.must(pyproject).path} is missing the following sections:\n" \ " * #{missing_keys.map { |key| "tool.poetry.#{key}" }.join("\n * ")}\n" ) end @@ -115,7 +115,11 @@ def pep621_dependencies dependencies end - sig { params(type: T.untyped, deps_hash: T.untyped).returns(Dependabot::FileParsers::Base::DependencySet) } + sig do + params(type: String, + deps_hash: T::Hash[String, + T.untyped]).returns(Dependabot::FileParsers::Base::DependencySet) + end def parse_poetry_dependency_group(type, deps_hash) dependencies = Dependabot::FileParsers::Base::DependencySet.new @@ -141,7 +145,7 @@ def normalised_name(name, extras) end # @param req can be an Array, Hash or String that represents the constraints for a dependency - sig { params(req: T.untyped, type: T.untyped).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } + sig { params(req: T.untyped, type: String).returns(T::Array[T::Hash[Symbol, T.nilable(String)]]) } def parse_requirements_from(req, type) [req].flatten.compact.filter_map do |requirement| next if requirement.is_a?(Hash) && UNSUPPORTED_DEPENDENCY_TYPES.intersect?(requirement.keys) @@ -151,14 +155,14 @@ def parse_requirements_from(req, type) if requirement.is_a?(String) { requirement: requirement, - file: pyproject.name, + file: T.must(pyproject).name, source: nil, groups: [type] } else { requirement: requirement["version"], - file: pyproject.name, + file: T.must(pyproject).name, source: requirement.fetch("source", nil), groups: [type] } @@ -178,14 +182,14 @@ def missing_poetry_keys required_keys.reject { |key| poetry_root.key?(key) } end - sig { returns(T.untyped) } + sig { returns(T::Boolean) } def using_pep621? !parsed_pyproject.dig("project", "dependencies").nil? || !parsed_pyproject.dig("project", "optional-dependencies").nil? || !parsed_pyproject.dig("build-system", "requires").nil? end - sig { returns(T.untyped) } + sig { returns(T::Hash[String, T.untyped]) } def poetry_root parsed_pyproject.dig("tool", "poetry") end @@ -232,7 +236,7 @@ def production_dependency_names sig { returns(T::Array[T.nilable(String)]) } def parse_production_dependency_names SharedHelpers.in_a_temporary_directory do - File.write(pyproject.name, pyproject.content) + File.write(T.must(pyproject).name, T.must(pyproject).content) File.write(lockfile.name, lockfile.content) begin @@ -252,7 +256,7 @@ def parse_production_dependency_names end end - sig { params(dep_name: T.untyped).returns(T.untyped) } + sig { params(dep_name: String).returns(T.untyped) } def version_from_lockfile(dep_name) return unless parsed_lockfile @@ -276,21 +280,22 @@ def normalise(name) sig { returns(T.untyped) } def parsed_pyproject - @parsed_pyproject ||= T.let(TomlRB.parse(pyproject.content), T.untyped) + @parsed_pyproject ||= T.let(TomlRB.parse(T.must(pyproject).content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError - raise Dependabot::DependencyFileNotParseable, pyproject.path + raise Dependabot::DependencyFileNotParseable, T.must(pyproject).path end sig { returns(T.untyped) } def parsed_poetry_lock - @parsed_poetry_lock ||= T.let(TomlRB.parse(poetry_lock.content), T.untyped) + @parsed_poetry_lock ||= T.let(TomlRB.parse(T.must(poetry_lock).content), T.untyped) rescue TomlRB::ParseError, TomlRB::ValueOverwriteError - raise Dependabot::DependencyFileNotParseable, poetry_lock.path + raise Dependabot::DependencyFileNotParseable, T.must(poetry_lock).path end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pyproject - @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, T.untyped) + @pyproject ||= T.let(dependency_files.find { |f| f.name == "pyproject.toml" }, + T.nilable(Dependabot::DependencyFile)) end sig { returns(T.untyped) } @@ -306,16 +311,16 @@ def parsed_pep621_dependencies SharedHelpers.run_helper_subprocess( command: "pyenv exec python3 #{NativeHelpers.python_helper_path}", function: "parse_pep621_dependencies", - args: [pyproject.name] + args: [T.must(pyproject).name] ) end end sig { returns(Integer) } def write_temporary_pyproject - path = pyproject.name + path = T.must(pyproject).name FileUtils.mkdir_p(Pathname.new(path).dirname) - File.write(path, pyproject.content) + File.write(path, T.must(pyproject).content) end sig { returns(T.untyped) } @@ -323,14 +328,16 @@ def parsed_lockfile parsed_poetry_lock if poetry_lock end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def poetry_lock - @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, T.untyped) + @poetry_lock ||= T.let(dependency_files.find { |f| f.name == "poetry.lock" }, + T.nilable(Dependabot::DependencyFile)) end - sig { returns(T.untyped) } + sig { returns(T.nilable(Dependabot::DependencyFile)) } def pdm_lock - @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, T.untyped) + @pdm_lock ||= T.let(dependency_files.find { |f| f.name == "pdm.lock" }, + T.nilable(Dependabot::DependencyFile)) end end end From 64be557030a3b31956ce6f4e510d5f3d6b1cf336 Mon Sep 17 00:00:00 2001 From: Randhir Date: Thu, 16 Jan 2025 23:38:37 +0000 Subject: [PATCH 54/72] Reduced untyped in pyproject_files_parser --- .../python/file_parser/pyproject_files_parser.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb index bfb0524332..2dcf33ba0f 100644 --- a/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb +++ b/python/lib/dependabot/python/file_parser/pyproject_files_parser.rb @@ -69,11 +69,11 @@ def parse_poetry_dependencies dependencies = Dependabot::FileParsers::Base::DependencySet.new POETRY_DEPENDENCY_TYPES.each do |type| - deps_hash = poetry_root[type] || {} + deps_hash = T.must(poetry_root)[type] || {} dependencies += parse_poetry_dependency_group(type, deps_hash) end - groups = poetry_root["group"] || {} + groups = T.must(poetry_root)["group"] || {} groups.each do |group, group_spec| dependencies += parse_poetry_dependency_group(group, group_spec["dependencies"]) end @@ -177,9 +177,9 @@ def using_poetry? sig { returns(T::Array[String]) } def missing_poetry_keys - package_mode = poetry_root.fetch("package-mode", true) + package_mode = T.must(poetry_root).fetch("package-mode", true) required_keys = package_mode ? %w(name version description authors) : [] - required_keys.reject { |key| poetry_root.key?(key) } + required_keys.reject { |key| T.must(poetry_root).key?(key) } end sig { returns(T::Boolean) } @@ -189,7 +189,7 @@ def using_pep621? !parsed_pyproject.dig("build-system", "requires").nil? end - sig { returns(T::Hash[String, T.untyped]) } + sig { returns(T.nilable(T::Hash[String, T.untyped])) } def poetry_root parsed_pyproject.dig("tool", "poetry") end From 410dbf1cc777965a3b262eda40b3d0a6ca9ae541 Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:29:27 +0000 Subject: [PATCH 55/72] Add Sorbet Type Checking for file: bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb (#11326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added sorbet typing * Added some typecheck. * Lint error fixes. * Updated as per the review comments --------- Co-authored-by: “Thavachelvam <“thavaahariharangit@git.com”> --- .../bundler/file_updater/gemspec_updater.rb | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb b/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb index 4bd5ab093d..deba3d3ddb 100644 --- a/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb +++ b/bundler/lib/dependabot/bundler/file_updater/gemspec_updater.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/bundler/file_updater" @@ -9,13 +9,17 @@ class FileUpdater class GemspecUpdater require_relative "requirement_replacer" + extend T::Sig + + sig { params(dependencies: T::Array[Dependabot::Dependency], gemspec: Dependabot::DependencyFile).void } def initialize(dependencies:, gemspec:) - @dependencies = dependencies - @gemspec = gemspec + @dependencies = T.let(dependencies, T::Array[Dependabot::Dependency]) + @gemspec = T.let(gemspec, Dependabot::DependencyFile) end + sig { returns(String) } def updated_gemspec_content - content = gemspec.content + content = T.let(T.must(gemspec.content), String) dependencies.each do |dependency| content = replace_gemspec_version_requirement( @@ -28,21 +32,28 @@ def updated_gemspec_content private + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependencies + + sig { returns(Dependabot::DependencyFile) } attr_reader :gemspec + sig do + params(gemspec: Dependabot::DependencyFile, dependency: Dependabot::Dependency, + content: String).returns(String) + end def replace_gemspec_version_requirement(gemspec, dependency, content) return content unless requirement_changed?(gemspec, dependency) updated_requirement = - dependency.requirements - .find { |r| r[:file] == gemspec.name } - .fetch(:requirement) + T.must(dependency.requirements + .find { |r| r[:file] == gemspec.name }) + .fetch(:requirement) previous_requirement = - dependency.previous_requirements - .find { |r| r[:file] == gemspec.name } - .fetch(:requirement) + T.must(T.must(dependency.previous_requirements) + .find { |r| r[:file] == gemspec.name }) + .fetch(:requirement) RequirementReplacer.new( dependency: dependency, @@ -52,9 +63,10 @@ def replace_gemspec_version_requirement(gemspec, dependency, content) ).rewrite(content) end + sig { params(file: Dependabot::DependencyFile, dependency: Dependabot::Dependency).returns(T::Boolean) } def requirement_changed?(file, dependency) changed_requirements = - dependency.requirements - dependency.previous_requirements + dependency.requirements - T.must(dependency.previous_requirements) changed_requirements.any? { |f| f[:file] == file.name } end From 0fce91e8c7e6c190765211b756f025d4f7caef12 Mon Sep 17 00:00:00 2001 From: Alfred Mazimbe Date: Fri, 17 Jan 2025 12:03:34 +0000 Subject: [PATCH 56/72] Add sorbet type checking for swift/file_parser/manifest_parser.rb (#11329) * Add sorbet type checking for swift/file_parser/manifest_parser.rb * Update manifest_parser.rb --- .../swift/file_parser/manifest_parser.rb | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/swift/lib/dependabot/swift/file_parser/manifest_parser.rb b/swift/lib/dependabot/swift/file_parser/manifest_parser.rb index 6f6b02054b..57289a2541 100644 --- a/swift/lib/dependabot/swift/file_parser/manifest_parser.rb +++ b/swift/lib/dependabot/swift/file_parser/manifest_parser.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strong # frozen_string_literal: true require "dependabot/file_parsers/base" @@ -8,25 +8,35 @@ module Dependabot module Swift class FileParser < Dependabot::FileParsers::Base class ManifestParser + extend T::Sig + extend T::Helpers + DEPENDENCY = /(?\.package\(\s* (?:name:\s+"[^"]+",\s*)?url:\s+"(?[^"]+)",\s*(?#{NativeRequirement::REGEXP})\s* \))/x + sig do + params( + manifest: Dependabot::DependencyFile, + source: T::Hash[Symbol, String] + ).void + end def initialize(manifest, source:) @manifest = manifest @source = source end + sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) } def requirements - found = manifest.content.scan(DEPENDENCY).find do |_declaration, url, _requirement| - SharedHelpers.scp_to_standard(url) == source[:url] + found = manifest.content&.scan(DEPENDENCY)&.find do |_declaration, url, _requirement| + SharedHelpers.scp_to_standard(url.to_s) == source[:url] end return [] unless found - declaration = found.first - requirement = NativeRequirement.new(found.last) + declaration = T.cast(found, T::Array[String]).first + requirement = NativeRequirement.new(T.cast(found, T::Array[String]).last) [ { @@ -41,7 +51,10 @@ def requirements private + sig { returns(Dependabot::DependencyFile) } attr_reader :manifest + + sig { returns(T::Hash[Symbol, String]) } attr_reader :source end end From 1328acbaaf01817bcd552b35fa39bbd6f9c3e0b0 Mon Sep 17 00:00:00 2001 From: "S.Sandhu" <167903774+sachin-sandhu@users.noreply.github.com> Date: Fri, 17 Jan 2025 09:56:32 -0500 Subject: [PATCH 57/72] Fixes [3.2k weekly errors] [pip] exception handlers for Python ecosystem (#11325) * Adds exception handler and test cases * Update python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb Co-authored-by: Alfred Mazimbe --------- Co-authored-by: Alfred Mazimbe --- .../pip_compile_version_resolver.rb | 21 ++++++++ .../pip_compile_version_resolver_spec.rb | 48 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb index 1b8accb8af..dcc98808a8 100644 --- a/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb +++ b/python/lib/dependabot/python/update_checker/pip_compile_version_resolver.rb @@ -37,6 +37,7 @@ class PipCompileVersionResolver attr_reader :dependency_files attr_reader :credentials attr_reader :repo_contents_path + attr_reader :error_handler def initialize(dependency:, dependency_files:, credentials:, repo_contents_path:) @dependency = dependency @@ -44,6 +45,7 @@ def initialize(dependency:, dependency_files:, credentials:, repo_contents_path: @credentials = credentials @repo_contents_path = repo_contents_path @build_isolation = true + @error_handler = PipCompileErrorHandler.new end def latest_resolvable_version(requirement: nil) @@ -186,6 +188,8 @@ def handle_pip_compile_errors(message) raise Dependabot::OutOfMemory if message.end_with?("MemoryError") + error_handler.handle_pipcompile_error(message) + raise end # rubocop:enable Metrics/AbcSize @@ -494,5 +498,22 @@ def setup_cfg_files end end end + + class PipCompileErrorHandler + SUBPROCESS_ERROR = /subprocess-exited-with-error/ + + INSTALLATION_ERROR = /InstallationError/ + + INSTALLATION_SUBPROCESS_ERROR = /InstallationSubprocessError/ + + HASH_MISMATCH = /HashMismatch/ + + def handle_pipcompile_error(error) + return unless error.match?(SUBPROCESS_ERROR) || error.match?(INSTALLATION_ERROR) || + error.match?(INSTALLATION_SUBPROCESS_ERROR) || error.match?(HASH_MISMATCH) + + raise DependencyFileNotResolvable, "Error resolving dependency" + end + end end end diff --git a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb index 300feb0385..1562c3372c 100644 --- a/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb +++ b/python/spec/dependabot/python/update_checker/pip_compile_version_resolver_spec.rb @@ -443,6 +443,54 @@ .to raise_error(Dependabot::OutOfMemory) end end + + context "when HelperSubprocessFailed exception is raised" do + let(:error_handler) { Dependabot::Python::PipCompileErrorHandler.new } + + let(:exception_message) { "HelperSubprocessFailed" } + + context "when dealing with subprocess-exited-with-error error" do + let(:exception_message) do + "Preparing metadata (setup.py): finished with status 'error' + error: subprocess-exited-with-error + + × python setup.py egg_info did not run successfully. + │ exit code: 1 + ╰─> [18 lines of output] + Traceback (most recent call last): + File \"\", line 2, in + exec(compile(''' + ~~~~^^^^^^^^^^^^ + # This is -- a caller that pip uses to run setup.py + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ...<31 lines>... + exec(compile(setup_py_code, filename, \"exec\")) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ''' % ('/tmp/pip-resolve-84tqp2g1/pillow_f28c34dffdb342a49c27519580a49fd3/setup.py',)" + end + + it "raises a helpful error" do + expect { error_handler.handle_pipcompile_error(exception_message) } + .to raise_error(Dependabot::DependencyFileNotResolvable) + end + end + + context "when dealing with an installation error" do + let(:exception_message) do + "pip._internal.exceptions.InstallationError: Could not install requirement" \ + " rugby-[FILTERED_REPO]@ https://****@github.com/compute-cloud/a300cfb3a4070c923246dd4.zip " \ + "from https://****@github.com/compute-cloud/[FILTERED_REPO]/archive/s.zip" \ + " (from -r requirements.in (line 23)) because of HTTP error 404 Client Error: Not" \ + " Found for url: https://github.com/compute-cloud/[FILTERED_REPO]/archive/a3246dd4.zip for" \ + " URL https://****@github.com/compute-cloud/[FILTERED_REPO]/archive/a3dd4.zip" + end + + it "raises a helpful error" do + expect { error_handler.handle_pipcompile_error(exception_message) } + .to raise_error(Dependabot::DependencyFileNotResolvable) + end + end + end end end end From aef668af8db9609ae9007e39d9dd43e738c50d80 Mon Sep 17 00:00:00 2001 From: Mark Allen Date: Fri, 17 Jan 2025 16:14:26 +0000 Subject: [PATCH 58/72] Handle enable_beta_ecosystems boolean --- bin/dry-run.rb | 4 ++++ common/lib/dependabot/file_fetchers/base.rb | 5 +++++ npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/bin/dry-run.rb b/bin/dry-run.rb index 653c0c1e68..cae15b736d 100755 --- a/bin/dry-run.rb +++ b/bin/dry-run.rb @@ -274,6 +274,10 @@ "Output pull request information metadata: title, description") do $options[:pull_request] = true end + + opts.on("--enable-beta-ecosystems", "Enable beta ecosystems") do |_value| + $options[:updater_options] = { enable_beta_ecosystems: true } + end end # rubocop:enable Metrics/BlockLength diff --git a/common/lib/dependabot/file_fetchers/base.rb b/common/lib/dependabot/file_fetchers/base.rb index 509687a964..566b21b1b8 100644 --- a/common/lib/dependabot/file_fetchers/base.rb +++ b/common/lib/dependabot/file_fetchers/base.rb @@ -128,6 +128,11 @@ def target_branch source.branch end + sig { returns(T::Boolean) } + def allow_beta_ecosystems? + !!options["enable_beta_ecosystems"] + end + sig { returns(T::Array[DependencyFile]) } def files return @files if @files.any? diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb index 289b20ccb2..a066c5527c 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb @@ -213,7 +213,7 @@ def pnpm_version sig { returns(T.nilable(T.any(Integer, String))) } def bun_version - return @bun_version = nil unless Experiments.enabled?(:bun_updates) + return @bun_version = nil unless allow_beta_ecosystems? || Experiments.enabled?(:bun_updates) @bun_version ||= T.let( package_manager_helper.setup(BunPackageManager::NAME), From 81dc28db309163374eed2b8d3c4e1bc6ded6e291 Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Sat, 18 Jan 2025 02:50:01 +0900 Subject: [PATCH 59/72] Bump Hex Elixir version to 1.16.3 (#11324) This bumps Elixir to the last version of 1.16. A couple quirks: 1. `Mix.Dep.load_on_environment/1` is gone, but under the hood it called `Mix.Dep.Converger.converge/1`anyways. 2. Charlist dependencies explicitly fail now, and I'm fairly sure they should be updated if charlists are still used. Co-authored-by: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> --- hex/Dockerfile | 4 ++-- hex/helpers/lib/parse_deps.exs | 2 +- hex/spec/dependabot/hex/file_parser_spec.rb | 6 ++--- .../hex/update_checker/file_preparer_spec.rb | 11 --------- .../mixfiles/git_source_with_charlist | 24 ------------------- 5 files changed, 6 insertions(+), 41 deletions(-) delete mode 100644 hex/spec/fixtures/mixfiles/git_source_with_charlist diff --git a/hex/Dockerfile b/hex/Dockerfile index 8a5bcf38a0..1ae9a8ad4c 100644 --- a/hex/Dockerfile +++ b/hex/Dockerfile @@ -15,8 +15,8 @@ RUN apt-get update \ # Install Elixir # https://github.com/elixir-lang/elixir/releases -ARG ELIXIR_VERSION=v1.15.8 -ARG ELIXIR_CHECKSUM=62d33c51417191e027c9b6f0c46e11daeb236a7dda6f0746ec4dd53263531092 +ARG ELIXIR_VERSION=v1.16.3 +ARG ELIXIR_CHECKSUM=e8e81771bc6acd62a2c1bf1b31c3aa3d0a469415de3b243b99f3e2e2d639f5ea RUN curl -sSLfO https://github.com/elixir-lang/elixir/releases/download/${ELIXIR_VERSION}/elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ && echo "$ELIXIR_CHECKSUM elixir-otp-${ERLANG_MAJOR_VERSION}.zip" | sha256sum -c - \ && unzip -d /usr/local/elixir -x elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ diff --git a/hex/helpers/lib/parse_deps.exs b/hex/helpers/lib/parse_deps.exs index d212c7c99a..950cce1685 100644 --- a/hex/helpers/lib/parse_deps.exs +++ b/hex/helpers/lib/parse_deps.exs @@ -3,7 +3,7 @@ defmodule Parser do # This is necessary because we can't specify :extra_applications to have :hex in other mixfiles. Mix.ensure_application!(:hex) - Mix.Dep.load_on_environment([]) + Mix.Dep.Converger.converge() |> Enum.flat_map(&parse_dep/1) |> Enum.map(&build_dependency(&1.opts[:lock], &1)) end diff --git a/hex/spec/dependabot/hex/file_parser_spec.rb b/hex/spec/dependabot/hex/file_parser_spec.rb index 539b575455..5c852b6dd0 100644 --- a/hex/spec/dependabot/hex/file_parser_spec.rb +++ b/hex/spec/dependabot/hex/file_parser_spec.rb @@ -255,8 +255,8 @@ end context "with a tag (rather than a ref)" do - let(:mixfile_fixture_name) { "git_source_with_charlist" } - let(:lockfile_fixture_name) { "git_source_with_charlist" } + let(:mixfile_fixture_name) { "git_source_tag_can_update" } + let(:lockfile_fixture_name) { "git_source_tag_can_update" } it "includes the git dependency" do expect(dependencies.length).to eq(2) @@ -476,7 +476,7 @@ it "returns the correct language" do expect(language.name).to eq "elixir" expect(language.requirement).to be_nil - expect(language.version.to_s).to eq "1.15.8" + expect(language.version.to_s).to eq "1.16.3" end end end diff --git a/hex/spec/dependabot/hex/update_checker/file_preparer_spec.rb b/hex/spec/dependabot/hex/update_checker/file_preparer_spec.rb index 43d0dcd0a4..8dba08f374 100644 --- a/hex/spec/dependabot/hex/update_checker/file_preparer_spec.rb +++ b/hex/spec/dependabot/hex/update_checker/file_preparer_spec.rb @@ -204,17 +204,6 @@ 'ref: "v1.2.1"}' ) end - - context "when there are single quotes" do - let(:mixfile_fixture_name) { "git_source_with_charlist" } - - it "updates the pin" do - expect(prepared_mixfile.content).to include( - '{:phoenix, ">= 0", github: "dependabot-fixtures/phoenix", ' \ - "ref: 'v1.2.1'}" - ) - end - end end end diff --git a/hex/spec/fixtures/mixfiles/git_source_with_charlist b/hex/spec/fixtures/mixfiles/git_source_with_charlist deleted file mode 100644 index f5eabfc02e..0000000000 --- a/hex/spec/fixtures/mixfiles/git_source_with_charlist +++ /dev/null @@ -1,24 +0,0 @@ -defmodule DependabotTest.Mixfile do - use Mix.Project - - def project do - [ - app: :dependabot_test, - version: "0.1.0", - elixir: "~> 1.5", - start_permanent: Mix.env == :prod, - deps: deps() - ] - end - - def application do - [extra_applications: [:logger]] - end - - defp deps do - [ - {:plug, "1.2.0"}, - {:phoenix, github: "dependabot-fixtures/phoenix", ref: 'v1.2.0'} - ] - end -end From 75e94bf506199a260f0c02a56aa1dbc044045733 Mon Sep 17 00:00:00 2001 From: Rob Aiken Date: Fri, 17 Jan 2025 18:49:03 +0000 Subject: [PATCH 60/72] Change to use update rather than install when updating pnpm packages (#11200) * Updating spec tests when switching from install to update --------- Co-authored-by: kbukum1 --- .../file_updater/pnpm_lockfile_updater.rb | 19 ++++-- .../pnpm_lockfile_updater_spec.rb | 62 ++++++------------- .../npm_and_yarn/file_updater_spec.rb | 2 +- .../package.json | 2 +- .../pnpm-lock.yaml | 10 +-- .../pnpm/unsupported_engine_pnpm/.npmrc | 2 +- .../pnpm/unsupported_engine_pnpm/package.json | 18 +++--- .../unsupported_engine_pnpm/pnpm-lock.yaml | 13 ++-- 8 files changed, 56 insertions(+), 72 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb index a7d1ffc725..ba336e88de 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb @@ -105,7 +105,7 @@ def run_pnpm_update(pnpm_lock:) File.write(".npmrc", npmrc_content(pnpm_lock)) SharedHelpers.with_git_configured(credentials: credentials) do - run_pnpm_updater + run_pnpm_update_packages write_final_package_json_files @@ -116,15 +116,22 @@ def run_pnpm_update(pnpm_lock:) end end - def run_pnpm_updater + def run_pnpm_update_packages dependency_updates = dependencies.map do |d| "#{d.name}@#{d.version}" end.join(" ") - Helpers.run_pnpm_command( - "install #{dependency_updates} --lockfile-only --ignore-workspace-root-check", - fingerprint: "install --lockfile-only --ignore-workspace-root-check" - ) + if Dependabot::Experiments.enabled?(:enable_fix_for_pnpm_no_change_error) + Helpers.run_pnpm_command( + "pnpm update #{dependency_updates} --lockfile-only --no-save -r", + fingerprint: "pnpm update --lockfile-only --no-save -r" + ) + else + Helpers.run_pnpm_command( + "install #{dependency_updates} --lockfile-only --ignore-workspace-root-check", + fingerprint: "install --lockfile-only --ignore-workspace-root-check" + ) + end end def run_pnpm_install diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater_spec.rb index 0cd5064af5..79e47977af 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater_spec.rb @@ -104,6 +104,9 @@ context "when there is a lockfile with tarball urls we don't have access to" do let(:project_name) { "pnpm/private_tarball_urls" } + let(:dependency_name) { "@dsp-testing/inner-source-top-secret-npm-2" } + let(:version) { "1.0.4" } + let(:previous_version) { "1.0.3" } it "raises a helpful error" do expect { updated_pnpm_lock_content } @@ -113,6 +116,9 @@ context "when there is a lockfile with tarball urls we don't have access to" do let(:project_name) { "pnpm/private_package_access" } + let(:dependency_name) { "@private-pkg/inner-source-top-secret-npm-2" } + let(:version) { "1.0.4" } + let(:previous_version) { "1.0.3" } it "raises a helpful error" do expect { updated_pnpm_lock_content } @@ -164,6 +170,9 @@ context "when there is a private registry we don't have access to" do let(:project_name) { "pnpm/private_package_access_with_package_name" } + let(:dependency_name) { "@private-pkg/inner-source-top-secret-npm-2" } + let(:version) { "1.0.4" } + let(:previous_version) { "1.0.3" } it "raises a helpful error" do expect { updated_pnpm_lock_content } @@ -172,13 +181,13 @@ end context "when there is a private registry we don't have access to and no package name is mentioned" do - let(:dependency_name) { "rollup" } - let(:version) { "3.29.5" } + let(:dependency_name) { "npm:rollup" } + let(:version) { "2.80.0" } let(:previous_version) { "^2.79.1" } let(:requirements) do [{ file: "package.json", - requirement: "3.29.5", + requirement: "2.80.0", groups: ["devDependencies"], source: nil }] @@ -200,18 +209,6 @@ end context "when there is a unsupported engine response (pnpm) from registry" do - let(:dependency_name) { "eslint" } - let(:version) { "9.9.0" } - let(:previous_version) { "8.32.0" } - let(:requirements) do - [{ - file: "package.json", - requirement: "9.9.0", - groups: ["devDependencies"], - source: nil - }] - end - let(:project_name) { "pnpm/unsupported_engine_pnpm" } it "raises a helpful error" do @@ -356,6 +353,9 @@ context "when there is a private repo we don't have access to and returns a 4xx error" do let(:project_name) { "pnpm/private_repo_no_access" } + let(:dependency_name) { "@dsp-testing/node" } + let(:version) { "1.0.4" } + let(:previous_version) { "1.0.3" } it "raises a helpful error" do expect { updated_pnpm_lock_content } @@ -401,6 +401,9 @@ context "when there is a private repo returns a 5xx error" do let(:project_name) { "pnpm/private_repo_with_server_error" } + let(:dependency_name) { "@dsp-testing/is-positive" } + let(:version) { "3.1.1" } + let(:previous_version) { "3.1.0" } it "raises a helpful error" do expect { updated_pnpm_lock_content } @@ -482,35 +485,6 @@ end end - context "with a registry resolution that returns err_pnpm_broken_metadata_json response" do - let(:dependency_name) { "nodemon" } - let(:version) { "3.3.3" } - let(:previous_version) { "^3.1.3" } - let(:requirements) do - [{ - file: "package.json", - requirement: "3.3.3", - groups: ["devDependencies"], - source: nil - }] - end - let(:previous_requirements) do - [{ - file: "package.json", - requirement: "^3.1.3", - groups: ["devDependencies"], - source: nil - }] - end - - let(:project_name) { "pnpm/broken_metadata" } - - it "raises a helpful error" do - expect { updated_pnpm_lock_content } - .to raise_error(Dependabot::DependencyFileNotResolvable) - end - end - context "with a registry resolution that returns missing_workspace_dir_package response" do let(:dependency_name) { "webpack" } let(:version) { "5.94.0" } diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater_spec.rb index 4401883567..4a6f179ab3 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater_spec.rb @@ -4013,7 +4013,7 @@ let(:previous_requirements) { [] } it "updates the version" do - expect(updated_pnpm_lock.content).to include("acorn@5.7.3:\n resolution").once + expect(updated_pnpm_lock.content).to include("acorn@5.2.1:\n resolution").once end end diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/package.json b/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/package.json index 617c9e0810..96f44d63dc 100644 --- a/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/package.json +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "@dsp-testing/is-positive": "3.1.0" + "@dsp-testing/is-positive": "^3.1.0" } } diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/pnpm-lock.yaml b/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/pnpm-lock.yaml index 85b377bf6e..5d50e3d6a7 100644 --- a/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/pnpm-lock.yaml +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/private_repo_with_server_error/pnpm-lock.yaml @@ -5,16 +5,12 @@ settings: excludeLinksFromLockfile: false dependencies: - '@dsp-testing/node-sass': - specifier: 1.0.3 - version: 1.0.3 + '@dsp-testing/is-positive@3.1.0': + specifier: ^3.1.0 + version: 3.1.0 packages: - /@dsp-testing/node-sass@2.0.3: - resolution: {integrity: sha512-5Kt5AHgt2qE9YFlRnqizh36k1lcuTdGQP3UsxJgxVUo1Uxh4Z7vDgr7wDBm2hp4PjZ6soE4zupSyfaCbYguQqg==, tarball: https://npm.pkg.github.com/download/@dsp-testing/inner-source-top-secret-npm-2/1.0.3/0a19a66110450848d0a88b1be211cadae740a86f0c1c1658ed89c9f391b8f605} - dev: false - /is-positive@3.1.0: resolution: {integrity: sha512-8ND1j3y9/HP94TOvGzr69/FgbkX2ruOldhLEsTWwcJVfo4oRjwemJmJxt7RJkKYH8tz7vYBP9JcKQY8CLuJ90Q==} engines: {node: '>=0.10.0'} diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/.npmrc b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/.npmrc index a423292be3..e27a59d3d8 100644 --- a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/.npmrc +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/.npmrc @@ -2,4 +2,4 @@ strict-peer-dependencies=false engine-strict=true registry=https://registry.npmjs.org/ hoist-pattern[]=*eslint* -use-node-version=18.0.0 + diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/package.json b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/package.json index f7103644e2..a31425be59 100644 --- a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/package.json +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/package.json @@ -1,10 +1,14 @@ { - "name": "abc", - "packageManager": "pnpm@8.15.1", - "engines": { - "node": "^14.16.0 || ^16.10.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^20.0.0" - }, + "name": "name", + "version": "0.0.0", + "private": true, "devDependencies": { - "eslint": "^8.32.0" + "@npmcli/fs": "3.1.0" + }, + "engines": { + "node": "20.11.1", + "npm": ">=9", + "pnpm": "use npm", + "yarn": "use npm" } -} +} \ No newline at end of file diff --git a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/pnpm-lock.yaml b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/pnpm-lock.yaml index 2b110ba965..c8dbf479ad 100644 --- a/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/pnpm-lock.yaml +++ b/npm_and_yarn/spec/fixtures/projects/pnpm/unsupported_engine_pnpm/pnpm-lock.yaml @@ -1,6 +1,9 @@ -lockfileVersion: '6.0' +lockfileVersion: '9.0' -devDependencies: - eslint: - specifier: ^8.32.0 - version: 8.57.0 +importers: + + .: + dependencies: + '@npmcli/fs@3.1.0': + resolution: {integrity: sha512-7kZUAaLscfgbwBQRbvdMYaZOWyMEcPTH/tJjnyAWJ/dvvs9Ef+CERx/qJb9GExJpl1qipaDGn7KqHnFGGixd0w==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} From f01d179880a32d3cf51b9759140560002efb79ec Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Wed, 15 Jan 2025 14:36:21 -0700 Subject: [PATCH 61/72] prefer `SafeVersions` property before considering other version restrictions --- .../Analyze/AnalyzeWorkerTests.cs | 55 +++++++++++++++++++ .../Analyze/VersionFinder.cs | 32 +++++++++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs index 0b546c97c3..669207d6f9 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Analyze/AnalyzeWorkerTests.cs @@ -478,6 +478,61 @@ await TestAnalyzeAsync( ); } + [Fact] + public async Task SafeVersionsPropertyIsHonored() + { + await TestAnalyzeAsync( + packages: + [ + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.0.0", "net8.0"), // initially this + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.1.0", "net8.0"), // should update to this due to `SafeVersions` + MockNuGetPackage.CreateSimplePackage("Some.Package", "1.2.0", "net8.0"), // this should not be considered + ], + discovery: new() + { + Path = "/", + Projects = [ + new() + { + FilePath = "./project.csproj", + TargetFrameworks = ["net8.0"], + Dependencies = [ + new("Some.Package", "1.0.0", DependencyType.PackageReference), + ], + ReferencedProjectPaths = [], + ImportedFiles = [], + AdditionalFiles = [], + }, + ], + }, + dependencyInfo: new() + { + Name = "Some.Package", + Version = "1.0.0", + IgnoredVersions = [], + IsVulnerable = false, + Vulnerabilities = [ + new() + { + DependencyName = "Some.Package", + PackageManager = "nuget", + VulnerableVersions = [Requirement.Parse(">= 1.0.0, < 1.1.0")], + SafeVersions = [Requirement.Parse("= 1.1.0")] + } + ], + }, + expectedResult: new() + { + UpdatedVersion = "1.1.0", + CanUpdate = true, + VersionComesFromMultiDependencyProperty = false, + UpdatedDependencies = [ + new("Some.Package", "1.1.0", DependencyType.Unknown, TargetFrameworks: ["net8.0"]), + ], + } + ); + } + [Fact] public async Task VersionFinderCanHandle404FromPackageSource_V2() { diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs index cf0555a9db..83cb185844 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs @@ -113,11 +113,33 @@ internal static Func CreateVersionFilter(DependencyInfo depe ? versionRange.MinVersion : null; - return version => (currentVersion is null || version > currentVersion) - && versionRange.Satisfies(version) - && (currentVersion is null || !currentVersion.IsPrerelease || !version.IsPrerelease || version.Version == currentVersion.Version) - && !dependencyInfo.IgnoredVersions.Any(r => r.IsSatisfiedBy(version)) - && !dependencyInfo.Vulnerabilities.Any(v => v.IsVulnerable(version)); + var safeVersions = dependencyInfo.Vulnerabilities.SelectMany(v => v.SafeVersions).ToList(); + return version => + { + if (safeVersions.Any()) + { + // only consider these + if (safeVersions.Any(s => s.IsSatisfiedBy(version))) + { + return true; + } + + return false; + } + else + { + var versionGreaterThanCurrent = currentVersion is null || version > currentVersion; + var rangeSatisfies = versionRange.Satisfies(version); + var prereleaseTypeMatches = currentVersion is null || !currentVersion.IsPrerelease || !version.IsPrerelease || version.Version == currentVersion.Version; + var isIgnoredVersion = dependencyInfo.IgnoredVersions.Any(i => i.IsSatisfiedBy(version)); + var isVulnerableVersion = dependencyInfo.Vulnerabilities.Any(v => v.IsVulnerable(version)); + return versionGreaterThanCurrent + && rangeSatisfies + && prereleaseTypeMatches + && !isIgnoredVersion + && !isVulnerableVersion; + } + }; } internal static Func CreateVersionFilter(NuGetVersion currentVersion) From 9d1e760cd7f3c4488bbd382e50305ad57a2a0931 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Wed, 15 Jan 2025 15:35:06 -0700 Subject: [PATCH 62/72] create dependency info object in end-to-end updater --- .../Run/MiscellaneousTests.cs | 61 +++++++++++++++++++ .../Run/ApiModel/Advisory.cs | 2 + .../NuGetUpdater.Core/Run/RunWorker.cs | 36 ++++++++--- 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MiscellaneousTests.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MiscellaneousTests.cs index 17db05d563..b21fe1d432 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MiscellaneousTests.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core.Test/Run/MiscellaneousTests.cs @@ -1,3 +1,5 @@ +using System.Text.Json; + using NuGet.Versioning; using NuGetUpdater.Core.Analyze; @@ -29,6 +31,16 @@ public void RequirementsFromIgnoredVersions(string dependencyName, Condition[] i Assert.Equal(expectedRequirementsStrings, actualRequirementsStrings); } + [Theory] + [MemberData(nameof(DependencyInfoFromJobData))] + public void DependencyInfoFromJob(Job job, Dependency dependency, DependencyInfo expectedDependencyInfo) + { + var actualDependencyInfo = RunWorker.GetDependencyInfo(job, dependency); + var expectedString = JsonSerializer.Serialize(expectedDependencyInfo, AnalyzeWorker.SerializerOptions); + var actualString = JsonSerializer.Serialize(actualDependencyInfo, AnalyzeWorker.SerializerOptions); + Assert.Equal(expectedString, actualString); + } + public static IEnumerable RequirementsFromIgnoredVersionsData() { yield return @@ -82,4 +94,53 @@ public void RequirementsFromIgnoredVersions(string dependencyName, Condition[] i } ]; } + + public static IEnumerable DependencyInfoFromJobData() + { + yield return + [ + // job + new Job() + { + Source = new() + { + Provider = "github", + Repo = "some/repo" + }, + SecurityAdvisories = [ + new() + { + DependencyName = "Some.Dependency", + AffectedVersions = [Requirement.Parse(">= 1.0.0, < 1.1.0")], + PatchedVersions = [Requirement.Parse("= 1.1.0")], + UnaffectedVersions = [Requirement.Parse("= 1.2.0")] + }, + new() + { + DependencyName = "Unrelated.Dependency", + AffectedVersions = [Requirement.Parse(">= 1.0.0, < 99.99.99")] + } + ] + }, + // dependency + new Dependency("Some.Dependency", "1.0.0", DependencyType.PackageReference), + // expectedDependencyInfo + new DependencyInfo() + { + Name = "Some.Dependency", + Version = "1.0.0", + IsVulnerable = true, + IgnoredVersions = [], + Vulnerabilities = [ + new() + { + DependencyName = "Some.Dependency", + PackageManager = "nuget", + VulnerableVersions = [Requirement.Parse(">= 1.0.0, < 1.1.0")], + SafeVersions = [Requirement.Parse("= 1.1.0"), Requirement.Parse("= 1.2.0")], + } + ] + } + ]; + } } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Advisory.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Advisory.cs index eb052eca40..1fddd91f71 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Advisory.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/ApiModel/Advisory.cs @@ -10,4 +10,6 @@ public record Advisory public ImmutableArray? AffectedVersions { get; init; } = null; public ImmutableArray? PatchedVersions { get; init; } = null; public ImmutableArray? UnaffectedVersions { get; init; } = null; + + public IEnumerable SafeVersions => (PatchedVersions ?? []).Concat(UnaffectedVersions ?? []); } diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs index 7bce84a7e6..35727ab217 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Run/RunWorker.cs @@ -4,6 +4,8 @@ using System.Text.Json; using System.Text.Json.Serialization; +using NuGet.Versioning; + using NuGetUpdater.Core.Analyze; using NuGetUpdater.Core.Discover; using NuGetUpdater.Core.Run.ApiModel; @@ -164,15 +166,7 @@ async Task TrackOriginalContentsAsync(string directory, string fileName) continue; } - var ignoredVersions = GetIgnoredRequirementsForDependency(job, dependency.Name); - var dependencyInfo = new DependencyInfo() - { - Name = dependency.Name, - Version = dependency.Version!, - IsVulnerable = false, - IgnoredVersions = ignoredVersions, - Vulnerabilities = [], - }; + var dependencyInfo = GetDependencyInfo(job, dependency); var analysisResult = await _analyzeWorker.RunAsync(repoContentsPath.FullName, discoveryResult, dependencyInfo); // TODO: log analysisResult if (analysisResult.CanUpdate) @@ -314,6 +308,30 @@ internal static ImmutableArray GetIgnoredRequirementsForDependency( return ignoredVersions; } + internal static DependencyInfo GetDependencyInfo(Job job, Dependency dependency) + { + var dependencyVersion = NuGetVersion.Parse(dependency.Version!); + var securityAdvisories = job.SecurityAdvisories.Where(s => s.DependencyName.Equals(dependency.Name, StringComparison.OrdinalIgnoreCase)).ToArray(); + var isVulnerable = securityAdvisories.Any(s => (s.AffectedVersions ?? []).Any(v => v.IsSatisfiedBy(dependencyVersion))); + var ignoredVersions = GetIgnoredRequirementsForDependency(job, dependency.Name); + var vulnerabilities = securityAdvisories.Select(s => new SecurityVulnerability() + { + DependencyName = dependency.Name, + PackageManager = "nuget", + VulnerableVersions = s.AffectedVersions ?? [], + SafeVersions = s.SafeVersions.ToImmutableArray(), + }).ToImmutableArray(); + var dependencyInfo = new DependencyInfo() + { + Name = dependency.Name, + Version = dependencyVersion.ToString(), + IsVulnerable = isVulnerable, + IgnoredVersions = ignoredVersions, + Vulnerabilities = vulnerabilities, + }; + return dependencyInfo; + } + internal static UpdatedDependencyList GetUpdatedDependencyListFromDiscovery(WorkspaceDiscoveryResult discoveryResult, string pathToContents) { string GetFullRepoPath(string path) From 57b57a5e4eec448ba54cd7fdb6858496b4b9c4e0 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Thu, 16 Jan 2025 14:43:57 -0700 Subject: [PATCH 63/72] combine update version check --- .../Analyze/VersionFinder.cs | 35 +++++++------------ 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs index 83cb185844..7a8f7160e4 100644 --- a/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs +++ b/nuget/helpers/lib/NuGetUpdater/NuGetUpdater.Core/Analyze/VersionFinder.cs @@ -116,29 +116,18 @@ internal static Func CreateVersionFilter(DependencyInfo depe var safeVersions = dependencyInfo.Vulnerabilities.SelectMany(v => v.SafeVersions).ToList(); return version => { - if (safeVersions.Any()) - { - // only consider these - if (safeVersions.Any(s => s.IsSatisfiedBy(version))) - { - return true; - } - - return false; - } - else - { - var versionGreaterThanCurrent = currentVersion is null || version > currentVersion; - var rangeSatisfies = versionRange.Satisfies(version); - var prereleaseTypeMatches = currentVersion is null || !currentVersion.IsPrerelease || !version.IsPrerelease || version.Version == currentVersion.Version; - var isIgnoredVersion = dependencyInfo.IgnoredVersions.Any(i => i.IsSatisfiedBy(version)); - var isVulnerableVersion = dependencyInfo.Vulnerabilities.Any(v => v.IsVulnerable(version)); - return versionGreaterThanCurrent - && rangeSatisfies - && prereleaseTypeMatches - && !isIgnoredVersion - && !isVulnerableVersion; - } + var versionGreaterThanCurrent = currentVersion is null || version > currentVersion; + var rangeSatisfies = versionRange.Satisfies(version); + var prereleaseTypeMatches = currentVersion is null || !currentVersion.IsPrerelease || !version.IsPrerelease || version.Version == currentVersion.Version; + var isIgnoredVersion = dependencyInfo.IgnoredVersions.Any(i => i.IsSatisfiedBy(version)); + var isVulnerableVersion = dependencyInfo.Vulnerabilities.Any(v => v.IsVulnerable(version)); + var isSafeVersion = !safeVersions.Any() || safeVersions.Any(s => s.IsSatisfiedBy(version)); + return versionGreaterThanCurrent + && rangeSatisfies + && prereleaseTypeMatches + && !isIgnoredVersion + && !isVulnerableVersion + && isSafeVersion; }; } From 6cf724548ab863cac95179002af3d3dda1d3e32f Mon Sep 17 00:00:00 2001 From: kbukum1 Date: Fri, 17 Jan 2025 15:24:31 -0800 Subject: [PATCH 64/72] Handle `NoChangeError` for Transitive Dependency Updates in `pnpm` Package Manager Behind Feature Flag (#11336) --- common/lib/dependabot/errors.rb | 57 +++++++++++++++++++ .../dependabot/npm_and_yarn/file_updater.rb | 18 ++++++ .../file_updater/bun_lockfile_updater_spec.rb | 2 + .../file_updater/npm_lockfile_updater_spec.rb | 2 + .../yarn_lockfile_updater_spec.rb | 2 + .../npm_and_yarn/update_checker_spec.rb | 2 + 6 files changed, 83 insertions(+) diff --git a/common/lib/dependabot/errors.rb b/common/lib/dependabot/errors.rb index 2171a9ea62..41069a15f1 100644 --- a/common/lib/dependabot/errors.rb +++ b/common/lib/dependabot/errors.rb @@ -33,6 +33,15 @@ def self.fetcher_error_details(error) "supported-versions": error.supported_versions } } + when Dependabot::ToolFeatureNotSupported + { + "error-type": "tool_feature_not_supported", + "error-detail": { + "tool-name": error.tool_name, + "tool-type": error.tool_type, + feature: error.feature + } + } when Dependabot::BranchNotFound { "error-type": "branch_not_found", @@ -103,6 +112,15 @@ def self.fetcher_error_details(error) sig { params(error: StandardError).returns(T.nilable(T::Hash[Symbol, T.untyped])) } def self.parser_error_details(error) case error + when Dependabot::ToolFeatureNotSupported + { + "error-type": "tool_feature_not_supported", + "error-detail": { + "tool-name": error.tool_name, + "tool-type": error.tool_type, + feature: error.feature + } + } when Dependabot::DependencyFileNotEvaluatable { "error-type": "dependency_file_not_evaluatable", @@ -170,6 +188,15 @@ def self.parser_error_details(error) sig { params(error: StandardError).returns(T.nilable(T::Hash[Symbol, T.untyped])) } def self.updater_error_details(error) case error + when Dependabot::ToolFeatureNotSupported + { + "error-type": "tool_feature_not_supported", + "error-detail": { + "tool-name": error.tool_name, + "tool-type": error.tool_type, + feature: error.feature + } + } when Dependabot::DependencyFileNotResolvable { "error-type": "dependency_file_not_resolvable", @@ -300,6 +327,7 @@ def self.updater_error_details(error) } end end + # rubocop:enable Metrics/MethodLength # rubocop:enable Metrics/CyclomaticComplexity # rubocop:enable Lint/RedundantCopDisableDirective @@ -490,6 +518,35 @@ def initialize(tool_name, detected_version, supported_versions) end end + class ToolFeatureNotSupported < DependabotError + extend T::Sig + + sig { returns(String) } + attr_reader :tool_name, :tool_type, :feature + + sig do + params( + tool_name: String, + tool_type: String, + feature: String + ).void + end + def initialize(tool_name:, tool_type:, feature:) + @tool_name = tool_name + @tool_type = tool_type + @feature = feature + super(build_message) + end + + private + + sig { returns(String) } + def build_message + "Dependabot doesn't support the feature '#{feature}' for #{tool_name} (#{tool_type}). " \ + "Please refer to the documentation for supported features." + end + end + class DependencyFileNotFound < DependabotError extend T::Sig diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater.rb index 838c48eb56..6846eaa517 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater.rb @@ -48,6 +48,7 @@ def self.updated_files_regex ] end + # rubocop:disable Metrics/PerceivedComplexity sig { override.returns(T::Array[DependencyFile]) } def updated_dependency_files updated_files = T.let([], T::Array[DependencyFile]) @@ -56,6 +57,22 @@ def updated_dependency_files updated_files += updated_lockfiles if updated_files.none? + + if Dependabot::Experiments.enabled?(:enable_fix_for_pnpm_no_change_error) + # when all dependencies are transitive + all_transitive = dependencies.none?(&:top_level?) + # when there is no update in package.json + no_package_json_update = package_files.empty? + # handle the no change error for transitive dependency updates + if pnpm_locks.any? && dependencies.length.positive? && all_transitive && no_package_json_update + raise ToolFeatureNotSupported.new( + tool_name: "pnpm", + tool_type: "package_manager", + feature: "updating transitive dependencies" + ) + end + end + raise NoChangeError.new( message: "No files were updated!", error_context: error_context(updated_files: updated_files) @@ -72,6 +89,7 @@ def updated_dependency_files vendor_updated_files(updated_files) end + # rubocop:enable Metrics/PerceivedComplexity private diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/bun_lockfile_updater_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/bun_lockfile_updater_spec.rb index 66604cd493..3f232210e0 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/bun_lockfile_updater_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/bun_lockfile_updater_spec.rb @@ -67,6 +67,8 @@ FileUtils.mkdir_p(tmp_path) allow(Dependabot::Experiments).to receive(:enabled?) .with(:enable_shared_helpers_command_timeout).and_return(true) + allow(Dependabot::Experiments).to receive(:enabled?) + .with(:enable_fix_for_pnpm_no_change_error).and_return(true) end after do diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater_spec.rb index 739cd770e5..9ed2baf7f4 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/npm_lockfile_updater_spec.rb @@ -76,6 +76,8 @@ .with(:enable_shared_helpers_command_timeout).and_return(true) allow(Dependabot::Experiments).to receive(:enabled?) .with(:npm_v6_deprecation_warning).and_return(true) + allow(Dependabot::Experiments).to receive(:enabled?) + .with(:enable_fix_for_pnpm_no_change_error).and_return(true) end after do diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater_spec.rb index af955adf0b..74acf56038 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater_spec.rb @@ -67,6 +67,8 @@ .with(:enable_corepack_for_npm_and_yarn).and_return(enable_corepack_for_npm_and_yarn) allow(Dependabot::Experiments).to receive(:enabled?) .with(:enable_shared_helpers_command_timeout).and_return(true) + allow(Dependabot::Experiments).to receive(:enabled?) + .with(:enable_fix_for_pnpm_no_change_error).and_return(true) end after do diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/update_checker_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/update_checker_spec.rb index cb2f35b6e1..8b0037b2a8 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/update_checker_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/update_checker_spec.rb @@ -75,6 +75,8 @@ .with(:enable_shared_helpers_command_timeout).and_return(true) allow(Dependabot::Experiments).to receive(:enabled?) .with(:npm_v6_deprecation_warning).and_return(true) + allow(Dependabot::Experiments).to receive(:enabled?) + .with(:enable_fix_for_pnpm_no_change_error).and_return(true) end after do From 0997a4f0e47d498dd11661f0400f761a5345627c Mon Sep 17 00:00:00 2001 From: kbukum1 Date: Fri, 17 Jan 2025 15:52:46 -0800 Subject: [PATCH 65/72] remove pnpm error (#11338) --- .../npm_and_yarn/file_updater/pnpm_lockfile_updater.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb index ba336e88de..3926c4461e 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/pnpm_lockfile_updater.rb @@ -123,8 +123,8 @@ def run_pnpm_update_packages if Dependabot::Experiments.enabled?(:enable_fix_for_pnpm_no_change_error) Helpers.run_pnpm_command( - "pnpm update #{dependency_updates} --lockfile-only --no-save -r", - fingerprint: "pnpm update --lockfile-only --no-save -r" + "update #{dependency_updates} --lockfile-only --no-save -r", + fingerprint: "update --lockfile-only --no-save -r" ) else Helpers.run_pnpm_command( From b3a0c1f86c20729494097ebc695067099f5b4ada Mon Sep 17 00:00:00 2001 From: "S.Sandhu" <167903774+sachin-sandhu@users.noreply.github.com> Date: Fri, 17 Jan 2025 20:14:41 -0500 Subject: [PATCH 66/72] Add version extract for corepack verbose and test cases (#11337) --- npm_and_yarn/lib/dependabot/npm_and_yarn/version.rb | 4 ++++ .../spec/dependabot/npm_and_yarn/version_spec.rb | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/version.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/version.rb index 7470120b79..87e5b962b3 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/version.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/version.rb @@ -80,6 +80,10 @@ def clean_version(version) # Matches @ followed by x.y.z (digits separated by dots) if (match = version.match(/@(\d+\.\d+\.\d+)/)) version = match[1] # Just "4.5.3" + + # Extract version in case the output contains Corepack verbose data + elsif version.include?("Corepack") + version = T.must(T.must(version.tr("\n", " ").match(/(\d+\.\d+\.\d+)/))[-1]) end version = version&.gsub(/^v/, "") end diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/version_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/version_spec.rb index 8a05ccea7d..973c0da643 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/version_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/version_spec.rb @@ -317,5 +317,15 @@ it { is_expected.to be(true) } end + + context "with corepack returning a additional info with version string" do + let(:requirement) { Gem::Requirement.new("4.6.0") } + let(:version_string) do + "Corepack is about to download " \ + "https://repo.yarnpkg.com/4.6.0/packages/yarnpkg-cli/bin/yarn.js\n4.6.0" + end + + it { is_expected.to be(true) } + end end end From 8fe5d1e6ac774a9bc92e88e7cd2a85d5abf54448 Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Mon, 20 Jan 2025 18:45:28 +0900 Subject: [PATCH 67/72] Bump Hex Elixir to 1.17.3 (#11339) This bumps Elixir to 1.17.3 There's only one required change here to switch the IO.read mode. --- hex/Dockerfile | 4 ++-- hex/helpers/lib/run.exs | 2 +- hex/spec/dependabot/hex/file_parser_spec.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hex/Dockerfile b/hex/Dockerfile index 1ae9a8ad4c..3750951ae2 100644 --- a/hex/Dockerfile +++ b/hex/Dockerfile @@ -15,8 +15,8 @@ RUN apt-get update \ # Install Elixir # https://github.com/elixir-lang/elixir/releases -ARG ELIXIR_VERSION=v1.16.3 -ARG ELIXIR_CHECKSUM=e8e81771bc6acd62a2c1bf1b31c3aa3d0a469415de3b243b99f3e2e2d639f5ea +ARG ELIXIR_VERSION=v1.17.3 +ARG ELIXIR_CHECKSUM=3fd0f58730f848b5650be2ef7c892b76c74324b8537181778117c43306a6e5f7 RUN curl -sSLfO https://github.com/elixir-lang/elixir/releases/download/${ELIXIR_VERSION}/elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ && echo "$ELIXIR_CHECKSUM elixir-otp-${ERLANG_MAJOR_VERSION}.zip" | sha256sum -c - \ && unzip -d /usr/local/elixir -x elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ diff --git a/hex/helpers/lib/run.exs b/hex/helpers/lib/run.exs index 71b0c2d91c..5212265745 100644 --- a/hex/helpers/lib/run.exs +++ b/hex/helpers/lib/run.exs @@ -1,6 +1,6 @@ defmodule DependencyHelper do def main() do - IO.read(:stdio, :all) + IO.read(:stdio, :eof) |> Jason.decode!() |> run() |> case do diff --git a/hex/spec/dependabot/hex/file_parser_spec.rb b/hex/spec/dependabot/hex/file_parser_spec.rb index 5c852b6dd0..8f99e0fbef 100644 --- a/hex/spec/dependabot/hex/file_parser_spec.rb +++ b/hex/spec/dependabot/hex/file_parser_spec.rb @@ -476,7 +476,7 @@ it "returns the correct language" do expect(language.name).to eq "elixir" expect(language.requirement).to be_nil - expect(language.version.to_s).to eq "1.16.3" + expect(language.version.to_s).to eq "1.17.3" end end end From 9b23d2355ec7c38ac4352415849dd47b36af6a8a Mon Sep 17 00:00:00 2001 From: Tyler Witt Date: Mon, 20 Jan 2025 22:24:32 +0900 Subject: [PATCH 68/72] Bump Hex OTP to 26, Elixir to 1.18.1 (#11350) This makes Elixir current. I included the OTP bump because it had no alterations besides version numbers. Only minor change required for 1.18.1 was re-ordering within the Dependabot::Dependency.requirements inside of a test. Hope that's fine. Co-authored-by: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> --- hex/Dockerfile | 8 ++++---- hex/spec/dependabot/hex/file_parser_spec.rb | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/hex/Dockerfile b/hex/Dockerfile index 3750951ae2..e9205cca05 100644 --- a/hex/Dockerfile +++ b/hex/Dockerfile @@ -4,9 +4,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ gnupg2 sudo wget -ARG ERLANG_MAJOR_VERSION=25 +ARG ERLANG_MAJOR_VERSION=26 -RUN echo "deb http://binaries2.erlang-solutions.com/ubuntu/ jammy-esl-erlang-25 contrib" >> /etc/apt/sources.list +RUN echo "deb http://binaries2.erlang-solutions.com/ubuntu/ jammy-esl-erlang-26 contrib" >> /etc/apt/sources.list RUN wget https://binaries2.erlang-solutions.com/GPG-KEY-pmanager.asc \ && sudo apt-key add GPG-KEY-pmanager.asc @@ -15,8 +15,8 @@ RUN apt-get update \ # Install Elixir # https://github.com/elixir-lang/elixir/releases -ARG ELIXIR_VERSION=v1.17.3 -ARG ELIXIR_CHECKSUM=3fd0f58730f848b5650be2ef7c892b76c74324b8537181778117c43306a6e5f7 +ARG ELIXIR_VERSION=v1.18.1 +ARG ELIXIR_CHECKSUM=aae4625102ba7020887918d1c0ac7c8ad972b65fe8103476765cc6b00ab16b5f RUN curl -sSLfO https://github.com/elixir-lang/elixir/releases/download/${ELIXIR_VERSION}/elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ && echo "$ELIXIR_CHECKSUM elixir-otp-${ERLANG_MAJOR_VERSION}.zip" | sha256sum -c - \ && unzip -d /usr/local/elixir -x elixir-otp-${ERLANG_MAJOR_VERSION}.zip \ diff --git a/hex/spec/dependabot/hex/file_parser_spec.rb b/hex/spec/dependabot/hex/file_parser_spec.rb index 8f99e0fbef..d8be6d3233 100644 --- a/hex/spec/dependabot/hex/file_parser_spec.rb +++ b/hex/spec/dependabot/hex/file_parser_spec.rb @@ -396,13 +396,13 @@ name: "plug", version: "1.3.6", requirements: [{ - requirement: "1.3.6", - file: "apps/dependabot_web/mix.exs", + requirement: "~> 1.3.0", + file: "apps/dependabot_business/mix.exs", groups: [], source: nil }, { - requirement: "~> 1.3.0", - file: "apps/dependabot_business/mix.exs", + requirement: "1.3.6", + file: "apps/dependabot_web/mix.exs", groups: [], source: nil }], @@ -476,7 +476,7 @@ it "returns the correct language" do expect(language.name).to eq "elixir" expect(language.requirement).to be_nil - expect(language.version.to_s).to eq "1.17.3" + expect(language.version.to_s).to eq "1.18.1" end end end From d6d0437fa387aef9aa5fda9c6d1e390615e35aef Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:14:47 +0000 Subject: [PATCH 69/72] Revert "Revert "Removing corepack usage from else clause (#11299)" (#11322)" (#11351) This reverts commit 2fd554a38f2fc9106d372fddbbad24e54d35f100. --- npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb index 1f028de1f8..7cd5ac3956 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb @@ -324,8 +324,8 @@ def self.run_npm_command(command, fingerprint: command) package_manager_run_command(NpmPackageManager::NAME, command, fingerprint: fingerprint) else Dependabot::SharedHelpers.run_shell_command( - "corepack npm #{command}", - fingerprint: "corepack npm #{fingerprint}" + "npm #{command}", + fingerprint: "npm #{fingerprint}" ) end end From 80315238145998ddfea4d3bfc0a5525973f231d5 Mon Sep 17 00:00:00 2001 From: Rob Aiken Date: Tue, 21 Jan 2025 11:23:45 +0000 Subject: [PATCH 70/72] Adding types to elm_json_updater.rb (#11331) * Adding types to elm_json_updater.rb * adding safe nav to dependency fetches --- .../elm/file_updater/elm_json_updater.rb | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/elm/lib/dependabot/elm/file_updater/elm_json_updater.rb b/elm/lib/dependabot/elm/file_updater/elm_json_updater.rb index c2d48cb115..808bb4a81f 100644 --- a/elm/lib/dependabot/elm/file_updater/elm_json_updater.rb +++ b/elm/lib/dependabot/elm/file_updater/elm_json_updater.rb @@ -1,4 +1,4 @@ -# typed: true +# typed: strict # frozen_string_literal: true require "dependabot/elm/file_updater" @@ -7,11 +7,15 @@ module Dependabot module Elm class FileUpdater class ElmJsonUpdater + extend T::Sig + + sig { params(elm_json_file: Dependabot::DependencyFile, dependencies: T::Array[Dependabot::Dependency]).void } def initialize(elm_json_file:, dependencies:) @elm_json_file = elm_json_file @dependencies = dependencies end + sig { returns(T.nilable(String)) } def updated_content dependencies .select { |dep| requirement_changed?(elm_json_file, dep) } @@ -32,34 +36,33 @@ def updated_content private + sig { returns(Dependabot::DependencyFile) } attr_reader :elm_json_file + + sig { returns(T::Array[Dependabot::Dependency]) } attr_reader :dependencies + sig { params(file: Dependabot::DependencyFile, dependency: Dependabot::Dependency).returns(T::Boolean) } def requirement_changed?(file, dependency) - changed_requirements = - dependency.requirements - dependency.previous_requirements + changed_requirements = dependency.requirements - T.must(dependency.previous_requirements) changed_requirements.any? { |f| f[:file] == file.name } end + sig { params(content: T.nilable(String), filename: String, dependency: Dependabot::Dependency).returns(String) } def update_requirement(content:, filename:, dependency:) - updated_req = - dependency.requirements - .find { |r| r.fetch(:file) == filename } - .fetch(:requirement) + updated_req = dependency.requirements.find { |r| r.fetch(:file) == filename } + &.fetch(:requirement) - old_req = - dependency.previous_requirements - .find { |r| r.fetch(:file) == filename } - .fetch(:requirement) + old_req = dependency.previous_requirements&.find { |r| r.fetch(:file) == filename } + &.fetch(:requirement) - return content unless old_req + return T.must(content) unless old_req dep = dependency - regex = - /"#{Regexp.quote(dep.name)}"\s*:\s+"#{Regexp.quote(old_req)}"/ + regex = /"#{Regexp.quote(dep.name)}"\s*:\s+"#{Regexp.quote(old_req)}"/ - content.gsub(regex) do |declaration| + T.must(content).gsub(regex) do |declaration| declaration.gsub(%("#{old_req}"), %("#{updated_req}")) end end From 9451c2e7d519ee0b8ca63bea5b208e107cf726a0 Mon Sep 17 00:00:00 2001 From: Hariharan Thavachelvam <164553783+thavaahariharangit@users.noreply.github.com> Date: Tue, 21 Jan 2025 15:40:18 +0000 Subject: [PATCH 71/72] Enabling alias parsing in pnpm_workspace_yaml (#11353) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * enabling alias parsing in pnpm_workspace_yaml * context message updated. * Adding bad alias check. * Updating context message. * Updating context message. --------- Co-authored-by: “Thavachelvam <“thavaahariharangit@git.com”> --- .../dependabot/npm_and_yarn/file_fetcher.rb | 4 +- .../npm_and_yarn/file_fetcher_spec.rb | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb index a066c5527c..6cbeb14509 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb @@ -645,8 +645,8 @@ def parsed_shrinkwrap def parsed_pnpm_workspace_yaml return {} unless pnpm_workspace_yaml - YAML.safe_load(T.must(T.must(pnpm_workspace_yaml).content)) - rescue Psych::SyntaxError + YAML.safe_load(T.must(T.must(pnpm_workspace_yaml).content), aliases: true) + rescue Psych::SyntaxError, Psych::BadAlias raise Dependabot::DependencyFileNotParseable, T.must(pnpm_workspace_yaml).path end diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_fetcher_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_fetcher_spec.rb index 3756b2c9c5..b3939dd784 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_fetcher_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_fetcher_spec.rb @@ -2021,6 +2021,61 @@ end end + context "with a pnpm_workspace_yaml" do + let(:source) do + Dependabot::Source.new( + provider: "github", + repo: "gocardless/bump", + directory: "/" + ) + end + let(:file_fetcher) { described_class.new(source: source, credentials: credentials) } + let(:pnpm_workspace_yaml) { Dependabot::DependencyFile.new(name: "pnpm-workspace.yaml", content: content) } + + before do + allow(file_fetcher).to receive(:pnpm_workspace_yaml).and_return(pnpm_workspace_yaml) + end + + context "when it's content is nil" do + let(:pnpm_workspace_yaml) { nil } + + it "returns an empty hash" do + expect(file_fetcher.send(:parsed_pnpm_workspace_yaml)).to eq({}) + end + end + + context "when it's content is valid YAML" do + let(:content) { "---\npackages:\n - 'packages/*'\n" } + + it "parses the YAML content" do + expect(file_fetcher.send(:parsed_pnpm_workspace_yaml)).to eq({ "packages" => ["packages/*"] }) + end + end + + context "when it's content contains valid alias" do + let(:content) { "---\npackages:\n - &default 'packages/*'\n - *default\n" } + let(:pnpm_workspace_yaml) { Dependabot::DependencyFile.new(name: "pnpm-workspace.yaml", content: content) } + + it "parses the YAML content with aliases" do + expect(file_fetcher.send(:parsed_pnpm_workspace_yaml)).to eq({ "packages" => ["packages/*", "packages/*"] }) + end + end + + context "when it's content contains invalid alias (BadAlias)" do + let(:content) { "---\npackages:\n - &id 'packages/*'\n - *id" } # Invalid alias reference + + before do + allow(YAML).to receive(:safe_load).and_raise(Psych::BadAlias) + end + + it "raises a DependencyFileNotParseable error" do + expect do + file_fetcher.send(:parsed_pnpm_workspace_yaml) + end.to raise_error(Dependabot::DependencyFileNotParseable) + end + end + end + context "with package.json file just including a dummy string" do before do allow(file_fetcher_instance).to receive(:commit).and_return("sha") From 46fcd61eede2786c4a14e476b11f2819aca567aa Mon Sep 17 00:00:00 2001 From: Rob Aiken Date: Tue, 21 Jan 2025 17:38:37 +0000 Subject: [PATCH 72/72] Skip catalog protocol dependencies if workspace file context is missing (#11361) --- .../dependabot/npm_and_yarn/file_parser.rb | 2 +- .../npm_and_yarn/file_parser_spec.rb | 6 +++++ .../package.json | 23 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 npm_and_yarn/spec/fixtures/projects/yarn/workspace_requirements_catalog/package.json diff --git a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb index 7f86e81eaf..3e22c29e55 100644 --- a/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb +++ b/npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb @@ -212,7 +212,7 @@ def manifest_dependencies next unless requirement.is_a?(String) # Skip dependencies using Yarn workspace cross-references as requirements - next if requirement.start_with?("workspace:") + next if requirement.start_with?("workspace:", "catalog:") requirement = "*" if requirement == "" dep = build_dependency( diff --git a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_parser_spec.rb b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_parser_spec.rb index 186cb4f959..13a6c2b31b 100644 --- a/npm_and_yarn/spec/dependabot/npm_and_yarn/file_parser_spec.rb +++ b/npm_and_yarn/spec/dependabot/npm_and_yarn/file_parser_spec.rb @@ -90,6 +90,12 @@ its(:length) { is_expected.to eq(0) } end + context "with pnpm `catalog:` requirements and no lockfile" do + let(:files) { project_dependency_files("yarn/workspace_requirements_catalog") } + + its(:length) { is_expected.to eq(0) } + end + context "with a package-lock.json" do let(:npm_fallback_version_above_v6_enabled) { false } diff --git a/npm_and_yarn/spec/fixtures/projects/yarn/workspace_requirements_catalog/package.json b/npm_and_yarn/spec/fixtures/projects/yarn/workspace_requirements_catalog/package.json new file mode 100644 index 0000000000..5b04f08281 --- /dev/null +++ b/npm_and_yarn/spec/fixtures/projects/yarn/workspace_requirements_catalog/package.json @@ -0,0 +1,23 @@ +{ + "name": "workspace_requirements_catalog", + "version": "0.0.1", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/gocardless/bump-test.git" + }, + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/gocardless/bump-test/issues" + }, + "homepage": "https://github.com/gocardless/bump-test#readme", + "dependencies": { + "ember-simple-charts": "catalog:", + "react": "catalog:react18" + } +}