From 553a1cfa4bfdf800a4756b6caf64682710234610 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 28 Jun 2024 00:30:37 -0400 Subject: [PATCH] Merge pull request #711 from ionite34/fixes Add setuptools version pin for base python (cherry picked from commit ef061d418579a365cf3d3e3ea3da22b66f2d3a6a) --- CHANGELOG.md | 2 + .../Dialogs/PythonPackagesItemViewModel.cs | 12 ++- .../Dialogs/PythonPackagesViewModel.cs | 24 +++++- .../Models/Packages/BaseGitPackage.cs | 24 +++--- .../Models/Packages/Fooocus.cs | 2 +- .../Python/MajorMinorVersion.cs | 3 + StabilityMatrix.Core/Python/PyBaseInstall.cs | 78 +++++++++++++++++-- StabilityMatrix.Core/Python/PyRunner.cs | 5 +- StabilityMatrix.Core/Python/PyVenvRunner.cs | 31 ++++---- 9 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 StabilityMatrix.Core/Python/MajorMinorVersion.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 012329d7..714514b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ## v2.11.4 +### Changed +- Base Python install will now use `setuptools==69.5.1` for compatibility with `torchsde`. Individual Packages can upgrade as required. ### Fixed - Fixed [#719](https://github.com/LykosAI/StabilityMatrix/issues/719) - Fix comments in Inference prompt not being ignored ### Supporters diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesItemViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesItemViewModel.cs index 81fcbc90..c21da257 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesItemViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesItemViewModel.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Avalonia.Controls; @@ -10,10 +9,11 @@ using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Python; +using StabilityMatrix.Core.Services; namespace StabilityMatrix.Avalonia.ViewModels.Dialogs; -public partial class PythonPackagesItemViewModel : ViewModelBase +public partial class PythonPackagesItemViewModel(ISettingsManager settingsManager) : ViewModelBase { [ObservableProperty] private PipPackageInfo package; @@ -101,7 +101,11 @@ public async Task LoadExtraInfo(DirectoryPath venvPath) } else { - await using var venvRunner = new PyVenvRunner(venvPath); + await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync( + venvPath, + workingDirectory: venvPath.Parent, + environmentVariables: settingsManager.Settings.EnvironmentVariables + ); PipShowResult = await venvRunner.PipShow(Package.Name); diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs index 68dd5ff5..45c284b2 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs @@ -4,6 +4,7 @@ using System.Reactive.Linq; using System.Threading.Tasks; using AsyncAwaitBestPractices; +using AutoCtor; using Avalonia.Controls; using Avalonia.Controls.Primitives; using Avalonia.Threading; @@ -12,6 +13,7 @@ using DynamicData; using DynamicData.Binding; using FluentAvalonia.UI.Controls; +using Microsoft.Extensions.Logging; using StabilityMatrix.Avalonia.Controls; using StabilityMatrix.Avalonia.Languages; using StabilityMatrix.Avalonia.ViewModels.Base; @@ -23,14 +25,19 @@ using StabilityMatrix.Core.Models.PackageModification; using StabilityMatrix.Core.Processes; using StabilityMatrix.Core.Python; +using StabilityMatrix.Core.Services; namespace StabilityMatrix.Avalonia.ViewModels.Dialogs; [View(typeof(PythonPackagesDialog))] [ManagedService] [Transient] +[AutoConstruct] public partial class PythonPackagesViewModel : ContentDialogViewModelBase { + private readonly ILogger logger; + private readonly ISettingsManager settingsManager; + public DirectoryPath? VenvPath { get; set; } [ObservableProperty] @@ -47,7 +54,8 @@ public partial class PythonPackagesViewModel : ContentDialogViewModelBase [ObservableProperty] private string searchQuery = string.Empty; - public PythonPackagesViewModel() + [AutoPostConstruct] + private void PostConstruct() { var searchPredicate = this.WhenPropertyChanged(vm => vm.SearchQuery) .Throttle(TimeSpan.FromMilliseconds(100)) @@ -65,7 +73,7 @@ public PythonPackagesViewModel() .Connect() .DeferUntilLoaded() .Filter(searchPredicate) - .Transform(p => new PythonPackagesItemViewModel { Package = p }) + .Transform(p => new PythonPackagesItemViewModel(settingsManager) { Package = p }) .SortBy(vm => vm.Package.Name) .Bind(Packages) .Subscribe(); @@ -86,7 +94,11 @@ private async Task Refresh() } else { - await using var venvRunner = new PyVenvRunner(VenvPath); + await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync( + VenvPath, + workingDirectory: VenvPath.Parent, + environmentVariables: settingsManager.Settings.EnvironmentVariables + ); var packages = await venvRunner.PipList(); packageSource.EditDiff(packages); @@ -104,7 +116,11 @@ private async Task RefreshBackground() if (VenvPath is null) return; - await using var venvRunner = new PyVenvRunner(VenvPath); + await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync( + VenvPath, + workingDirectory: VenvPath.Parent, + environmentVariables: settingsManager.Settings.EnvironmentVariables + ); var packages = await venvRunner.PipList(); diff --git a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs index 8c2347db..999569b9 100644 --- a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs @@ -1,4 +1,5 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.IO.Compression; using NLog; using Octokit; @@ -152,27 +153,22 @@ public async Task SetupVenv( Action? onConsoleOutput = null ) { - if (VenvRunner != null) + if (Interlocked.Exchange(ref VenvRunner, null) is { } oldRunner) { - await VenvRunner.DisposeAsync().ConfigureAwait(false); + await oldRunner.DisposeAsync().ConfigureAwait(false); } - VenvRunner = await PyBaseInstall - .Default.CreateVenvRunnerAsync( - Path.Combine(installedPackagePath, venvName), - workingDirectory: installedPackagePath, - environmentVariables: SettingsManager.Settings.EnvironmentVariables, - withDefaultTclTkEnv: Compat.IsWindows, - withQueriedTclTkEnv: Compat.IsUnix - ) + var venvRunner = await SetupVenvPure(installedPackagePath, venvName, forceRecreate, onConsoleOutput) .ConfigureAwait(false); - if (forceRecreate || !VenvRunner.Exists()) + if (Interlocked.Exchange(ref VenvRunner, venvRunner) is { } oldRunner2) { - await VenvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false); + await oldRunner2.DisposeAsync().ConfigureAwait(false); } - return VenvRunner; + Debug.Assert(VenvRunner != null, "VenvRunner != null"); + + return venvRunner; } /// diff --git a/StabilityMatrix.Core/Models/Packages/Fooocus.cs b/StabilityMatrix.Core/Models/Packages/Fooocus.cs index 33d0fd0b..d74f7528 100644 --- a/StabilityMatrix.Core/Models/Packages/Fooocus.cs +++ b/StabilityMatrix.Core/Models/Packages/Fooocus.cs @@ -274,7 +274,7 @@ public override async Task InstallPackage( progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true)); // Pip version 24.1 deprecated numpy requirement spec used by torchsde 0.2.5 - await venvRunner.PipInstall(["pip>=23.3.2,<24.1"], onConsoleOutput).ConfigureAwait(false); + await venvRunner.PipInstall(["pip==23.3.2"], onConsoleOutput).ConfigureAwait(false); var pipArgs = new PipInstallArgs(); diff --git a/StabilityMatrix.Core/Python/MajorMinorVersion.cs b/StabilityMatrix.Core/Python/MajorMinorVersion.cs new file mode 100644 index 00000000..79b73233 --- /dev/null +++ b/StabilityMatrix.Core/Python/MajorMinorVersion.cs @@ -0,0 +1,3 @@ +namespace StabilityMatrix.Core.Python; + +public readonly record struct MajorMinorVersion(int Major, int Minor); diff --git a/StabilityMatrix.Core/Python/PyBaseInstall.cs b/StabilityMatrix.Core/Python/PyBaseInstall.cs index e0a8bd67..bb12ff5f 100644 --- a/StabilityMatrix.Core/Python/PyBaseInstall.cs +++ b/StabilityMatrix.Core/Python/PyBaseInstall.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Text.RegularExpressions; using NLog; using StabilityMatrix.Core.Exceptions; using StabilityMatrix.Core.Helper; @@ -8,11 +9,14 @@ namespace StabilityMatrix.Core.Python; -public class PyBaseInstall(DirectoryPath rootPath) +public class PyBaseInstall(DirectoryPath rootPath, MajorMinorVersion? version = null) { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); - public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir); + private readonly Lazy _lazyVersion = + version != null + ? new Lazy(version.Value) + : new Lazy(() => FindPythonVersion(rootPath)); /// /// Root path of the Python installation. @@ -25,15 +29,17 @@ public class PyBaseInstall(DirectoryPath rootPath) /// public bool IsWindowsPortable { get; init; } - private int MajorVersion { get; init; } - - private int MinorVersion { get; init; } + /// + /// Major and minor version of the Python installation. + /// Set in the constructor or lazily queried via . + /// + public MajorMinorVersion Version => _lazyVersion.Value; public FilePath PythonExePath => Compat.Switch( (PlatformKind.Windows, RootPath.JoinFile("python.exe")), - (PlatformKind.Linux, RootPath.JoinFile("bin", "python3")), - (PlatformKind.MacOS, RootPath.JoinFile("bin", "python3")) + (PlatformKind.Linux, RootPath.JoinFile("bin", $"python{Version.Major}")), + (PlatformKind.MacOS, RootPath.JoinFile("bin", $"python{Version.Major}")) ); public string DefaultTclTkPath => @@ -43,6 +49,64 @@ public class PyBaseInstall(DirectoryPath rootPath) (PlatformKind.MacOS, RootPath.JoinFile("lib", "tcl8.6")) ); + public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir, new MajorMinorVersion(3, 10)); + + // Attempt to find the major and minor version of the Python installation. + private static MajorMinorVersion FindPythonVersion( + DirectoryPath rootPath, + PlatformKind platform = default + ) + { + if (platform == default) + { + platform = Compat.Platform; + } + + var searchPath = rootPath; + string glob; + Regex regex; + + if (platform.HasFlag(PlatformKind.Windows)) + { + glob = "python*.dll"; + regex = new Regex(@"python(\d)(\d+)\.dll"); + } + else if (platform.HasFlag(PlatformKind.MacOS)) + { + searchPath = rootPath.JoinDir("lib"); + glob = "libpython*.*.dylib"; + regex = new Regex(@"libpython(\d+)\.(\d+).dylib"); + } + else if (platform.HasFlag(PlatformKind.Linux)) + { + searchPath = rootPath.JoinDir("lib"); + glob = "libpython*.*.so"; + regex = new Regex(@"libpython(\d+)\.(\d+).so"); + } + else + { + throw new NotSupportedException("Unsupported platform"); + } + + var globResults = rootPath.EnumerateFiles(glob).ToList(); + if (globResults.Count == 0) + { + throw new FileNotFoundException("Python library file not found", searchPath + glob); + } + + // Get first matching file + var match = globResults.Select(path => regex.Match(path.Name)).FirstOrDefault(x => x.Success); + if (match is null) + { + throw new FileNotFoundException( + $"Python library file not found with pattern '{regex}'", + searchPath + glob + ); + } + + return new(int.Parse(match.Groups[1].Value), int.Parse(match.Groups[2].Value)); + } + /// /// Creates a new virtual environment runner. /// diff --git a/StabilityMatrix.Core/Python/PyRunner.cs b/StabilityMatrix.Core/Python/PyRunner.cs index 830e7d93..ebc8e47f 100644 --- a/StabilityMatrix.Core/Python/PyRunner.cs +++ b/StabilityMatrix.Core/Python/PyRunner.cs @@ -124,7 +124,10 @@ await ProcessRunner // Pip version 24.1 deprecated numpy star requirement spec used by some packages // So make the base pip less than that for compatibility, venvs can upgrade themselves if needed await ProcessRunner - .GetProcessResultAsync(PythonExePath, ["-m", "pip", "install", "pip>=23.3.2,<24.1"]) + .GetProcessResultAsync( + PythonExePath, + ["-m", "pip", "install", "pip==23.3.2", "setuptools==69.5.1"] + ) .EnsureSuccessExitCode() .ConfigureAwait(false); } diff --git a/StabilityMatrix.Core/Python/PyVenvRunner.cs b/StabilityMatrix.Core/Python/PyVenvRunner.cs index fe8678f3..b984f7f9 100644 --- a/StabilityMatrix.Core/Python/PyVenvRunner.cs +++ b/StabilityMatrix.Core/Python/PyVenvRunner.cs @@ -20,6 +20,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + private string? lastSetPyvenvCfgPath; + /// /// Relative path to the site-packages folder from the venv root. /// This is platform specific. @@ -102,13 +104,6 @@ internal PyVenvRunner(PyBaseInstall baseInstall, DirectoryPath rootPath) EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); } - [Obsolete("Use `PyBaseInstall.CreateVenvRunner` instead.")] - public PyVenvRunner(DirectoryPath rootPath) - { - RootPath = rootPath; - EnvironmentVariables = EnvironmentVariables.SetItem("VIRTUAL_ENV", rootPath.FullPath); - } - public void UpdateEnvironmentVariables( Func, ImmutableDictionary> env ) @@ -171,12 +166,16 @@ public async Task Setup( /// Set current python path to pyvenv.cfg /// This should be called before using the venv, in case user moves the venv directory. /// - private void SetPyvenvCfg(string pythonDirectory) + private void SetPyvenvCfg(string pythonDirectory, bool force = false) { // Skip if we are not created yet if (!Exists()) return; + // Skip if already set to same value + if (lastSetPyvenvCfgPath == pythonDirectory && !force) + return; + // Path to pyvenv.cfg var cfgPath = Path.Combine(RootPath, "pyvenv.cfg"); if (!File.Exists(cfgPath)) @@ -205,6 +204,9 @@ private void SetPyvenvCfg(string pythonDirectory) // Convert to string for writing, strip the top section var cfgString = cfg.ToString()!.Replace(topSection, ""); File.WriteAllText(cfgPath, cfgString); + + // Update last set path + lastSetPyvenvCfgPath = pythonDirectory; } /// @@ -230,7 +232,6 @@ public async Task PipInstall(ProcessArgs args, Action? outputData outputDataReceived?.Invoke(s); }); - SetPyvenvCfg(PyRunner.PythonDir); RunDetached(args.Prepend("-m pip install").Concat("--exists-action s"), outputAction); await Process.WaitForExitAsync().ConfigureAwait(false); @@ -266,7 +267,6 @@ public async Task PipUninstall(string args, Action? outputDataRec outputDataReceived?.Invoke(s); }); - SetPyvenvCfg(PyRunner.PythonDir); RunDetached($"-m pip uninstall -y {args}", outputAction); await Process.WaitForExitAsync().ConfigureAwait(false); @@ -289,7 +289,7 @@ public async Task> PipList() throw new FileNotFoundException("pip not found", PipPath); } - SetPyvenvCfg(PyRunner.PythonDir); + SetPyvenvCfg(BaseInstall.RootPath); var result = await ProcessRunner .GetProcessResultAsync( @@ -342,7 +342,7 @@ public async Task> PipList() throw new FileNotFoundException("pip not found", PipPath); } - SetPyvenvCfg(PyRunner.PythonDir); + SetPyvenvCfg(BaseInstall.RootPath); var result = await ProcessRunner .GetProcessResultAsync( @@ -379,7 +379,7 @@ public async Task> PipList() throw new FileNotFoundException("pip not found", PipPath); } - SetPyvenvCfg(PyRunner.PythonDir); + SetPyvenvCfg(BaseInstall.RootPath); var args = new ProcessArgsBuilder( "-m", @@ -442,7 +442,6 @@ public async Task CustomInstall(ProcessArgs args, Action? outputD outputDataReceived(s); }); - SetPyvenvCfg(PyRunner.PythonDir); RunDetached(args, outputAction); await Process.WaitForExitAsync().ConfigureAwait(false); @@ -472,7 +471,7 @@ public async Task Run(ProcessArgs arguments) output.Append(s); }); - SetPyvenvCfg(PyRunner.PythonDir); + SetPyvenvCfg(BaseInstall.RootPath); using var process = ProcessRunner.StartProcess( PythonPath, arguments, @@ -499,7 +498,7 @@ public void RunDetached( { throw new FileNotFoundException("Venv python not found", PythonPath); } - SetPyvenvCfg(PyRunner.PythonDir); + SetPyvenvCfg(BaseInstall.RootPath); Logger.Info( "Launching venv process [{PythonPath}] "