Merge pull request #713 from ionite34/backport/main/pr-711

[dev to main] backport: Add setuptools version pin for base python (711)

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2
 
 
 ## v2.11.4
+### Changed
+- Base Python install will now use `setuptools==69.5.1` for compatibility with `torchsde`. Individual Packages can upgrade as required.
 ### Fixed
 - Fixed [#719](https://github.com/LykosAI/StabilityMatrix/issues/719) - Fix comments in Inference prompt not being ignored
 ### Supporters

StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesItemViewModel.cs

@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Avalonia.Controls;
@@ -10,10 +9,11 @@
 using StabilityMatrix.Core.Helper;
 using StabilityMatrix.Core.Models.FileInterfaces;
 using StabilityMatrix.Core.Python;
+using StabilityMatrix.Core.Services;
 
 namespace StabilityMatrix.Avalonia.ViewModels.Dialogs;
 
-public partial class PythonPackagesItemViewModel : ViewModelBase
+public partial class PythonPackagesItemViewModel(ISettingsManager settingsManager) : ViewModelBase
 {
     [ObservableProperty]
     private PipPackageInfo package;
@@ -101,7 +101,11 @@ public async Task LoadExtraInfo(DirectoryPath venvPath)
             }
             else
             {
-                await using var venvRunner = new PyVenvRunner(venvPath);
+                await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync(
+                    venvPath,
+                    workingDirectory: venvPath.Parent,
+                    environmentVariables: settingsManager.Settings.EnvironmentVariables
+                );
 
                 PipShowResult = await venvRunner.PipShow(Package.Name);
 

StabilityMatrix.Avalonia/ViewModels/Dialogs/PythonPackagesViewModel.cs

@@ -4,6 +4,7 @@
 using System.Reactive.Linq;
 using System.Threading.Tasks;
 using AsyncAwaitBestPractices;
+using AutoCtor;
 using Avalonia.Controls;
 using Avalonia.Controls.Primitives;
 using Avalonia.Threading;
@@ -12,6 +13,7 @@
 using DynamicData;
 using DynamicData.Binding;
 using FluentAvalonia.UI.Controls;
+using Microsoft.Extensions.Logging;
 using StabilityMatrix.Avalonia.Controls;
 using StabilityMatrix.Avalonia.Languages;
 using StabilityMatrix.Avalonia.ViewModels.Base;
@@ -23,14 +25,19 @@
 using StabilityMatrix.Core.Models.PackageModification;
 using StabilityMatrix.Core.Processes;
 using StabilityMatrix.Core.Python;
+using StabilityMatrix.Core.Services;
 
 namespace StabilityMatrix.Avalonia.ViewModels.Dialogs;
 
 [View(typeof(PythonPackagesDialog))]
 [ManagedService]
 [Transient]
+[AutoConstruct]
 public partial class PythonPackagesViewModel : ContentDialogViewModelBase
 {
+    private readonly ILogger<PythonPackagesViewModel> logger;
+    private readonly ISettingsManager settingsManager;
+
     public DirectoryPath? VenvPath { get; set; }
 
     [ObservableProperty]
@@ -47,7 +54,8 @@ public partial class PythonPackagesViewModel : ContentDialogViewModelBase
     [ObservableProperty]
     private string searchQuery = string.Empty;
 
-    public PythonPackagesViewModel()
+    [AutoPostConstruct]
+    private void PostConstruct()
     {
         var searchPredicate = this.WhenPropertyChanged(vm => vm.SearchQuery)
             .Throttle(TimeSpan.FromMilliseconds(100))
@@ -65,7 +73,7 @@ public PythonPackagesViewModel()
             .Connect()
             .DeferUntilLoaded()
             .Filter(searchPredicate)
-            .Transform(p => new PythonPackagesItemViewModel { Package = p })
+            .Transform(p => new PythonPackagesItemViewModel(settingsManager) { Package = p })
             .SortBy(vm => vm.Package.Name)
             .Bind(Packages)
             .Subscribe();
@@ -86,7 +94,11 @@ private async Task Refresh()
             }
             else
             {
-                await using var venvRunner = new PyVenvRunner(VenvPath);
+                await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync(
+                    VenvPath,
+                    workingDirectory: VenvPath.Parent,
+                    environmentVariables: settingsManager.Settings.EnvironmentVariables
+                );
 
                 var packages = await venvRunner.PipList();
                 packageSource.EditDiff(packages);
@@ -104,7 +116,11 @@ private async Task RefreshBackground()
         if (VenvPath is null)
             return;
 
-        await using var venvRunner = new PyVenvRunner(VenvPath);
+        await using var venvRunner = await PyBaseInstall.Default.CreateVenvRunnerAsync(
+            VenvPath,
+            workingDirectory: VenvPath.Parent,
+            environmentVariables: settingsManager.Settings.EnvironmentVariables
+        );
 
         var packages = await venvRunner.PipList();
 

StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs

@@ -1,4 +1,5 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.IO.Compression;
 using NLog;
 using Octokit;
@@ -152,27 +153,22 @@ public async Task<PyVenvRunner> SetupVenv(
         Action<ProcessOutput>? onConsoleOutput = null
     )
     {
-        if (VenvRunner != null)
+        if (Interlocked.Exchange(ref VenvRunner, null) is { } oldRunner)
         {
-            await VenvRunner.DisposeAsync().ConfigureAwait(false);
+            await oldRunner.DisposeAsync().ConfigureAwait(false);
         }
 
-        VenvRunner = await PyBaseInstall
-            .Default.CreateVenvRunnerAsync(
-                Path.Combine(installedPackagePath, venvName),
-                workingDirectory: installedPackagePath,
-                environmentVariables: SettingsManager.Settings.EnvironmentVariables,
-                withDefaultTclTkEnv: Compat.IsWindows,
-                withQueriedTclTkEnv: Compat.IsUnix
-            )
+        var venvRunner = await SetupVenvPure(installedPackagePath, venvName, forceRecreate, onConsoleOutput)
             .ConfigureAwait(false);
 
-        if (forceRecreate || !VenvRunner.Exists())
+        if (Interlocked.Exchange(ref VenvRunner, venvRunner) is { } oldRunner2)
         {
-            await VenvRunner.Setup(true, onConsoleOutput).ConfigureAwait(false);
+            await oldRunner2.DisposeAsync().ConfigureAwait(false);
         }
 
-        return VenvRunner;
+        Debug.Assert(VenvRunner != null, "VenvRunner != null");
+
+        return venvRunner;
     }
 
     /// <summary>

StabilityMatrix.Core/Models/Packages/Fooocus.cs

@@ -274,7 +274,7 @@ public override async Task InstallPackage(
         progress?.Report(new ProgressReport(-1f, "Installing requirements...", isIndeterminate: true));
 
         // Pip version 24.1 deprecated numpy requirement spec used by torchsde 0.2.5
-        await venvRunner.PipInstall(["pip>=23.3.2,<24.1"], onConsoleOutput).ConfigureAwait(false);
+        await venvRunner.PipInstall(["pip==23.3.2"], onConsoleOutput).ConfigureAwait(false);
 
         var pipArgs = new PipInstallArgs();
 

StabilityMatrix.Core/Python/MajorMinorVersion.cs

@@ -0,0 +1,3 @@
+namespace StabilityMatrix.Core.Python;
+
+public readonly record struct MajorMinorVersion(int Major, int Minor);

StabilityMatrix.Core/Python/PyBaseInstall.cs

@@ -1,4 +1,5 @@
 using System.Text.Json;
+using System.Text.RegularExpressions;
 using NLog;
 using StabilityMatrix.Core.Exceptions;
 using StabilityMatrix.Core.Helper;
@@ -8,11 +9,14 @@
 
 namespace StabilityMatrix.Core.Python;
 
-public class PyBaseInstall(DirectoryPath rootPath)
+public class PyBaseInstall(DirectoryPath rootPath, MajorMinorVersion? version = null)
 {
     private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
 
-    public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir);
+    private readonly Lazy<MajorMinorVersion> _lazyVersion =
+        version != null
+            ? new Lazy<MajorMinorVersion>(version.Value)
+            : new Lazy<MajorMinorVersion>(() => FindPythonVersion(rootPath));
 
     /// <summary>
     /// Root path of the Python installation.
@@ -25,15 +29,17 @@ public class PyBaseInstall(DirectoryPath rootPath)
     /// </summary>
     public bool IsWindowsPortable { get; init; }
 
-    private int MajorVersion { get; init; }
-
-    private int MinorVersion { get; init; }
+    /// <summary>
+    /// Major and minor version of the Python installation.
+    /// Set in the constructor or lazily queried via <see cref="FindPythonVersion"/>.
+    /// </summary>
+    public MajorMinorVersion Version => _lazyVersion.Value;
 
     public FilePath PythonExePath =>
         Compat.Switch(
             (PlatformKind.Windows, RootPath.JoinFile("python.exe")),
-            (PlatformKind.Linux, RootPath.JoinFile("bin", "python3")),
-            (PlatformKind.MacOS, RootPath.JoinFile("bin", "python3"))
+            (PlatformKind.Linux, RootPath.JoinFile("bin", $"python{Version.Major}")),
+            (PlatformKind.MacOS, RootPath.JoinFile("bin", $"python{Version.Major}"))
         );
 
     public string DefaultTclTkPath =>
@@ -43,6 +49,64 @@ public class PyBaseInstall(DirectoryPath rootPath)
             (PlatformKind.MacOS, RootPath.JoinFile("lib", "tcl8.6"))
         );
 
+    public static PyBaseInstall Default { get; } = new(PyRunner.PythonDir, new MajorMinorVersion(3, 10));
+
+    // Attempt to find the major and minor version of the Python installation.
+    private static MajorMinorVersion FindPythonVersion(
+        DirectoryPath rootPath,
+        PlatformKind platform = default
+    )
+    {
+        if (platform == default)
+        {
+            platform = Compat.Platform;
+        }
+
+        var searchPath = rootPath;
+        string glob;
+        Regex regex;
+
+        if (platform.HasFlag(PlatformKind.Windows))
+        {
+            glob = "python*.dll";
+            regex = new Regex(@"python(\d)(\d+)\.dll");
+        }
+        else if (platform.HasFlag(PlatformKind.MacOS))
+        {
+            searchPath = rootPath.JoinDir("lib");
+            glob = "libpython*.*.dylib";
+            regex = new Regex(@"libpython(\d+)\.(\d+).dylib");
+        }
+        else if (platform.HasFlag(PlatformKind.Linux))
+        {
+            searchPath = rootPath.JoinDir("lib");
+            glob = "libpython*.*.so";
+            regex = new Regex(@"libpython(\d+)\.(\d+).so");
+        }
+        else
+        {
+            throw new NotSupportedException("Unsupported platform");
+        }
+
+        var globResults = rootPath.EnumerateFiles(glob).ToList();
+        if (globResults.Count == 0)
+        {
+            throw new FileNotFoundException("Python library file not found", searchPath + glob);
+        }
+
+        // Get first matching file
+        var match = globResults.Select(path => regex.Match(path.Name)).FirstOrDefault(x => x.Success);
+        if (match is null)
+        {
+            throw new FileNotFoundException(
+                $"Python library file not found with pattern '{regex}'",
+                searchPath + glob
+            );
+        }
+
+        return new(int.Parse(match.Groups[1].Value), int.Parse(match.Groups[2].Value));
+    }
+
     /// <summary>
     /// Creates a new virtual environment runner.
     /// </summary>

StabilityMatrix.Core/Python/PyRunner.cs

@@ -124,7 +124,10 @@ await ProcessRunner
         // Pip version 24.1 deprecated numpy star requirement spec used by some packages
         // So make the base pip less than that for compatibility, venvs can upgrade themselves if needed
         await ProcessRunner
-            .GetProcessResultAsync(PythonExePath, ["-m", "pip", "install", "pip>=23.3.2,<24.1"])
+            .GetProcessResultAsync(
+                PythonExePath,
+                ["-m", "pip", "install", "pip==23.3.2", "setuptools==69.5.1"]
+            )
             .EnsureSuccessExitCode()
             .ConfigureAwait(false);
     }