chore: use poetry from upstream nixpkgs

nix-community · Oct 24, 2023 · 806eb84 · 806eb84
1 parent 705cf90
commit 806eb84
Show file tree

Hide file tree

Showing 35 changed files with 4,057 additions and 4,913 deletions.
diff --git a/.envrc b/.envrc
@@ -1,2 +1,2 @@
-use nix
+use flake
 watch_file ./nix/*
diff --git a/README.md b/README.md
@@ -61,8 +61,6 @@ poetry2nix.mkPoetryApplication {
 }
 ```
 
-See [./pkgs/poetry/default.nix](./pkgs/poetry/default.nix) for a working example.
-
 #### Dependency environment
 
 The resulting derivation also has the passthru attribute `dependencyEnv`, which is an environment with a python interpreter, all non-development dependencies and your application.

diff --git a/default.nix b/default.nix
@@ -150,8 +150,6 @@ lib.makeScope pkgs.newScope (self: {
       };
       getFunctorFn = fn: if builtins.typeOf fn == "set" then fn.__functor else fn;
 
-      poetryPkg = pkgs.callPackage ./pkgs/poetry { inherit python; poetry2nix = self; };
-
       scripts = pyProject.tool.poetry.scripts or { };
       hasScripts = scripts != { };
       scriptsPackage = self.mkPoetryScriptsPackage {
@@ -252,8 +250,7 @@ lib.makeScope pkgs.newScope (self: {
                   };
 
                   # # Use poetry-core from the poetry build (pep517/518 build-system)
-                  poetry-core = poetryPkg.passthru.python.pkgs.poetry-core;
-                  poetry = poetryPkg;
+                  # inherit (self.pkgs.poetry.passthru.python.pkgs) poetry-core;
 
                   __toPluginAble = toPluginAble self;
                 } // lib.optionalAttrs (! super ? setuptools-scm) {

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -1,11 +1,15 @@
 {
   description = "Poetry2nix flake";
 
-  inputs.flake-utils.url = "github:numtide/flake-utils";
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable-small";
+  inputs = {
+    flake-utils.url = "github:numtide/flake-utils";
+    nixpkgs.url = "github:cpcloud/nixpkgs/test-poetry-fixes";
 
-  inputs.nix-github-actions.url = "github:nix-community/nix-github-actions";
-  inputs.nix-github-actions.inputs.nixpkgs.follows = "nixpkgs";
+    nix-github-actions = {
+      url = "github:nix-community/nix-github-actions";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
 
   outputs = { self, nixpkgs, flake-utils, nix-github-actions }:
     {
@@ -16,7 +20,7 @@
           mkPkgs = system: import nixpkgs {
             config = {
               allowAliases = false;
-              allowInsecurePredicate = x: true;
+              allowInsecurePredicate = _: true;
             };
             overlays = [ self.overlay ];
             inherit system;
@@ -40,7 +44,7 @@
                 # Aggregate all tests into one derivation so that only one GHA runner is scheduled for all darwin jobs
                 aggregate = pkgs.runCommand "darwin-aggregate"
                   {
-                    env.TEST_INPUTS = (lib.concatStringsSep " " (lib.attrValues (lib.filterAttrs (n: v: lib.isDerivation v) tests)));
+                    env.TEST_INPUTS = lib.concatStringsSep " " (lib.attrValues (lib.filterAttrs (_: v: lib.isDerivation v) tests));
                   } "touch $out";
               };
           };
@@ -53,24 +57,56 @@
         };
         default = self.templates.app;
       };
-
     } // (flake-utils.lib.eachDefaultSystem (system:
       let
-        pkgs = nixpkgs.legacyPackages.${system};
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [
+            (_: _: {
+              p2nix-tools = pkgs.callPackage ./tools { };
+            })
+          ];
+          config = {
+            allowAliases = false;
+            permittedInsecurePackages = [
+              "python3.8-requests-2.29.0"
+              "python3.8-cryptography-40.0.2"
+              "python3.9-requests-2.29.0"
+              "python3.9-cryptography-40.0.2"
+              "python3.10-requests-2.29.0"
+              "python3.10-cryptography-40.0.2"
+              "python3.11-requests-2.29.0"
+              "python3.11-cryptography-40.0.2"
+            ];
+          };
+        };
+
         poetry2nix = import ./default.nix { inherit pkgs; };
-        poetry = pkgs.callPackage ./pkgs/poetry { python = pkgs.python3; inherit poetry2nix; };
       in
       rec {
         packages = {
-          inherit poetry;
           poetry2nix = poetry2nix.cli;
           default = poetry2nix.cli;
         };
 
-        legacyPackages = poetry2nix;
+        devShells = {
+          default = pkgs.mkShell {
+            nativeBuildInputs = with pkgs; [
+              p2nix-tools.env
+              p2nix-tools.flamegraph
+              nixpkgs-fmt
+              poetry
+              niv
+              jq
+              nix-prefetch-git
+              nix-eval-jobs
+              nix-build-uncached
+            ];
+          };
+        };
 
         apps = {
-          poetry = flake-utils.lib.mkApp { drv = packages.poetry; };
+          inherit (pkgs) poetry;
           poetry2nix = flake-utils.lib.mkApp { drv = packages.poetry2nix; };
           default = apps.poetry2nix;
         };

diff --git a/overlay.nix b/overlay.nix
@@ -1,4 +1,3 @@
 final: prev: {
   poetry2nix = import ./default.nix { pkgs = final; };
-  poetry = prev.callPackage ./pkgs/poetry { python = final.python3; inherit (final) poetry2nix; };
 }
diff --git a/overrides/build-systems.json b/overrides/build-systems.json
@@ -1065,6 +1065,7 @@
     "setuptools"
   ],
   "arrow": [
+    "flit-core",
     "setuptools"
   ],
   "arsenic": [
@@ -8491,6 +8492,7 @@
     "setuptools"
   ],
   "isoduration": [
+    "flit-core",
     "setuptools"
   ],
   "isort": [
@@ -12144,6 +12146,7 @@
   ],
   "pandas": [
     "cython",
+    "meson-python",
     "oldest-supported-numpy",
     "setuptools",
     {
@@ -12192,6 +12195,8 @@
     "setuptools"
   ],
   "param": [
+    "hatch-vcs",
+    "hatchling",
     "setuptools"
   ],
   "parameter-expansion-patched": [
@@ -16795,6 +16800,9 @@
     "setuptools-scm"
   ],
   "pyviz-comms": [
+    "hatch-jupyter-builder",
+    "hatch-nodejs-version",
+    "hatchling",
     "setuptools"
   ],
   "pyvizio": [

diff --git a/overrides/default.nix b/overrides/default.nix
@@ -3,7 +3,7 @@
 }:
 
 let
-  addBuildSystem =
+  addBuildSystem' =
     { self
     , drv
     , attr
@@ -44,6 +44,20 @@ let
           # We do not need the build system for wheels.
           if old ? format && old.format == "wheel" then
             { }
+          else if attr == "poetry" then
+            {
+              # replace poetry
+              postPatch = (old.postPatch or "") + ''
+                if [ -f pyproject.toml ]; then
+                  toml="$(mktemp)"
+                  yj -tj < pyproject.toml | jq --from-file ${./poetry-to-poetry-core.jq} | yj -jt > "$toml"
+                  mv "$toml" pyproject.toml
+                fi
+              '';
+              nativeBuildInputs = old.nativeBuildInputs or [ ]
+                ++ [ self.poetry-core self.pkgs.yj self.pkgs.jq ]
+                ++ map (a: self.${a}) extraAttrs;
+            }
           else
             {
               nativeBuildInputs =
@@ -59,10 +73,10 @@ in
 lib.composeManyExtensions [
   # NixOps
   (self: super:
-    lib.mapAttrs (_: v: addBuildSystem { inherit self; drv = v; attr = "poetry"; }) (lib.filterAttrs (n: _: lib.strings.hasPrefix "nixops" n) super)
+    lib.mapAttrs (_: v: addBuildSystem' { inherit self; drv = v; attr = "poetry"; }) (lib.filterAttrs (n: _: lib.strings.hasPrefix "nixops" n) super)
     // {
       # NixOps >=2 dependency
-      nixos-modules-contrib = addBuildSystem { inherit self; drv = super.nixos-modules-contrib; attr = "poetry"; };
+      nixos-modules-contrib = addBuildSystem' { inherit self; drv = super.nixos-modules-contrib; attr = "poetry"; };
     }
   )
 
@@ -73,7 +87,7 @@ lib.composeManyExtensions [
     in
     lib.mapAttrs
       (attr: systems: builtins.foldl'
-        (drv: attr: addBuildSystem {
+        (drv: attr: addBuildSystem' {
           inherit drv self attr;
         })
         (super.${attr} or null)
@@ -117,6 +131,8 @@ lib.composeManyExtensions [
     in
 
     {
+      addBuildSystem = attr: drv: addBuildSystem' { inherit self drv attr; };
+
       #### BEGIN bootstrapping pkgs
       installer = bootstrappingBase.installer.override {
         inherit (self) buildPythonPackage flit-core;
@@ -147,6 +163,8 @@ lib.composeManyExtensions [
       };
       #### END bootstrapping pkgs
 
+      poetry = self.poetry-core;
+
       automat = super.automat.overridePythonAttrs (
         old: lib.optionalAttrs (lib.versionOlder old.version "22.10.0") {
           propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ [ self.m2r ];
@@ -232,7 +250,7 @@ lib.composeManyExtensions [
 
       argon2-cffi =
         if (lib.versionAtLeast super.argon2-cffi.version "21.2.0") then
-          addBuildSystem
+          addBuildSystem'
             {
               inherit self;
               drv = super.argon2-cffi;
@@ -1832,10 +1850,12 @@ lib.composeManyExtensions [
       );
 
 
-      pandas = super.pandas.overridePythonAttrs (old: {
-
+      pandas = super.pandas.overridePythonAttrs (old: lib.optionalAttrs (!(old.src.isWheel or false)) {
+        nativeBuildInputs = old.nativeBuildInputs or [ ] ++ [ pkg-config ];
         buildInputs = old.buildInputs or [ ] ++ lib.optional stdenv.isDarwin pkgs.libcxx;
 
+        dontUseMesonConfigure = true;
+
         # Doesn't work with -Werror,-Wunused-command-line-argument
         # https://github.com/NixOS/nixpkgs/issues/39687
         hardeningDisable = lib.optional stdenv.cc.isClang "strictoverflow";
@@ -1850,7 +1870,6 @@ lib.composeManyExtensions [
                       "['pandas/src/klib', 'pandas/src', '$cpp_sdk']"
         '';
 
-
         enableParallelBuilding = true;
       });
 
@@ -1865,7 +1884,7 @@ lib.composeManyExtensions [
         propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [ self.pyutilib ];
       });
 
-      paramiko = super.paramiko.overridePythonAttrs (old: {
+      paramiko = super.paramiko.overridePythonAttrs (_: {
         doCheck = false; # requires networking
       });
 
@@ -1940,29 +1959,6 @@ lib.composeManyExtensions [
         propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.poetry ];
       });
 
-      poetry-core = super.poetry-core.overridePythonAttrs (old:
-        let
-          initFile =
-            if lib.versionOlder super.poetry-core.version "1.1"
-            then "poetry/__init__.py"
-            else "./src/poetry/core/__init__.py";
-        in
-        {
-          # "Vendor" dependencies (for build-system support)
-          postPatch = ''
-            echo "import sys" >> ${initFile}
-            for path in $propagatedBuildInputs; do
-              echo "sys.path.insert(0, \"$path\")" >> ${initFile}
-            done
-          '';
-
-          # Propagating dependencies leads to issues downstream
-          # We've already patched poetry to prefer "vendored" dependencies
-          postFixup = ''
-            rm $out/nix-support/propagated-build-inputs
-          '';
-        });
-
       pkgutil-resolve-name = super.pkgutil-resolve-name.overridePythonAttrs (
         old: lib.optionalAttrs (!(old.src.isWheel or false)) {
           postPatch = ''

diff --git a/overrides/poetry-to-poetry-core.jq b/overrides/poetry-to-poetry-core.jq
@@ -0,0 +1,4 @@
+# replace poetry.masonry.api with poetry.core.masonry.api if it's the build backend
+(.["build-system"]["build-backend"] |= (select(. == "poetry.masonry.api") |= "poetry.core.masonry.api")) |
+# replace build-system.requires with poetry-core if it's poetry
+(.["build-system"]["requires"] |= map(if test("^\\s*poetry\\s*(>=|<=|==|!=|~=)\\s*([0-9]|\\.)*") then "poetry-core" else . end))
-Original file line number
+Diff line change
@@ Expand Up / @@ -61,8 +61,6 @@ poetry2nix.mkPoetryApplication { @@
     }
     ```
-    See [./pkgs/poetry/default.nix](./pkgs/poetry/default.nix) for a working example.
     #### Dependency environment
     The resulting derivation also has the passthru attribute `dependencyEnv`, which is an environment with a python interpreter, all non-development dependencies and your application.
@@ Expand Down @@