From da5a9c6dda63db6e53c7bc27295f7566995fc849 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A1szl=C3=B3=20Vask=C3=B3?= <1771332+vlaci@users.noreply.github.com> Date: Tue, 29 Oct 2024 17:24:39 +0100 Subject: [PATCH] python: remove `LD_LIBRARY_PATH` hack Injecting `LD_LIBRARY_PATH` to the Python runtime environment is great to bypass the need of having to patch non-nix binaries loaded into that environment, however it breaks down, when Python executes any other program not compiled for the given Nix system, e.g. a shell script via `subprocess`. To work this around, `devenv` could inject a `pth`[^1] file to the virtual environment it creates, to undo our changes, but it breaks down in the other ways. One example is `pre-commit` (when not using git-hooks.nix), which creates its own virtual environments, where our `pth` file change won't propagate to. The most durable solution is to just ~accept defeat~, use `patchelf` via the obscure `auto-patchelf`[^2] helper script which powers the `autoPatchelfHook` helper in `nixpkgs`. This is a bit more 'static' approach than what we had before, so any changes in packages will get patched only at the next `direnv` invocation. Fixes #1111 [^1]: https://docs.python.org/3/library/site.html [^2]: https://github.com/NixOS/nixpkgs/blob/725bc5338587583da77f31afb55c41043bbf84df/pkgs/by-name/au/auto-patchelf/source/auto-patchelf.py --- src/modules/languages/python.nix | 62 ++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/src/modules/languages/python.nix b/src/modules/languages/python.nix index c1b7195be..54cef109a 100644 --- a/src/modules/languages/python.nix +++ b/src/modules/languages/python.nix @@ -10,21 +10,28 @@ let ); readlink = "${pkgs.coreutils}/bin/readlink -f "; - package = pkgs.callPackage ../../python-wrapper.nix { - python = cfg.package; - requiredPythonModules = cfg.package.pkgs.requiredPythonModules; - makeWrapperArgs = [ - "--prefix" - "LD_LIBRARY_PATH" - ":" - libraries - ] ++ lib.optionals pkgs.stdenv.isDarwin [ - "--prefix" - "DYLD_LIBRARY_PATH" - ":" - libraries - ]; - }; + + auto-patchelf = + venv: + let + drv = pkgs.buildEnv { + name = "patchelf"; + paths = [ + pkgs.patchelf + pkgs.auto-patchelf + ]; + }; + librariesArgs = lib.replaceStrings [ ":" ] [ " " ] libraries; + in + '' + ${drv}/bin/auto-patchelf \ + --paths ${venv}/lib ${venv}/bin \ + --libs ${librariesArgs} \ + --runtime-dependencies \ + --append-rpaths \ + --ignore-missing \ + --extra-args + ''; requirements = pkgs.writeText "requirements.txt" (toString ( if lib.isPath cfg.venv.requirements @@ -52,7 +59,7 @@ let VENV_PATH="${config.env.DEVENV_STATE}/venv" - profile_python="$(${readlink} ${package.interpreter})" + profile_python="$(${readlink} ${cfg.package.interpreter})" devenv_interpreter_path="$(${pkgs.coreutils}/bin/cat "$VENV_PATH/.devenv_interpreter" 2> /dev/null || echo false )" venv_python="$(${readlink} "$devenv_interpreter_path")" @@ -71,15 +78,16 @@ let [ -f "${config.env.DEVENV_STATE}/poetry.lock.checksum" ] && rm ${config.env.DEVENV_STATE}/poetry.lock.checksum ''} ${if cfg.uv.enable then '' - echo uv venv -p ${package.interpreter} "$VENV_PATH" - uv venv -p ${package.interpreter} "$VENV_PATH" + echo uv venv -p ${cfg.package.interpreter} "$VENV_PATH" + uv venv -p ${cfg.package.interpreter} "$VENV_PATH" '' else '' - echo ${package.interpreter} -m venv ${if builtins.isNull cfg.version || lib.versionAtLeast cfg.version "3.9" then "--upgrade-deps" else ""} "$VENV_PATH" - ${package.interpreter} -m venv ${if builtins.isNull cfg.version || lib.versionAtLeast cfg.version "3.9" then "--upgrade-deps" else ""} "$VENV_PATH" + echo ${cfg.package.interpreter} -m venv ${if builtins.isNull cfg.version || lib.versionAtLeast cfg.version "3.9" then "--upgrade-deps" else ""} "$VENV_PATH" + ${cfg.package.interpreter} -m venv ${if builtins.isNull cfg.version || lib.versionAtLeast cfg.version "3.9" then "--upgrade-deps" else ""} "$VENV_PATH" '' } - echo "${package.interpreter}" > "$VENV_PATH/.devenv_interpreter" + echo "${cfg.package.interpreter}" > "$VENV_PATH/.devenv_interpreter" + ${auto-patchelf "$VENV_PATH"} fi source "$VENV_PATH"/bin/activate @@ -143,7 +151,7 @@ let # Avoid running "uv sync" for every shell. # Only run it when the "pyproject.toml" file or Python interpreter has changed. - local ACTUAL_UV_CHECKSUM="${package.interpreter}:$(${pkgs.nix}/bin/nix-hash --type sha256 pyproject.toml):''${UV_SYNC_COMMAND[@]}" + local ACTUAL_UV_CHECKSUM="${cfg.package.interpreter}:$(${pkgs.nix}/bin/nix-hash --type sha256 pyproject.toml):''${UV_SYNC_COMMAND[@]}" local UV_CHECKSUM_FILE="$VENV_PATH/uv.sync.checksum" if [ -f "$UV_CHECKSUM_FILE" ] then @@ -157,6 +165,7 @@ let if "''${UV_SYNC_COMMAND[@]}" then echo "$ACTUAL_UV_CHECKSUM" > "$UV_CHECKSUM_FILE" + ${auto-patchelf "$VENV_PATH"} else echo "uv sync failed. Run 'uv sync' manually." >&2 exit 1 @@ -185,7 +194,7 @@ let unset VIRTUAL_ENV # Make sure poetry's venv uses the configured Python executable. - ${cfg.poetry.package}/bin/poetry env use --no-interaction --quiet ${package.interpreter} + ${cfg.poetry.package}/bin/poetry env use --no-interaction --quiet ${cfg.package.interpreter} } function _devenv_poetry_install @@ -194,7 +203,7 @@ let # Avoid running "poetry install" for every shell. # Only run it when the "poetry.lock" file or Python interpreter has changed. # We do this by storing the interpreter path and a hash of "poetry.lock" in venv. - local ACTUAL_POETRY_CHECKSUM="${package.interpreter}:$(${pkgs.nix}/bin/nix-hash --type sha256 pyproject.toml):$(${pkgs.nix}/bin/nix-hash --type sha256 poetry.lock):''${POETRY_INSTALL_COMMAND[@]}" + local ACTUAL_POETRY_CHECKSUM="${cfg.package.interpreter}:$(${pkgs.nix}/bin/nix-hash --type sha256 pyproject.toml):$(${pkgs.nix}/bin/nix-hash --type sha256 poetry.lock):''${POETRY_INSTALL_COMMAND[@]}" local POETRY_CHECKSUM_FILE=".venv/poetry.lock.checksum" if [ -f "$POETRY_CHECKSUM_FILE" ] then @@ -208,6 +217,7 @@ let if ''${POETRY_INSTALL_COMMAND[@]} then echo "$ACTUAL_POETRY_CHECKSUM" > "$POETRY_CHECKSUM_FILE" + ${auto-patchelf ".venv"} else echo "Poetry install failed. Run 'poetry install' manually." exit 1 @@ -435,7 +445,7 @@ in cachix.pull = lib.mkIf (cfg.version != null) [ "nixpkgs-python" ]; - packages = [ package ] + packages = [ cfg.package ] ++ (lib.optional cfg.poetry.enable cfg.poetry.package) ++ (lib.optional cfg.uv.enable cfg.uv.package); @@ -484,7 +494,7 @@ in }; enterShell = '' - export PYTHONPATH="$DEVENV_PROFILE/${package.sitePackages}''${PYTHONPATH:+:$PYTHONPATH}" + export PYTHONPATH="$DEVENV_PROFILE/${cfg.package.sitePackages}''${PYTHONPATH:+:$PYTHONPATH}" ''; }; }