-
-
Notifications
You must be signed in to change notification settings - Fork 455
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
40192db
commit 15185ad
Showing
23 changed files
with
396 additions
and
97 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,22 +1,5 @@ | ||
| { lib }: | ||
| let | ||
| inherit (builtins) mapAttrs; | ||
| inherit (lib) fix; | ||
| in | ||
|
|
||
| fix (self: mapAttrs (_: path: import path ({ inherit lib; } // self)) { | ||
| pip = ./pip.nix; | ||
| pypa = ./pypa.nix; | ||
| project = ./project.nix; | ||
| renderers = ./renderers.nix; | ||
| validators = ./validators.nix; | ||
| poetry = ./poetry.nix; | ||
|
|
||
| pep427 = ./pep427.nix; | ||
| pep440 = ./pep440.nix; | ||
| pep508 = ./pep508.nix; | ||
| pep518 = ./pep518.nix; | ||
| pep599 = ./pep599.nix; | ||
| pep600 = ./pep600.nix; | ||
| pep621 = ./pep621.nix; | ||
| }) | ||
| { pkgs, lib }: | ||
| { | ||
| lib = import ./lib { inherit lib; }; | ||
| fetchers = import ./fetchers { inherit pkgs lib; }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| { pkgs | ||
| , lib | ||
| , | ||
| }: | ||
| let | ||
| inherit (builtins) substring filter head nixPath; | ||
| inherit (lib) toLower; | ||
|
|
||
| # Predict URL from the PyPI index. | ||
| # Args: | ||
| # pname: package name | ||
| # file: filename including extension | ||
| # hash: SRI hash | ||
| # kind: Language implementation and version tag | ||
| predictURLFromPypi = | ||
| { | ||
| # package name | ||
| pname | ||
| , # filename including extension | ||
| file | ||
| , # Language implementation and version tag | ||
| kind | ||
| , | ||
| }: "https://files.pythonhosted.org/packages/${kind}/${toLower (substring 0 1 file)}/${pname}/${file}"; | ||
| in | ||
| lib.mapAttrs (_: func: lib.makeOverridable func) { | ||
| /* | ||
| Fetch from the PyPI index. | ||
| At first we try to fetch the predicated URL but if that fails we | ||
| will use the Pypi API to determine the correct URL. | ||
| Type: fetchFromPypi :: AttrSet -> derivation | ||
| */ | ||
| fetchFromPypi = | ||
| { | ||
| # package name | ||
| pname | ||
| , # filename including extension | ||
| file | ||
| , # the version string of the dependency | ||
| version | ||
| , # SRI hash | ||
| hash | ||
| , # Language implementation and version tag | ||
| kind | ||
| , # Options to pass to `curl` | ||
| curlOpts ? "" | ||
| , | ||
| }: | ||
| let | ||
| predictedURL = predictURLFromPypi { inherit pname file kind; }; | ||
| in | ||
| pkgs.stdenvNoCC.mkDerivation { | ||
| name = file; | ||
| nativeBuildInputs = [ | ||
| pkgs.buildPackages.curl | ||
| pkgs.buildPackages.jq | ||
| ]; | ||
| isWheel = lib.strings.hasSuffix "whl" file; | ||
| system = "builtin"; | ||
|
|
||
| preferLocalBuild = true; | ||
| impureEnvVars = | ||
| lib.fetchers.proxyImpureEnvVars | ||
| ++ [ | ||
| "NIX_CURL_FLAGS" | ||
| ]; | ||
|
|
||
| inherit pname file version curlOpts predictedURL; | ||
|
|
||
| builder = ./fetch-from-pypi.sh; | ||
|
|
||
| outputHashMode = "flat"; | ||
| outputHashAlgo = "sha256"; | ||
| outputHash = hash; | ||
|
|
||
| passthru = { | ||
| urls = [ predictedURL ]; # retain compatibility with nixpkgs' fetchurl | ||
| }; | ||
| }; | ||
|
|
||
| /* | ||
| Fetch from the PyPI legacy API. | ||
| Some repositories (such as Devpi) expose the Pypi legacy API (https://warehouse.pypa.io/api-reference/legacy.html). | ||
| Type: fetchFromLegacy :: AttrSet -> derivation | ||
| */ | ||
| fetchFromLegacy = | ||
| { | ||
| # package name | ||
| pname | ||
| , # URL to package index | ||
| url | ||
| , # filename including extension | ||
| file | ||
| , # SRI hash | ||
| hash | ||
| , | ||
| }: | ||
| let | ||
| pathParts = filter ({ prefix, path }: "NETRC" == prefix) nixPath; # deadnix: skip | ||
| netrc_file = | ||
| if (pathParts != [ ]) | ||
| then (head pathParts).path | ||
| else ""; | ||
| in | ||
| pkgs.runCommand file | ||
| { | ||
| nativeBuildInputs = [ pkgs.buildPackages.python3 ]; | ||
| impureEnvVars = lib.fetchers.proxyImpureEnvVars; | ||
| outputHashMode = "flat"; | ||
| outputHashAlgo = "sha256"; | ||
| outputHash = hash; | ||
| NETRC = netrc_file; | ||
| passthru.isWheel = lib.strings.hasSuffix "whl" file; | ||
| } '' | ||
| python ${./fetch-from-legacy.py} ${url} ${pname} ${file} | ||
| mv ${file} $out | ||
| ''; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| # Some repositories (such as Devpi) expose the Pypi legacy API | ||
| # (https://warehouse.pypa.io/api-reference/legacy.html). | ||
| # | ||
| # Note it is not possible to use pip | ||
| # https://discuss.python.org/t/pip-download-just-the-source-packages-no-building-no-metadata-etc/4651/12 | ||
|
|
||
| import netrc | ||
| import os | ||
| import shutil | ||
| import ssl | ||
| import sys | ||
| import urllib.request | ||
| from html.parser import HTMLParser | ||
| from os.path import normpath | ||
| from typing import Optional | ||
| from urllib.parse import urlparse, urlunparse | ||
|
|
||
|
|
||
| # Parse the legacy index page to extract the href and package names | ||
| class Pep503(HTMLParser): | ||
| def __init__(self) -> None: | ||
| super().__init__() | ||
| self.sources: dict[str, str] = {} | ||
| self.url: Optional[str] = None | ||
| self.name: Optional[str] = None | ||
|
|
||
| def handle_data(self, data: str) -> None: | ||
| if self.url is not None: | ||
| self.name = data | ||
|
|
||
| def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None: | ||
| if tag == "a": | ||
| for name, value in attrs: | ||
| if name == "href": | ||
| self.url = value | ||
|
|
||
| def handle_endtag(self, tag: str) -> None: | ||
| if self.url is not None: | ||
| if not self.name: | ||
| raise ValueError("Name not set") | ||
|
|
||
| self.sources[self.name] = self.url | ||
| self.url = None | ||
|
|
||
|
|
||
| url = sys.argv[1] | ||
| package_name = sys.argv[2] | ||
| index_url = url + "/" + package_name + "/" | ||
| package_filename = sys.argv[3] | ||
|
|
||
| # Parse username and password for this host from the netrc file if given. | ||
| username: Optional[str] = None | ||
| password: Optional[str] = None | ||
| if os.environ["NETRC"]: | ||
| netrc_obj = netrc.netrc(os.environ["NETRC"]) | ||
| host = urlparse(index_url).netloc | ||
| # Strip port number if present | ||
| if ":" in host: | ||
| host = host.split(":")[0] | ||
| authenticators = netrc_obj.authenticators(host) | ||
| if authenticators: | ||
| username, _, password = authenticators | ||
|
|
||
| print("Reading index %s" % index_url) | ||
|
|
||
| context = ssl.create_default_context() | ||
| context.check_hostname = False | ||
| context.verify_mode = ssl.CERT_NONE | ||
|
|
||
| # Extract out username/password from index_url, if present. | ||
| parsed_url = urlparse(index_url) | ||
| username = parsed_url.username or username | ||
| password = parsed_url.password or password | ||
| index_url = parsed_url._replace(netloc=parsed_url.netloc.rpartition("@")[-1]).geturl() | ||
|
|
||
| req = urllib.request.Request(index_url) | ||
| if username and password: | ||
| import base64 | ||
|
|
||
| password_b64 = base64.b64encode(":".join((username, password)).encode()).decode("utf-8") | ||
| req.add_header("Authorization", "Basic {}".format(password_b64)) | ||
| response = urllib.request.urlopen(req, context=context) | ||
| index = response.read() | ||
|
|
||
| parser = Pep503() | ||
| parser.feed(str(index, "utf-8")) | ||
| if package_filename not in parser.sources: | ||
| print("The file %s has not be found in the index %s" % (package_filename, index_url)) | ||
| exit(1) | ||
|
|
||
| package_file = open(package_filename, "wb") | ||
| # Sometimes the href is a relative or absolute path within the index's domain. | ||
| indicated_url = urlparse(parser.sources[package_filename]) | ||
| if indicated_url.netloc == "": | ||
| parsed_url = urlparse(index_url) | ||
|
|
||
| if indicated_url.path.startswith("/"): | ||
| # An absolute path within the index's domain. | ||
| path = parser.sources[package_filename] | ||
| else: | ||
| # A relative path. | ||
| path = parsed_url.path + "/" + parser.sources[package_filename] | ||
|
|
||
| package_url = urlunparse( | ||
| ( | ||
| parsed_url.scheme, | ||
| parsed_url.netloc, | ||
| path, | ||
| None, | ||
| None, | ||
| None, | ||
| ) | ||
| ) | ||
| else: | ||
| package_url = parser.sources[package_filename] | ||
|
|
||
| # Handle urls containing "../" | ||
| parsed_url = urlparse(package_url) | ||
| real_package_url = urlunparse( | ||
| ( | ||
| parsed_url.scheme, | ||
| parsed_url.netloc, | ||
| normpath(parsed_url.path), | ||
| parsed_url.params, | ||
| parsed_url.query, | ||
| parsed_url.fragment, | ||
| ) | ||
| ) | ||
| print("Downloading %s" % real_package_url) | ||
|
|
||
| req = urllib.request.Request(real_package_url) | ||
| if username and password: | ||
| req.add_unredirected_header("Authorization", "Basic {}".format(password_b64)) | ||
| response = urllib.request.urlopen(req, context=context) | ||
|
|
||
| with response as r: | ||
| shutil.copyfileobj(r, package_file) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # shellcheck disable=SC1091,SC2154 | ||
| source "$stdenv/setup" | ||
| set -euo pipefail | ||
|
|
||
| curl="curl \ | ||
| --location \ | ||
| --max-redirs 20 \ | ||
| --retry 2 \ | ||
| --disable-epsv \ | ||
| --cookie-jar cookies \ | ||
| --insecure \ | ||
| --speed-time 5 \ | ||
| --progress-bar \ | ||
| --fail \ | ||
| $curlOpts \ | ||
| $NIX_CURL_FLAGS" | ||
|
|
||
| echo "Trying to fetch with predicted URL: $predictedURL" | ||
|
|
||
| $curl "$predictedURL" --output "$out" && exit 0 | ||
|
|
||
| echo "Predicted URL '$predictedURL' failed, querying pypi.org" | ||
| $curl "https://pypi.org/pypi/$pname/json" | jq -r ".releases.\"$version\"[] | select(.filename == \"$file\") | .url" > url | ||
| url=$(cat url) | ||
| $curl "$url" --output "$out" |
Oops, something went wrong.