Find packages from a custom index

pythonbrewer/brew.py

@@ -41,7 +41,7 @@ def install
 end"""
 
 
-def calculate_dep_params(dep, required_suffixes=None):
+def calculate_dep_params(dep, index=None, required_suffixes=None):
     logger.info(
         " - Calculating SHA256 hash of dependency: %s, version: %s" % (
             dep['package_name'],
@@ -51,6 +51,7 @@ def calculate_dep_params(dep, required_suffixes=None):
     package_files = fetch_pypi_package_files(
         dep['package_name'],
         dep['installed_version'],
+        index=index,
         required_suffixes=required_suffixes or ["py2.py3-none-any.whl", ".tar.gz", ".zip"]
     )
     if len(package_files) == 0:
@@ -71,7 +72,7 @@ def get_release_file_sha256(url):
 
 
 def generate_homebrew_formula(python_package_name, formula_name, description, homepage, git_repo_url,
-                              release_url=None, required_suffixes=None):
+                              index=None, release_url=None, required_suffixes=None):
     """Attempts to generate a Homebrew formula template from the given information.
 
     Args:
@@ -80,6 +81,7 @@ def generate_homebrew_formula(python_package_name, formula_name, description, ho
         description: The full text description of the formula.
         homepage: The homepage URL for the package.
         git_repo_url: The Git repo URL for the package.
+        index: URL of a PyPI index to search for packages in.
         release_url: A custom release URL for the Python package, which will override getting the release
             from PyPI.
         required_suffixes: A precedence-ordered list of possible suffixes for the desired packages.
@@ -100,7 +102,7 @@ def generate_homebrew_formula(python_package_name, formula_name, description, ho
 
     dep_string = ""
     for dep in deps[:-1]:
-        package_name, url, sha256 = calculate_dep_params(dep, required_suffixes=required_suffixes)
+        package_name, url, sha256 = calculate_dep_params(dep, index=index, required_suffixes=required_suffixes)
         dep_string += ('\n\n  resource "{package_name}" do\n'
                        '    url "{url}"\n'
                        '    sha256 "{sha256}"\n'

pythonbrewer/cmdline.py

@@ -26,6 +26,11 @@ def main():
         "output_file",
         help="Where to write the formula template (Ruby file)"
     )
+    parser.add_argument(
+        "-i", "--index",
+        default=None,
+        help="The PyPI index to use to find packages"
+    )
     parser.add_argument(
         "-n", "--formula-name",
         default=None,
@@ -78,8 +83,8 @@ def main():
     output_file = os.path.abspath(args.output_file)
     with open(output_file, "wt", encoding="utf-8") as f:
         template = generate_homebrew_formula(args.package_name, formula_name, args.description,
-                                             args.homepage, args.git_repo, release_url=args.release_url,
-                                             required_suffixes=args.suffixes.split(","))
+                                             args.homepage, args.git_repo, index=args.index, 
+                                             release_url=args.release_url, required_suffixes=args.suffixes.split(","))
         f.write(template)
 
     logger.info("Wrote template to %s" % output_file)

pythonbrewer/pypi.py

@@ -7,6 +7,7 @@
 
 import requests
 import html5lib
+from urllib.parse import urljoin
 
 from pythonbrewer.errors import *
 
@@ -18,6 +19,7 @@
     "get_pypi_sha256"
 ]
 
+PYPI_INDEX = "https://pypi.python.org/simple"
 
 def get_html_namespace(tree):
     p = re.compile(r"\{(?P<namespace>[^\}]+)\}.+")
@@ -29,7 +31,7 @@ def html_findall(tree, namespace, xpath):
     return tree.findall(xpath.format(ns="{%s}" % namespace))
 
 
-def fetch_pypi_package_files(package_name, package_version, required_suffixes=None):
+def fetch_pypi_package_files(package_name, package_version, required_suffixes=None, index=None):
     """Attempts to fetch a list of files for the given package/version from PyPI.
 
     Args:
@@ -38,11 +40,14 @@ def fetch_pypi_package_files(package_name, package_version, required_suffixes=No
         required_suffixes: The required suffixes for the package files we want. Leave as None if you want all files for
             the particular version. If supplied, it must be a list and must specify, in order of precedence,
             the suffixes.
+        index: URL of a PyPI index to search for packages in.
 
     Returns:
         A list of URLs to different distributions for the specified package.
     """
-    response = requests.get("https://pypi.python.org/simple/%s" % package_name)
+    if not index:
+        index = PYPI_INDEX
+    response = requests.get("{index}/{name}/".format(index=index, name=package_name))
     if response.status_code >= 300:
         raise PackageNotFoundError(package_name)
 
@@ -54,13 +59,13 @@ def fetch_pypi_package_files(package_name, package_version, required_suffixes=No
     # parse the incoming HTML
     tree = html5lib.parse(response.text)
     namespace = get_html_namespace(tree)
-    links = html_findall(tree, namespace, "./{ns}body/{ns}a")
+    links = html_findall(tree, namespace, "./{ns}body//{ns}a")
 
     urls_for_suffix = dict([(suffix, []) for suffix in required_suffixes])
     urls = []
     for link in links:
         filename = re.sub(r"\.[0]+(\d+)", r".\1", link.text.strip().lower())
-        url = link.attrib["href"]
+        url = urljoin(index, link.attrib["href"])
 
         if filename.startswith(expected_prefix) or filename.startswith(expected_prefix_underscore):
             if required_suffixes is not None: