Skip to content

Commit

Permalink
Simplify pyinstaller.spec. Solve daemon/chia name collision.
Browse files Browse the repository at this point in the history
  • Loading branch information
richardkiss committed Apr 6, 2021
1 parent 660970f commit 7906d02
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 100 deletions.
2 changes: 1 addition & 1 deletion build_scripts/build_linux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then
fi

electron-packager . chia-blockchain --asar.unpack="**/daemon/**" --platform=linux \
--icon=chia/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
--icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
--appVersion=$CHIA_INSTALLER_VERSION
LAST_EXIT_CODE=$?
if [ "$LAST_EXIT_CODE" -ne 0 ]; then
Expand Down
2 changes: 1 addition & 1 deletion build_scripts/build_macos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ if [ "$LAST_EXIT_CODE" -ne 0 ]; then
fi

electron-packager . Chia --asar.unpack="**/daemon/**" --platform=darwin \
--icon=chia/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
--icon=src/assets/img/Chia.icns --overwrite --app-bundle-id=net.chia.blockchain \
--appVersion=$CHIA_INSTALLER_VERSION
LAST_EXIT_CODE=$?
if [ "$LAST_EXIT_CODE" -ne 0 ]; then
Expand Down
2 changes: 1 addition & 1 deletion build_scripts/build_windows.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ Write-Output "packageName is $packageName"

Write-Output " ---"
Write-Output "electron-packager"
electron-packager . Chia --asar.unpack="**\daemon\**" --overwrite --icon=.\chia\assets\img\chia.ico --app-version=$packageVersion
electron-packager . Chia --asar.unpack="**\daemon\**" --overwrite --icon=.\src\assets\img\chia.ico --app-version=$packageVersion
Write-Output " ---"

Write-Output " ---"
Expand Down
5 changes: 4 additions & 1 deletion chia/cmds/start_funcs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import asyncio
import os
import subprocess
import sys

from pathlib import Path
from typing import Optional

Expand All @@ -11,7 +13,8 @@
def launch_start_daemon(root_path: Path) -> subprocess.Popen:
os.environ["CHIA_ROOT"] = str(root_path)
# TODO: use startupinfo=subprocess.DETACHED_PROCESS on windows
process = subprocess.Popen("chia run_daemon".split(), stdout=subprocess.PIPE)
chia = sys.argv[0]
process = subprocess.Popen(f"{chia} run_daemon".split(), stdout=subprocess.PIPE)
return process


Expand Down
165 changes: 69 additions & 96 deletions chia/pyinstaller.spec
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,43 @@ import platform

from pkg_resources import get_distribution

from os import listdir
from os.path import isfile, join
from PyInstaller.utils.hooks import collect_submodules, copy_metadata

THIS_IS_WINDOWS = platform.system().lower().startswith("win")


def dir_for_module(mod_name):
"""
This returns a path to a directory
"""
mod = importlib.import_module(mod_name)
return pathlib.Path(mod.__file__).parent
ROOT = pathlib.Path(importlib.import_module("chia").__file__).absolute().parent.parent


def path_for_file(mod_name, filename=None):
def solve_name_collision_problem(analysis):
"""
This returns a path to a file (__init__.py by default)
"""
mod = importlib.import_module(mod_name)
There is a collision between the `chia` file name (which is the executable)
and the `chia` directory, which contains non-code resources like `english.txt`.
We move all the resources in the zipped area so there is no
need to create the `chia` directory, since the names collide.
# some modules, like `chia.ssl` don't set mod.__file__ because there isn't actually
# any code in there. We have to look at mod.__path__ instead, which is a list.
# for now, we just take the first item, since this function isn't expected to
# return a list of paths, just one path.
# BRAIN DAMAGE
Fetching data now requires going into a zip file, so it will be slower.
It's best if files that are used frequently are cached.
if mod.__file__ is None:
path = pathlib.Path(mod.__path__._path[0])
if filename is None:
raise ValueError("no file __init__.py in this module")
return path / filename
A sample large compressible file (1 MB of `/dev/zero`), seems to be
about eight times slower.
path = pathlib.Path(mod.__file__)
if filename is not None:
path = path.parent / filename
return path
Note that this hack isn't documented, but seems to work.
"""

zipped = []
datas = []
for data in analysis.datas:
if str(data[0]).startswith("chia/"):
zipped.append(data)
else:
datas.append(data)

# Include all files that end with clvm.hex
puzzles_path = dir_for_module("chia.wallet.puzzles")
# items in this field are included in the binary
analysis.zipped_data = zipped

puzzle_dist_path = "./data/chia/wallet/puzzles"
onlyfiles = [f for f in listdir(puzzles_path) if isfile(join(puzzles_path, f))]
# these items will be dropped in the root folder uncompressed
analysis.datas = datas

root = pathlib.Path().absolute()

keyring_imports = collect_submodules("keyring.backends")

Expand All @@ -61,8 +52,6 @@ version_data = copy_metadata(get_distribution("chia-blockchain"))[0]

block_cipher = None

other = ["pkg_resources.py2_warn"]

SERVERS = [
"wallet",
"full_node",
Expand All @@ -72,28 +61,31 @@ SERVERS = [
"timelord",
]

if THIS_IS_WINDOWS:
other.extend(["win32timezone", "win32cred", "pywintypes", "win32ctypes.pywin32"])

# TODO: collapse all these entry points into one `chia_exec` entrypoint that accepts the server as a parameter

entry_points = ["chia.cmds.chia"] + [f"chia.server.start_{s}" for s in SERVERS]


if THIS_IS_WINDOWS:
# this probably isn't necessary
entry_points.extend(["aiohttp", "chia.util.bip39"])

hiddenimports = []
hiddenimports.extend(other)
hiddenimports.extend(entry_points)
hiddenimports.extend(keyring_imports)

binaries = []


if THIS_IS_WINDOWS:
hiddenimports.extend(["win32timezone", "win32cred", "pywintypes", "win32ctypes.pywin32"])

# this probably isn't necessary
if THIS_IS_WINDOWS:
entry_points.extend(["aiohttp", "chia.util.bip39"])

if THIS_IS_WINDOWS:
chia_mod = importlib.import_module("chia")
dll_paths = ROOT / "*.dll"

binaries = [
(
dir_for_module("chia").parent / "*.dll",
dll_paths,
".",
),
(
Expand All @@ -107,58 +99,21 @@ if THIS_IS_WINDOWS:
]


datas = [
(puzzles_path, puzzle_dist_path),
(path_for_file("mozilla-ca", "cacert.pem"), f"./mozilla-ca/"),
(path_for_file("chia.ssl", "dst_root_ca.pem"), f"./data/chia/ssl/"),
(path_for_file("chia.ssl", "chia_ca.key"), f"./data/chia/ssl/"),
(path_for_file("chia.ssl", "chia_ca.crt"), f"./data/chia/ssl/"),
(path_for_file("chia.util", "english.txt"), f"./data/chia/util/"),
version_data,
]
datas = []

datas.append((f"{ROOT}/chia/util/english.txt", "chia/util"))
datas.append((f"{ROOT}/chia/util/initial-config.yaml", "chia/util"))
datas.append((f"{ROOT}/chia/wallet/puzzles/*.hex", "chia/wallet/puzzles"))
datas.append((f"{ROOT}/chia/ssl/*", "chia/ssl"))
datas.append((f"{ROOT}/mozilla-ca/*", "mozilla-ca"))
datas.append(version_data)

pathex = [root]

chia = Analysis(
[path_for_file("chia.cmds.chia")],
pathex=pathex,
binaries=binaries,
datas=datas,
hiddenimports=hiddenimports,
hookspath=[],
runtime_hooks=[],
excludes=[],
win_no_prefer_redirects=False,
win_private_assemblies=False,
cipher=block_cipher,
noarchive=False,
)

chia_pyz = PYZ(chia.pure, chia.zipped_data, cipher=block_cipher)

chia_exe = EXE(
chia_pyz,
chia.scripts,
[],
exclude_binaries=True,
name="chia",
debug=False,
bootloader_ignore_signals=False,
strip=False,
)

pathex = []

COLLECT_ARGS = [
chia_exe,
chia.binaries,
chia.zipfiles,
chia.datas,
]

for server in SERVERS:
def add_binary(name, path_to_script, collect_args):
analysis = Analysis(
[path_for_file(f"chia.server.start_{server}")],
[path_to_script],
pathex=pathex,
binaries=binaries,
datas=datas,
Expand All @@ -172,20 +127,38 @@ for server in SERVERS:
noarchive=False,
)

pyz = PYZ(analysis.pure, analysis.zipped_data, cipher=block_cipher)
solve_name_collision_problem(analysis)

binary_pyz = PYZ(analysis.pure, analysis.zipped_data, cipher=block_cipher)

exe = EXE(
pyz,
binary_exe = EXE(
binary_pyz,
analysis.scripts,
[],
exclude_binaries=True,
name=f"start_{server}",
name=name,
debug=False,
bootloader_ignore_signals=False,
strip=False,
)

COLLECT_ARGS.extend([exe, analysis.binaries, analysis.zipfiles, analysis.datas])
collect_args.extend(
[
binary_exe,
analysis.binaries,
analysis.zipfiles,
analysis.datas,
]
)


COLLECT_ARGS = []

add_binary("chia", f"{ROOT}/chia/cmds/chia.py", COLLECT_ARGS)
add_binary("daemon", f"{ROOT}/chia/daemon/server.py", COLLECT_ARGS)

for server in SERVERS:
add_binary(f"start_{server}", f"{ROOT}/chia/server/start_{server}.py", COLLECT_ARGS)

COLLECT_KWARGS = dict(
strip=False,
Expand Down

0 comments on commit 7906d02

Please sign in to comment.