From 67af41d71f3500f62ff8201bdd69eaad8fffcd70 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 7 Nov 2018 22:47:32 -0800 Subject: [PATCH 1/8] Update find_libpython.py --- deps/find_libpython.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deps/find_libpython.py b/deps/find_libpython.py index c5156179..dd00f3f5 100755 --- a/deps/find_libpython.py +++ b/deps/find_libpython.py @@ -112,7 +112,6 @@ def _linked_libpython_windows(): return None - def library_name(name, suffix=SHLIB_SUFFIX, is_windows=is_windows): """ Convert a file basename `name` to a library name (no "lib" and ".so" etc.) @@ -197,7 +196,6 @@ def candidate_names(suffix=SHLIB_SUFFIX): yield dlprefix + stem + suffix - @uniquified def candidate_paths(suffix=SHLIB_SUFFIX): """ @@ -263,8 +261,13 @@ def normalize_path(path, suffix=SHLIB_SUFFIX, is_apple=is_apple): Parameters ---------- - path : str ot None + path : str or None A candidate path to a shared library. + + Returns + ------- + path : str or None + Normalized existing path or `None`. """ if not path: return None From 3b15e6efd84c5d1dd391d0c20a7814ee08a613f5 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 7 Nov 2018 23:04:40 -0800 Subject: [PATCH 2/8] Use python binary instead of libpython if it's a PIE ...and not dynamically linked to libpython. --- deps/build.jl | 36 +++++++++++++++++++++++++++++++----- deps/find_libpython.py | 9 +++++++++ src/pyinit.jl | 2 +- src/startup.jl | 2 +- 4 files changed, 42 insertions(+), 7 deletions(-) diff --git a/deps/build.jl b/deps/build.jl index a464b069..f1a2fce9 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -69,14 +69,39 @@ function show_dlopen_error(lib, e) end end -# return libpython name, libpython pointer +# return libpython name, libpython pointer, is_pie function find_libpython(python::AbstractString) - dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL + dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL + python = something(Compat.Sys.which(python)) + + # If libpython is dynamically linked, use it: + lib, = try + exec_find_libpython(python, `--dynamic`) + catch + [nothing] + end + if lib !== nothing + try + return (Libdl.dlopen(lib, dlopen_flags), lib, false) + catch e + show_dlopen_error(lib, e) + end + end + + # If `python` is a position independent executable, use it as a + # shared library: + try + return (Libdl.dlopen(python, dlopen_flags), python, true) + catch e + show_dlopen_error(python, e) + end + + # Otherwise, look for common locations of libpython: libpaths = exec_find_libpython(python, `--list-all`) for lib in libpaths try - return (Libdl.dlopen(lib, dlopen_flags), lib) + return (Libdl.dlopen(lib, dlopen_flags), lib, false) catch e show_dlopen_error(lib, e) end @@ -95,7 +120,7 @@ function find_libpython(python::AbstractString) # it easier for users to investigate Python setup # PyCall.jl trying to use. It also helps PyJulia to # compare libpython. - return (libpython, Libdl.dlpath(libpython)) + return (libpython, Libdl.dlpath(libpython), false) catch e show_dlopen_error(lib, e) end @@ -191,7 +216,7 @@ try # make sure deps.jl file is removed on error Conda.add("numpy") end - (libpython, libpy_name) = find_libpython(python) + (libpython, libpy_name, is_pie) = find_libpython(python) programname = pysys(python, "executable") # Get PYTHONHOME, either from the environment or from Python @@ -227,6 +252,7 @@ try # make sure deps.jl file is removed on error const pyversion_build = $(repr(pyversion)) const PYTHONHOME = "$(escape_string(PYTHONHOME))" const wPYTHONHOME = $(wstringconst(PYTHONHOME)) + const is_pie = $(repr(is_pie)) "True if we are using the Python distribution in the Conda package." const conda = $use_conda diff --git a/deps/find_libpython.py b/deps/find_libpython.py index dd00f3f5..9ea82b5c 100755 --- a/deps/find_libpython.py +++ b/deps/find_libpython.py @@ -360,6 +360,11 @@ def cli_find_libpython(cli_op, verbose): print_all(candidate_names()) elif cli_op == "candidate-paths": print_all(p for p in candidate_paths() if p and os.path.isabs(p)) + elif cli_op == "dynamic": + path = normalize_path(linked_libpython()) + if path is None: + return 1 + print(path, end="") else: path = find_libpython() if path is None: @@ -388,6 +393,10 @@ def main(args=None): "--candidate-paths", action="store_const", dest="cli_op", const="candidate-paths", help="Print list of candidate paths of libpython.") + group.add_argument( + "--dynamic", + action="store_const", dest="cli_op", const="dynamic", + help="Print the path to dynamically linked libpython.") ns = parser.parse_args(args) parser.exit(cli_find_libpython(**vars(ns))) diff --git a/src/pyinit.jl b/src/pyinit.jl index 363de859..e80af150 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -88,7 +88,7 @@ function __init__() # issue #189 libpy_handle = libpython === nothing ? C_NULL : - Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) + Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ()) diff --git a/src/startup.jl b/src/startup.jl index daf3d615..59219ecb 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -44,7 +44,7 @@ if !symbols_present isfile(depfile) || error("PyCall not properly installed. Please run Pkg.build(\"PyCall\")") include(depfile) # generated by Pkg.build("PyCall") # Only to be used at top-level - pointer will be invalid after reload - libpy_handle = Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) + libpy_handle = Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) # need SetPythonHome to avoid warning, #299 Py_SetPythonHome(libpy_handle, PYTHONHOME, wPYTHONHOME, pyversion_build) else From a7c976ae761667408c75d059441350cbf11852a4 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 8 Nov 2018 20:36:31 -0800 Subject: [PATCH 3/8] Log PyCall.is_pie in test --- test/runtests.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 7de01831..429c3a3a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,14 @@ filter(f, d::AbstractDict) = Base.filter(f, d) PYTHONPATH=get(ENV,"PYTHONPATH","") PYTHONHOME=get(ENV,"PYTHONHOME","") PYTHONEXECUTABLE=get(ENV,"PYTHONEXECUTABLE","") -Compat.@info "Python version $pyversion from $(PyCall.libpython), PYTHONHOME=$(PyCall.PYTHONHOME)\nENV[PYTHONPATH]=$PYTHONPATH\nENV[PYTHONHOME]=$PYTHONHOME\nENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE" +Compat.@info """ +Python version $pyversion from $(PyCall.libpython), +PYTHONHOME=$(PyCall.PYTHONHOME) +is_pie=$(PyCall.is_pie) +ENV[PYTHONPATH]=$PYTHONPATH +ENV[PYTHONHOME]=$PYTHONHOME +ENV[PYTHONEXECUTABLE]=$PYTHONEXECUTABLE +""" roundtrip(T, x) = convert(T, PyObject(x)) roundtrip(x) = roundtrip(PyAny, x) From 5669d620ebc80defd201f6d435c7ae4af70797c3 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sat, 10 Nov 2018 16:12:21 -0600 Subject: [PATCH 4/8] Use libpy_handle instead of libpython --- src/pyinit.jl | 2 ++ src/startup.jl | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pyinit.jl b/src/pyinit.jl index e80af150..e234a619 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -86,6 +86,8 @@ function __init__() error("Using Conda.jl python, but location of $python seems to have moved to $(Conda.PYTHONDIR). Re-run Pkg.build(\"PyCall\") and restart Julia.") end + global libpy_handle + # issue #189 libpy_handle = libpython === nothing ? C_NULL : Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) diff --git a/src/startup.jl b/src/startup.jl index 59219ecb..4d6499c6 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -125,15 +125,15 @@ if libpython == nothing end else macro pysym(func) - :(($(esc(func)), libpython)) + :(Libdl.dlsym(libpy_handle, $(esc(func)))) end macro pyglobal(name) - :(cglobal(($(esc(name)), libpython))) + :(convert(Ptr{Cvoid}, Libdl.dlsym(libpy_handle, $(esc(name))))) end macro pyglobalobj(name) - :(cglobal(($(esc(name)), libpython), PyObject_struct)) + :(convert(Ptr{PyObject_struct}, Libdl.dlsym(libpy_handle, $(esc(name))))) end macro pyglobalobjptr(name) - :(unsafe_load(cglobal(($(esc(name)), libpython), Ptr{PyObject_struct}))) + :(unsafe_load(convert(Ptr{Ptr{PyObject_struct}}, Libdl.dlsym(libpy_handle, $(esc(name)))))) end end From e01ee984766688c3f8286c5dc06b9f88686370ff Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 10 Nov 2018 16:03:38 -0800 Subject: [PATCH 5/8] Revert "Use libpy_handle instead of libpython" This reverts commit 5669d620ebc80defd201f6d435c7ae4af70797c3. --- src/pyinit.jl | 2 -- src/startup.jl | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/pyinit.jl b/src/pyinit.jl index e234a619..e80af150 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -86,8 +86,6 @@ function __init__() error("Using Conda.jl python, but location of $python seems to have moved to $(Conda.PYTHONDIR). Re-run Pkg.build(\"PyCall\") and restart Julia.") end - global libpy_handle - # issue #189 libpy_handle = libpython === nothing ? C_NULL : Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) diff --git a/src/startup.jl b/src/startup.jl index 4d6499c6..59219ecb 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -125,15 +125,15 @@ if libpython == nothing end else macro pysym(func) - :(Libdl.dlsym(libpy_handle, $(esc(func)))) + :(($(esc(func)), libpython)) end macro pyglobal(name) - :(convert(Ptr{Cvoid}, Libdl.dlsym(libpy_handle, $(esc(name))))) + :(cglobal(($(esc(name)), libpython))) end macro pyglobalobj(name) - :(convert(Ptr{PyObject_struct}, Libdl.dlsym(libpy_handle, $(esc(name))))) + :(cglobal(($(esc(name)), libpython), PyObject_struct)) end macro pyglobalobjptr(name) - :(unsafe_load(convert(Ptr{Ptr{PyObject_struct}}, Libdl.dlsym(libpy_handle, $(esc(name)))))) + :(unsafe_load(cglobal(($(esc(name)), libpython), Ptr{PyObject_struct}))) end end From b86f0e4bd3de09cf4cc437bc253df92825fdfdce Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 10 Nov 2018 16:05:27 -0800 Subject: [PATCH 6/8] Put Libdl.RTLD_DEEPBIND back --- deps/build.jl | 2 +- src/pyinit.jl | 2 +- src/startup.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/build.jl b/deps/build.jl index f1a2fce9..d5dfd64d 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -71,7 +71,7 @@ end # return libpython name, libpython pointer, is_pie function find_libpython(python::AbstractString) - dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL + dlopen_flags = Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL python = something(Compat.Sys.which(python)) diff --git a/src/pyinit.jl b/src/pyinit.jl index e80af150..363de859 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -88,7 +88,7 @@ function __init__() # issue #189 libpy_handle = libpython === nothing ? C_NULL : - Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) + Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ()) diff --git a/src/startup.jl b/src/startup.jl index 59219ecb..daf3d615 100644 --- a/src/startup.jl +++ b/src/startup.jl @@ -44,7 +44,7 @@ if !symbols_present isfile(depfile) || error("PyCall not properly installed. Please run Pkg.build(\"PyCall\")") include(depfile) # generated by Pkg.build("PyCall") # Only to be used at top-level - pointer will be invalid after reload - libpy_handle = Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_GLOBAL) + libpy_handle = Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) # need SetPythonHome to avoid warning, #299 Py_SetPythonHome(libpy_handle, PYTHONHOME, wPYTHONHOME, pyversion_build) else From 7b42704ea6e87f8c04219a02dc50812c3e8daa43 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 10 Nov 2018 16:06:31 -0800 Subject: [PATCH 7/8] Reset stdio in python namespace --- src/pyinit.jl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/pyinit.jl b/src/pyinit.jl index 363de859..565c81f1 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -89,6 +89,17 @@ function __init__() # issue #189 libpy_handle = libpython === nothing ? C_NULL : Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) + if is_pie + unsafe_store!( + cglobal((@pysym :stdin), Ptr{Cvoid}), + unsafe_load(cglobal(:stdin, Ptr{Cvoid}))) + unsafe_store!( + cglobal((@pysym :stdout), Ptr{Cvoid}), + unsafe_load(cglobal(:stdout, Ptr{Cvoid}))) + unsafe_store!( + cglobal((@pysym :stderr), Ptr{Cvoid}), + unsafe_load(cglobal(:stderr, Ptr{Cvoid}))) + end already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ()) From 14d9a232fd1d8fa1aebadf982f0c026f19c89d53 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 10 Nov 2018 18:52:56 -0800 Subject: [PATCH 8/8] Check if std streams exists before copying them to PIE --- src/pyinit.jl | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/pyinit.jl b/src/pyinit.jl index 565c81f1..8fa42bbd 100644 --- a/src/pyinit.jl +++ b/src/pyinit.jl @@ -78,6 +78,23 @@ function Py_Finalize() ccall(@pysym(:Py_Finalize), Cvoid, ()) end +macro _copy_global_to_pie(libpy_handle, symbols...) + exprs = [ + quote + if hassym($(esc(libpy_handle)), $sym) + unsafe_store!(cglobal((@pysym $sym), Ptr{Cvoid}), + unsafe_load(cglobal($sym, Ptr{Cvoid}))) + end + end + for sym in symbols + ] + quote + if is_pie + $(exprs...) + end + end +end + function __init__() # sanity check: in Pkg for Julia 0.7+, the location of Conda can change # if e.g. you checkout Conda master, and we'll need to re-build PyCall @@ -89,17 +106,7 @@ function __init__() # issue #189 libpy_handle = libpython === nothing ? C_NULL : Libdl.dlopen(libpython, Libdl.RTLD_LAZY|Libdl.RTLD_DEEPBIND|Libdl.RTLD_GLOBAL) - if is_pie - unsafe_store!( - cglobal((@pysym :stdin), Ptr{Cvoid}), - unsafe_load(cglobal(:stdin, Ptr{Cvoid}))) - unsafe_store!( - cglobal((@pysym :stdout), Ptr{Cvoid}), - unsafe_load(cglobal(:stdout, Ptr{Cvoid}))) - unsafe_store!( - cglobal((@pysym :stderr), Ptr{Cvoid}), - unsafe_load(cglobal(:stderr, Ptr{Cvoid}))) - end + @_copy_global_to_pie(libpy_handle, :stdin, :stdout, :stderr) already_inited = 0 != ccall((@pysym :Py_IsInitialized), Cint, ())