diff --git a/Artifacts.toml b/Artifacts.toml new file mode 100644 index 0000000..5f72efb --- /dev/null +++ b/Artifacts.toml @@ -0,0 +1,6 @@ +[pyjulia_src_julia] +git-tree-sha1 = "8654eeff06dfbf1d593f2551a0f4807a011d2d27" + + [[pyjulia_src_julia.download]] + url = "https://github.com/JuliaPy/pyjulia/archive/822d1e3ea7e90d37691724e1bb471dc143a1ca0a.tar.gz" + sha256 = "b4ebc1945d3a76414c6a6f2c178ff9adda7e142745fd6875793687dd4bafd4ae" diff --git a/Project.toml b/Project.toml index d6bf56b..cb39221 100644 --- a/Project.toml +++ b/Project.toml @@ -1,11 +1,18 @@ +authors = ["Takafumi Arakaki and contributors"] name = "PyPreferences" uuid = "cc9521c6-0242-4dda-8d66-c47a9d9eec02" -authors = ["Takafumi Arakaki and contributors"] version = "0.1.0" [compat] julia = "1" +[deps] +CompilePreferences = "21216c6a-2e73-6563-6e65-726566657250" +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" +Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +VersionParsing = "81def892-9a0e-5fdd-b105-ffc91e053289" + [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/src/PyPreferences.jl b/src/PyPreferences.jl index 2681fdb..4719d9d 100644 --- a/src/PyPreferences.jl +++ b/src/PyPreferences.jl @@ -1,5 +1,32 @@ -module PyPreferences +baremodule PyPreferences -# Write your package code here. +function use_system end +function use_conda end +# function use_jll end +function use_inprocess end +# API to be used from PyCall +function assert_configured end +function instruction_message end + +# function diagnose end +function status end + +module Implementations +include("core.jl") +end + +let prefs = Implementations.setup_non_failing() + global const python = prefs.python + global const inprocess = prefs.inprocess + global const conda = prefs.conda + global const python_fullpath = prefs.python_fullpath + global const libpython = prefs.libpython + global const python_version = prefs.python_version + global const PYTHONHOME = prefs.PYTHONHOME end + +const pyprogramname = python_fullpath +const pyversion_build = python_version + +end # baremodule PyPreferences diff --git a/src/core.jl b/src/core.jl new file mode 100644 index 0000000..a09d4f0 --- /dev/null +++ b/src/core.jl @@ -0,0 +1,353 @@ +using ..PyPreferences: PyPreferences, use_system + +import Conda +import Libdl +using CompilePreferences +using Pkg.Artifacts +using VersionParsing + +struct PythonPreferences + python::Union{Nothing,String} + inprocess::Bool + conda::Bool + # jll::Bool +end + +# Fix the environment for running `python`, and setts IO encoding to UTF-8. +# If cmd is the Conda python, then additionally removes all PYTHON* and +# CONDA* environment variables. +function pythonenv(cmd::Cmd) + @assert cmd.env === nothing # TODO: handle non-nothing case + env = copy(ENV) + if dirname(cmd.exec[1]) == abspath(Conda.PYTHONDIR) + pythonvars = String[] + for var in keys(env) + if startswith(var, "CONDA") || startswith(var, "PYTHON") + push!(pythonvars, var) + end + end + for var in pythonvars + pop!(env, var) + end + end + # set PYTHONIOENCODING when running python executable, so that + # we get UTF-8 encoded text as output (this is not the default on Windows). + env["PYTHONIOENCODING"] = "UTF-8" + setenv(cmd, env) +end + +pyvar(python::AbstractString, mod::AbstractString, var::AbstractString) = + chomp(read(pythonenv(`$python -c "import $mod; print($mod.$(var))"`), String)) + +pyconfigvar(python::AbstractString, var::AbstractString) = + pyvar(python, "distutils.sysconfig", "get_config_var('$(var)')") +pyconfigvar(python, var, default) = + let v = pyconfigvar(python, var) + v == "None" ? default : v + end + +function pythonhome_of(pyprogramname::AbstractString) + if Sys.iswindows() + # PYTHONHOME tells python where to look for both pure python + # and binary modules. When it is set, it replaces both + # `prefix` and `exec_prefix` and we thus need to set it to + # both in case they differ. This is also what the + # documentation recommends. However, they are documented + # to always be the same on Windows, where it causes + # problems if we try to include both. + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.exec_prefix) + """ + else + script = """ + import sys + if hasattr(sys, "base_exec_prefix"): + sys.stdout.write(sys.base_prefix) + sys.stdout.write(":") + sys.stdout.write(sys.base_exec_prefix) + else: + sys.stdout.write(sys.prefix) + sys.stdout.write(":") + sys.stdout.write(sys.exec_prefix) + """ + # https://docs.python.org/3/using/cmdline.html#envvar-PYTHONHOME + end + return read(pythonenv(`$pyprogramname -c $script`), String) +end +# To support `venv` standard library (as well as `virtualenv`), we +# need to use `sys.base_prefix` and `sys.base_exec_prefix` here. +# Otherwise, initializing Python in `__init__` below fails with +# unrecoverable error: +# +# Fatal Python error: initfsencoding: unable to load the file system codec +# ModuleNotFoundError: No module named 'encodings' +# +# This is because `venv` does not symlink standard libraries like +# `virtualenv`. For example, `lib/python3.X/encodings` does not +# exist. Rather, `venv` relies on the behavior of Python runtime: +# +# If a file named "pyvenv.cfg" exists one directory above +# sys.executable, sys.prefix and sys.exec_prefix are set to that +# directory and it is also checked for site-packages +# --- https://docs.python.org/3/library/venv.html +# +# Thus, we need point `PYTHONHOME` to `sys.base_prefix` and +# `sys.base_exec_prefix`. If the virtual environment is created by +# `virtualenv`, those `sys.base_*` paths point to the virtual +# environment. Thus, above code supports both use cases. +# +# See also: +# * https://docs.python.org/3/library/venv.html +# * https://docs.python.org/3/library/site.html +# * https://docs.python.org/3/library/sys.html#sys.base_exec_prefix +# * https://github.com/JuliaPy/PyCall.jl/issues/410 + +python_version_of(python) = vparse(pyvar(python, "platform", "python_version()")) + +function find_libpython_py_path() + dir = artifact"pyjulia_src_julia" + return joinpath(dir, only(readdir(dir)), "find_libpython.py") +end + +function exec_find_libpython(python::AbstractString, options, verbose::Bool) + # Do not inline `@__DIR__` into the backticks to expand correctly. + # See: https://github.com/JuliaLang/julia/issues/26323 + script = find_libpython_py_path() + cmd = `$python $script $options` + if verbose + cmd = `$cmd --verbose` + end + return readlines(pythonenv(cmd)) +end + +# return libpython path, libpython pointer +function find_libpython( + python::AbstractString; + _dlopen = Libdl.dlopen, + verbose::Bool = false, +) + dlopen_flags = Libdl.RTLD_LAZY | Libdl.RTLD_DEEPBIND | Libdl.RTLD_GLOBAL + + libpaths = exec_find_libpython(python, `--list-all`, verbose) + for lib in libpaths + try + return (lib, _dlopen(lib, dlopen_flags)) + catch e + @warn "Failed to `dlopen` $lib" exception = (e, catch_backtrace()) + end + end + @warn """ + Python (`find_libpython.py`) failed to find `libpython`. + Falling back to `Libdl`-based discovery. + """ + + # Try all candidate libpython names and let Libdl find the path. + # We do this *last* because the libpython in the system + # library path might be the wrong one if multiple python + # versions are installed (we prefer the one in LIBDIR): + libs = exec_find_libpython(python, `--candidate-names`, verbose) + for lib in libs + lib = splitext(lib)[1] + try + libpython = _dlopen(lib, dlopen_flags) + return (Libdl.dlpath(libpython), libpython) + catch e + @debug "Failed to `dlopen` $lib" exception = (e, catch_backtrace()) + end + end + + return nothing, nothing +end + +function PyPreferences.use_system(python::AbstractString = "python3") + return set(python = python) +end + +function PyPreferences.use_conda() + Conda.add("numpy") + return set(conda = true) +end + +function PyPreferences.use_inprocess() + return set(inprocess = true) +end + +conda_python_fullpath() = + abspath(Conda.PYTHONDIR, "python" * (Sys.iswindows() ? ".exe" : "")) + +#= +function use_jll() +end +=# + +set(; python = nothing, inprocess = false, conda = false) = + set(PythonPreferences(python, inprocess, conda)) + +function set(prefs::PythonPreferences) + @save_preferences!(Dict(prefs)) + recompile() + return prefs +end + +function Base.Dict(prefs::PythonPreferences) + rawprefs = Dict{String,Any}() + if prefs.python !== nothing + rawprefs["python"] = prefs.python + end + if prefs.inprocess + rawprefs["inprocess"] = true + end + if prefs.conda + rawprefs["conda"] = true + end + return rawprefs +end + +PythonPreferences(rawprefs::AbstractDict) = PythonPreferences( + get(rawprefs, "python", nothing), + get(rawprefs, "inprocess", false), + get(rawprefs, "conda", false), +) + +function _load_python_preferences() + # TODO: lookup v#.#? + rawprefs = @load_preferences() + isempty(rawprefs) && return nothing + return PythonPreferences(rawprefs) +end + +function load_pypreferences_code() + return """ + $(Base.load_path_setup_code()) + PyPreferences = Base.require(Base.PkgId( + Base.UUID("cc9521c6-0242-4dda-8d66-c47a9d9eec02"), + "PyPreferences", + )) + """ +end + +function include_stdin_cmd() + return ``` + $(Base.julia_cmd()) + --startup-file=no + -e "include_string(Main, read(stdin, String))" + ``` +end + +function recompile() + code = """ + $(load_pypreferences_code()) + PyPreferences.assert_configured() + """ + cmd = include_stdin_cmd() + open(cmd; write = true) do io + write(io, code) + end + return +end + +function PyPreferences.instruction_message() + return """ + PyPreferences.jl is not configured properly. Run: + using Pkg + Pkg.add("PyPreferences") + using PyPreferences + @doc PyPreferences + for usage. + """ +end + +function PyPreferences.assert_configured() + if ( + PyPreferences.python === nothing || + PyPreferences.python_fullpath === nothing || + PyPreferences.libpython === nothing || + PyPreferences.python_version === nothing || + PyPreferences.PYTHONHOME === nothing + ) + error(PyPreferences.instruction_message()) + end +end + +function setup_non_failing() + python = nothing + inprocess = false + conda = false + python_fullpath = nothing + libpython = nothing + python_version = nothing + PYTHONHOME = nothing + + prefs = _load_python_preferences() + if prefs !== nothing + python = prefs.python + inprocess = prefs.inprocess + conda = prefs.conda + end + if !inprocess + if conda + python = python_fullpath = conda_python_fullpath() + elseif python === nothing + # TODO: mimic PyCall's deps/build.jl + python = "python3" + end + + try + if python !== nothing + python_fullpath = Sys.which(python) + end + if python_fullpath !== nothing + libpython, = find_libpython(python_fullpath) + python_version = python_version_of(python_fullpath) + PYTHONHOME = pythonhome_of(python_fullpath) + end + catch err + @error( + "Failed to configure for `$python`", + exception = (err, catch_backtrace()) + ) + end + end + if python === nothing + python = python_fullpath + end + + return ( + python = python, + inprocess = inprocess, + conda = conda, + python_fullpath = python_fullpath, + libpython = libpython, + python_version = python_version, + PYTHONHOME = PYTHONHOME, + ) +end + +function status_inprocess() + print(""" + python : $(PyPreferences.python) + inprocess : $(PyPreferences.inprocess) + conda : $(PyPreferences.conda) + python_fullpath: $(PyPreferences.python_fullpath) + libpython : $(PyPreferences.libpython) + python_version : $(PyPreferences.python_version) + PYTHONHOME : $(PyPreferences.PYTHONHOME) + """) +end + +function PyPreferences.status() + # TODO: compare with in-process values + code = """ + $(load_pypreferences_code()) + PyPreferences.Implementations.status_inprocess() + """ + cmd = include_stdin_cmd() + open(pipeline(cmd; stdout = stdout, stderr = stderr); write = true) do io + write(io, code) + end + return +end