Skip to content

Commit

Permalink
inspection: use pep517 metadata build
Browse files Browse the repository at this point in the history
This change replaces setup.py explicit execution in favour of pep517
metadata builds.

In addition to improving handling of PEP 517 metadata builds, error
handling when reading setup files have also been improved.
  • Loading branch information
abn committed Jul 24, 2020
1 parent 435739a commit 4250a5f
Show file tree
Hide file tree
Showing 5 changed files with 268 additions and 88 deletions.
164 changes: 111 additions & 53 deletions poetry/inspection/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,17 @@

logger = logging.getLogger(__name__)

PEP517_META_BUILD = """\
import pep517.build
import pep517.meta
path='{source}'
system=pep517.build.compat_system(path)
pep517.meta.build(source_dir=path, dest='{dest}', system=system)
"""

PEP517_META_BUILD_DEPS = ["pep517===0.8.2", "toml==0.10.1"]


class PackageInfoError(ValueError):
def __init__(self, path): # type: (Union[Path, str]) -> None
Expand Down Expand Up @@ -256,17 +267,27 @@ def _from_sdist_file(cls, path): # type: (Path) -> PackageInfo

return info.update(new_info)

@staticmethod
def has_setup_files(path): # type: (Path) -> bool
return any((path / f).exists() for f in SetupReader.FILES)

@classmethod
def from_setup_py(cls, path): # type: (Union[str, Path]) -> PackageInfo
def from_setup_files(cls, path): # type: (Path) -> PackageInfo
"""
Mechanism to parse package information from a `setup.py` file. This uses the implentation
Mechanism to parse package information from a `setup.[py|cfg]` file. This uses the implementation
at `poetry.utils.setup_reader.SetupReader` in order to parse the file. This is not reliable for
complex setup files and should only attempted as a fallback.
:param path: Path to `setup.py` file
:return:
"""
result = SetupReader.read_from_directory(Path(path))
if not cls.has_setup_files(path):
raise PackageInfoError(path)

try:
result = SetupReader.read_from_directory(path)
except Exception:
raise PackageInfoError(path)

python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
Expand All @@ -288,14 +309,20 @@ def from_setup_py(cls, path): # type: (Union[str, Path]) -> PackageInfo

requirements = parse_requires(requires)

return cls(
info = cls(
name=result.get("name"),
version=result.get("version"),
summary=result.get("description", ""),
requires_dist=requirements or None,
requires_python=python_requires,
)

if not (info.name and info.version) and not info.requires_dist:
# there is nothing useful here
raise PackageInfoError(path)

return info

@staticmethod
def _find_dist_info(path): # type: (Path) -> Iterator[Path]
"""
Expand All @@ -308,22 +335,20 @@ def _find_dist_info(path): # type: (Path) -> Iterator[Path]
# Sometimes pathlib will fail on recursive symbolic links, so we need to workaround it
# and use the glob module instead. Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
directories = glob.iglob(Path(path, pattern).as_posix(), recursive=True)
directories = glob.iglob(path.joinpath(pattern).as_posix(), recursive=True)
else:
directories = path.glob(pattern)

for d in directories:
yield Path(d)

@classmethod
def from_metadata(cls, path): # type: (Union[str, Path]) -> Optional[PackageInfo]
def from_metadata(cls, path): # type: (Path) -> Optional[PackageInfo]
"""
Helper method to parse package information from an unpacked metadata directory.
:param path: The metadata directory to parse information from.
"""
path = Path(path)

if path.suffix in {".dist-info", ".egg-info"}:
directories = [path]
else:
Expand Down Expand Up @@ -392,10 +417,79 @@ def _get_poetry_package(path): # type: (Path) -> Optional[ProjectPackage]
except RuntimeError:
pass

@classmethod
def _pep517_metadata(cls, path): # type (Path) -> PackageInfo
"""
Helper method to use PEP-517 library to build and read package metadata.
:param path: Path to package source to build and read metadata for.
"""
info = None
try:
info = cls.from_setup_files(path)
if info.requires_dist is not None:
return info
except PackageInfoError:
pass

with temporary_directory() as tmp_dir:
# TODO: cache PEP 517 build environment corresponding to each project venv
venv_dir = Path(tmp_dir) / ".venv"
EnvManager.build_venv(venv_dir.as_posix())
venv = VirtualEnv(venv_dir, venv_dir)

dest_dir = Path(tmp_dir) / "dist"
dest_dir.mkdir()

try:
venv.run(
"python",
"-m",
"pip",
"install",
"--disable-pip-version-check",
"--ignore-installed",
*PEP517_META_BUILD_DEPS
)
venv.run(
"python",
"-",
input_=PEP517_META_BUILD.format(
source=path.as_posix(), dest=dest_dir.as_posix()
),
)
return cls.from_metadata(dest_dir)
except EnvCommandError as e:
# something went wrong while attempting pep517 metadata build
# fallback to egg_info if setup.py available
cls._log("PEP517 build failed: {}".format(e), level="debug")
setup_py = path / "setup.py"
if not setup_py.exists():
raise PackageInfoError(path)

cwd = Path.cwd()
os.chdir(path.as_posix())
try:
venv.run("python", "setup.py", "egg_info")
return cls.from_metadata(path)
except EnvCommandError:
raise PackageInfoError(path)
finally:
os.chdir(cwd.as_posix())

if info:
cls._log(
"Falling back to parsed setup.py file for {}".format(path), "debug"
)
return info

# if we reach here, everything has failed and all hope is lost
raise PackageInfoError(path)

@classmethod
def from_directory(
cls, path, allow_build=False
): # type: (Union[str, Path], bool) -> PackageInfo
): # type: (Path, bool) -> PackageInfo
"""
Generate package information from a package source directory. When `allow_build` is enabled and
introspection of all available metadata fails, the package is attempted to be build in an isolated
Expand All @@ -404,57 +498,28 @@ def from_directory(
:param path: Path to generate package information from.
:param allow_build: If enabled, as a fallback, build the project to gather metadata.
"""
path = Path(path)

current_dir = os.getcwd()

info = cls.from_metadata(path)

if info and info.requires_dist is not None:
# return only if requirements are discovered
return info

setup_py = path.joinpath("setup.py")

project_package = cls._get_poetry_package(path)
if project_package:
return cls.from_package(project_package)

if not setup_py.exists():
if not allow_build and info:
# we discovered PkgInfo but no requirements were listed
return info
# this means we cannot do anything else here
raise PackageInfoError(path)

if not allow_build:
return cls.from_setup_py(path=path)

try:
# TODO: replace with PEP517
# we need to switch to the correct path in order for egg_info command to work
os.chdir(str(path))

# Execute egg_info
cls._execute_setup()
except EnvCommandError:
cls._log(
"Falling back to parsing setup.py file for {}".format(path), "debug"
)
# egg_info could not be generated, we fallback to ast parser
return cls.from_setup_py(path=path)
else:
info = cls.from_metadata(path)
if not allow_build:
return cls.from_setup_files(path)
return cls._pep517_metadata(path)
except PackageInfoError as e:
if info:
# we discovered PkgInfo but no requirements were listed
return info
finally:
os.chdir(current_dir)

# if we reach here, everything has failed and all hope is lost
raise PackageInfoError(path)
raise e

@classmethod
def from_sdist(cls, path): # type: (Union[Path, pkginfo.SDist]) -> PackageInfo
def from_sdist(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from an sdist file, packed or unpacked.
Expand Down Expand Up @@ -508,10 +573,3 @@ def from_path(cls, path): # type: (Path) -> PackageInfo
return cls.from_bdist(path=path)
except PackageInfoError:
return cls.from_sdist(path=path)

@classmethod
def _execute_setup(cls):
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
8 changes: 6 additions & 2 deletions tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

from poetry.config.config import Config as BaseConfig
from poetry.config.dict_config_source import DictConfigSource
from poetry.inspection.info import PackageInfo
from poetry.utils._compat import Path
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
Expand Down Expand Up @@ -79,8 +80,11 @@ def download_mock(mocker):


@pytest.fixture(autouse=True)
def execute_setup_mock(mocker):
mocker.patch("poetry.inspection.info.PackageInfo._execute_setup")
def pep517_metadata_mock(mocker):
mocker.patch(
"poetry.inspection.info.PackageInfo._pep517_metadata",
return_value=PackageInfo(name="demo", version="0.1.2"),
)


@pytest.fixture
Expand Down
23 changes: 0 additions & 23 deletions tests/fixtures/inspection/demo_only_setup/setup.py

This file was deleted.

Loading

0 comments on commit 4250a5f

Please sign in to comment.