Issues/782/pyenv detour

.travis.yml

@@ -1,33 +1,6 @@
-x-pyenv-shard: &x-pyenv-shard
-  language: generic
-  env: &env >
-    PYENV_ROOT="${HOME}/.pyenv_pex"
-    PATH="${PYENV_ROOT}/shims:${PATH}"
-  cache: &cache
-    # The default is 3 minutes (180).
-    timeout: 300
-    directories:
-      - .pyenv_test
-      - "${PYENV_ROOT}"
-  install: |
-    PYENV="${PYENV_ROOT}/bin/pyenv"
-    if [ ! -x "${PYENV}" ]; then
-      rm -rf ${PYENV_ROOT}
-      git clone https://github.com/pyenv/pyenv "${PYENV_ROOT}";
-    fi
-    "${PYENV}" install --keep --skip-existing ${PYENV_VERSION}
-    "${PYENV}" global ${PYENV_VERSION}
-    pip install -U tox
-
-x-py27: &x-py27 PYENV_VERSION=2.7.17
-
-x-py38: &x-py38 PYENV_VERSION=3.8.0
-
-x-pypy: &x-pypy PYENV_VERSION=pypy2.7-7.2.0
-
 x-linux-shard: &x-linux-shard
   os: linux
-  dist: bionic 
+  dist: bionic
   language: python
   install: |
     pip install -U tox
@@ -45,9 +18,27 @@ x-linux-38-shard: &x-linux-38-shard
   python: 3.8
 
 x-osx-shard: &x-osx-shard
-  <<: *x-pyenv-shard
   os: osx
-  osx_image: xcode9.4
+  osx_image: xcode10.1
+  language: generic
+  env: &env >
+    PYENV_ROOT="${HOME}/.pyenv_pex"
+    PATH="${PYENV_ROOT}/shims:${PATH}"
+  cache:
+    # The default is 3 minutes (180).
+    timeout: 300
+    directories:
+      - .pyenv_test
+      - "${PYENV_ROOT}"
+  install: |
+    PYENV="${PYENV_ROOT}/bin/pyenv"
+    if [ ! -x "${PYENV}" ]; then
+      rm -rf ${PYENV_ROOT}
+      git clone https://github.com/pyenv/pyenv "${PYENV_ROOT}";
+    fi
+    "${PYENV}" install --keep --skip-existing ${PYENV_VERSION}
+    "${PYENV}" global ${PYENV_VERSION}
+    pip install -U tox
 
 x-osx-ssl: &x-osx-ssl >
   CPPFLAGS=-I/usr/local/opt/openssl/include
@@ -57,15 +48,15 @@ x-osx-27-shard: &x-osx-27-shard
   <<: *x-osx-shard
   env:
     - *env
-    - *x-py27
     - *x-osx-ssl
+    - PYENV_VERSION=2.7.17
 
 x-osx-38-shard: &x-osx-38-shard
   <<: *x-osx-shard
   env:
     - *env
-    - *x-py38
     - *x-osx-ssl
+    - PYENV_VERSION=3.8.0
 
 # NB: Travis partitions caches using a combination of os, language amd env vars. As such, we do not
 # use TOXENV and instead pass the toxenv via -e on the command line. This helps ensure we share

pex/interpreter.py

@@ -46,6 +46,7 @@ def get(cls):
     supported_tags = tuple(tags.sys_tags())
     preferred_tag = supported_tags[0]
     return cls(
+      binary=sys.executable,
       python_tag=preferred_tag.interpreter,
       abi_tag=preferred_tag.abi,
       platform_tag=preferred_tag.platform,
@@ -58,8 +59,8 @@ def get(cls):
   def decode(cls, encoded):
     TRACER.log('creating PythonIdentity from encoded: %s' % encoded, V=9)
     values = json.loads(encoded)
-    if len(values) != 6:
-      raise cls.InvalidError("Invalid id string: %s" % encoded)
+    if len(values) != 7:
+      raise cls.InvalidError("Invalid interpreter identity: %s" % encoded)
 
     supported_tags = values.pop('supported_tags')
 
@@ -80,11 +81,21 @@ def _find_interpreter_name(cls, python_tag):
         return interpreter
     raise ValueError('Unknown interpreter: {}'.format(python_tag))
 
-  def __init__(self, python_tag, abi_tag, platform_tag, version, supported_tags, env_markers):
+  def __init__(
+      self,
+      binary,
+      python_tag,
+      abi_tag,
+      platform_tag,
+      version,
+      supported_tags,
+      env_markers
+  ):
     # N.B.: We keep this mapping to support historical values for `distribution` and `requirement`
     # properties.
     self._interpreter_name = self._find_interpreter_name(python_tag)
 
+    self._binary = binary
     self._python_tag = python_tag
     self._abi_tag = abi_tag
     self._platform_tag = platform_tag
@@ -94,6 +105,7 @@ def __init__(self, python_tag, abi_tag, platform_tag, version, supported_tags, e
 
   def encode(self):
     values = dict(
+      binary=self._binary,
       python_tag=self._python_tag,
       abi_tag=self._abi_tag,
       platform_tag=self._platform_tag,
@@ -106,6 +118,10 @@ def encode(self):
     )
     return json.dumps(values)
 
+  @property
+  def binary(self):
+    return self._binary
+
   @property
   def python_tag(self):
     return self._python_tag
@@ -185,15 +201,27 @@ def python(self):
     return '%d.%d' % (self.version[0:2])
 
   def __str__(self):
-    return '%s-%s.%s.%s' % (
-      self._interpreter_name,
-      self._version[0],
-      self._version[1],
-      self._version[2]
+    # N.B.: Kept as distinct from __repr__ to support legacy str(identity) used by Pants v1 when
+    # forming cache locations.
+    return '{interpreter_name}-{major}.{minor}.{patch}'.format(
+      interpreter_name=self._interpreter_name,
+      major=self._version[0],
+      minor=self._version[1],
+      patch=self._version[2]
+    )
+
+  def __repr__(self):
+    return '{type}({binary!r}, {python_tag!r}, {abi_tag!r}, {platform_tag!r}, {version!r})'.format(
+      type=self.__class__.__name__,
+      binary=self._binary,
+      python_tag=self._python_tag,
+      abi_tag=self._abi_tag,
+      platform_tag=self._platform_tag,
+      version=self._version
     )
 
   def _tup(self):
-    return self._python_tag, self._abi_tag, self._platform_tag, self._version
+    return self._binary, self._python_tag, self._abi_tag, self._platform_tag, self._version
 
   def __eq__(self, other):
     if type(other) is not type(self):
@@ -313,7 +341,7 @@ def create_interpreter(stdout):
       identity = stdout.decode('utf-8').strip()
       if not identity:
         raise cls.IdentificationError('Could not establish identity of %s' % binary)
-      return cls(binary, PythonIdentity.decode(identity))
+      return cls(PythonIdentity.decode(identity))
 
     return SpawnedJob.stdout(job, result_func=create_interpreter)
 
@@ -322,7 +350,7 @@ def _expand_path(cls, path):
     if os.path.isfile(path):
       return [path]
     elif os.path.isdir(path):
-      return [os.path.join(path, fn) for fn in os.listdir(path)]
+      return sorted(os.path.join(path, fn) for fn in os.listdir(path))
     return []
 
   @classmethod
@@ -349,7 +377,7 @@ def _spawn_from_binary(cls, binary):
     if cached_interpreter is not None:
       return SpawnedJob.completed(cached_interpreter)
     if normalized_binary == cls._normalize_path(sys.executable):
-      current_interpreter = cls(sys.executable, PythonIdentity.get())
+      current_interpreter = cls(PythonIdentity.get())
       return SpawnedJob.completed(current_interpreter)
     return cls._spawn_from_binary_external(normalized_binary)
 
@@ -412,16 +440,15 @@ def _sanitized_environment(cls, env=None):
     env_copy.pop('MACOSX_DEPLOYMENT_TARGET', None)
     return env_copy
 
-  def __init__(self, binary, identity):
+  def __init__(self, identity):
     """Construct a PythonInterpreter.
 
     You should probably use `PythonInterpreter.from_binary` instead.
 
-    :param binary: The full path of the python binary.
     :param identity: The :class:`PythonIdentity` of the PythonInterpreter.
     """
-    self._binary = self._normalize_path(binary)
     self._identity = identity
+    self._binary = self._normalize_path(self.identity.binary)
 
     self._PYTHON_INTERPRETER_BY_NORMALIZED_PATH[self._binary] = self
 
@@ -475,7 +502,11 @@ def __lt__(self, other):
     return self.version < other.version
 
   def __repr__(self):
-    return '%s(%r, %r)' % (self.__class__.__name__, self._binary, self._identity)
+    return '{type}({binary!r}, {identity!r})'.format(
+      type=self.__class__.__name__,
+      binary=self._binary,
+      identity=self._identity
+    )
 
 
 def spawn_python_job(args, env=None, interpreter=None, expose=None, **subprocess_kwargs):

pex/jobs.py

@@ -229,7 +229,7 @@ def spawn_jobs():
                      .format(item.item, spawn_func, item.error))
       elif error is not None:  # I.E.: `item` is not an exception, but there was a prior exception.
         item.kill()
-      else:
+      elif isinstance(item, SpawnedJob):
         try:
           yield item.await_result()
         except Job.Error as e:
@@ -240,5 +240,8 @@ def spawn_jobs():
           else:
             # Otherwise, if we were given no `raise_type`, we continue on and just log the failure.
             TRACER.log('{} raised {}'.format(item, e))
+      else:
+        raise AssertionError('Unexpected item on spawned job queue {} of type {}'
+                             .format(item, type(item)))
     finally:
       job_slots.release()

pex/testing.py

@@ -12,8 +12,6 @@
 from collections import namedtuple
 from textwrap import dedent
 
-import pytest
-
 from pex.common import open_zip, safe_mkdir, safe_mkdtemp, safe_rmtree, temporary_dir, touch
 from pex.compatibility import PY3, nested
 from pex.distribution_target import DistributionTarget
@@ -35,13 +33,6 @@
 NOT_CPYTHON36_OR_LINUX = NOT_CPYTHON36 or IS_LINUX
 
 
-skip_for_pyenv_use_under_pypy = pytest.mark.skipif(
-  IS_PYPY,
-  reason='Our pyenv interpreter setup fails under pypy: '
-         'https://github.com/pantsbuild/pex/issues/477'
-)
-
-
 @contextlib.contextmanager
 def temporary_filename():
   """Creates a temporary filename.