Validate and maybe prune interpreter cache run over run

src/python/pants/backend/python/interpreter_cache.py

@@ -118,6 +118,10 @@ def select_interpreter_for_targets(self, targets):
   def _interpreter_from_path(self, path, filters=()):
     try:
       executable = os.readlink(os.path.join(path, 'python'))
+      if not os.path.exists(executable):
+        if os.path.dirname(path) == self._cache_dir:
+          self._purge_interpreter(path)
+        return None
     except OSError:
       return None
     interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
@@ -251,3 +255,14 @@ def _resolve_and_link(self, interpreter, requirement, target_link):
     _safe_link(target_location, target_link)
     logger.debug('    installed {}'.format(target_location))
     return Package.from_href(target_location)
+
+  def _purge_interpreter(self, interpreter_dir):
+    try:
+      logger.info('Detected stale interpreter `{}` in the interpreter cache, purging.'
+                  .format(interpreter_dir))
+      shutil.rmtree(interpreter_dir, ignore_errors=True)
+    except Exception as e:
+      logger.warn(
+        'Caught exception {!r} during interpreter purge. Please run `./pants clean-all`!'
+        .format(e)
+      )

src/python/pants/backend/python/tasks/select_interpreter.py

@@ -78,6 +78,9 @@ def execute(self):
       interpreter_path_file = self._interpreter_path_file(target_set_id)
       if not os.path.exists(interpreter_path_file):
         self._create_interpreter_path_file(interpreter_path_file, python_tgts)
+      else:
+        if self._detect_and_purge_invalid_interpreter(interpreter_path_file):
+          self._create_interpreter_path_file(interpreter_path_file, python_tgts)
 
     interpreter = self._get_interpreter(interpreter_path_file)
     self.context.products.register_data(PythonInterpreter, interpreter)
@@ -95,6 +98,17 @@ def _create_interpreter_path_file(self, interpreter_path_file, targets):
   def _interpreter_path_file(self, target_set_id):
     return os.path.join(self.workdir, target_set_id, 'interpreter.info')
 
+  def _detect_and_purge_invalid_interpreter(self, interpreter_path_file):
+    with open(interpreter_path_file, 'r') as infile:
+      lines = infile.readlines()
+    binary = lines[0].strip()
+    if not os.path.exists(binary):
+      self.context.log.info('Stale interpreter reference detected: {}, removing reference and '
+                            'selecting a new interpreter.'.format(binary))
+      os.remove(interpreter_path_file)
+      return True
+    return False
+
   @staticmethod
   def _get_interpreter(interpreter_path_file):
     with open(interpreter_path_file, 'r') as infile: