Windows embedable support

docs/changelog/1774.feature.rst

@@ -0,0 +1,2 @@
+Support for Windows embeddable Python package: includes ``python<VERSION>.zip``
+in the creator sources.

docs/changelog/2348.bugfix.rst

@@ -0,0 +1 @@
+Upgrade embedded setuptools to ``62.3.2`` from ``62.1.0`` and pip to ``22.1.2`` from ``22.0.4`` - by :user:`gaborbernat`.

docs/changelog/2351.bugfix.rst

@@ -0,0 +1 @@
+Use ``shlex.quote`` instead of deprecated ``pipes.quote`` in Python 3. - by :user:`frenzymadness`.

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -1,6 +1,9 @@
 from __future__ import absolute_import, unicode_literals
 
 import abc
+import fnmatch
+from itertools import chain
+from operator import methodcaller as method
 from textwrap import dedent
 
 from six import add_metaclass
@@ -53,11 +56,20 @@ def setup_meta(cls, interpreter):
 
     @classmethod
     def sources(cls, interpreter):
-        for src in super(CPython3Windows, cls).sources(interpreter):
-            yield src
-        if not cls.has_shim(interpreter):
-            for src in cls.include_dll_and_pyd(interpreter):
-                yield src
+        if cls.has_shim(interpreter):
+            refs = cls.executables(interpreter)
+        else:
+            refs = chain(
+                cls.executables(interpreter),
+                cls.dll_and_pyd(interpreter),
+                cls.python_zip(interpreter),
+            )
+        for ref in refs:
+            yield ref
+
+    @classmethod
+    def executables(cls, interpreter):
+        return super(CPython3Windows, cls).sources(interpreter)
 
     @classmethod
     def has_shim(cls, interpreter):
@@ -79,13 +91,32 @@ def host_python(cls, interpreter):
         return super(CPython3Windows, cls).host_python(interpreter)
 
     @classmethod
-    def include_dll_and_pyd(cls, interpreter):
+    def dll_and_pyd(cls, interpreter):
         dll_folder = Path(interpreter.system_prefix) / "DLLs"
         host_exe_folder = Path(interpreter.system_executable).parent
         for folder in [host_exe_folder, dll_folder]:
             for file in folder.iterdir():
                 if file.suffix in (".pyd", ".dll"):
-                    yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
+                    yield PathRefToDest(file, cls.to_bin)
 
-    def to_dll_and_pyd(self, src):
-        return self.bin_dir / src.name
+    @classmethod
+    def python_zip(cls, interpreter):
+        """
+        "python{VERSION}.zip" contains compiled *.pyc std lib packages, where
+        "VERSION" is `py_version_nodot` var from the `sysconfig` module.
+        :see: https://docs.python.org/3/using/windows.html#the-embeddable-package
+        :see: `discovery.py_info.PythonInfo` class (interpreter).
+        :see: `python -m sysconfig` output.
+
+        :note: The embeddable Python distribution for Windows includes
+        "python{VERSION}.zip" and "python{VERSION}._pth" files. User can
+        move/rename *zip* file and edit `sys.path` by editing *_pth* file.
+        Here the `pattern` is used only for the default *zip* file name!
+        """
+        pattern = "*python{}.zip".format(interpreter.version_nodot)
+        matches = fnmatch.filter(interpreter.path, pattern)
+        matched_paths = map(Path, matches)
+        existing_paths = filter(method("exists"), matched_paths)
+        path = next(existing_paths, None)
+        if path is not None:
+            yield PathRefToDest(path, cls.to_bin)

src/virtualenv/discovery/cached_py_info.py

@@ -8,7 +8,6 @@
 
 import logging
 import os
-import pipes
 import sys
 from collections import OrderedDict
 
@@ -19,6 +18,11 @@
 from virtualenv.util.six import ensure_text
 from virtualenv.util.subprocess import Popen, subprocess
 
+if PY2:
+    from pipes import quote
+else:
+    from shlex import quote
+
 _CACHE = OrderedDict()
 _CACHE[Path(sys.executable)] = PythonInfo()
 
@@ -126,7 +130,7 @@ def __repr__(self):
         def e(v):
             return v.decode("utf-8") if isinstance(v, bytes) else v
 
-        cmd_repr = e(" ").join(pipes.quote(e(c)) for c in self.cmd)
+        cmd_repr = e(" ").join(quote(e(c)) for c in self.cmd)
         if self.env is not None:
             cmd_repr += e(" env of {!r}").format(self.env)
         if PY2:

src/virtualenv/discovery/py_info.py

@@ -47,6 +47,10 @@ def abs_path(v):
         self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
         self.architecture = 64 if sys.maxsize > 2**32 else 32
 
+        # Used to determine some file names.
+        # See `CPython3Windows.python_zip()`.
+        self.version_nodot = sysconfig.get_config_var("py_version_nodot")
+
         self.version = u(sys.version)
         self.os = u(os.name)
 

src/virtualenv/seed/wheels/embed/__init__.py

@@ -6,28 +6,28 @@
 BUNDLE_FOLDER = Path(__file__).absolute().parent
 BUNDLE_SUPPORT = {
     "3.11": {
-        "pip": "pip-22.0.4-py3-none-any.whl",
-        "setuptools": "setuptools-62.1.0-py3-none-any.whl",
+        "pip": "pip-22.1.2-py3-none-any.whl",
+        "setuptools": "setuptools-62.3.2-py3-none-any.whl",
         "wheel": "wheel-0.37.1-py2.py3-none-any.whl",
     },
     "3.10": {
-        "pip": "pip-22.0.4-py3-none-any.whl",
-        "setuptools": "setuptools-62.1.0-py3-none-any.whl",
+        "pip": "pip-22.1.2-py3-none-any.whl",
+        "setuptools": "setuptools-62.3.2-py3-none-any.whl",
         "wheel": "wheel-0.37.1-py2.py3-none-any.whl",
     },
     "3.9": {
-        "pip": "pip-22.0.4-py3-none-any.whl",
-        "setuptools": "setuptools-62.1.0-py3-none-any.whl",
+        "pip": "pip-22.1.2-py3-none-any.whl",
+        "setuptools": "setuptools-62.3.2-py3-none-any.whl",
         "wheel": "wheel-0.37.1-py2.py3-none-any.whl",
     },
     "3.8": {
-        "pip": "pip-22.0.4-py3-none-any.whl",
-        "setuptools": "setuptools-62.1.0-py3-none-any.whl",
+        "pip": "pip-22.1.2-py3-none-any.whl",
+        "setuptools": "setuptools-62.3.2-py3-none-any.whl",
         "wheel": "wheel-0.37.1-py2.py3-none-any.whl",
     },
     "3.7": {
-        "pip": "pip-22.0.4-py3-none-any.whl",
-        "setuptools": "setuptools-62.1.0-py3-none-any.whl",
+        "pip": "pip-22.1.2-py3-none-any.whl",
+        "setuptools": "setuptools-62.3.2-py3-none-any.whl",
         "wheel": "wheel-0.37.1-py2.py3-none-any.whl",
     },
     "3.6": {

tasks/make_zipapp.py

@@ -3,7 +3,6 @@
 import io
 import json
 import os
-import pipes
 import shutil
 import subprocess
 import sys
@@ -18,6 +17,11 @@
 from packaging.markers import Marker
 from packaging.requirements import Requirement
 
+if sys.version_info[0] == 2:
+    from pipes import quote
+else:
+    from shlex import quote
+
 HERE = Path(__file__).parent.absolute()
 
 VERSIONS = ["3.{}".format(i) for i in range(10, 4, -1)] + ["2.7"]
@@ -227,7 +231,7 @@ def run_suppress_output(cmd, stop_print_on_fail=False):
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
     out, err = process.communicate()
     if stop_print_on_fail and process.returncode != 0:
-        print("exit with {} of {}".format(process.returncode, " ".join(pipes.quote(i) for i in cmd)), file=sys.stdout)
+        print("exit with {} of {}".format(process.returncode, " ".join(quote(i) for i in cmd)), file=sys.stdout)
         if out:
             print(out, file=sys.stdout)
         if err:

tests/unit/activation/conftest.py

@@ -1,7 +1,6 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
-import pipes
 import re
 import shutil
 import subprocess
@@ -11,12 +10,17 @@
 import pytest
 import six
 
-from virtualenv.info import IS_PYPY, WIN_CPYTHON_2
+from virtualenv.info import IS_PYPY, PY2, WIN_CPYTHON_2
 from virtualenv.run import cli_run
 from virtualenv.util.path import Path
 from virtualenv.util.six import ensure_str, ensure_text
 from virtualenv.util.subprocess import Popen
 
+if PY2:
+    from pipes import quote
+else:
+    from shlex import quote
+
 
 class ActivationTester(object):
     def __init__(self, of_class, session, cmd, activate_script, extension):
@@ -157,7 +161,7 @@ def assert_output(self, out, raw, tmp_path):
         assert out[-1] == "None", raw
 
     def quote(self, s):
-        return pipes.quote(s)
+        return quote(s)
 
     def python_cmd(self, cmd):
         return "{} -c {}".format(os.path.basename(sys.executable), self.quote(cmd))

tests/unit/activation/test_batch.py

@@ -1,8 +1,12 @@
 from __future__ import absolute_import, unicode_literals
 
-import pipes
-
 from virtualenv.activation import BatchActivator
+from virtualenv.info import PY2
+
+if PY2:
+    from pipes import quote
+else:
+    from shlex import quote
 
 
 def test_batch(activation_tester_class, activation_tester, tmp_path, activation_python):
@@ -25,7 +29,7 @@ def _get_test_lines(self, activate_script):
 
         def quote(self, s):
             """double quotes needs to be single, and single need to be double"""
-            return "".join(("'" if c == '"' else ('"' if c == "'" else c)) for c in pipes.quote(s))
+            return "".join(("'" if c == '"' else ('"' if c == "'" else c)) for c in quote(s))
 
         def print_prompt(self):
             return "echo %PROMPT%"

tests/unit/activation/test_powershell.py

@@ -1,11 +1,16 @@
 from __future__ import absolute_import, unicode_literals
 
-import pipes
 import sys
 
 import pytest
 
 from virtualenv.activation import PowerShellActivator
+from virtualenv.info import PY2
+
+if PY2:
+    from pipes import quote
+else:
+    from shlex import quote
 
 
 @pytest.mark.slow
@@ -23,7 +28,7 @@ def __init__(self, session):
 
         def quote(self, s):
             """powershell double double quote needed for quotes within single quotes"""
-            return pipes.quote(s).replace('"', '""')
+            return quote(s).replace('"', '""')
 
         def _get_test_lines(self, activate_script):
             # for BATCH utf-8 support need change the character code page to 650001

tests/unit/create/via_global_ref/builtin/conftest.py

@@ -0,0 +1,25 @@
+import sys
+
+import pytest
+from testing import path
+from testing.py_info import read_fixture
+
+from virtualenv.util.path import Path
+
+# Allows to import from `testing` into test submodules.
+sys.path.append(Path(__file__).parent)
+
+
+@pytest.fixture
+def py_info(py_info_name):
+    return read_fixture(py_info_name)
+
+
+@pytest.fixture
+def mock_files(mocker):
+    return lambda paths, files: path.mock_files(mocker, paths, files)
+
+
+@pytest.fixture
+def mock_pypy_libs(mocker):
+    return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs)

tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json

@@ -0,0 +1,61 @@
+{
+  "platform": "win32",
+  "implementation": "CPython",
+  "version_info": {
+    "major": 3,
+    "minor": 10,
+    "micro": 4,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version_nodot": "310",
+  "version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]",
+  "os": "nt",
+  "prefix": "c:\\path\\to\\python",
+  "base_prefix": "c:\\path\\to\\python",
+  "real_prefix": null,
+  "base_exec_prefix": "c:\\path\\to\\python",
+  "exec_prefix": "c:\\path\\to\\python",
+  "executable": "c:\\path\\to\\python\\python.exe",
+  "original_executable": "c:\\path\\to\\python\\python.exe",
+  "system_executable": "c:\\path\\to\\python\\python.exe",
+  "has_venv": false,
+  "path": [
+    "c:\\path\\to\\python\\Scripts\\virtualenv.exe",
+    "c:\\path\\to\\python\\python310.zip",
+    "c:\\path\\to\\python",
+    "c:\\path\\to\\python\\Lib\\site-packages"
+  ],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "utf-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{installed_base}/Lib",
+    "platstdlib": "{base}/Lib",
+    "purelib": "{base}/Lib/site-packages",
+    "platlib": "{base}/Lib/site-packages",
+    "include": "{installed_base}/Include",
+    "scripts": "{base}/Scripts",
+    "data": "{base}"
+  },
+  "distutils_install": {
+    "purelib": "Lib\\site-packages",
+    "platlib": "Lib\\site-packages",
+    "headers": "Include\\UNKNOWN",
+    "scripts": "Scripts",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile"
+  },
+  "sysconfig_vars": {
+    "PYTHONFRAMEWORK": "",
+    "installed_base": "c:\\path\\to\\python",
+    "base": "c:\\path\\to\\python"
+  },
+  "system_stdlib": "c:\\path\\to\\python\\Lib",
+  "system_stdlib_platform": "c:\\path\\to\\python\\Lib",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}