ylib2to3 -- Fork lib2to3 with some blib2to3 backports; add structured pattern matching (match) and parenthesized-context-managers statement support

.github/workflows/ci.yml

@@ -20,9 +20,12 @@ jobs:
       uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b  # v4.6.0
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Update pip
       run: |
         python -m pip install --upgrade pip
+    - name: Install YAPF
+      run: |
+        pip install .
     - name: Lint with flake8
       run: |
         python -m pip install flake8

.gitignore

@@ -20,6 +20,7 @@
 .sw?
 # OS X specific files.
 .DS_store
+*.pickle
 
 #==============================================================================#
 # Files to ignore
@@ -34,4 +35,5 @@
 /yapf.egg-info
 
 /.idea
+.vscode/settings.json
 /.venv*/

setup.py

@@ -17,9 +17,9 @@
 import sys
 import unittest
 
-from setuptools import find_packages, setup, Command
-
-import yapf
+from setuptools import Command
+from setuptools import find_packages
+from setuptools import setup
 
 
 class RunTests(Command):
@@ -42,7 +42,7 @@ def run(self):
 with codecs.open('README.rst', 'r', 'utf-8') as fd:
   setup(
       name='yapf',
-      version=yapf.__version__,
+      version='0.33.0',
       description='A formatter for Python code.',
       url='https://github.com/google/yapf',
       long_description=fd.read(),
@@ -74,12 +74,25 @@ def run(self):
       entry_points={
           'console_scripts': [
               'yapf = yapf:run_main',
-              'yapf-diff = third_party.yapf_diff.yapf_diff:main',
+              'yapf-diff = yapf.third_party.yapf_diff.yapf_diff:main',
           ],
       },
       cmdclass={
           'test': RunTests,
       },
+      package_data={
+          'yapf': [
+              'third_party/yapf_diff/LICENSE',
+              'third_party/_ylib2to3/Grammar.txt',
+              'third_party/_ylib2to3/PatternGrammar.txt',
+              'third_party/_ylib2to3/LICENSE',
+          ]
+      },
+      include_package_data=True,
       python_requires='>=3.7',
-      install_requires=['tomli>=2.0.1'],
+      install_requires=[
+          'importlib-metadata>=6.6.0',
+          'platformdirs>=3.5.1',
+          'tomli>=2.0.1',
+      ],
   )

yapf/__init__.py

@@ -33,12 +33,14 @@
 import os
 import sys
 
+from importlib_metadata import metadata
+
 from yapf.yapflib import errors
 from yapf.yapflib import file_resources
 from yapf.yapflib import style
 from yapf.yapflib import yapf_api
 
-__version__ = '0.33.0'
+__version__ = metadata('yapf')['Version']
 
 
 def _raw_input():
@@ -200,8 +202,8 @@ def FormatFiles(filenames,
   """
   changed = False
   if parallel:
-    import multiprocessing  # pylint: disable=g-import-not-at-top
     import concurrent.futures  # pylint: disable=g-import-not-at-top
+    import multiprocessing  # pylint: disable=g-import-not-at-top
     workers = min(multiprocessing.cpu_count(), len(filenames))
     with concurrent.futures.ProcessPoolExecutor(workers) as executor:
       future_formats = [

yapf/pytree/blank_line_calculator.py

@@ -22,10 +22,9 @@
   newlines: The number of newlines required before the node.
 """
 
-from lib2to3.pgen2 import token as grammar_token
-
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
+from yapf.third_party._ylib2to3.pgen2 import token as grammar_token
 from yapf.yapflib import style
 
 _NO_BLANK_LINES = 1
@@ -174,5 +173,4 @@ def _StartsInZerothColumn(node):
 
 
 def _AsyncFunction(node):
-  return (node.prev_sibling and
-          node.prev_sibling.type == grammar_token.ASYNC)
+  return (node.prev_sibling and node.prev_sibling.type == grammar_token.ASYNC)

yapf/pytree/comment_splicer.py

@@ -21,11 +21,10 @@
   SpliceComments(): the main function exported by this module.
 """
 
-from lib2to3 import pygram
-from lib2to3 import pytree
-from lib2to3.pgen2 import token
-
 from yapf.pytree import pytree_utils
+from yapf.third_party._ylib2to3 import pygram
+from yapf.third_party._ylib2to3 import pytree
+from yapf.third_party._ylib2to3.pgen2 import token
 
 
 def SpliceComments(tree):

yapf/pytree/continuation_splicer.py

@@ -19,8 +19,7 @@
   SpliceContinuations(): the main function exported by this module.
 """
 
-from lib2to3 import pytree
-
+from yapf.third_party._ylib2to3 import pytree
 from yapf.yapflib import format_token
 
 

yapf/pytree/pytree_unwrapper.py

@@ -26,14 +26,13 @@
 For most uses, the convenience function UnwrapPyTree should be sufficient.
 """
 
-# The word "token" is overloaded within this module, so for clarity rename
-# the imported pgen2.token module.
-from lib2to3 import pytree
-from lib2to3.pgen2 import token as grammar_token
-
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
 from yapf.pytree import split_penalty
+# The word "token" is overloaded within this module, so for clarity rename
+# the imported pgen2.token module.
+from yapf.third_party._ylib2to3 import pytree
+from yapf.third_party._ylib2to3.pgen2 import token as grammar_token
 from yapf.yapflib import format_token
 from yapf.yapflib import logical_line
 from yapf.yapflib import object_state
@@ -125,6 +124,8 @@ def _StartNewLine(self):
       'try_stmt',
       'expect_clause',
       'with_stmt',
+      'match_stmt',
+      'case_block',
       'funcdef',
       'classdef',
   })
@@ -144,11 +145,13 @@ def Visit_simple_stmt(self, node):
     single_stmt_suite = (
         node.parent and pytree_utils.NodeName(node.parent) in self._STMT_TYPES)
     is_comment_stmt = pytree_utils.IsCommentStatement(node)
-    if single_stmt_suite and not is_comment_stmt:
+    is_inside_match = node.parent and pytree_utils.NodeName(
+        node.parent) == 'match_stmt'
+    if (single_stmt_suite and not is_comment_stmt) or is_inside_match:
       self._cur_depth += 1
     self._StartNewLine()
     self.DefaultNodeVisit(node)
-    if single_stmt_suite and not is_comment_stmt:
+    if (single_stmt_suite and not is_comment_stmt) or is_inside_match:
       self._cur_depth -= 1
 
   def _VisitCompoundStatement(self, node, substatement_names):
@@ -253,6 +256,20 @@ def Visit_decorated(self, node):  # pylint: disable=invalid-name
   def Visit_with_stmt(self, node):  # pylint: disable=invalid-name
     self._VisitCompoundStatement(node, self._WITH_STMT_ELEMS)
 
+  _MATCH_STMT_ELEMS = frozenset({'match', 'case'})
+
+  def Visit_match_stmt(self, node):  # pylint: disable=invalid-name
+    self._VisitCompoundStatement(node, self._MATCH_STMT_ELEMS)
+
+  # case_block refers to the grammar element name in Grammar.txt
+  _CASE_BLOCK_ELEMS = frozenset({'case'})
+
+  def Visit_case_block(self, node):
+    self._cur_depth += 1
+    self._StartNewLine()
+    self._VisitCompoundStatement(node, self._CASE_BLOCK_ELEMS)
+    self._cur_depth -= 1
+
   def Visit_suite(self, node):  # pylint: disable=invalid-name
     # A 'suite' starts a new indentation level in Python.
     self._cur_depth += 1

yapf/pytree/pytree_utils.py

@@ -27,11 +27,11 @@
 import ast
 import os
 
-from lib2to3 import pygram
-from lib2to3 import pytree
-from lib2to3.pgen2 import driver
-from lib2to3.pgen2 import parse
-from lib2to3.pgen2 import token
+from yapf.third_party._ylib2to3 import pygram
+from yapf.third_party._ylib2to3 import pytree
+from yapf.third_party._ylib2to3.pgen2 import driver
+from yapf.third_party._ylib2to3.pgen2 import parse
+from yapf.third_party._ylib2to3.pgen2 import token
 
 # TODO(eliben): We may want to get rid of this filtering at some point once we
 # have a better understanding of what information we need from the tree. Then,

yapf/pytree/pytree_visitor.py

@@ -26,9 +26,8 @@
 
 import sys
 
-from lib2to3 import pytree
-
 from yapf.pytree import pytree_utils
+from yapf.third_party._ylib2to3 import pytree
 
 
 class PyTreeVisitor(object):

yapf/pytree/split_penalty.py

@@ -15,11 +15,10 @@
 
 import re
 
-from lib2to3 import pytree
-from lib2to3.pgen2 import token as grammar_token
-
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
+from yapf.third_party._ylib2to3 import pytree
+from yapf.third_party._ylib2to3.pgen2 import token as grammar_token
 from yapf.yapflib import style
 from yapf.yapflib import subtypes
 

yapf/pytree/subtype_assigner.py

@@ -24,12 +24,11 @@
       subtypes.
 """
 
-from lib2to3 import pytree
-from lib2to3.pgen2 import token as grammar_token
-from lib2to3.pygram import python_symbols as syms
-
 from yapf.pytree import pytree_utils
 from yapf.pytree import pytree_visitor
+from yapf.third_party._ylib2to3 import pytree
+from yapf.third_party._ylib2to3.pgen2 import token as grammar_token
+from yapf.third_party._ylib2to3.pygram import python_symbols as syms
 from yapf.yapflib import style
 from yapf.yapflib import subtypes