[SPARK-33002][PYTHON] Remove non-API annotations

### What changes were proposed in this pull request? This PR: - removes annotations for modules which are not part of the public API. - removes `__init__.pyi` files, if no annotations, beyond exports, are present. ### Why are the changes needed? Primarily to reduce maintenance overhead and as requested in the comments to #29591 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Existing tests and additional MyPy checks: ``` mypy --no-incremental --config python/mypy.ini python/pyspark MYPYPATH=python/ mypy --no-incremental --config python/mypy.ini examples/src/main/python/ml examples/src/main/python/sql examples/src/main/python/sql/streaming ``` Closes #29879 from zero323/SPARK-33002. Authored-by: zero323 <mszymkiewicz@gmail.com> Signed-off-by: HyukjinKwon <gurwls223@apache.org>
apache · Oct 7, 2020 · 72da6f8 · 72da6f8
1 parent 4e1ded6
commit 72da6f8
Show file tree

Hide file tree

Showing 30 changed files with 14 additions and 1,039 deletions.
diff --git a/python/mypy.ini b/python/mypy.ini
@@ -32,5 +32,8 @@ ignore_missing_imports = True
 [mypy-pandas.*]
 ignore_missing_imports = True
 
-[mypy-pyarrow]
+[mypy-pyarrow.*]
+ignore_missing_imports = True
+
+[mypy-psutil.*]
 ignore_missing_imports = True
diff --git a/python/pyspark/_globals.pyi b/python/pyspark/_globals.pyi
diff --git a/python/pyspark/accumulators.pyi b/python/pyspark/accumulators.pyi
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Callable, Generic, Tuple, Type, TypeVar
+from typing import Callable, Dict, Generic, Tuple, Type, TypeVar
 
 import socketserver.BaseRequestHandler  # type: ignore
 
@@ -27,6 +27,8 @@ U = TypeVar("U", bound=SupportsIAdd)
 
 import socketserver as SocketServer
 
+_accumulatorRegistry: Dict[int, Accumulator]
+
 class Accumulator(Generic[T]):
     aid: int
     accum_param: AccumulatorParam[T]

diff --git a/python/pyspark/broadcast.pyi b/python/pyspark/broadcast.pyi
@@ -17,10 +17,12 @@
 # under the License.
 
 import threading
-from typing import Any, Generic, Optional, TypeVar
+from typing import Any, Dict, Generic, Optional, TypeVar
 
 T = TypeVar("T")
 
+_broadcastRegistry: Dict[int, Broadcast]
+
 class Broadcast(Generic[T]):
     def __init__(
         self,

diff --git a/python/pyspark/daemon.pyi b/python/pyspark/daemon.pyi
diff --git a/python/pyspark/find_spark_home.pyi b/python/pyspark/find_spark_home.pyi
diff --git a/python/pyspark/java_gateway.pyi b/python/pyspark/java_gateway.pyi
diff --git a/python/pyspark/join.pyi b/python/pyspark/join.pyi
diff --git a/python/pyspark/ml/__init__.pyi b/python/pyspark/ml/__init__.pyi
diff --git a/python/pyspark/mllib/__init__.pyi b/python/pyspark/mllib/__init__.pyi
diff --git a/python/pyspark/rddsampler.pyi b/python/pyspark/rddsampler.pyi
diff --git a/python/pyspark/resource/__init__.pyi b/python/pyspark/resource/__init__.pyi
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
@@ -342,7 +342,7 @@ def dumps(self, obj):
 
 # Hack namedtuple, make it picklable
 
-__cls = {}
+__cls = {}  # type: ignore
 
 
 def _restore(name, fields, value):