requirements.txt

# PySpark dependencies (required)
py4j

# PySpark dependencies (optional)
numpy
pyarrow<13.0.0
pandas
scipy
plotly
mlflow>=1.0
scikit-learn
matplotlib<3.3.0
memory-profiler==0.60.0

# PySpark test dependencies
unittest-xml-reporting
openpyxl

# PySpark test dependencies (optional)
coverage

# Linter
mypy==0.920
pytest-mypy-plugins==1.9.3
flake8==3.9.0
# See SPARK-38680.
pandas-stubs<1.2.0.54

# Documentation (SQL)
mkdocs

# Documentation (Python)
pydata_sphinx_theme
ipython
nbsphinx
numpydoc
jinja2<3.0.0
sphinx<3.1.0
sphinx-plotly-directive
sphinx-copybutton<0.5.3
docutils<0.18.0
# See SPARK-38279.
markupsafe==2.0.1

# Development scripts
jira
PyGithub

# pandas API on Spark Code formatter.
black==22.6.0
py

# Spark Connect (required)
grpcio==1.48.1
grpcio-status==1.48.1
protobuf==3.19.5
googleapis-common-protos==1.56.4

# Spark Connect python proto generation plugin (optional)
mypy-protobuf==3.3.0
googleapis-common-protos-stubs==2.2.0
grpc-stubs==1.24.11

# TorchDistributor dependencies
torch==1.13.1
torchvision==0.14.1