[BEAM-12000] Add Python 3.9 support. #16008

.github/workflows/python_tests.yml

@@ -100,12 +100,8 @@ jobs:
           {"py_ver": "3.6", "tox_env": "py36"},
           {"py_ver": "3.7", "tox_env": "py37"},
           {"py_ver": "3.8", "tox_env": "py38"},
+          {"py_ver": "3.9", "tox_env": "py39"},
         ]
-        exclude:
-          # TODO remove exclusion after issue with protobuf is solved
-          # https://github.com/protocolbuffers/protobuf/issues/7765
-          - os: windows-latest
-            params: {"py_ver": "3.8", "tox_env": "py38"}
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -140,12 +136,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: [3.6, 3.7, 3.8]
-        exclude:
-          # TODO remove exclusion after issue with protobuf is solved
-          # https://github.com/protocolbuffers/protobuf/issues/7765
-          - os: windows-latest
-            python: 3.8
+        python: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -173,12 +164,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python: [3.6, 3.7, 3.8]
-        exclude:
-          # TODO remove exclusion after issue with protobuf is solved
-          # https://github.com/protocolbuffers/protobuf/issues/7765
-          - os: windows-latest
-            python: 3.8
+        python: [3.6, 3.7, 3.8, 3.9]
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

.test-infra/jenkins/PythonTestProperties.groovy

@@ -22,7 +22,8 @@ class PythonTestProperties {
   final static List<String> ALL_SUPPORTED_VERSIONS = [
     '3.6',
     '3.7',
-    '3.8'
+    '3.8',
+    '3.9'
   ]
   final static List<String> SUPPORTED_CONTAINER_TASKS = ALL_SUPPORTED_VERSIONS.collect {
     "py${it.replace('.', '')}"

.test-infra/jenkins/build.gradle

@@ -22,6 +22,7 @@ plugins {
 applyGroovyNature()
 applyPythonNature()
 
+// TODO(BEAM-9980): Don't hardcode this version, take the value provided by Python nature.
 pythonVersion = '3.8'
 
 task generateMetricsReport {

.test-infra/jenkins/job_PreCommit_Portable_Python.groovy

@@ -48,7 +48,7 @@ builder.build {
     }
     gradle {
       rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(':sdks:python:test-suites:portable:py38:preCommitPy38')
+      tasks(':sdks:python:test-suites:portable:py39:preCommitPy38')
       commonJobProperties.setGradleSwitches(delegate)
     }
   }

.test-infra/jenkins/metrics_report/tox.ini

@@ -14,7 +14,7 @@
 ;    See the License for the specific language governing permissions and
 ;    limitations under the License.
 ;
-
+; TODO(BEAM-9980): Don't hardcode Py3.8 version.
 [tox]
 skipsdist = True
 envlist = py38-test,py38-generate-report

build.gradle.kts

@@ -267,6 +267,7 @@ tasks.register("pythonPreCommit") {
   dependsOn(":sdks:python:test-suites:tox:py36:preCommitPy36")
   dependsOn(":sdks:python:test-suites:tox:py37:preCommitPy37")
   dependsOn(":sdks:python:test-suites:tox:py38:preCommitPy38")
+  dependsOn(":sdks:python:test-suites:tox:py39:preCommitPy39")
   dependsOn(":sdks:python:test-suites:dataflow:preCommitIT")
   dependsOn(":sdks:python:test-suites:dataflow:preCommitIT_V2")
 }
@@ -279,9 +280,11 @@ tasks.register("pythonDockerBuildPreCommit") {
   dependsOn(":sdks:python:container:py36:docker")
   dependsOn(":sdks:python:container:py37:docker")
   dependsOn(":sdks:python:container:py38:docker")
+  dependsOn(":sdks:python:container:py39:docker")
 }
 
 tasks.register("pythonLintPreCommit") {
+  // TODO(BEAM-9980): Find a better way to specify lint and formatter tasks without hardcoding py version.
   dependsOn(":sdks:python:test-suites:tox:py37:lint")
 }
 
@@ -314,15 +317,24 @@ tasks.register("python38PostCommit") {
   dependsOn(":sdks:python:test-suites:portable:py38:postCommitPy38")
 }
 
+tasks.register("python39PostCommit") {
+  // TODO(BEAM-12920): Enable DF suite here.
+  // dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py39:postCommitIT")
+  dependsOn(":sdks:python:test-suites:direct:py39:hdfsIntegrationTest")
+  dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
+}
+
 tasks.register("portablePythonPreCommit") {
   dependsOn(":sdks:python:test-suites:portable:py36:preCommitPy36")
-  dependsOn(":sdks:python:test-suites:portable:py37:preCommitPy37")
+  dependsOn(":sdks:python:test-suites:portable:py39:preCommitPy39")
 }
 
 tasks.register("pythonSparkPostCommit") {
   dependsOn(":sdks:python:test-suites:portable:py36:sparkValidatesRunner")
   dependsOn(":sdks:python:test-suites:portable:py37:sparkValidatesRunner")
   dependsOn(":sdks:python:test-suites:portable:py38:sparkValidatesRunner")
+  dependsOn(":sdks:python:test-suites:portable:py39:sparkValidatesRunner")
 }
 
 tasks.register("websitePreCommit") {
@@ -348,12 +360,14 @@ tasks.register("runBeamDependencyCheck") {
 }
 
 tasks.register("whitespacePreCommit") {
+  // TODO(BEAM-9980): Find a better way to specify the tasks without hardcoding py version.
   dependsOn(":sdks:python:test-suites:tox:py38:archiveFilesToLint")
   dependsOn(":sdks:python:test-suites:tox:py38:unpackFilesToLint")
   dependsOn(":sdks:python:test-suites:tox:py38:whitespacelint")
 }
 
 tasks.register("typescriptPreCommit") {
+  // TODO(BEAM-9980): Find a better way to specify the tasks without hardcoding py version.
   dependsOn(":sdks:python:test-suites:tox:py38:eslint")
   dependsOn(":sdks:python:test-suites:tox:py38:jest")
 }

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -2445,6 +2445,7 @@ class BeamModulePlugin implements Plugin<Project> {
       // Python interpreter version for virtualenv setup and test run. This value can be
       // set from commandline with -PpythonVersion, or in build script of certain project.
       // If none of them applied, version set here will be used as default value.
+      // TODO(BEAM-12000): Move default value to Py3.9.
       project.ext.pythonVersion = project.hasProperty('pythonVersion') ?
           project.pythonVersion : '3.8'
 
@@ -2621,6 +2622,7 @@ class BeamModulePlugin implements Plugin<Project> {
             ':sdks:python:container:py36:docker',
             ':sdks:python:container:py37:docker',
             ':sdks:python:container:py38:docker',
+            ':sdks:python:container:py39:docker',
           ]
           doLast {
             // TODO: Figure out GCS credentials and use real GCS input and output.

release/src/main/scripts/build_release_candidate.sh

@@ -363,6 +363,7 @@ if [[ $confirmation = "y" ]]; then
   git clone --branch "${RC_TAG}" --depth 1 ${GIT_REPO_URL}
   cd ${BEAM_ROOT_DIR}
   RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}")
+  # TODO(BEAM-9980): Don't hardcode py version in this file.
   cd sdks/python && pip install -r build-requirements.txt && tox -e py38-docs
   GENERATED_PYDOC=~/${LOCAL_WEBSITE_UPDATE_DIR}/${LOCAL_PYTHON_DOC}/${BEAM_ROOT_DIR}/sdks/python/target/docs/_build
   rm -rf ${GENERATED_PYDOC}/.doctrees

sdks/python/apache_beam/__init__.py

@@ -75,7 +75,7 @@
 import warnings
 
 if sys.version_info.major == 3:
-  if sys.version_info.minor <= 5 or sys.version_info.minor >= 9:
+  if sys.version_info.minor <= 5 or sys.version_info.minor >= 10:
     warnings.warn(
         'This version of Apache Beam has not been sufficiently tested on '
         'Python %s.%s. You may encounter bugs or missing features.' %

sdks/python/apache_beam/runners/common.py

@@ -558,7 +558,7 @@ def __init__(self,
     default_arg_values = signature.process_method.defaults
     self.has_windowed_inputs = (
         not all(si.is_globally_windowed() for si in side_inputs) or
-        (core.DoFn.WindowParam in default_arg_values) or
+        any(core.DoFn.WindowParam == arg for arg in default_arg_values) or
         signature.is_stateful_dofn())
     self.user_state_context = user_state_context
     self.is_splittable = signature.is_splittable_dofn()
@@ -592,7 +592,7 @@ class ArgPlaceholder(object):
       def __init__(self, placeholder):
         self.placeholder = placeholder
 
-    if core.DoFn.ElementParam not in default_arg_values:
+    if all(core.DoFn.ElementParam != arg for arg in default_arg_values):
       # TODO(BEAM-7867): Handle cases in which len(arg_names) ==
       #   len(default_arg_values).
       args_to_pick = len(arg_names) - len(default_arg_values) - 1

sdks/python/apache_beam/runners/dataflow/internal/apiclient.py

@@ -76,7 +76,7 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.6', '3.7', '3.8']
+_PYTHON_VERSIONS_SUPPORTED_BY_DATAFLOW = ['3.6', '3.7', '3.8', '3.9']
 
 
 class Step(object):

sdks/python/apache_beam/runners/dataflow/internal/apiclient_test.py

@@ -973,7 +973,7 @@ def test_interpreter_version_check_passes_py38(self):
 
   @mock.patch(
       'apache_beam.runners.dataflow.internal.apiclient.sys.version_info',
-      (3, 9, 0))
+      (3, 10, 0))
   @mock.patch(
       'apache_beam.runners.dataflow.internal.apiclient.'
       'beam_version.__version__',

sdks/python/container/build.gradle

@@ -39,18 +39,21 @@ task buildAll {
   dependsOn ':sdks:python:container:py36:docker'
   dependsOn ':sdks:python:container:py37:docker'
   dependsOn ':sdks:python:container:py38:docker'
+  dependsOn ':sdks:python:container:py39:docker'
 }
 
 task pushAll {
   dependsOn ':sdks:python:container:py36:dockerPush'
   dependsOn ':sdks:python:container:py37:dockerPush'
   dependsOn ':sdks:python:container:py38:dockerPush'
+  dependsOn ':sdks:python:container:py39:dockerPush'
 }
 
 task generatePythonRequirementsAll {
   dependsOn ':sdks:python:container:py36:generatePythonRequirements'
   dependsOn ':sdks:python:container:py37:generatePythonRequirements'
   dependsOn ':sdks:python:container:py38:generatePythonRequirements'
+  dependsOn ':sdks:python:container:py39:generatePythonRequirements'
 }
 
 artifacts {

sdks/python/container/py39/base_image_requirements.txt

@@ -0,0 +1,156 @@
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+# Autogenerated requirements file for Apache Beam py39 container image.
+# Run ./gradlew :sdks:python:container:generatePythonRequirementsAll to update.
+# Do not edit manually, adjust ../base_image_requirements_manual.txt or
+# Apache Beam's setup.py instead, and regenerate the list.
+# You will need Python intepreters for all versions supported by Beam, see:
+# https://s.apache.org/beam-python-dev-wiki
+# Reach out to a committer if you need help.
+
+absl-py==1.0.0
+astunparse==1.6.3
+atomicwrites==1.4.0
+attrs==21.4.0
+beautifulsoup4==4.10.0
+bs4==0.0.1
+cachetools==4.2.4
+certifi==2021.10.8
+charset-normalizer==2.0.11
+click==8.0.3
+cloudpickle==2.0.0
+crcmod==1.7
+Cython==0.29.27
+deprecation==2.1.0
+dill==0.3.1.1
+docker==5.0.3
+docopt==0.6.2
+execnet==1.9.0
+fastavro==1.4.9
+fasteners==0.17.3
+flatbuffers==2.0
+freezegun==1.1.0
+future==0.18.2
+gast==0.5.3
+google-api-core==1.31.5
+google-api-python-client==2.36.0
+google-apitools==0.5.31
+google-auth==1.35.0
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.6
+google-cloud-bigquery==2.32.0
+google-cloud-bigquery-storage==2.11.0
+google-cloud-bigtable==1.7.0
+google-cloud-core==1.7.2
+google-cloud-datastore==1.15.3
+google-cloud-dlp==3.6.0
+google-cloud-language==1.3.0
+google-cloud-profiler==3.0.7
+google-cloud-pubsub==2.9.0
+google-cloud-pubsublite==1.3.0
+google-cloud-recommendations-ai==0.2.0
+google-cloud-spanner==1.19.1
+google-cloud-videointelligence==1.16.1
+google-cloud-vision==1.0.0
+google-crc32c==1.3.0
+google-pasta==0.2.0
+google-python-cloud-debugger==2.18
+google-resumable-media==2.2.0
+googleapis-common-protos==1.54.0
+greenlet==1.1.2
+grpc-google-iam-v1==0.12.3
+grpcio==1.43.0
+grpcio-gcp==0.2.2
+grpcio-status==1.43.0
+guppy3==3.1.2
+h5py==3.6.0
+hdfs==2.6.0
+httplib2==0.19.1
+idna==3.3
+importlib-metadata==4.10.1
+joblib==1.1.0
+keras==2.8.0
+Keras-Preprocessing==1.1.2
+libclang==13.0.0
+libcst==0.4.1
+Markdown==3.3.6
+mmh3==3.0.0
+mock==2.0.0
+more-itertools==8.12.0
+mypy-extensions==0.4.3
+nltk==3.6.7
+nose==1.3.7
+numpy==1.21.5
+oauth2client==4.1.3
+oauthlib==3.2.0
+opt-einsum==3.3.0
+orjson==3.6.6
+overrides==6.1.0
+packaging==21.3
+pandas==1.4.0
+parameterized==0.7.5
+pbr==5.8.1
+pluggy==0.13.1
+proto-plus==1.20.0
+protobuf==3.19.4
+psycopg2-binary==2.9.3
+py==1.11.0
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pydot==1.4.2
+PyHamcrest==1.10.1
+pymongo==3.12.3
+pyparsing==2.4.7
+pytest==4.6.11
+pytest-forked==1.4.0
+pytest-timeout==1.4.2
+pytest-xdist==1.34.0
+python-dateutil==2.8.2
+python-snappy==0.6.0
+pytz==2021.3
+PyYAML==6.0
+regex==2022.1.18
+requests==2.27.1
+requests-mock==1.9.3
+requests-oauthlib==1.3.1
+rsa==4.8
+scikit-learn==1.0.2
+scipy==1.8.0
+six==1.16.0
+soupsieve==2.3.1
+SQLAlchemy==1.4.31
+tenacity==5.1.5
+tensorboard==2.8.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.8.0
+tensorflow-io-gcs-filesystem==0.24.0
+termcolor==1.1.0
+testcontainers==3.4.2
+tf-estimator-nightly==2.8.0.dev2021122109
+threadpoolctl==3.1.0
+tqdm==4.62.3
+typing-inspect==0.7.1
+typing-utils==0.1.0
+typing_extensions==4.0.1
+uritemplate==4.1.1
+urllib3==1.26.8
+wcwidth==0.2.5
+websocket-client==1.2.3
+Werkzeug==2.0.3
+wrapt==1.13.3
+zipp==3.7.0