Move SNOWFLAKE provider to new structure (apache#46059)

* moving snowflake provider

* fix static checks

* moving common-compat provider

* moving snowflake provider

* fix static checks

* resolving conflict

.github/boring-cyborg.yml

@@ -385,10 +385,7 @@ labelPRBasedOnFilePath:
     - providers/tests/smtp/**/*
 
   provider:snowflake:
-    - providers/src/airflow/providers/snowflake/**/*
-    - docs/apache-airflow-providers-snowflake/**/*
-    - providers/tests/snowflake/**/*
-    - providers/tests/system/snowflake/**/*
+    - providers/snowflake/**
 
   provider:sqlite:
     - providers/src/airflow/providers/sqlite/**/*

docs/.gitignore

@@ -36,6 +36,7 @@ apache-airflow-providers-samba
 apache-airflow-providers-segment
 apache-airflow-providers-singularity
 apache-airflow-providers-ssh
+apache-airflow-providers-snowflake
 apache-airflow-providers-standard
 apache-airflow-providers-teradata
 apache-airflow-providers-tableau

generated/provider_dependencies.json

@@ -1239,7 +1239,7 @@
       "pandas>=2.1.2,<2.2",
       "pyarrow>=14.0.1",
       "snowflake-connector-python>=3.7.1",
-      "snowflake-snowpark-python>=1.17.0;python_version<\"3.12\"",
+      "snowflake-snowpark-python>=1.17.0;python_version<'3.12'",
       "snowflake-sqlalchemy>=1.4.0"
     ],
     "devel-deps": [],

providers/snowflake/README.rst

@@ -0,0 +1,89 @@
+
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+ .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+ .. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE
+    `PROVIDER_README_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+
+
+Package ``apache-airflow-providers-snowflake``
+
+Release: ``6.0.0``
+
+
+`Snowflake <https://www.snowflake.com/>`__
+
+
+Provider package
+----------------
+
+This is a provider package for ``snowflake`` provider. All classes for this provider package
+are in ``airflow.providers.snowflake`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.0.0/>`_.
+
+Installation
+------------
+
+You can install this package on top of an existing Airflow 2 installation (see ``Requirements`` below
+for the minimum Airflow version supported) via
+``pip install apache-airflow-providers-snowflake``
+
+The package supports the following python versions: 3.9,3.10,3.11,3.12
+
+Requirements
+------------
+
+==========================================  =====================================
+PIP package                                 Version required
+==========================================  =====================================
+``apache-airflow``                          ``>=2.9.0``
+``apache-airflow-providers-common-compat``  ``>=1.1.0``
+``apache-airflow-providers-common-sql``     ``>=1.20.0``
+``pandas``                                  ``>=2.1.2,<2.2``
+``pyarrow``                                 ``>=14.0.1``
+``snowflake-connector-python``              ``>=3.7.1``
+``snowflake-sqlalchemy``                    ``>=1.4.0``
+``snowflake-snowpark-python``               ``>=1.17.0; python_version < "3.12"``
+==========================================  =====================================
+
+Cross provider package dependencies
+-----------------------------------
+
+Those are dependencies that might be needed in order to use all the features of the package.
+You need to install the specified provider packages in order to use them.
+
+You can install such cross-provider dependencies when installing from PyPI. For example:
+
+.. code-block:: bash
+
+    pip install apache-airflow-providers-snowflake[common.compat]
+
+
+==================================================================================================================  =================
+Dependent package                                                                                                   Extra
+==================================================================================================================  =================
+`apache-airflow-providers-common-compat <https://airflow.apache.org/docs/apache-airflow-providers-common-compat>`_  ``common.compat``
+`apache-airflow-providers-common-sql <https://airflow.apache.org/docs/apache-airflow-providers-common-sql>`_        ``common.sql``
+`apache-airflow-providers-openlineage <https://airflow.apache.org/docs/apache-airflow-providers-openlineage>`_      ``openlineage``
+==================================================================================================================  =================
+
+The changelog for the provider package can be found in the
+`changelog <https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.0.0/changelog.html>`_.

docs/apache-airflow-providers-snowflake/decorators/snowpark.rst → providers/snowflake/docs/decorators/snowpark.rst

@@ -51,7 +51,7 @@ Use the ``snowflake_conn_id`` argument to specify connection used. If not specif
 
 An example usage of the ``@task.snowpark`` is as follows:
 
-.. exampleinclude:: /../../providers/tests/system/snowflake/example_snowpark_decorator.py
+.. exampleinclude:: /../../providers/snowflake/tests/system/snowflake/example_snowpark_decorator.py
     :language: python
     :start-after: [START howto_decorator_snowpark]
     :end-before: [END howto_decorator_snowpark]

docs/apache-airflow-providers-snowflake/operators/copy_into_snowflake.rst → providers/snowflake/docs/operators/copy_into_snowflake.rst

@@ -43,7 +43,7 @@ a file format (see `docs <https://docs.snowflake.com/en/sql-reference/sql/create
 
 An example usage of the CopyFromExternalStageToSnowflakeOperator is as follows:
 
-.. exampleinclude:: /../../providers/tests/system/snowflake/example_copy_into_snowflake.py
+.. exampleinclude:: /../../providers/snowflake/tests/system/snowflake/example_copy_into_snowflake.py
     :language: python
     :start-after: [START howto_operator_s3_copy_into_snowflake]
     :end-before: [END howto_operator_s3_copy_into_snowflake]

docs/apache-airflow-providers-snowflake/operators/snowflake.rst → providers/snowflake/docs/operators/snowflake.rst

@@ -50,7 +50,7 @@ the connection metadata is structured as follows:
 
 An example usage of the SQLExecuteQueryOperator to connect to Snowflake is as follows:
 
-.. exampleinclude:: /../../providers/tests/system/snowflake/example_snowflake.py
+.. exampleinclude:: /../../providers/snowflake/tests/system/snowflake/example_snowflake.py
     :language: python
     :start-after: [START howto_operator_snowflake]
     :end-before: [END howto_operator_snowflake]
@@ -95,7 +95,7 @@ the connection metadata is structured as follows:
 
 An example usage of the SnowflakeSqlApiHook is as follows:
 
-.. exampleinclude:: /../../providers/tests/system/snowflake/example_snowflake.py
+.. exampleinclude:: /../../providers/snowflake/tests/system/snowflake/example_snowflake.py
     :language: python
     :start-after: [START howto_snowflake_sql_api_operator]
     :end-before: [END howto_snowflake_sql_api_operator]

docs/apache-airflow-providers-snowflake/operators/snowpark.rst → providers/snowflake/docs/operators/snowpark.rst

@@ -55,7 +55,7 @@ Use the ``snowflake_conn_id`` argument to specify connection used. If not specif
 
 An example usage of the ``@task.snowpark`` is as follows:
 
-.. exampleinclude:: /../../providers/tests/system/snowflake/example_snowpark_operator.py
+.. exampleinclude:: /../../providers/snowflake/tests/system/snowflake/example_snowpark_operator.py
     :language: python
     :start-after: [START howto_operator_snowpark]
     :end-before: [END howto_operator_snowpark]

providers/src/airflow/providers/snowflake/provider.yaml → providers/snowflake/provider.yaml

@@ -80,27 +80,13 @@ versions:
   - 1.1.0
   - 1.0.0
 
-dependencies:
-  - apache-airflow>=2.9.0
-  - apache-airflow-providers-common-compat>=1.1.0
-  - apache-airflow-providers-common-sql>=1.20.0
-  # In pandas 2.2 minimal version of the sqlalchemy is 2.0
-  # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
-  # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
-  # In addition FAB also limit sqlalchemy to < 2.0
-  - pandas>=2.1.2,<2.2
-  - pyarrow>=14.0.1
-  - snowflake-connector-python>=3.7.1
-  - snowflake-sqlalchemy>=1.4.0
-  - snowflake-snowpark-python>=1.17.0;python_version<"3.12"
-
 integrations:
   - integration-name: Snowflake
     external-doc-url: https://snowflake.com/
     how-to-guide:
       - /docs/apache-airflow-providers-snowflake/operators/snowflake.rst
       - /docs/apache-airflow-providers-snowflake/operators/snowpark.rst
-    logo: /integration-logos/snowflake/Snowflake.png
+    logo: /docs/integration-logos/Snowflake.png
     tags: [service]
 
 operators:

providers/snowflake/pyproject.toml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN!
+
+# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE
+# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY
+[build-system]
+requires = ["flit_core==3.10.1"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "apache-airflow-providers-snowflake"
+version = "6.0.0"
+description = "Provider package apache-airflow-providers-snowflake for Apache Airflow"
+readme = "README.rst"
+authors = [
+    {name="Apache Software Foundation", email="dev@airflow.apache.org"},
+]
+maintainers = [
+    {name="Apache Software Foundation", email="dev@airflow.apache.org"},
+]
+keywords = [ "airflow-provider", "snowflake", "airflow", "integration" ]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Environment :: Console",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Intended Audience :: System Administrators",
+    "Framework :: Apache Airflow",
+    "Framework :: Apache Airflow :: Provider",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: System :: Monitoring",
+]
+requires-python = "~=3.9"
+
+# The dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+dependencies = [
+    "apache-airflow>=2.9.0",
+    "apache-airflow-providers-common-compat>=1.1.0",
+    "apache-airflow-providers-common-sql>=1.20.0",
+    # In pandas 2.2 minimal version of the sqlalchemy is 2.0
+    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
+    # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
+    # In addition FAB also limit sqlalchemy to < 2.0
+    "pandas>=2.1.2,<2.2",
+    "pyarrow>=14.0.1",
+    "snowflake-connector-python>=3.7.1",
+    "snowflake-sqlalchemy>=1.4.0",
+    "snowflake-snowpark-python>=1.17.0;python_version<'3.12'",
+]
+
+# The optional dependencies should be modified in place in the generated file
+# Any change in the dependencies is preserved when the file is regenerated
+[project.optional-dependencies]
+"openlineage" = [
+    "apache-airflow-providers-openlineage"
+]
+
+[project.urls]
+"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.0.0"
+"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-snowflake/6.0.0/changelog.html"
+"Bug Tracker" = "https://github.com/apache/airflow/issues"
+"Source Code" = "https://github.com/apache/airflow"
+"Slack Chat" = "https://s.apache.org/airflow-slack"
+"Twitter" = "https://x.com/ApacheAirflow"
+"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/"
+
+[project.entry-points."apache_airflow_provider"]
+provider_info = "airflow.providers.snowflake.get_provider_info:get_provider_info"
+
+[tool.flit.module]
+name = "airflow.providers.snowflake"
+
+[tool.pytest.ini_options]
+ignore = "tests/system/"