[DOP-13337] Add Pydantic v2 support

MobileTeleSystems · Mar 5, 2024 · ac8ed3a · ac8ed3a
1 parent a994486
commit ac8ed3a
Show file tree

Hide file tree

Showing 106 changed files with 656 additions and 246 deletions.
diff --git a/.github/workflows/data/base/tracked.txt b/.github/workflows/data/base/tracked.txt
@@ -3,4 +3,5 @@
 .github/workflows/data/base/**
 requirements/core.txt
 requirements/tests/base.txt
+requirements/tests/pydantic-*.txt
 .env.local
diff --git a/.github/workflows/data/clickhouse/matrix.yml b/.github/workflows/data/clickhouse/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/core/matrix.yml b/.github/workflows/data/core/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/ftp/matrix.yml b/.github/workflows/data/ftp/matrix.yml
@@ -1,8 +1,15 @@
 min: &min
+  pydantic-version: 1
   python-version: '3.7'
   os: ubuntu-latest
 
 max: &max
+  pydantic-version: 2
+  python-version: '3.12'
+  os: ubuntu-latest
+
+latest: &latest
+  pydantic-version: latest
   python-version: '3.12'
   os: ubuntu-latest
 
@@ -11,9 +18,13 @@ matrix:
   # chonjay21/ftps image has only latest tag
     - ftp-version: latest
       <<: *max
-  full: &full
+  full:
     - ftp-version: latest
       <<: *min
     - ftp-version: latest
       <<: *max
-  nightly: *full
+  nightly:
+    - ftp-version: latest
+      <<: *min
+    - ftp-version: latest
+      <<: *latest
diff --git a/.github/workflows/data/ftps/matrix.yml b/.github/workflows/data/ftps/matrix.yml
@@ -1,8 +1,15 @@
 min: &min
+  pydantic-version: 1
   python-version: '3.7'
   os: ubuntu-latest
 
 max: &max
+  pydantic-version: 2
+  python-version: '3.12'
+  os: ubuntu-latest
+
+latest: &latest
+  pydantic-version: latest
   python-version: '3.12'
   os: ubuntu-latest
 
@@ -11,9 +18,13 @@ matrix:
   # chonjay21/ftps image has only latest tag
     - ftps-version: latest
       <<: *max
-  full: &full
+  full:
     - ftps-version: latest
       <<: *min
     - ftps-version: latest
       <<: *max
-  nightly: *full
+  nightly:
+    - ftps-version: latest
+      <<: *min
+    - ftps-version: latest
+      <<: *latest
diff --git a/.github/workflows/data/greenplum/matrix.yml b/.github/workflows/data/greenplum/matrix.yml
@@ -1,13 +1,23 @@
 min: &min
   # Spark 2.3.0 does not support passing ivysettings.xml
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   # Greenplum connector does not support Spark 3.3+
   spark-version: 3.2.4
+  pydantic-version: 2
+  python-version: '3.10'
+  java-version: 11
+  os: ubuntu-latest
+
+latest: &latest
+  # Greenplum connector does not support Spark 3.3+
+  spark-version: 3.2.4
+  pydantic-version: latest
   python-version: '3.10'
   java-version: 11
   os: ubuntu-latest
@@ -16,9 +26,13 @@ matrix:
   small:
     - greenplum-version: 7.0.0
       <<: *max
-  full: &full
+  full:
     - greenplum-version: 6.25.3
       <<: *min
     - greenplum-version: 7.0.0
       <<: *max
-  nightly: *full
+  nightly:
+    - greenplum-version: 6.25.3
+      <<: *min
+    - greenplum-version: 7.0.0
+      <<: *latest
diff --git a/.github/workflows/data/hdfs/matrix.yml b/.github/workflows/data/hdfs/matrix.yml
@@ -1,20 +1,23 @@
 min: &min
   hadoop-version: hadoop2-hdfs
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   hadoop-version: hadoop3-hdfs
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   hadoop-version: hadoop3-hdfs
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/hive/matrix.yml b/.github/workflows/data/hive/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/kafka/matrix.yml b/.github/workflows/data/kafka/matrix.yml
@@ -1,20 +1,23 @@
 min: &min
   # kafka_version: 0.10.2-1-r3
   kafka-version: 3.5.1
+  pydantic-version: 1
   spark-version: 2.4.8
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   kafka-version: 3.5.1
+  pydantic-version: 2
   spark-version: 3.5.0
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   kafka-version: latest
+  pydantic-version: latest
   spark-version: latest
   python-version: '3.12'
   java-version: 20

diff --git a/.github/workflows/data/local-fs/matrix.yml b/.github/workflows/data/local-fs/matrix.yml
@@ -1,29 +1,34 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 min_avro: &min_avro
   spark-version: 2.4.8
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 min_excel: &min_excel
   spark-version: 3.2.4
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/mongodb/matrix.yml b/.github/workflows/data/mongodb/matrix.yml
@@ -1,18 +1,21 @@
 min: &min
   # MongoDB connector does not support Spark 2
   spark-version: 3.2.4
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.4.2
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/mssql/matrix.yml b/.github/workflows/data/mssql/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/mysql/matrix.yml b/.github/workflows/data/mysql/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/oracle/matrix.yml b/.github/workflows/data/oracle/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/postgres/matrix.yml b/.github/workflows/data/postgres/matrix.yml
@@ -1,17 +1,20 @@
 min: &min
   spark-version: 2.3.1
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest

diff --git a/.github/workflows/data/s3/matrix.yml b/.github/workflows/data/s3/matrix.yml
@@ -3,20 +3,23 @@ min: &min
   minio-version: 2021.3.17
   # Minimal Spark version with Hadoop 3.x support
   spark-version: 3.2.4
+  pydantic-version: 1
   python-version: '3.7'
   java-version: 8
   os: ubuntu-latest
 
 max: &max
   minio-version: 2023.7.18
   spark-version: 3.5.0
+  pydantic-version: 2
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest
 
 latest: &latest
   minio-version: latest
   spark-version: latest
+  pydantic-version: latest
   python-version: '3.12'
   java-version: 20
   os: ubuntu-latest