Release 1.5.1 (#17)

* Extracted Java Code out for DP Module * Upgraded Java Compatibility to 17+ * @Nojipiz added feature to stream large datasets from BigQuery using ZIO Streams --------- Co-authored-by: Orlando Vargas <nojipiz@protonmail.com>
tharwaninitin · May 17, 2024 · d2c6c5a · d2c6c5a
1 parent 8642073
commit d2c6c5a
Show file tree

Hide file tree

Showing 28 changed files with 453 additions and 250 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,8 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 11, 14 ]
-#       removed version 8 temporarily to avoid sbt-updates failing because of the latest version of LogbackVersion
+        java-version: [ 17, 21 ]
       fail-fast: false
     steps:
     - uses: actions/checkout@v2
@@ -21,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 8, 11, 14 ]
+        java-version: [ 17, 21 ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -36,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 8, 11, 14 ]
+        java-version: [ 17, 21 ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -51,7 +50,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 8, 11, 14 ]
+        java-version: [ 17, 21 ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -66,7 +65,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 8, 11, 14 ]
+        java-version: [ 17, 21 ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml
@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 11, 8, 14 ]
+        java-version: [ 17 ]
       fail-fast: false
     steps:
       - name: Checkout Code

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -1,4 +1,4 @@
-name: Scala Code Linting Check
+name: Scala/Java Code Linting Check
 on: [push]
 jobs:
   scalafmt:
@@ -10,7 +10,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: 11
+          java-version: 21
       - name: Check scalafmt
         run: sbt scalafmtCheckAll
   scalastyle:
@@ -22,6 +22,18 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: 11
+          java-version: 21
       - name: Check scalastyle
         run: sbt "scalastyle; Test/scalastyle"
+  javafmt:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+      - name: Set up JDK
+        uses: actions/setup-java@v2
+        with:
+          distribution: 'adopt'
+          java-version: 21
+      - name: Check javafmt
+        run: sbt javafmtCheckAll
diff --git a/.github/workflows/mdoc.yml b/.github/workflows/mdoc.yml
@@ -10,6 +10,6 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: 11
+          java-version: 21
       - name: Check Documentation
         run: sbt "docs/mdoc --check"
diff --git a/.github/workflows/publishLocal.yml b/.github/workflows/publishLocal.yml
@@ -0,0 +1,37 @@
+name: Publish Local CI
+on: [push]
+jobs:
+  publish213:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [ 21 ]
+      fail-fast: false
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: ${{ matrix.java-version }}
+          cache: 'sbt'
+      - name: Publish Local for 2.13
+        run: sbt "++ 2.13.12 publishLocal"
+  publish3:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        java-version: [21]
+      fail-fast: false
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'adopt'
+          java-version: ${{ matrix.java-version }}
+          cache: 'sbt'
+      - name: Publish Local for 3
+        run: sbt "++ 3.3.1 publishLocal"
diff --git a/.github/workflows/semver.yml b/.github/workflows/semver.yml
@@ -9,7 +9,7 @@ jobs:
       uses: actions/setup-java@v3
       with:
         distribution: 'adopt'
-        java-version: 11
+        java-version: 21
         cache: 'sbt'
 #    - name: Run Semantic Versioning Policy Check for BQ
 #      run: sbt bq/versionPolicyCheck

diff --git a/.github/workflows/updates.yml b/.github/workflows/updates.yml
@@ -10,6 +10,6 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: 11
+          java-version: 21
       - name: Check Dependency Updates
         run: sbt dependencyUpdates
diff --git a/README.md b/README.md
@@ -8,32 +8,32 @@
 
 Add the latest release as a dependency to your project
 
-| Module               | Latest Version                                                                                                                                                                                                   |                                                                                                                                                                 Documentation | Scala Versions                                                                                                                                                                                                          | 
-|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| Google Cloud Storage | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-gcs_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-gcs)               |               [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-gcs_2.12/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-gcs_2.12) | [![gcp4zio-gcs Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-gcs/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-gcs)                      |
-| Dataproc             | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-dp_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-dp)                 |                 [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-dp_2.12/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-dp_2.12) | [![gcp4zio-dp Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-dp/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-dp)                         |
-| BigQuery             | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-bq_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-bq)                 |                 [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-bq_2.12/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-bq_2.12) | [![gcp4zio-bq Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-bq/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-bq)                         |
-| PubSub               | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-pubsub_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-pubsub)         |         [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-pubsub_2.12/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-pubsub_2.12) | [![gcp4zio-pubsub Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-pubsub/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-pubsub)             |
-| Cloud Monitoring     | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-monitoring_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-monitoring) | [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-monitoring_2.12/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-monitoring_2.12) | [![gcp4zio-monitoring Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-monitoring/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-monitoring) |
+| Module               | Latest Version                                                                                                                                                                                                   |                                                                                                                                                           Documentation | Scala Versions                                                                                                                                                                                                          | 
+|----------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Google Cloud Storage | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-gcs_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-gcs)               |               [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-gcs_3/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-gcs_3) | [![gcp4zio-gcs Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-gcs/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-gcs)                      |
+| Dataproc             | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-dp_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-dp)                 |                 [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-dp_3/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-dp_3) | [![gcp4zio-dp Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-dp/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-dp)                         |
+| BigQuery             | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-bq_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-bq)                 |                 [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-bq_3/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-bq_3) | [![gcp4zio-bq Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-bq/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-bq)                         |
+| PubSub               | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-pubsub_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-pubsub)         |         [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-pubsub_3/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-pubsub_3) | [![gcp4zio-pubsub Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-pubsub/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-pubsub)             |
+| Cloud Monitoring     | [![Latest Version](https://maven-badges.herokuapp.com/maven-central/com.github.tharwaninitin/gcp4zio-monitoring_2.12/badge.svg)](https://mvnrepository.com/artifact/com.github.tharwaninitin/gcp4zio-monitoring) | [![javadoc](https://javadoc.io/badge2/com.github.tharwaninitin/gcp4zio-monitoring_3/javadoc.svg)](https://javadoc.io/doc/com.github.tharwaninitin/gcp4zio-monitoring_3) | [![gcp4zio-monitoring Scala version support](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-monitoring/latest-by-scala-version.svg)](https://index.scala-lang.org/tharwaninitin/gcp4zio/gcp4zio-monitoring) |
 
-This project is compiled with scala versions 2.12.18, 2.13.12, 3.3.1
+This project is tested with scala versions 2.13.12, 3.3.1 and java versions 17, 21
 
 __SBT__
 ``` scala mdoc
 libraryDependencies ++= List(
-      "com.github.tharwaninitin" %% "gcp4zio-gcs" % "1.5.0",
-      "com.github.tharwaninitin" %% "gcp4zio-dp"  % "1.5.0",
-      "com.github.tharwaninitin" %% "gcp4zio-bq"  % "1.5.0",
-      "com.github.tharwaninitin" %% "gcp4zio-pubsub"  % "1.5.0",
-      "com.github.tharwaninitin" %% "gcp4zio-monitoring"  % "1.5.0"
+      "com.github.tharwaninitin" %% "gcp4zio-gcs" % "1.5.1",
+      "com.github.tharwaninitin" %% "gcp4zio-dp"  % "1.5.1",
+      "com.github.tharwaninitin" %% "gcp4zio-bq"  % "1.5.1",
+      "com.github.tharwaninitin" %% "gcp4zio-pubsub"  % "1.5.1",
+      "com.github.tharwaninitin" %% "gcp4zio-monitoring"  % "1.5.1"
    )
 ```
 __Maven__
 ```
 <dependency>
     <groupId>com.github.tharwaninitin</groupId>
     <artifactId>gcp4zio-gcs_2.12</artifactId>
-    <version>1.5.0</version>
+    <version>1.5.1</version>
 </dependency>
 ```
 # GCP4ZIO API's
@@ -123,14 +123,14 @@ GCS.copyObjectsGCStoGCS(
 import gcp4zio.dp._
 
 // Create Dataproc Cluster Properties
-val dpProps = ClusterProps(bucketName = "dpLogBucket")
+val props = new ClusterProps("dataproc-logs")
 // Create Dataproc Cluster
-val createTask = DPCluster.createDataproc("dpCluster", dpProps)
+val createTask = DPCluster.createDataproc("cluster1", props)
 
 // Delete Dataproc Cluster
 val deleteTask = DPCluster.deleteDataproc("dpCluster")
 
-(createTask *> deleteTask).provide(DPCluster.live("gcpProject", "gcpRegion", "dpEndpoint"))
+(createTask *> deleteTask).provide(DPCluster.live("gcpProject", "gcpRegion"))
 ```  
 
 ### Dataproc Job

diff --git a/build.sbt b/build.sbt
@@ -8,6 +8,9 @@ lazy val commonSettings = Seq(
   scalaVersion               := scala212,
   crossScalaVersions         := allScalaVersions,
   dependencyUpdatesFailBuild := true,
+  compileOrder               := CompileOrder.JavaThenScala,
+//  scalacOptions ++= Seq("-target:11"),
+//  scalacOptions ++= Seq("-release", "25"),
   dependencyUpdatesFilter -= moduleFilter(organization = "org.scala-lang"),
   scalacOptions ++= {
     CrossVersion.partialVersion(scalaVersion.value) match {
@@ -35,7 +38,7 @@ lazy val gcp4zio = (project in file("."))
     crossScalaVersions := Nil, // crossScalaVersions must be set to Nil on the aggregating project
     publish / skip     := true
   )
-  .aggregate(bq, dp, gcs, pubsub, batch, monitoring)
+  .aggregate(bq, dp, gcs, pubsub, batch, monitoring, examples)
 
 lazy val bq = (project in file("modules/bq"))
   .settings(commonSettings)
@@ -76,7 +79,7 @@ lazy val docs = project
   .settings(
     name           := "gcp4zio-docs",
     publish / skip := true,
-    mdocVariables  := Map("VERSION" -> version.value, "Scala212" -> scala212, "Scala213" -> scala213, "Scala3" -> scala3),
+    mdocVariables  := Map("VERSION" -> version.value, "Scala212" -> scala212, "Scala213" -> scala213, "Scala3" -> scala3, "JavaVersions" -> "17, 21"),
     mdocIn         := new File("docs/readme.template.md"),
     mdocOut        := new File("README.md")
   )