diff --git a/.bsp/sbt.json b/.bsp/sbt.json new file mode 100644 index 0000000..00eb150 --- /dev/null +++ b/.bsp/sbt.json @@ -0,0 +1 @@ +{"name":"sbt","version":"1.8.3","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/home/maxim/.sdkman/candidates/java/11.0.22-tem/bin/java","-Xms100m","-Xmx100m","-classpath","/home/maxim/.sdkman/candidates/sbt/1.7.1/bin/sbt-launch.jar","-Dsbt.script=/home/maxim/.sdkman/candidates/sbt/current/bin/sbt","xsbt.boot.Boot","-bsp"]} \ No newline at end of file diff --git a/.github/workflows/codeql-check.yml b/.github/workflows/codeql-check.yml index 4b043b8..31cc64b 100644 --- a/.github/workflows/codeql-check.yml +++ b/.github/workflows/codeql-check.yml @@ -18,68 +18,22 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Cache SBT - uses: actions/cache@v3 - with: - path: | - ~/.ivy2/cache - ~/.sbt - key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }} - - name: Setup JDK 8 uses: actions/setup-java@v4 with: java-version: '8' distribution: 'temurin' - - name: Setup Scala and SBT - uses: olafurpg/setup-scala@v14 - with: - java-version: '8' - - - name: Check and format code with Scalafmt - run: sbt scalafmtAll - - - name: Check and fix code with Scalafix - run: sbt scalafixAll + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 - - name: Check for changes and commit - id: git-check - run: | - if [ -n "$(git status --porcelain | grep -v '^??')" ]; then - echo "changes=true" >> $GITHUB_ENV # Use environment file to set output - git config --global user.name 'GitHub Actions Bot' - git config --global user.email 'actions@github.com' - git add -u - git commit -m "Apply code quality checks (auto-formatted)" - git fetch --prune - git pull --rebase origin ${{ github.head_ref }} - git push origin HEAD:${{ github.head_ref }} - else - echo "No changes to commit." - fi - env: - CI: true - - - name: Post retry comment - if: env.changes == 'true' - uses: actions/github-script@v7 + - name: Cache Gradle + uses: actions/cache@v3 with: - script: | - const prNumber = context.payload.pull_request ? context.payload.pull_request.number : null; - if (prNumber) { - const { data: pr } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - const newBody = pr.body + '\n\nAuto-commit applied. A retry was triggered due to a failure in formatting/linting checks.'; - await github.rest.pulls.update({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - body: newBody - }); - } else { - console.log('No pull request context found, unable to update PR description.'); - } \ No newline at end of file + path: | + ~/.ivy2/cache + ~/.gradle + key: ${{ runner.os }}-gradle-${{ hashFiles('**/build.gradle') }} + + - name: Run all checks + run: ./gradlew check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ca96b5..9b456a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,21 +11,24 @@ jobs: with: fetch-depth: 0 - - uses: actions/setup-java@v4 + - name: Setup JDK 8 + uses: actions/setup-java@v4 with: - distribution: temurin - java-version: 8 - cache: sbt + java-version: '8' + distribution: 'temurin' - - name: Cache SBT + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 + + - name: Cache Gradle uses: actions/cache@v3 with: path: | ~/.ivy2/cache - ~/.sbt - key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }} + ~/.gradle + key: ${{ runner.os }}-gradle-${{ hashFiles('**/build.gradle') }} - - run: sbt ci-release + - run: ./gradlew publishToSonatype env: PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} PGP_SECRET: ${{ secrets.PGP_SECRET }} @@ -41,4 +44,4 @@ jobs: prerelease: false token: ${{ secrets.GITHUB_TOKEN }} files: | - target/scala-2.12/*.jar + build/libs/*.jar diff --git a/.github/workflows/tests_clickhouse.yml b/.github/workflows/tests_clickhouse.yml index d8b6948..a423245 100644 --- a/.github/workflows/tests_clickhouse.yml +++ b/.github/workflows/tests_clickhouse.yml @@ -24,36 +24,30 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Cache SBT - uses: actions/cache@v3 - with: - path: | - ~/.ivy2/cache - ~/.sbt - key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt') }} - - name: Setup JDK 8 uses: actions/setup-java@v4 with: java-version: '8' distribution: 'temurin' - - name: Setup Scala and SBT - uses: olafurpg/setup-scala@v14 - with: - java-version: '8' + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v3 - - name: Compile .jar package with SBT - run: | - sbt package + - name: Cache Gradle + uses: actions/cache@v3 + with: + path: | + ~/.ivy2/cache + ~/.gradle + key: ${{ runner.os }}-gradle-${{ hashFiles('**/build.gradle') }} - name: Run Integration Tests with Coverage run: | - sbt clean coverage test coverageReport + ./gradlew test - name: Upload Coverage to Codecov uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: target/scala-2.12/scoverage-report/scoverage.xml + files: build/reports/jacoco/test/*.xml fail_ci_if_error: true diff --git a/.gitignore b/.gitignore index b158582..d7706d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,19 +3,6 @@ *.log *~ -# sbt specific -dist/* -target/ -lib_managed/ -src_managed/ -project/boot/ -project/plugins/project/ -project/local-plugins.sbt -.history -.bsp -.metals -.bloop - # Scala-IDE specific .scala_dependencies .cache @@ -35,3 +22,8 @@ classes/ # Visual Studio Code .vscode + +# Gradle-specific +.gradle/ +gradle/ +build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c92dfd..cbefbc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,18 +6,17 @@ This document provides detailed steps to build the Spark Dialect Extension from Before you start, ensure you have the following installed: - **Java**: Java 8 or higher. [Java Installation Guide](https://adoptopenjdk.net/) -- **Scala**: [Scala Installation Guide](https://scala-lang.org/download/) -- **SBT**: [SBT Installation Guide](https://www.scala-sbt.org/download.html) +- **Gradle**: [Gradle Installation Guide](https://docs.gradle.org/current/userguide/installation.html) ### Compile the Project To compile the project and generate a JAR file, run the following command in the project's root directory: ```bash -sbt package +./gradlew crossBuildV212Jar crossBuildV213Jar ``` -This command compiles the source code and packages it into a .jar file located in the ``target/scala-2.12`` directory. +This command compiles the source code and packages it into a .jar files located in the ``build/libs`` directory. ## Running Scala Tests @@ -33,20 +32,14 @@ docker-compose -f docker-compose.test.yml up -d ``` ### Execute Tests -To run the Scala tests, execute: -```bash -sbt test -``` - -### With Coverage Report -To run the tests with coverage and generate a report, use: +To run the Scala tests, execute: ```bash -sbt clean coverage test coverageReport +./gradlew test ``` -After the tests, you can view the coverage report by opening the ``target/scala-2.12/scoverage-report/index.html`` file in your web browser. +After the tests, you can view the coverage report by opening the ``build/reports/tests/test/index.html`` file in your web browser. ### Stopping Docker Containers After completing the tests, you can stop the Docker containers with: @@ -61,14 +54,14 @@ docker-compose -f docker-compose.test.yml down To format all Scala source files in the project, execute the following command from the project's root directory: ```bash -sbt scalafmtAll +./gradlew scalafmtAll ``` ## Using Scalafix for Linting and Refactoring To lint and refactor the code, run Scalafix using the following command: ```bash -sbt scalafixAll +./gradlew scalafix ``` This command checks the code against various rules specified in the ```.scalafix.conf``` file and applies fixes where possible. @@ -86,7 +79,7 @@ git pull -p 2. Copy version (it must start with **v**, e.g. **v1.0.0**) ```bash -VERSION=$(cat VERSION) +VERSION=$(./gradlew -q printVersion) ``` 3. Commit and push changes to ``develop`` branch @@ -118,7 +111,7 @@ git push origin "$VERSION" ```bash git checkout develop NEXT_VERSION=$(echo "$VERSION" | awk -F. '/[0-9]+\./{$NF++;print}' OFS=.) -echo "$NEXT_VERSION" > VERSION +sed -i "s/version = \".*\"/version = \"$NEXT_VERSION\"/" build.gradle git add . git commit -m "Bump version" git push diff --git a/VERSION b/VERSION deleted file mode 100644 index 45c7a58..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -v0.0.1 diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..f7537d8 --- /dev/null +++ b/build.gradle @@ -0,0 +1,156 @@ +plugins { + id "scala" + id "java-library" + id "maven-publish" + id "signing" + id "jacoco" + id "io.github.gradle-nexus.publish-plugin" version "2.0.0" + id "net.nemerosa.versioning" version "3.1.0" + // Scala support + id "com.github.maiflai.scalatest" version "0.32" + id "cz.augi.gradle.scalafmt" version "1.21.3" + id "io.github.cosmicsilence.scalafix" version "0.2.2" +} + +group = "io.github.mtsongithub.doetl" +// TODO: crossbuild for Scala 2.13 +archivesBaseName = "spark-dialect-extension_2.12" +version = "0.0.1" +description = "Spark dialect extension for enhanced type handling" + +repositories { + mavenCentral() +} + +dependencies { + // Scala support + implementation "org.scala-lang:scala-library:2.12.19" + // core Spark dependencies + // TODO: crossbuild for Spark 3.2-3.5 + implementation "org.apache.spark:spark-core_2.12:3.5.1" + implementation "org.apache.spark:spark-sql_2.12:3.5.1" + // latest compatible with Java 8 + implementation "ch.qos.logback:logback-classic:1.3.14" + + // test dependencies + testImplementation "org.scalatest:scalatest_2.12:3.2.19" + testImplementation "com.vladsch.flexmark:flexmark-all:0.64.8" + testImplementation "org.mockito:mockito-scala_2.12:1.17.37" + // latest compatible with Java 8 + testImplementation "io.github.cdimascio:dotenv-java:2.3.2" + + // Clickhouse dependency + testImplementation "com.clickhouse:clickhouse-jdbc:0.6.0-patch5" + testImplementation "org.apache.httpcomponents.client5:httpclient5:5.3.1" + testImplementation "org.apache.httpcomponents.core5:httpcore5:5.1.5" +} + +ext { + isReleaseVersion = !version.endsWith("SNAPSHOT") +} + +test { + finalizedBy jacocoTestReport +} + +jacocoTestReport { + dependsOn test + + reports { + xml.required = true + } +} + +tasks.withType(ScalaCompile) { + scalaCompileOptions.additionalParameters = ["-Ywarn-unused"] +} + +getTasksByName("test", true).each { task -> + task.dependsOn(jar) +} + +task printVersion { + doLast { + println project.version + } +} + +task scaladocJar(type: Jar, dependsOn: scaladoc) { + archiveClassifier = "scaladoc" + from scaladoc.destinationDir +} + +java { + targetCompatibility = JavaVersion.VERSION_1_8 + withSourcesJar() +} + +jar { + manifest { + attributes( + "Implementation-Title": project.name, + "Implementation-Version": project.version, + "Implementation-Vendor": "MTS PJSC", + "Created-By": "Gradle ${gradle.gradleVersion}", + "Built-By": System.properties["user.name"], + "Build-Timestamp": versioning.info.time, + "Build-Revision": versioning.info.commit, + "Build-Jdk": "${System.properties["java.version"]} (${System.properties["java.vendor"]} ${System.properties["java.vm.version"]})", + "Build-OS": "${System.properties["os.name"]} ${System.properties["os.arch"]} ${System.properties["os.version"]}", + ) + } +} + +publishing { + publications { + mavenCentral(MavenPublication) { + from(components.java) + artifactId = "spark-dialect-extension_2.12" + + artifact scaladocJar + + pom { + name = "spark-dialect-extension" + url = "https://theplatform.ru/platforms/dataops" + licenses { + license { + name = "Apache-2.0" + url = "http://www.apache.org/licenses/LICENSE-2.0" + } + } + developers { + developer { + id = "doetl" + name = "DataOps.ETL" + email = "onetools@mts.ru" + } + } + scm { + connection = "scm:git:git://github.com/MobileTeleSystems/spark-dialect-extension.git" + developerConnection = "scm:git:ssh://github.com/MobileTeleSystems/spark-dialect-extension.git" + url = "https://github.com/MobileTeleSystems/spark-dialect-extension" + } + } + + withBuildIdentifier() + } + } +} + + +nexusPublishing { + repositories { + sonatype { + username = System.getenv("SONATYPE_USERNAME") + password = System.getenv("SONATYPE_PASSWORD") + } + } +} + +signing { + required { isReleaseVersion } + def signingKey = System.getenv("PGP_SECRET") + def signingPassword = System.getenv("PGP_PASSPHRASE") + useInMemoryPgpKeys(signingKey, signingPassword) + sign publishing.publications.mavenCentral +} diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 5e8cf3f..0000000 --- a/build.sbt +++ /dev/null @@ -1,46 +0,0 @@ -import xerial.sbt.Sonatype.sonatypeCentralHost - -ThisBuild / organization := "io.github.mtsongithub.doetl" -ThisBuild / organizationName := "MTS PJSC" -ThisBuild / description := "Spark dialect extension for enhanced type handling." -ThisBuild / homepage := Some(url("https://theplatform.ru/platforms/dataops")) -ThisBuild / startYear := Some(2024) -ThisBuild / licenses := List("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0")) -ThisBuild / sonatypeCredentialHost := sonatypeCentralHost - -// minimum compatible version with scalafix -ThisBuild / scalaVersion := "2.12.19" - -lazy val commonSettings = Seq( - semanticdbEnabled := true, - semanticdbVersion := scalafixSemanticdb.revision -) - -lazy val root = (project in file(".")) - .settings(commonSettings) - .settings( - name := "spark-dialect-extension", - - // warn unused params (needed for scalafix) - scalacOptions += "-Ywarn-unused", - - libraryDependencies ++= Seq( - // core Spark dependencies - "org.apache.spark" %% "spark-core" % "3.5.1", - "org.apache.spark" %% "spark-sql" % "3.5.1", - - // latest compatible with Java 8 - "ch.qos.logback" % "logback-classic" % "1.3.14", - - // test dependencies - "org.scalatest" %% "scalatest" % "3.2.19" % "test", - "org.mockito" %% "mockito-scala" % "1.17.37" % "test", - // latest compatible with Java 8 - "io.github.cdimascio" % "dotenv-java" % "2.3.2" % "test", - - // clickhouse dependency - "com.clickhouse" % "clickhouse-jdbc" % "0.6.0-patch5" % "test", - "org.apache.httpcomponents.client5" % "httpclient5" % "5.3.1" % "test", - "org.apache.httpcomponents.core5" % "httpcore5" % "5.1.5" % "test", - ) - ) diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..f5feea6 --- /dev/null +++ b/gradlew @@ -0,0 +1,252 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..9b42019 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,94 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/project/build.properties b/project/build.properties deleted file mode 100644 index 72413de..0000000 --- a/project/build.properties +++ /dev/null @@ -1 +0,0 @@ -sbt.version=1.8.3 diff --git a/project/plugins.sbt b/project/plugins.sbt deleted file mode 100644 index f54cb7c..0000000 --- a/project/plugins.sbt +++ /dev/null @@ -1,4 +0,0 @@ -addSbtPlugin("org.scalameta" %% "sbt-scalafmt" % "2.5.2") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.1.0") -addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.12.1") -addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.6.1") diff --git a/src/test/scala/io/github/mtsongithub/doetl/sparkdialectextensions/SharedSparkSession.scala b/src/test/scala/io/github/mtsongithub/doetl/sparkdialectextensions/SharedSparkSession.scala index f780599..724c482 100644 --- a/src/test/scala/io/github/mtsongithub/doetl/sparkdialectextensions/SharedSparkSession.scala +++ b/src/test/scala/io/github/mtsongithub/doetl/sparkdialectextensions/SharedSparkSession.scala @@ -18,7 +18,7 @@ trait SharedSparkSession extends BeforeAndAfterAll { self: Suite => val scalaVersion: String = scala.util.Properties.versionNumberString val majorScalaVersion: String = scalaVersion.split('.').take(2).mkString(".") val jarFiles = - Paths.get("target", s"scala-$majorScalaVersion").toFile.listFiles().filter(_.isFile) + Paths.get("build", "libs").toFile.listFiles().filter(_.getName.contains(s"_$majorScalaVersion-")).filter(_.isFile) val jarPaths = jarFiles.map(_.getAbsolutePath).mkString(",") _spark = SparkSession @@ -27,7 +27,6 @@ trait SharedSparkSession extends BeforeAndAfterAll { self: Suite => .appName("Spark Test Session") .config("spark.ui.enabled", "false") // disable UI to reduce overhead .config("spark.jars", jarPaths) // include the JAR file containing the custom dialect - .config("spark.driver.bindAddress", "127.0.0.1") .getOrCreate() // register custom Clickhouse dialect