From 466655a8b20576f9486022f10263948edbf98f7c Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 6 Mar 2023 15:12:29 -0800 Subject: [PATCH 1/5] Upgrade to 22.04 and publish jars upon commit --- .github/workflows/docs.yml | 2 +- .github/workflows/example.yml | 2 +- .github/workflows/java.yml | 18 ++++++++++++++---- .github/workflows/python.yml | 2 +- .github/workflows/r.yml | 2 +- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index b6cba5bdb1..9211cde651 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml index dd1bcb0515..eccc97ddc9 100644 --- a/.github/workflows/example.yml +++ b/.github/workflows/example.yml @@ -10,7 +10,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 6f25ce9092..76ddbb4080 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -11,7 +11,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: fail-fast: true matrix: @@ -62,11 +62,21 @@ jobs: SKIP_TESTS: ${{ matrix.skipTests }} run: mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} ${SKIP_TESTS} - run: mkdir staging - - run: cp core/target/sedona-*.jar staging - - run: cp sql/target/sedona-*.jar staging - run: cp viz/target/sedona-*.jar staging - - run: cp python-adapter/target/sedona-*.jar staging + - run: cp spark-shaded/target/sedona-*.jar staging + - run: cp flink-shaded/target/sedona-*.jar staging - uses: actions/upload-artifact@v2 with: name: generated-jars path: staging + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: true + title: "Latest Development Build" + files: | + LICENSE + viz/target/sedona-*.jar + spark-shaded/target/sedona-*.jar + flink-shaded/target/sedona-*.jar \ No newline at end of file diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 41e9b16a11..fe33e9646d 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -11,7 +11,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: include: diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml index 32acdfe306..e5d46b5843 100644 --- a/.github/workflows/r.yml +++ b/.github/workflows/r.yml @@ -11,7 +11,7 @@ on: jobs: build: - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: fail-fast: true matrix: From 4b635db85cd9e247a9574d9ba2be8dae5db5c360 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 6 Mar 2023 15:29:21 -0800 Subject: [PATCH 2/5] Fix GitHub Actions --- .github/workflows/java.yml | 13 +------------ .github/workflows/python.yml | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 76ddbb4080..42177f0cad 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -68,15 +68,4 @@ jobs: - uses: actions/upload-artifact@v2 with: name: generated-jars - path: staging - - uses: "marvinpinto/action-automatic-releases@latest" - with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "latest" - prerelease: true - title: "Latest Development Build" - files: | - LICENSE - viz/target/sedona-*.jar - spark-shaded/target/sedona-*.jar - flink-shaded/target/sedona-*.jar \ No newline at end of file + path: staging \ No newline at end of file diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index fe33e9646d..85a89117e0 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -71,7 +71,7 @@ jobs: SPARK_VERSION: ${{ matrix.spark }} HADOOP_VERSION: ${{ matrix.hadoop }} run: tar -xzf spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz - - run: sudo apt-get -y install python3-pip python-dev + - run: sudo apt-get -y install python3-pip python-dev-is-python3 - run: sudo pip3 install -U setuptools - run: sudo pip3 install -U wheel - run: sudo pip3 install -U virtualenvwrapper From fd296bfd07d14a64c6eeae5c1f724b24c66003a4 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 6 Mar 2023 21:25:35 -0800 Subject: [PATCH 3/5] Fix the github action --- .github/workflows/java.yml | 6 +++--- .github/workflows/python.yml | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 42177f0cad..f89c55eb00 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -27,7 +27,7 @@ jobs: - spark: 3.3.0 scala: 2.12.15 jdk: '11' - skipTests: '-DskipTests' + skipTests: '' - spark: 3.2.3 scala: 2.12.15 jdk: '8' @@ -55,7 +55,6 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - - run: git submodule update --init --recursive # Checkout Git submodule if necessary - env: SPARK_VERSION: ${{ matrix.spark }} SCALA_VERSION: ${{ matrix.scala }} @@ -64,7 +63,8 @@ jobs: - run: mkdir staging - run: cp viz/target/sedona-*.jar staging - run: cp spark-shaded/target/sedona-*.jar staging - - run: cp flink-shaded/target/sedona-*.jar staging + - run: | + [ -d "flink-shaded" ] && cp flink-shaded/target/sedona-*.jar staging - uses: actions/upload-artifact@v2 with: name: generated-jars diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 85a89117e0..4412d9420f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -58,7 +58,6 @@ jobs: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2 - - run: git submodule update --init --recursive # Checkout Git submodule if necessary - env: SPARK_VERSION: ${{ matrix.spark }} SCALA_VERSION: ${{ matrix.scala }} From 96c3f52eb0a265a516021b1ee1d8dc50531cdbf1 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 6 Mar 2023 22:04:20 -0800 Subject: [PATCH 4/5] Fix the action --- .github/workflows/java.yml | 2 +- python/tests/utils/test_crs_transformation.py | 24 ------------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index f89c55eb00..d768e16c3a 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -64,7 +64,7 @@ jobs: - run: cp viz/target/sedona-*.jar staging - run: cp spark-shaded/target/sedona-*.jar staging - run: | - [ -d "flink-shaded" ] && cp flink-shaded/target/sedona-*.jar staging + [ -d "flink-shaded/target/" ] && cp flink-shaded/target/sedona-*.jar staging - uses: actions/upload-artifact@v2 with: name: generated-jars diff --git a/python/tests/utils/test_crs_transformation.py b/python/tests/utils/test_crs_transformation.py index 9c7748b198..71b638b3d8 100644 --- a/python/tests/utils/test_crs_transformation.py +++ b/python/tests/utils/test_crs_transformation.py @@ -159,27 +159,3 @@ def test_spatial_join_query_with_polygon_rdd_using_index(self): if data[1].__len__() != 0: for right_data in data[1]: assert right_data.getUserData() is not None - - def test_polygon_distance_join_with_crs_transformation(self): - query_rdd = PolygonRDD( - self.sc, - input_location_query_polygon, splitter, True, - num_partitions, StorageLevel.MEMORY_ONLY, "epsg:4326", "epsg:3857" - ) - window_rdd = CircleRDD(query_rdd, 0.1) - - object_rdd = PolygonRDD( - self.sc, input_location_query_polygon, splitter, True, num_partitions, StorageLevel.MEMORY_ONLY, - "epsg:4326", "epsg:3857") - - object_rdd.rawJvmSpatialRDD.jsrdd.repartition(4) - object_rdd.spatialPartitioning(GridType.KDBTREE) - object_rdd.buildIndex(IndexType.RTREE, True) - window_rdd.spatialPartitioning(object_rdd.getPartitioner()) - - results = JoinQuery.DistanceJoinQuery(object_rdd, window_rdd, True, False).collect() - assert 5467 == results.__len__() - - for data in results: - for polygon_data in data[1]: - assert Circle(data[0].geom, 0.1).covers(polygon_data.geom) From 2fa54c8ac9983ffb6e7528d9b6284b7f16be187d Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Mon, 6 Mar 2023 22:34:15 -0800 Subject: [PATCH 5/5] Suppress the error message --- .github/workflows/java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index d768e16c3a..4b5daf0bfc 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -64,7 +64,7 @@ jobs: - run: cp viz/target/sedona-*.jar staging - run: cp spark-shaded/target/sedona-*.jar staging - run: | - [ -d "flink-shaded/target/" ] && cp flink-shaded/target/sedona-*.jar staging + [ -d "flink-shaded/target/" ] && cp flink-shaded/target/sedona-*.jar staging 2>/dev/null || true - uses: actions/upload-artifact@v2 with: name: generated-jars