Merge pull request #95 from jiayuasu/GeoSpark-for-Spark-1.X

Push GeoSpark 0.7.0

.travis.yml

@@ -2,6 +2,8 @@ os:
   - linux
 
 language: java
+jdk:
+  - oraclejdk8
 sudo: required
 git:
   submodules: false
@@ -30,4 +32,4 @@ notifications:
     recipients:
       - jiayu2@asu.edu
     on_success: always
-    on_failure: always
+    on_failure: always

README.md

@@ -3,7 +3,7 @@
 
 | Version   |      GeoSpark Core      |  [Babylon Viz System](https://github.com/DataSystemsLab/GeoSpark/tree/master/babylon)|
 |:----------:|:-------------:|:------:|
-| Stable |  0.6.2 | 0.2.0 |
+| Stable |  0.7.0 | 0.2.0 |
 | Latest |  [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.datasyslab/geospark/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.datasyslab/geospark)   |   [![Maven Central](https://maven-badges.herokuapp.com/maven-central/org.datasyslab/babylon/badge.svg)](https://maven-badges.herokuapp.com/maven-central/org.datasyslab/babylon) |
 | Source code |  [![Build Status](https://travis-ci.org/jiayuasu/GeoSpark.svg?branch=master)](https://travis-ci.org/jiayuasu/GeoSpark) | [![Build Status](https://travis-ci.org/jiayuasu/GeoSpark.svg?branch=master)](https://travis-ci.org/jiayuasu/GeoSpark) |
 
@@ -48,6 +48,11 @@ Inside, Overlap, DatasetBoundary, Minimum Bounding Rectangl, Polygon Union
 
 ## Spatial Operation
 Spatial Range Query, Spatial Join Query, and Spatial K Nearest Neighbors Query.
+
+## Coordinate Reference System (CRS) Transformation (aka. Coordinate projection)
+
+GeoSpark allows users to transform the original CRS (e.g., degree based coordinates such as EPSG:4326 and WGS84) to any other CRS (e.g., meter based coordinates such as EPSG:3005) so that it can accurately process both geographic data and geometrical data. Please specify your desired CRS in GeoSpark Spatial RDD constructor ([Example](https://github.com/DataSystemsLab/GeoSpark/blob/master/core/src/main/scala/org/datasyslab/geospark/showcase/ScalaExample.scala#L221)).
+
 ## Users
 
 ### Companies that are using GeoSpark (incomplete list)
@@ -107,4 +112,4 @@ Please visit [GeoSpark project wesbite](http://geospark.datasyslab.org) for late
 GeoSpark is one of the projects under [Data Systems Lab](http://www.datasyslab.org/) at Arizona State University. The mission of Data Systems Lab is designing and developing experimental data management systems (e.g., database systems).
 
 # Thanks for the help from GeoSpark community
-We appreciate the help and suggestions from GeoSpark users: [**Thanks List**](https://github.com/DataSystemsLab/GeoSpark/wiki/GeoSpark-Community-Thanks-List)
+We appreciate the help and suggestions from GeoSpark users: [**Thanks List**](https://github.com/DataSystemsLab/GeoSpark/wiki/GeoSpark-Community-Thanks-List)

core/pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.datasyslab</groupId>
 	<artifactId>geospark</artifactId>
-	<version>0.6.2-spark-1.x</version>
+	<version>0.7.0-spark-1.x</version>
 
 	<name>${project.groupId}:${project.artifactId}</name>
 	<description>Geospatial extension for Apache Spark</description>
@@ -41,14 +41,15 @@
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <scala.version>2.10.6</scala.version>
         <scala.compat.version>2.10</scala.compat.version>
+        <geotools.version>17.0</geotools.version>        
 	</properties>
 
 	<dependencies>
         <dependency>
 			<groupId>org.apache.spark</groupId>
 			<artifactId>spark-core_2.10</artifactId>
 			<version>1.6.3</version>
-			<scope>provided</scope>
+            <scope>provided</scope>
 		</dependency>
 
 		<dependency>
@@ -64,7 +65,7 @@
 			<exclusions>
 				<exclusion>
 			<groupId>com.vividsolutions</groupId>
-			<artifactId>jts</artifactId>
+			<artifactId>jts-core</artifactId>
 				</exclusion>
 			</exclusions>
 		</dependency>
@@ -73,8 +74,34 @@
   			<groupId>org.datasyslab</groupId>
   			<artifactId>sernetcdf</artifactId>
   			<version>0.1.0</version>
-		</dependency>
-
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-main</artifactId>
+            <version>${geotools.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.vividsolutions</groupId>
+                    <artifactId>jts</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-referencing</artifactId>
+            <version>${geotools.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-epsg-hsql</artifactId>
+            <version>${geotools.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.geotools</groupId>
+            <artifactId>gt-epsg-extension</artifactId>
+            <version>${geotools.version}</version>
+        </dependency>
         <!-- Test -->
         <dependency>
             <groupId>junit</groupId>
@@ -101,7 +128,18 @@
             <scope>test</scope>
         </dependency>
 	</dependencies>
-
+    <repositories>
+      <repository>
+          <id>maven2-repository.dev.java.net</id>
+          <name>Java.net repository</name>
+          <url>http://download.java.net/maven/2</url>
+      </repository>
+      <repository>
+          <id>osgeo</id>
+          <name>Open Source Geospatial Foundation Repository</name>
+          <url>http://download.osgeo.org/webdav/geotools/</url>
+      </repository>
+    </repositories>
 	<build>
         <sourceDirectory>src/main/java</sourceDirectory>
 		<plugins>

core/src/main/java/org/datasyslab/geospark/formatMapper/EarthdataHDFPointMapper.java

@@ -226,10 +226,13 @@ public List<Geometry> call(String hdfAddress) throws Exception {
 	    		}
 	    		Point observation = geometryFactory.createPoint(coordinate);
 	    		String userData = "";
-	    		for(int k=0;k<dataVariableList.length;k++)
+
+	    		for(int k=0;k<dataVariableList.length-1;k++)
 	    		{
-	    			userData+=SerNetCDFUtils.getDataAsym(dataArrayList[k], j, i, offset, increment);
+	    			userData+=SerNetCDFUtils.getDataAsym(dataArrayList[k], j, i, offset, increment)+" ";
 	    		}
+				userData+=SerNetCDFUtils.getDataAsym(dataArrayList[dataVariableList.length-1], j, i, offset, increment);
+
 	    		observation.setUserData(userData);
 	    		hdfData.add(observation);
 	    	}

core/src/main/java/org/datasyslab/geospark/formatMapper/RectangleFormatMapper.java

@@ -23,6 +23,7 @@
 import com.vividsolutions.jts.geom.Polygon;
 import com.vividsolutions.jts.io.WKTReader;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class RectangleFormatMapper.
  */

core/src/main/java/org/datasyslab/geospark/geometryObjects/Circle.java

@@ -16,6 +16,7 @@
 import com.vividsolutions.jts.geom.GeometryFactory;
 import com.vividsolutions.jts.geom.GeometryFilter;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class Circle.
  */

core/src/main/java/org/datasyslab/geospark/joinJudgement/GeometryByCircleJudgement.java

@@ -18,6 +18,7 @@
 import scala.Tuple2;
 
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class GeometryByCircleJudgement.
  */

core/src/main/java/org/datasyslab/geospark/joinJudgement/GeometryByCircleJudgementUsingIndex.java

@@ -22,6 +22,7 @@
 
 import scala.Tuple2;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class GeometryByCircleJudgementUsingIndex.
  */

core/src/main/java/org/datasyslab/geospark/joinJudgement/GeometryByPolygonJudgement.java

@@ -17,6 +17,7 @@
 
 import scala.Tuple2;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class GeometryByPolygonJudgement.
  */

core/src/main/java/org/datasyslab/geospark/joinJudgement/GeometryByPolygonJudgementUsingIndex.java

@@ -22,6 +22,7 @@
 
 import scala.Tuple2;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class GeometryByPolygonJudgementUsingIndex.
  */

core/src/main/java/org/datasyslab/geospark/knnJudgement/RectangleDistanceComparator.java

@@ -12,6 +12,7 @@
 import com.vividsolutions.jts.geom.Point;
 import com.vividsolutions.jts.geom.Polygon;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class RectangleDistanceComparator.
  */

core/src/main/java/org/datasyslab/geospark/rangeJudgement/RangeFilterUsingIndex.java

@@ -20,6 +20,7 @@
 import com.vividsolutions.jts.index.quadtree.Quadtree;
 import com.vividsolutions.jts.index.strtree.STRtree;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class RangeFilterUsingIndex.
  */

core/src/main/java/org/datasyslab/geospark/showcase/Example.java

@@ -129,9 +129,12 @@ public static void main(String[] args) {
 			testSpatialJoinQueryUsingIndex();
 			testDistanceJoinQuery();
 			testDistanceJoinQueryUsingIndex();
+			testCRSTransformationSpatialRangeQuery();
+			testCRSTransformationSpatialRangeQueryUsingIndex();
 		} catch (Exception e) {
 			e.printStackTrace();
 			System.out.println("GeoSpark DEMOs failed!");
+			return;
 		}		
         sc.stop();
 		System.out.println("All GeoSpark DEMOs passed!");
@@ -294,6 +297,40 @@ public static void testDistanceJoinQueryUsingIndex() throws Exception {
     		assert resultSize>0;
     	}
     }
-
+
+
+	/**
+	 * Test CRS transformation spatial range query.
+	 *
+	 * @throws Exception the exception
+	 */
+	public static void testCRSTransformationSpatialRangeQuery() throws Exception {
+		objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY(),"epsg:4326","epsg:3005");
+		objectRDD.rawSpatialRDD.persist(StorageLevel.MEMORY_ONLY());
+		for(int i=0;i<eachQueryLoopTimes;i++)
+		{
+			long resultSize = RangeQuery.SpatialRangeQuery(objectRDD, rangeQueryWindow, false,false).count();
+			assert resultSize>-1;
+		}
+	}
+
+
+
+	/**
+	 * Test CRS transformation spatial range query using index.
+	 *
+	 * @throws Exception the exception
+	 */
+	public static void testCRSTransformationSpatialRangeQueryUsingIndex() throws Exception {
+		objectRDD = new PointRDD(sc, PointRDDInputLocation, PointRDDOffset, PointRDDSplitter, true, StorageLevel.MEMORY_ONLY(), "epsg:4326","epsg:3005");
+		objectRDD.buildIndex(PointRDDIndexType,false);
+		objectRDD.indexedRawRDD.persist(StorageLevel.MEMORY_ONLY());
+		for(int i=0;i<eachQueryLoopTimes;i++)
+		{
+			long resultSize = RangeQuery.SpatialRangeQuery(objectRDD, rangeQueryWindow, false,true).count();
+			assert resultSize>-1;
+		}
+
+	}
 
 }

core/src/main/java/org/datasyslab/geospark/showcase/UserSuppliedLineStringMapper.java

@@ -23,6 +23,7 @@
 import com.vividsolutions.jts.geom.Polygon;
 import com.vividsolutions.jts.io.WKTReader;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class UserSuppliedLineStringMapper.
  */

core/src/main/java/org/datasyslab/geospark/showcase/UserSuppliedPointMapper.java

@@ -20,6 +20,7 @@
 
 
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class UserSuppliedPointMapper.
  */

core/src/main/java/org/datasyslab/geospark/showcase/UserSuppliedPolygonMapper.java

@@ -21,6 +21,7 @@
 import com.vividsolutions.jts.geom.Polygon;
 import com.vividsolutions.jts.io.WKTReader;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class UserSuppliedPolygonMapper.
  */

core/src/main/java/org/datasyslab/geospark/showcase/UserSuppliedRectangleMapper.java

@@ -22,6 +22,7 @@
 import com.vividsolutions.jts.geom.Polygon;
 import com.vividsolutions.jts.io.WKTReader;
 
+// TODO: Auto-generated Javadoc
 /**
  * The Class UserSuppliedRectangleMapper.
  */