Use Java enum for SaveMode.

apache · Feb 9, 2015 · 99950a2 · 99950a2
1 parent 4679665
commit 99950a2
Show file tree

Hide file tree

Showing 12 changed files with 36 additions and 98 deletions.
diff --git a/...g/apache/spark/sql/sources/SaveModes.java → ...rg/apache/spark/sql/sources/SaveMode.java b/...g/apache/spark/sql/sources/SaveModes.java → ...rg/apache/spark/sql/sources/SaveMode.java
@@ -16,19 +16,23 @@
  */
 package org.apache.spark.sql.sources;
 
-public class SaveModes {
-
+/**
+ * SaveMode is used to specify the expected behavior of saving a DataFrame to a data source.
+ */
+public enum SaveMode {
   /**
-   * Gets the Append object.
+   * Append mode means that when saving a DataFrame to a data source, if data already exists,
+   * contents of the DataFrame are expected to be appended to existing data.
    */
-  public static final SaveMode Append = Append$.MODULE$;
+  Append,
   /**
-   * Gets the Overwrite object.
+   * Overwrite mode means that when saving a DataFrame to a data source, if data already exists,
+   * existing data is expected to be overwritten by the contents of the DataFrame.
    */
-  public static final SaveMode Overwrite = Overwrite$.MODULE$;
-
+  Overwrite,
   /**
-   * Gets the ErrorIfExists object.
+   * ErrorIfExists mode means that when saving a DataFrame to a data source, if data already exists,
+   * an exception is expected to be thrown.
    */
-  public static final SaveMode ErrorIfExists = ErrorIfExists$.MODULE$;
+  ErrorIfExists
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameImpl.scala
@@ -308,7 +308,7 @@ private[sql] class DataFrameImpl protected[sql](
 
   override def saveAsParquetFile(path: String): Unit = {
     if (sqlContext.conf.parquetUseDataSourceApi) {
-      save("org.apache.spark.sql.parquet", SaveModes.ErrorIfExists, Map("path" -> path))
+      save("org.apache.spark.sql.parquet", SaveMode.ErrorIfExists, Map("path" -> path))
     } else {
       sqlContext.executePlan(WriteToFile(path, logicalPlan)).toRdd
     }
@@ -396,7 +396,7 @@ private[sql] class DataFrameImpl protected[sql](
   }
 
   override def save(path: String): Unit = {
-    save(path, SaveModes.ErrorIfExists)
+    save(path, SaveMode.ErrorIfExists)
   }
 
   override def save(path: String, mode: SaveMode): Unit = {
@@ -405,7 +405,7 @@ private[sql] class DataFrameImpl protected[sql](
   }
 
   override def save(path: String, dataSourceName: String): Unit = {
-    save(dataSourceName, SaveModes.ErrorIfExists, Map("path" -> path))
+    save(dataSourceName, SaveMode.ErrorIfExists, Map("path" -> path))
   }
 
   override def save(path: String, dataSourceName: String, mode: SaveMode): Unit = {

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/IncomputableColumn.scala b/sql/core/src/main/scala/org/apache/spark/sql/IncomputableColumn.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.sql
 
-import org.apache.spark.sql.sources.SaveMode
-
 import scala.reflect.ClassTag
 
 import org.apache.spark.api.java.JavaRDD
@@ -27,9 +25,9 @@ import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedSt
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.sql.sources.SaveMode
 import org.apache.spark.sql.types.StructType
 
-
 private[sql] class IncomputableColumn(protected[sql] val expr: Expression) extends Column {
 
   def this(name: String) = this(name match {

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JSONRelation.scala
@@ -62,13 +62,13 @@ private[sql] class DefaultSource
     val filesystemPath = new Path(path)
     val fs = filesystemPath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration)
     mode match {
-      case Append =>
+      case SaveMode.Append =>
         sys.error(s"Append mode is not supported by ${this.getClass.getCanonicalName}")
-      case Overwrite =>
+      case SaveMode.Overwrite =>
         if (fs.exists(filesystemPath)) {
           fs.delete(filesystemPath, true)
         }
-      case ErrorIfExists =>
+      case SaveMode.ErrorIfExists =>
         if (fs.exists(filesystemPath)) {
           sys.error(s"path $path already exists.")
         }

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/newParquet.scala
@@ -87,13 +87,13 @@ class DefaultSource
     val filesystemPath = new Path(path)
     val fs = filesystemPath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration)
     mode match {
-      case Append =>
+      case SaveMode.Append =>
         sys.error(s"Append mode is not supported by ${this.getClass.getCanonicalName}")
-      case Overwrite =>
+      case SaveMode.Overwrite =>
         if (fs.exists(filesystemPath)) {
           fs.delete(filesystemPath, true)
         }
-      case ErrorIfExists =>
+      case SaveMode.ErrorIfExists =>
         if (fs.exists(filesystemPath)) {
           sys.error(s"path $path already exists.")
         }

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala
@@ -366,7 +366,7 @@ private [sql] case class CreateTempTableUsingAsSelect(
 
   def run(sqlContext: SQLContext) = {
     val df = DataFrame(sqlContext, query)
-    val resolved = ResolvedDataSource(sqlContext, provider, SaveModes.ErrorIfExists, options, df)
+    val resolved = ResolvedDataSource(sqlContext, provider, SaveMode.ErrorIfExists, options, df)
     sqlContext.registerRDDAsTable(
       DataFrame(sqlContext, LogicalRelation(resolved.relation)), tableName)
 

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala
@@ -91,8 +91,6 @@ trait CreatableRelationProvider {
     * ErrorIfExists mode means that when saving a DataFrame to a data source,
     * if data already exists, an exception is expected to be thrown.
     *
-    * For Java users, mode can be inspected through equality check (e.g. mode == SaveModes.Append).
-    *
     * @param sqlContext
     * @param mode
     * @param parameters

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/modes.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/modes.scala
diff --git a/sql/core/src/test/java/org/apache/spark/sql/sources/JavaSaveLoadSuite.java b/sql/core/src/test/java/org/apache/spark/sql/sources/JavaSaveLoadSuite.java
@@ -70,29 +70,11 @@ public void setUp() throws IOException {
     df.registerTempTable("jsonTable");
   }
 
-  @Test
-  public void getSaveMode() {
-    SaveMode append = SaveModes.Append;
-    Assert.assertTrue(append == SaveModes.Append);
-    Assert.assertTrue(append != SaveModes.Overwrite);
-    Assert.assertTrue(append != SaveModes.ErrorIfExists);
-
-    SaveMode overwrite = SaveModes.Overwrite;
-    Assert.assertTrue(overwrite == SaveModes.Overwrite);
-    Assert.assertTrue(overwrite != SaveModes.Append);
-    Assert.assertTrue(overwrite != SaveModes.ErrorIfExists);
-
-    SaveMode errorIfExists = SaveModes.ErrorIfExists;
-    Assert.assertTrue(errorIfExists == SaveModes.ErrorIfExists);
-    Assert.assertTrue(errorIfExists != SaveModes.Append);
-    Assert.assertTrue(errorIfExists != SaveModes.Overwrite);
-  }
-
   @Test
   public void saveAndLoad() {
     Map<String, String> options = new HashMap<String, String>();
     options.put("path", path.toString());
-    df.save("org.apache.spark.sql.json", SaveModes.ErrorIfExists, options);
+    df.save("org.apache.spark.sql.json", SaveMode.ErrorIfExists, options);
 
     DataFrame loadedDF = sqlContext.load("org.apache.spark.sql.json", options);
 
@@ -103,7 +85,7 @@ public void saveAndLoad() {
   public void saveAndLoadWithSchema() {
     Map<String, String> options = new HashMap<String, String>();
     options.put("path", path.toString());
-    df.save("org.apache.spark.sql.json", SaveModes.ErrorIfExists, options);
+    df.save("org.apache.spark.sql.json", SaveMode.ErrorIfExists, options);
 
     List<StructField> fields = new ArrayList<>();
     fields.add(DataTypes.createStructField("b", DataTypes.StringType, true));

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/SaveLoadSuite.scala
@@ -83,7 +83,7 @@ class SaveLoadSuite extends DataSourceTest with BeforeAndAfterAll {
 
   test("save with data source and options, and load") {
     conf.setConf(SQLConf.DEFAULT_DATA_SOURCE_NAME, "not a source name")
-    df.save("org.apache.spark.sql.json", SaveModes.ErrorIfExists, Map("path" -> path.toString))
+    df.save("org.apache.spark.sql.json", SaveMode.ErrorIfExists, Map("path" -> path.toString))
     checkLoad
   }
 
@@ -103,16 +103,15 @@ class SaveLoadSuite extends DataSourceTest with BeforeAndAfterAll {
     df.save(path.toString, "org.apache.spark.sql.json")
     checkLoad
 
-    df.save("org.apache.spark.sql.json", SaveModes.Overwrite, Map("path" -> path.toString))
+    df.save("org.apache.spark.sql.json", SaveMode.Overwrite, Map("path" -> path.toString))
     checkLoad
 
     message = intercept[RuntimeException] {
-      df.save("org.apache.spark.sql.json", SaveModes.Append, Map("path" -> path.toString))
+      df.save("org.apache.spark.sql.json", SaveMode.Append, Map("path" -> path.toString))
     }.getMessage
 
     assert(
       message.contains("Append mode is not supported"),
       "We should complain that 'Append mode is not supported' for JSON source.")
-
   }
 }
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.sql.hive.execution
 
 import org.apache.spark.annotation.DeveloperApi
-import org.apache.spark.sql.sources.{SaveModes, ResolvedDataSource}
+import org.apache.spark.sql.sources.{SaveMode, ResolvedDataSource}
 import org.apache.spark.sql.{DataFrame, SQLContext}
 import org.apache.spark.sql.catalyst.expressions.Row
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
@@ -168,7 +168,7 @@ case class CreateMetastoreDataSourceAsSelect(
       }
 
     // Create the relation based on the data of df.
-    ResolvedDataSource(sqlContext, provider, SaveModes.ErrorIfExists, optionsWithPath, df)
+    ResolvedDataSource(sqlContext, provider, SaveMode.ErrorIfExists, optionsWithPath, df)
 
     hiveContext.catalog.createDataSourceTable(
       tableName,

diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaMetastoreDataSourcesSuite.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/JavaMetastoreDataSourcesSuite.java
@@ -44,6 +44,7 @@ public class JavaMetastoreDataSourcesSuite {
 
   String originalDefaultSource;
   File path;
+  File hiveManagedPath;
   DataFrame df;
 
   private void checkAnswer(DataFrame actual, List<Row> expected) {
@@ -64,6 +65,10 @@ public void setUp() throws IOException {
     if (path.exists()) {
       path.delete();
     }
+    hiveManagedPath = new File(sqlContext.catalog().hiveDefaultTableFilePath("javaSavedTable"));
+    if (hiveManagedPath.exists()) {
+      hiveManagedPath.delete();
+    }
 
     List<String> jsonObjects = new ArrayList<String>(10);
     for (int i = 0; i < 10; i++) {