add license header

gnehil · Jul 29, 2024 · 3b11805 · 3b11805
1 parent 9752217
commit 3b11805
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 66 deletions.
diff --git a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSparkConnector.scala b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestSparkConnector.scala
@@ -17,19 +17,19 @@
 
 package org.apache.doris.spark.sql
 
-import org.apache.spark.sql.types.{StringType, StructField, StructType}
-import org.apache.spark.sql.{SaveMode, SparkSession}
+import org.apache.spark.sql.SparkSession
 import org.apache.spark.{SparkConf, SparkContext}
-import org.junit.{Ignore, Test}
+import org.junit.Ignore
+import org.junit.Test
 
 // This test need real connect info to run.
 // Set the connect info before comment out this @Ignore
 @Ignore
 class TestSparkConnector {
-  val dorisFeNodes = "10.16.10.6:8939"
+  val dorisFeNodes = "your_fe_host:8030"
   val dorisUser = "root"
   val dorisPwd = ""
-  val dorisTable = "test.dwd_test"
+  val dorisTable = "test.test_tbl"
 
   val kafkaServers = ""
   val kafkaTopics = ""
@@ -111,70 +111,11 @@ class TestSparkConnector {
       .option("doris.fenodes", dorisFeNodes)
       .option("user", dorisUser)
       .option("password", dorisPwd)
-      .option("sink.batch.size", 2)
-      .option("sink.max-retries", 2)
+      .option("sink.batch.size",2)
+      .option("sink.max-retries",2)
       .start().awaitTermination()
     spark.stop()
   }
 
-  @Test
-  def sqlReadTest(): Unit = {
-
-    val spark = SparkSession.builder()
-      .master("local")
-      .getOrCreate()
-    spark.sql(
-      s"""
-         |CREATE TEMPORARY VIEW t
-         |USING doris
-         |OPTIONS(
-         | "table.identifier"="${dorisTable}",
-         | "fenodes"="${dorisFeNodes}",
-         | "user"="${dorisUser}",
-         | "password"="${dorisPwd}"
-         |)
-         |""".stripMargin)
-
-    spark.sql(
-        """
-          |select * from t where dt = '2023-06-15'
-          |""".stripMargin)
-      // .explain()
-      .show(false)
-
-  }
-
-  @Test
-  def jsonDataWriteTest(): Unit = {
-    val schema = StructType(Array(
-      StructField("batch_id", StringType, true),
-      StructField("gen_uuid", StringType, true),
-      StructField("keyword", StringType, true),
-      StructField("step", StringType, true),
-      StructField("title", StringType, true),
-      StructField("original_keyword", StringType, true),
-      StructField("host_ip", StringType, true),
-      StructField("modify_at", StringType, true)
-    ))
-    val sparkSession = SparkSession.builder().appName("JSON DATA READ").master("local[*]").getOrCreate()
-    val df = sparkSession.read.schema(schema).json("/Users/gnehil/Downloads/social_google_trends_keyword_v2_fdc.json").coalesce(1)
-    // df.show(2)
-    df.write.format("doris").mode(SaveMode.Append).option(
-      "doris.table.identifier", "test.social_google_trends_keyword_v2_fdc_20240506"
-    ).option(
-      "doris.fenodes", "10.16.10.6:48733"
-    ).option("user", "root").option("password", ""
-      //      ).option("doris.write.fields", fieldsString
-    ).option("sink.properties.format", "json"
-    ).option("sink.batch.size", 100000
-      //      ).option("doris.request.connect.timeout.ms", DORIS_REQUEST_CONNECT_TIMEOUT_MS
-    ).option(
-      "doris.query.port", 49733
-    ).option(
-      "sink.max-retries", "1"
-    ).save()
-    sparkSession.stop()
-  }
-
 }
 
diff --git a/spark-load/spark-load-common/pom.xml b/spark-load/spark-load-common/pom.xml
@@ -1,4 +1,23 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+-->
+
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">