TsFile V4 for Table Model

cpp/bench_mark/bench_mark_src/bench_conf.h

@@ -16,11 +16,13 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+
 #include <vector>
 
 namespace bench {
 int LOOP_NUM = 100000;
 int THREAD_NUM = 1;
 int TIMESERIES_NUM = 50;
 std::vector<int> TYPE_LIST = {0, 0, 1, 0, 1};
-}  // namespace bench
+}  // namespace bench
+

cpp/examples/c_examples/c_examples.h

@@ -17,7 +17,6 @@
  * under the License.
  */
 
-
 #include "cwrapper/TsFile-cwrapper.h"
 
 #ifdef __cplusplus
@@ -29,4 +28,5 @@ ErrorCode read_tsfile();
 
 #ifdef __cplusplus
 }
-#endif
+#endif
+

cpp/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.apache.tsfile</groupId>
         <artifactId>tsfile-parent</artifactId>
-        <version>1.0.1-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
     <artifactId>tsfile-cpp</artifactId>
     <packaging>pom</packaging>

cpp/src/reader/query_data_set.h

@@ -34,3 +34,4 @@ class QueryDataSet {
 }  // namespace storage
 
 #endif  // READER_QUERY_DATA_SET_H
+

java/common/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.tsfile</groupId>
         <artifactId>tsfile-java</artifactId>
-        <version>1.0.1-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
     <artifactId>common</artifactId>
     <name>TsFile: Java: Common</name>

java/common/src/main/java/org/apache/tsfile/block/column/Column.java

@@ -23,6 +23,8 @@
 import org.apache.tsfile.utils.Binary;
 import org.apache.tsfile.utils.TsPrimitiveType;
 
+import java.util.Arrays;
+
 public interface Column {
 
   /** Get the data type. */
@@ -124,6 +126,14 @@ default TsPrimitiveType getTsPrimitiveType(int position) {
   /** Returns the array to determine whether each position of the column is null or not. */
   boolean[] isNull();
 
+  /**
+   * Set the given range as null.
+   *
+   * @param start start position (inclusive)
+   * @param end end position (exclusive)
+   */
+  void setNull(int start, int end);
+
   /** Returns the number of positions in this block. */
   int getPositionCount();
 
@@ -164,4 +174,14 @@ default TsPrimitiveType getTsPrimitiveType(int position) {
   void reverse();
 
   int getInstanceSize();
+
+  void setPositionCount(int count);
+
+  default void reset() {
+    setPositionCount(0);
+    final boolean[] isNulls = isNull();
+    if (isNulls != null) {
+      Arrays.fill(isNulls, false);
+    }
+  }
 }

java/examples/pom.xml

@@ -24,7 +24,7 @@
     <parent>
         <groupId>org.apache.tsfile</groupId>
         <artifactId>tsfile-java</artifactId>
-        <version>1.0.1-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
     <artifactId>examples</artifactId>
     <packaging>pom</packaging>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>org.apache.tsfile</groupId>
             <artifactId>tsfile</artifactId>
-            <version>1.0.1-SNAPSHOT</version>
+            <version>1.2.0-SNAPSHOT</version>
         </dependency>
     </dependencies>
     <build>

java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTSRecord.java

@@ -55,7 +55,7 @@ public static void main(String[] args) throws IOException {
     }
 
     try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
-      List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+      List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
       measurementSchemas.add(
           new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
       measurementSchemas.add(
@@ -84,7 +84,7 @@ public static void main(String[] args) throws IOException {
   private static void writeAligned(
       TsFileWriter tsFileWriter,
       String deviceId,
-      List<MeasurementSchema> schemas,
+      List<IMeasurementSchema> schemas,
       long rowSize,
       long startTime,
       long startValue)

java/examples/src/main/java/org/apache/tsfile/TsFileWriteAlignedWithTablet.java

@@ -26,6 +26,7 @@
 import org.apache.tsfile.read.common.Path;
 import org.apache.tsfile.write.TsFileWriter;
 import org.apache.tsfile.write.record.Tablet;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 
 import org.slf4j.Logger;
@@ -56,7 +57,8 @@ public static void main(String[] args) throws IOException {
     }
 
     try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
-      List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+
+      List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
       measurementSchemas.add(
           new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.PLAIN));
       measurementSchemas.add(
@@ -87,7 +89,7 @@ public static void main(String[] args) throws IOException {
   private static void writeAlignedWithTablet(
       TsFileWriter tsFileWriter,
       String deviceId,
-      List<MeasurementSchema> schemas,
+      List<IMeasurementSchema> schemas,
       long rowNum,
       long startTime,
       long startValue)
@@ -127,7 +129,7 @@ private static void writeNonAlignedWithTablet(TsFileWriter tsFileWriter)
     tsFileWriter.registerTimeseries(
         new Path(DEVICE_2), new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
     // construct Tablet
-    List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+    List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
     measurementSchemas.add(new MeasurementSchema(SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
     measurementSchemas.add(new MeasurementSchema(SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
     Tablet tablet = new Tablet(DEVICE_2, measurementSchemas);

java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTSRecord.java

@@ -56,7 +56,7 @@ public static void main(String[] args) {
       }
 
       try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
-        List<MeasurementSchema> schemas = new ArrayList<>();
+        List<IMeasurementSchema> schemas = new ArrayList<>();
         schemas.add(new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.RLE));
         schemas.add(new MeasurementSchema(Constant.SENSOR_2, TSDataType.INT64, TSEncoding.RLE));
         schemas.add(new MeasurementSchema(Constant.SENSOR_3, TSDataType.INT64, TSEncoding.RLE));
@@ -80,7 +80,7 @@ public static void main(String[] args) {
   private static void write(
       TsFileWriter tsFileWriter,
       String deviceId,
-      List<MeasurementSchema> schemas,
+      List<IMeasurementSchema> schemas,
       long rowSize,
       long startTime,
       long startValue)

java/examples/src/main/java/org/apache/tsfile/TsFileWriteWithTablet.java

@@ -51,7 +51,7 @@ public static void main(String[] args) {
       }
 
       try (TsFileWriter tsFileWriter = new TsFileWriter(f)) {
-        List<MeasurementSchema> measurementSchemas = new ArrayList<>();
+        List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
         measurementSchemas.add(
             new MeasurementSchema(Constant.SENSOR_1, TSDataType.INT64, TSEncoding.PLAIN));
         measurementSchemas.add(
@@ -81,7 +81,7 @@ public static void main(String[] args) {
   private static void writeWithTablet(
       TsFileWriter tsFileWriter,
       String deviceId,
-      List<MeasurementSchema> schemas,
+      List<IMeasurementSchema> schemas,
       long rowNum,
       long startTime,
       long startValue)

java/pom.xml

@@ -24,16 +24,17 @@
     <parent>
         <groupId>org.apache.tsfile</groupId>
         <artifactId>tsfile-parent</artifactId>
-        <version>1.0.1-SNAPSHOT</version>
+        <version>1.2.0-SNAPSHOT</version>
     </parent>
     <artifactId>tsfile-java</artifactId>
-    <version>1.0.1-SNAPSHOT</version>
+    <version>1.2.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>TsFile: Java</name>
     <modules>
         <module>common</module>
         <module>tsfile</module>
         <module>examples</module>
+        <module>tools</module>
     </modules>
     <dependencyManagement>
         <dependencies>

java/tools/README-zh.md

@@ -0,0 +1,126 @@
+<!--
+
+    Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+
+[English](./README.md) | [中文](./README-zh.md)
+# TsFile Tools 手册
+## 简介
+
+## 开发
+
+### 前置条件
+
+构建 Java 版的 TsFile Tools，必须要安装以下依赖:
+
+1. Java >= 1.8 (1.8, 11 到 17 都经过验证. 请确保设置了环境变量).
+2. Maven >= 3.6 (如果要从源代码编译TsFile).
+
+
+### 使用 maven 构建
+
+```
+mvn clean package -P with-java -DskipTests
+```
+
+### 安装到本地机器
+
+```
+mvn install -P with-java -DskipTests
+```
+
+## schema 定义
+
+| 参数         | 说明                       | 是否必填 | 默认值  |
+|------------|--------------------------|------|------|
+| table_name | 表名                       | 是    |      |
+| time_precision | 时间精度（可选值有：ms/us/ns）      | 否    | ms   |
+| has_header | 是否包含表头 (可选值有：true/false) | 否    | true |
+| separator | 行内分隔符（可选值有：, /tab/ ;）    | 否    | ,    |
+| null_format | 空值                       | 否    |    |
+| id_columns | 主键列，支持cvs中不存在的列做为层级      | 否    |      |
+| time_column | 时间列                      | 是    |      |
+| csv_columns | 按照顺序与csv列一一对应            | 是    |      |
+
+说明：
+
+id_columns 按照顺序进行设置值，支持csv 文件中不存在的列作为层级
+例如csv 只有a,b,c,d,time五列则
+id_columns
+a1 default aa
+a
+其中a1 不在csv列，为虚拟列，默认值为aa
+
+csv_columns 之后的内容为值列的定义，每一行的第一个字段为在tsfile中的测点名，第二个字段为类型
+当csv中某一列不需要写入 tsfile时，可以设置为 SKIP
+例：
+csv_columns
+地区 TEXT，
+厂号 TEXT,
+设备号 TEXT,
+SKIP,
+SKIP,
+时间 INT64,
+温度 FLOAT,
+排量 DOUBLE,
+
+### 数据示例
+csv 文件内容
+```
+地区,  厂号, 设备号,  型号, 维修周期, 时间,   温度,   排量
+河北, 1001,     1,    10,       1,    1,   80.0, 1000.0
+河北, 1001,     1,    10,       1,    4,   80.0, 1000.0
+河北, 1002,     7,     5,       2,    1,   90.0, 1200.0
+```
+schema 定义
+
+```
+table_name=root.db1
+time_precision=ms
+has_header=true
+separator=,
+null_format=\N
+
+
+id_columns
+集团 DEFAULT 大唐
+地区
+厂号
+设备号
+
+time_column=时间
+
+csv_columns
+地区 TEXT，
+厂号 TEXT,
+设备号 TEXT,
+SKIP,
+SKIP,
+时间 INT64,
+温度 FLOAT,
+排量 DOUBLE,
+```
+## 命令
+
+```
+csv2tsfile.sh --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx 
+csv2tsfile.bat --source ./xxx/xxx --target /xxx/xxx --fail_dir /xxx/xxx 
+```
+
+