Merge pull request #823 from taosdata/feat/TS-5195-3.0

enh: support varbinary and geometry
taosdata · Nov 25, 2024 · 8baf16e · 8baf16e
2 parents d453692 + dc0c78d
commit 8baf16e
Show file tree

Hide file tree

Showing 11 changed files with 678 additions and 21 deletions.
diff --git a/.github/workflows/3.0-macos-debug.yml b/.github/workflows/3.0-macos-debug.yml
@@ -88,7 +88,7 @@ jobs:
           || github.event_name == 'schedule'
         run: |
           brew update
-          brew install -f argp-standalone pkg-config
+          brew install --overwrite argp-standalone pkg-config
           brew info argp-standalone
 
       - name: Set up Go

diff --git a/.github/workflows/3.0-macos-release.yml b/.github/workflows/3.0-macos-release.yml
@@ -88,7 +88,7 @@ jobs:
           || github.event_name == 'schedule'
         run: |
           brew update
-          brew install -f argp-standalone pkg-config
+          brew install --overwrite argp-standalone pkg-config
           brew info argp-standalone
 
       - name: Set up Go

diff --git a/.github/workflows/3.0-taosdump-release-ws.yml b/.github/workflows/3.0-taosdump-release-ws.yml
@@ -215,6 +215,10 @@ jobs:
                             done
           fi
 
+          export TDENGINE_CLOUD_DSN="http://127.0.0.1:6041"
+          python3 ./test.py -f taosdump/native/taosdumpTypeGeometry.py 
+          python3 ./test.py -f taosdump/native/taosdumpTypeVarbinary.py
+
       - name: taosdump Cmd line Test
         if:
           (steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/src/benchData.c b/src/benchData.c
@@ -547,7 +547,7 @@ int tmpGeometry(char *tmp, int iface, Field *field, int64_t k) {
 
     // gen point count
     int32_t cnt = field->length / 24;
-    if(cnt == 0) {
+    if(cnt < 2) {
         snprintf(tmp, field->length, "POINT(%d %d)", tmpUint16(field), tmpUint16(field));
         return 0;
     }

diff --git a/src/benchInsert.c b/src/benchInsert.c
@@ -3237,6 +3237,7 @@ static void initStmtData(char dataType, void **data, uint32_t length) {
 
         case TSDB_DATA_TYPE_BINARY:
         case TSDB_DATA_TYPE_NCHAR:
+        case TSDB_DATA_TYPE_VARBINARY:
         case TSDB_DATA_TYPE_GEOMETRY:
             tmpP = calloc(1, g_arguments->prepared_rand * length);
             assert(tmpP);
@@ -3252,7 +3253,7 @@ static void initStmtData(char dataType, void **data, uint32_t length) {
             break;
 
         default:
-            errorPrint("Unknown data type: %s\n",
+            errorPrint("Unknown data type on initStmtData: %s\n",
                        convertDatatypeToString(dataType));
             exit(EXIT_FAILURE);
     }

diff --git a/src/benchUtil.c b/src/benchUtil.c
@@ -898,6 +898,8 @@ char *convertDatatypeToString(int type) {
             return "double";
         case TSDB_DATA_TYPE_JSON:
             return "json";    
+        case TSDB_DATA_TYPE_VARBINARY:
+            return "varbinary";
         case TSDB_DATA_TYPE_GEOMETRY:
             return "geometry";
         default:
@@ -946,6 +948,7 @@ int64_t convertDatatypeToDefaultMin(uint8_t type) {
     int64_t ret = 0;
     switch (type) {
         case TSDB_DATA_TYPE_BOOL:
+        case TSDB_DATA_TYPE_GEOMETRY:
             ret = 0;
             break;
         case TSDB_DATA_TYPE_TINYINT:
@@ -979,6 +982,7 @@ int64_t convertDatatypeToDefaultMax(uint8_t type) {
             ret = 254;
             break;
         case TSDB_DATA_TYPE_SMALLINT:
+        case TSDB_DATA_TYPE_GEOMETRY:
             ret = 32767;
             break;
         case TSDB_DATA_TYPE_USMALLINT:

diff --git a/src/taosdump.c b/src/taosdump.c
@@ -714,6 +714,10 @@ static char *typeToStr(int type) {
             return "bigint unsigned";
         case TSDB_DATA_TYPE_JSON:
             return "JSON";
+        case TSDB_DATA_TYPE_VARBINARY:
+            return "varbinary";
+        case TSDB_DATA_TYPE_GEOMETRY:
+            return "geometry";
         default:
             break;
     }
@@ -757,6 +761,10 @@ static int typeStrToType(const char *type_str) {
         return TSDB_DATA_TYPE_UBIGINT;
     } else if (0 == strcasecmp(type_str, "JSON")) {
         return TSDB_DATA_TYPE_JSON;
+    } else if (0 == strcasecmp(type_str, "varbinary")) {
+        return TSDB_DATA_TYPE_VARBINARY;
+    } else if (0 == strcasecmp(type_str, "geometry")) {
+        return TSDB_DATA_TYPE_GEOMETRY;
     } else {
         errorPrint("%s() LN%d Unknown type: %s\n",
                 __func__, __LINE__, type_str);
@@ -2026,8 +2034,10 @@ static int dumpCreateMTableClause(
 
     for (; counter < numColsAndTags; counter++) {
         if (counter != count_temp) {
-            if ((TSDB_DATA_TYPE_BINARY == tableDes->cols[counter].type)
-                    || (TSDB_DATA_TYPE_NCHAR == tableDes->cols[counter].type)) {
+            if ((TSDB_DATA_TYPE_BINARY    == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_VARBINARY == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_GEOMETRY  == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_NCHAR     == tableDes->cols[counter].type)) {
                 if (tableDes->cols[counter].var_value) {
                     pstr += sprintf(pstr, ",\'%s\'",
                             tableDes->cols[counter].var_value);
@@ -2039,8 +2049,10 @@ static int dumpCreateMTableClause(
                 pstr += sprintf(pstr, ",%s", tableDes->cols[counter].value);
             }
         } else {
-            if ((TSDB_DATA_TYPE_BINARY == tableDes->cols[counter].type)
-                    || (TSDB_DATA_TYPE_NCHAR == tableDes->cols[counter].type)) {
+            if ((TSDB_DATA_TYPE_BINARY    == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_VARBINARY == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_GEOMETRY  == tableDes->cols[counter].type) ||
+                (TSDB_DATA_TYPE_NCHAR     == tableDes->cols[counter].type)) {
                 if (tableDes->cols[counter].var_value) {
                     pstr += sprintf(pstr, "\'%s\'",
                                     tableDes->cols[counter].var_value);
@@ -2301,6 +2313,8 @@ static int processFieldsValueV3(
 
         case TSDB_DATA_TYPE_NCHAR:
         case TSDB_DATA_TYPE_JSON:
+        case TSDB_DATA_TYPE_VARBINARY:
+        case TSDB_DATA_TYPE_GEOMETRY:
             {
                 if (g_args.avro) {
                     if (len < (COL_VALUEBUF_LEN - 1)) {
@@ -2513,6 +2527,8 @@ static int processFieldsValueV2(
 
         case TSDB_DATA_TYPE_NCHAR:
         case TSDB_DATA_TYPE_JSON:
+        case TSDB_DATA_TYPE_VARBINARY:
+        case TSDB_DATA_TYPE_GEOMETRY:
             {
                 if (g_args.avro) {
                     if (len < (COL_VALUEBUF_LEN - 1)) {
@@ -3990,6 +4006,8 @@ static int convertTbDesToJsonImplMore(
 
         case TSDB_DATA_TYPE_NCHAR:
         case TSDB_DATA_TYPE_JSON:
+        case TSDB_DATA_TYPE_VARBINARY:
+        case TSDB_DATA_TYPE_GEOMETRY:
             ret = sprintf(pstr,
                     "{\"name\":\"%s%d\",\"type\":[\"null\",\"%s\"]",
                     colOrTag, i-adjust, "bytes");
@@ -4975,6 +4993,8 @@ static int processValueToAvro(
 
         case TSDB_DATA_TYPE_NCHAR:
         case TSDB_DATA_TYPE_JSON:
+        case TSDB_DATA_TYPE_VARBINARY:
+        case TSDB_DATA_TYPE_GEOMETRY:
             if (NULL == value) {
                 avro_value_set_branch(&avro_value, 0, &branch);
                 avro_value_set_null(&branch);
@@ -5330,8 +5350,10 @@ static void freeBindArray(char *bindArray, int elements) {
         bind = (TAOS_MULTI_BIND *)((char *)bindArray
                 + (sizeof(TAOS_MULTI_BIND) * j));
         if ((TSDB_DATA_TYPE_BINARY != bind->buffer_type)
-                && (TSDB_DATA_TYPE_NCHAR != bind->buffer_type)
-                && (TSDB_DATA_TYPE_JSON != bind->buffer_type)) {
+                && (TSDB_DATA_TYPE_VARBINARY != bind->buffer_type)
+                && (TSDB_DATA_TYPE_GEOMETRY  != bind->buffer_type)
+                && (TSDB_DATA_TYPE_NCHAR     != bind->buffer_type)
+                && (TSDB_DATA_TYPE_JSON      != bind->buffer_type)) {
             tfree(bind->buffer);
         }
     }
@@ -6142,6 +6164,8 @@ static int64_t dumpInAvroTbTagsImpl(
                             break;
                         case TSDB_DATA_TYPE_NCHAR:
                         case TSDB_DATA_TYPE_JSON:
+                        case TSDB_DATA_TYPE_VARBINARY:
+                        case TSDB_DATA_TYPE_GEOMETRY:
                             curr_sqlstr_len = dumpInAvroTagNChar(
                                          field, &field_value, sqlstr,
                                            curr_sqlstr_len);
@@ -6626,20 +6650,41 @@ static void dumpInAvroDataNChar(FieldStruct *field,
                               avro_value_t *value,
                               TAOS_MULTI_BIND *bind,
                               char *is_null) {
-    size_t bytessize;
+    size_t bytessize = 0;
     void *bytesbuf = NULL;
 
     avro_value_t nchar_branch;
     avro_value_get_current_branch(value, &nchar_branch);
 
     avro_value_get_bytes(&nchar_branch,
         (const void **)&bytesbuf, &bytessize);
-    if (NULL == bytesbuf) {
+    if (NULL == bytesbuf || bytessize == 0) {
         debugPrint2("%s | ", "NULL");
         bind->is_null = is_null;
     } else {
         debugPrint2("%s | ", (char*)bytesbuf);
-        bind->buffer_length = strlen((char*)bytesbuf);
+        bind->buffer_length = bytessize;
+    }
+    bind->buffer = bytesbuf;
+}
+
+static void dumpInAvroDataBytes(FieldStruct *field,
+                              avro_value_t *value,
+                              TAOS_MULTI_BIND *bind,
+                              char *is_null) {
+    size_t bytessize = 0;
+    void *bytesbuf = NULL;
+
+    avro_value_t branch;
+    avro_value_get_current_branch(value, &branch);
+
+    avro_value_get_bytes(&branch, (const void **)&bytesbuf, &bytessize);
+    if (NULL == bytesbuf || bytessize == 0) {
+        debugPrint2("%s | ", "NULL");
+        bind->is_null = is_null;
+    } else {
+        debugPrint2("bytes len =%ld | ", bytessize);
+        bind->buffer_length = bytessize;
     }
     bind->buffer = bytesbuf;
 }
@@ -6655,7 +6700,7 @@ static void dumpInAvroDataBinary(FieldStruct *field,
     size_t size;
     avro_value_get_string(&branch, (const char **)&buf, &size);
 
-    if (NULL == buf) {
+    if (NULL == buf || size == 0) {
         debugPrint2("%s | ", "NULL");
         bind->is_null = is_null;
     } else {
@@ -7327,12 +7372,22 @@ static int64_t dumpInAvroDataImpl(
                         break;
                     case TSDB_DATA_TYPE_JSON:
                     case TSDB_DATA_TYPE_NCHAR:
-                        if (field->type != TSDB_DATA_TYPE_NCHAR) {
-                            warnPrint("field[%d] type is not nchar/json!\n", i);
-                            bind->is_null = &is_null;
+                        // RecordSchema bytes only covert to nchar type
+                        if (field->type == TSDB_DATA_TYPE_NCHAR) {
+                            dumpInAvroDataNChar(field, &field_value, bind, &is_null);    
                         } else {
-                            dumpInAvroDataNChar(field, &field_value,
-                                    bind, &is_null);
+                            warnPrint("field[%d] type is not nchar/json! field->type=%d\n", i, field->type);
+                            bind->is_null = &is_null;                            
+                        }
+                        break;
+                    case TSDB_DATA_TYPE_VARBINARY:
+                    case TSDB_DATA_TYPE_GEOMETRY:
+                        // RecordSchema bytes only covert to nchar type
+                        if (field->type == TSDB_DATA_TYPE_NCHAR) {
+                            dumpInAvroDataBytes(field, &field_value, bind, &is_null);
+                        } else {
+                            warnPrint("field[%d] type is not varbinary/geometry! field->type=%d\n", i, field->type);
+                            bind->is_null = &is_null;
                         }
                         break;
                     case TSDB_DATA_TYPE_BOOL:
@@ -7468,9 +7523,19 @@ static int64_t dumpInAvroDataImpl(
 
     // last batch execute
     if (0 != (count % g_args.data_batch)) {
-        if (0 != (code = taos_stmt_execute(stmt))) {
-            errorPrint("error last=%s count=%" PRId64 " batch=%d\n", taos_stmt_errstr(stmt), count, g_args.data_batch);
+#ifdef WEBSOCKET
+        if (g_args.cloud || g_args.restful) {
+            // nothing to do
+        } else {
+#endif
+            if (0 != (code = taos_stmt_execute(stmt))) {
+                errorPrint("error last=%s count=%" PRId64 " batch=%d\n", taos_stmt_errstr(stmt), count, g_args.data_batch);
+            } else {
+                success ++;
+            }
+#ifdef WEBSOCKET
         }
+#endif
     }
 
     free(tbName);
@@ -7994,6 +8059,8 @@ static int processResultValue(
                     GET_DOUBLE_VAL(value));
 
         case TSDB_DATA_TYPE_BINARY:
+        case TSDB_DATA_TYPE_VARBINARY:
+        case TSDB_DATA_TYPE_GEOMETRY:
             {
                 char *bbuf = calloc(1, TSDB_MAX_ALLOWED_SQL_LEN);
                 if (NULL == bbuf) {
@@ -9161,6 +9228,8 @@ static int createMTableAvroHeadImp(
 
             case TSDB_DATA_TYPE_NCHAR:
             case TSDB_DATA_TYPE_JSON:
+            case TSDB_DATA_TYPE_VARBINARY:
+            case TSDB_DATA_TYPE_GEOMETRY:
                 if (0 == strncmp(
                             subTableDes->cols[subTableDes->columns+tag].note,
                             "NUL", 3)) {
@@ -9785,6 +9854,8 @@ static int writeTagsToAvro(
 
             case TSDB_DATA_TYPE_NCHAR:
             case TSDB_DATA_TYPE_JSON:
+            case TSDB_DATA_TYPE_VARBINARY:
+            case TSDB_DATA_TYPE_GEOMETRY:
                 if (0 == strncmp(
                             tbDes->cols[tbDes->columns+tag].note,
                             "NUL", 3)) {

diff --git a/tests/taosdump/native/json/geometry.json b/tests/taosdump/native/json/geometry.json
@@ -0,0 +1,47 @@
+{
+    "filetype": "insert",
+    "cfgdir": "/etc/taos",
+    "host": "127.0.0.1",
+    "port": 6030,
+    "user": "root",
+    "password": "taosdata",
+    "num_of_records_per_req": 500,
+    "create_table_thread_count": 1,
+    "thread_count": 2,
+    "prepared_rand": 1000,
+    "confirm_parameter_prompt": "no",
+    "databases": [
+        {
+            "dbinfo": {
+                "name": "geodb",
+                "drop": "yes",
+                "vgroups": 2,
+                "replica": 1,
+                "precision": "ms"
+            },
+            "super_tables": [
+                {
+                    "name": "meters",
+                    "child_table_exists": "no",
+                    "childtable_count": 10,
+                    "insert_rows": 1000,
+                    "childtable_prefix": "d",
+                    "insert_mode": "taosc",
+                    "timestamp_step": 10,
+                    "start_timestamp":1600000000000,
+                    "columns": [
+                        { "type": "int",         "name": "ic", "min": 10, "max":10000 },
+                        { "type": "usmallint",   "name": "usi" },
+                        { "type": "geometry",   "name": "geo1", "len": 32},
+                        { "type": "geometry",   "name": "geo2", "len": 100},
+                        { "type": "geometry",   "name": "geo3", "len": 1024}
+                    ],
+                    "tags": [
+                        {"type": "geometry", "name": "tgeo1", "len":30},
+                        {"type": "geometry", "name": "tgeo2", "len":256}
+                    ]
+                }
+            ]
+        }
+    ]
+}