From 3caa56ca734832185f948cd93aec91055dc8a5e6 Mon Sep 17 00:00:00 2001 From: 80597928 <673421862@qq.com> Date: Wed, 18 Oct 2023 20:32:36 +0800 Subject: [PATCH 1/5] improve http e2e test #5446 --- .../seatunnel/e2e/connector/http/HttpIT.java | 10 +++ .../http_formrequestbody_to_assert.conf | 78 +++++++++++++++++++ .../http_jsonrequestbody_to_assert.conf | 78 +++++++++++++++++++ .../src/test/resources/mockserver-config.json | 55 +++++++++++++ 4 files changed, 221 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java index 402abd5915e..4a3c256da16 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java @@ -135,6 +135,16 @@ public void testSourceToAssertSink(TestContainer container) Container.ExecResult execResult12 = container.executeJob("/http_multilinejson_to_assert.conf"); Assertions.assertEquals(0, execResult12.getExitCode()); + + // http httpFormrequestbody + Container.ExecResult execResult13 = + container.executeJob("/http_formrequestbody_to_assert.conf"); + Assertions.assertEquals(0, execResult13.getExitCode()); + + // http httpJsonRequestBody + Container.ExecResult execResult14 = + container.executeJob("/http_jsonrequestbody_to_assert.conf"); + Assertions.assertEquals(0, execResult14.getExitCode()); } public String getMockServerConfig() { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf new file mode 100644 index 00000000000..b5370aae712 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Http { + result_table_name = "http" + url = "http://mockserver:1080/example/formBody" + method = "POST" + params ={id = 1} + format = "json" + schema = { + fields { + name = string + age = int + } + } + } +} + +sink { + Console { + source_table_name = "http" + } + Assert { + source_table_name = "http" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 3 + }, + { + rule_type = MIN_ROW + rule_value = 2 + } + ], + field_rules = [ + { + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = age + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf new file mode 100644 index 00000000000..f722e1ca316 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Http { + result_table_name = "http" + url = "http://mockserver:1080/example/jsonBody" + method = "POST" + body="{"id":1}" + format = "json" + schema = { + fields { + name = string + age = int + } + } + } +} + +sink { + Console { + source_table_name = "http" + } + Assert { + source_table_name = "http" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 2 + }, + { + rule_type = MIN_ROW + rule_value = 2 + } + ], + field_rules = [ + { + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = age + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } + } +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json index 622c4a1d9f9..dad309d27f9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json @@ -4447,5 +4447,60 @@ ] } } + }, + { + "httpRequest": { + "method" : "POST", + "path": "/example/jsonBody", + "body": { + "type": "JSON", + "json": { + "id": 1 + }, + "matchType": "STRICT" + } + }, + "httpResponse": { + "body": [ + { + "name": "lzl", + "age": 18 + }, + { + "name": "pizz", + "age": 19 + } + ], + "headers": { + "Content-Type": "application/json" + } + } + }, + { + "httpRequest": { + "path": "/example/formBody", + "method": "POST", + "body": { + "type": "PARAMETERS", + "parameters": { + "id": "1" + } + } + }, + "httpResponse": { + "body": [ + { + "name": "lzl", + "age": 18 + }, + { + "name": "pizz", + "age": 19 + } + ], + "headers": { + "Content-Type": "application/json" + } + } } ] From 7a32ee47425f7a817c3229181aaa3432d044d816 Mon Sep 17 00:00:00 2001 From: 80597928 <673421862@qq.com> Date: Wed, 18 Oct 2023 20:52:13 +0800 Subject: [PATCH 2/5] fix doc --- .../java/org/apache/seatunnel/e2e/connector/http/HttpIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java index 4a3c256da16..d65617bb555 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/java/org/apache/seatunnel/e2e/connector/http/HttpIT.java @@ -136,7 +136,7 @@ public void testSourceToAssertSink(TestContainer container) container.executeJob("/http_multilinejson_to_assert.conf"); Assertions.assertEquals(0, execResult12.getExitCode()); - // http httpFormrequestbody + // http httpFormRequestbody Container.ExecResult execResult13 = container.executeJob("/http_formrequestbody_to_assert.conf"); Assertions.assertEquals(0, execResult13.getExitCode()); From 9e6316dd7a9d579ba1b625b1826522843a2987e3 Mon Sep 17 00:00:00 2001 From: 80597928 <673421862@qq.com> Date: Thu, 19 Oct 2023 10:44:13 +0800 Subject: [PATCH 3/5] modify equals --- .../src/test/resources/http_formrequestbody_to_assert.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf index b5370aae712..010cb4278ea 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf @@ -46,7 +46,7 @@ sink { row_rules = [ { rule_type = MAX_ROW - rule_value = 3 + rule_value = 2 }, { rule_type = MIN_ROW From 7ec0705842d03f5306bb0e940d47498acfc972ad Mon Sep 17 00:00:00 2001 From: 80597928 <673421862@qq.com> Date: Fri, 20 Oct 2023 11:23:33 +0800 Subject: [PATCH 4/5] fix doc --- docs/en/connector-v2/sink/Http.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/en/connector-v2/sink/Http.md b/docs/en/connector-v2/sink/Http.md index 099943ce86e..55a69e93334 100644 --- a/docs/en/connector-v2/sink/Http.md +++ b/docs/en/connector-v2/sink/Http.md @@ -36,7 +36,6 @@ They can be downloaded via install-plugin.sh or from the Maven central repositor |-----------------------------|--------|----------|---------|-----------------------------------------------------------------------------------------------------| | url | String | Yes | - | Http request url | | headers | Map | No | - | Http headers | -| params | Map | No | - | Http params | | retry | Int | No | - | The max retry times if request http return to `IOException` | | retry_backoff_multiplier_ms | Int | No | 100 | The retry-backoff times(millis) multiplier if request http failed | | retry_backoff_max_ms | Int | No | 10000 | The maximum retry-backoff times(millis) if request http failed | From c75bd7352322e1467c71ec76212ac56149b29b39 Mon Sep 17 00:00:00 2001 From: 80597928 <673421862@qq.com> Date: Mon, 23 Oct 2023 16:35:06 +0800 Subject: [PATCH 5/5] improve e2e remove HttpSinkFactory.java params and add sink e2e case --- .../seatunnel/http/sink/HttpSinkFactory.java | 1 - .../http_formrequestbody_to_assert.conf | 7 ++++++ .../http_jsonrequestbody_to_assert.conf | 7 ++++++ .../src/test/resources/mockserver-config.json | 24 +++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkFactory.java b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkFactory.java index 0d961f711ce..8411001fff8 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkFactory.java +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/http/sink/HttpSinkFactory.java @@ -36,7 +36,6 @@ public OptionRule optionRule() { return OptionRule.builder() .required(HttpConfig.URL) .optional(HttpConfig.HEADERS) - .optional(HttpConfig.PARAMS) .optional(HttpConfig.RETRY) .optional(HttpConfig.RETRY_BACKOFF_MULTIPLIER_MS) .optional(HttpConfig.RETRY_BACKOFF_MAX_MS) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf index 010cb4278ea..facebc81ef8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_formrequestbody_to_assert.conf @@ -75,4 +75,11 @@ sink { ] } } + Http { + source_table_name = "http" + url = "http://mockserver:1080/example/webhook" + headers { + token = "9e32e859ef044462a257e1fc76730066" + } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf index f722e1ca316..5fde4d48729 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_jsonrequestbody_to_assert.conf @@ -75,4 +75,11 @@ sink { ] } } + Http { + source_table_name = "http" + url = "http://mockserver:1080/example/webhook" + headers { + token = "9e32e859ef044462a257e1fc76730066" + } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json index dad309d27f9..2c419277e04 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/mockserver-config.json @@ -4502,5 +4502,29 @@ "Content-Type": "application/json" } } + }, + { + "httpRequest": { + "path": "/example/webhook", + "method": "POST", + "headers": { + "token": ["9e32e859ef044462a257e1fc76730066"] + } + }, + "httpResponse": { + "body": [ + { + "name": "lzl", + "age": 18 + }, + { + "name": "pizz", + "age": 19 + } + ], + "headers": { + "Content-Type": "application/json" + } + } } ]