Skip to content

Commit

Permalink
[Bugfix][Connector-v2] fix file sink isPartitionFieldWriteInFile oc…
Browse files Browse the repository at this point in the history
…curred exception when no columns are give for the sink
  • Loading branch information
wei.zhao committed Sep 16, 2023
1 parent 0ce1712 commit f0b2981
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import lombok.Data;
import lombok.NonNull;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
Expand Down Expand Up @@ -83,7 +84,8 @@ public FileSinkConfig(@NonNull Config config, @NonNull SeaTunnelRowType seaTunne

// if the config sink_columns is empty, all fields in SeaTunnelRowTypeInfo will being write
if (CollectionUtils.isEmpty(this.sinkColumnList)) {
this.sinkColumnList = Arrays.asList(seaTunnelRowTypeInfo.getFieldNames());
this.sinkColumnList =
new ArrayList<>(Arrays.asList(seaTunnelRowTypeInfo.getFieldNames()));
}

if (config.hasPath(BaseSinkConfig.PARTITION_BY.key())) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.seatunnel.connectors.seatunnel.file.writer;

import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;

import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
import org.apache.seatunnel.connectors.seatunnel.file.sink.config.FileSinkConfig;

import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.net.URL;
import java.nio.file.Paths;

public class FileSinkConfigTest {

@Test
public void testConfigInit() throws Exception {
URL conf = OrcReadStrategyTest.class.getResource("/test_write_hdfs.conf");
Assertions.assertNotNull(conf);
String confPath = Paths.get(conf.toURI()).toString();
Config config = ConfigFactory.parseFile(new File(confPath));

SeaTunnelRowType rowType =
new SeaTunnelRowType(
new String[] {"data", "ts"},
new SeaTunnelDataType[] {BasicType.STRING_TYPE, BasicType.STRING_TYPE});
Assertions.assertDoesNotThrow(() -> new FileSinkConfig(config, rowType));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
fs.defaultFS = "hdfs://hadoop01:9000"
have_partition = true
partition_by = ["ts"]
partition_dir_expression = "${v0}"
is_partition_field_write_in_file = false
path = "/data/test"
file_format_type = "json"
batch_size=10
}

0 comments on commit f0b2981

Please sign in to comment.