[Feature] split transform lib to a special folder

liunaijie · Jul 12, 2024 · 6676e20 · 6676e20
1 parent 562712c
commit 6676e20
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 23 deletions.
diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/config/Common.java
@@ -139,19 +139,38 @@ public static Path libDir() {
         return Paths.get(getSeaTunnelHome(), "lib");
     }
 
+    /** transform Dir */
+    public static Path transformsDir() {
+        return Paths.get(getSeaTunnelHome(), "transforms");
+    }
+
     /** return lib jars, which located in 'lib/*' or 'lib/{dir}/*'. */
     public static List<Path> getLibJars() {
         Path libRootDir = Common.libDir();
+        Path transformsRootDir = Common.transformsDir();
         if (!Files.exists(libRootDir) || !Files.isDirectory(libRootDir)) {
             return Collections.emptyList();
         }
+        List<Path> libPaths;
         try (Stream<Path> stream = Files.walk(libRootDir, APP_LIB_DIR_DEPTH, FOLLOW_LINKS)) {
-            return stream.filter(it -> !it.toFile().isDirectory())
-                    .filter(it -> it.getFileName().toString().endsWith(".jar"))
-                    .collect(Collectors.toList());
+            libPaths =
+                    stream.filter(it -> !it.toFile().isDirectory())
+                            .filter(it -> it.getFileName().toString().endsWith(".jar"))
+                            .collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        List<Path> transformPaths;
+        try (Stream<Path> stream = Files.walk(transformsRootDir, APP_LIB_DIR_DEPTH, FOLLOW_LINKS)) {
+            transformPaths =
+                    stream.filter(it -> !it.toFile().isDirectory())
+                            .filter(it -> it.getFileName().toString().endsWith(".jar"))
+                            .collect(Collectors.toList());
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
+        libPaths.addAll(transformPaths);
+        return libPaths;
     }
 
     /** return the jar package configured in env jars */

diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
@@ -167,25 +167,8 @@
             <unpack>false</unpack>
             <includes>
                 <include>org.apache.seatunnel:seatunnel-transforms-v2:jar</include>
-                <include>org.apache.hadoop:hadoop-aws:jar</include>
-                <include>com.amazonaws:aws-java-sdk-bundle:jar</include>
-                <include>org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional</include>
-                <!--Add hadoop aliyun jar -->
-                <include>org.apache.hadoop:hadoop-aliyun:jar</include>
-                <include>com.aliyun.oss:aliyun-sdk-oss:jar</include>
-                <include>org.jdom:jdom:jar</include>
-
-                <!--Add netty buffer jar -->
-                <include>io.netty:netty-buffer:jar</include>
-                <include>io.netty:netty-common:jar</include>
-
-                <!--Add hive exec jar -->
-                <include>org.apache.hive:hive-exec:jar</include>
-                <include>org.apache.hive:hive-service:jar</include>
-                <include>org.apache.thrift:libfb303:jar</include>
             </includes>
-            <outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>
-            <outputDirectory>/lib</outputDirectory>
+            <outputDirectory>/transforms</outputDirectory>
             <scope>provided</scope>
         </dependencySet>
 
@@ -209,6 +192,20 @@
                 <include>com.amazon.redshift:redshift-jdbc42:jar</include>
                 <include>net.snowflake.snowflake-jdbc:jar</include>
                 <include>com.xugudb:xugu-jdbc:jar</include>
+                <include>org.apache.hadoop:hadoop-aws:jar</include>
+                <include>com.amazonaws:aws-java-sdk-bundle:jar</include>
+                <include>org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional</include>
+                <!--Add hadoop aliyun jar -->
+                <include>org.apache.hadoop:hadoop-aliyun:jar</include>
+                <include>com.aliyun.oss:aliyun-sdk-oss:jar</include>
+                <include>org.jdom:jdom:jar</include>
+                <!--Add netty buffer jar -->
+                <include>io.netty:netty-buffer:jar</include>
+                <include>io.netty:netty-common:jar</include>
+                <!--Add hive exec jar -->
+                <include>org.apache.hive:hive-exec:jar</include>
+                <include>org.apache.hive:hive-service:jar</include>
+                <include>org.apache.thrift:libfb303:jar</include>
             </includes>
             <outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>
             <outputDirectory>/lib</outputDirectory>

diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml b/seatunnel-dist/src/main/assembly/assembly-bin.xml
@@ -161,13 +161,24 @@
             <scope>provided</scope>
         </dependencySet>
 
-        <!-- ============ SeaTunnel Transforms V2 Jars And SeaTunnel Hadoop3 Uber Jar============  -->
+        <!-- ============ SeaTunnel Transforms V2 Jars ============  -->
         <dependencySet>
             <useProjectArtifact>false</useProjectArtifact>
             <useTransitiveDependencies>true</useTransitiveDependencies>
             <unpack>false</unpack>
             <includes>
                 <include>org.apache.seatunnel:seatunnel-transforms-v2:jar</include>
+            </includes>
+            <outputDirectory>/transforms</outputDirectory>
+            <scope>provided</scope>
+        </dependencySet>
+
+        <!-- ============ Lib Jars ============  -->
+        <dependencySet>
+            <useProjectArtifact>false</useProjectArtifact>
+            <useTransitiveDependencies>true</useTransitiveDependencies>
+            <unpack>false</unpack>
+            <includes>
                 <include>org.apache.seatunnel:seatunnel-hadoop3-3.1.4-uber:jar:*:optional</include>
             </includes>
             <outputFileNameMapping>${artifact.file.name}</outputFileNameMapping>

diff --git a/...va/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelTransformPluginDiscovery.java b/...va/org/apache/seatunnel/plugin/discovery/seatunnel/SeaTunnelTransformPluginDiscovery.java
@@ -34,7 +34,7 @@
 public class SeaTunnelTransformPluginDiscovery extends AbstractPluginDiscovery<SeaTunnelTransform> {
 
     public SeaTunnelTransformPluginDiscovery() {
-        super(Common.libDir());
+        super(Common.transformsDir());
     }
 
     @Override