Skip to content

Commit af631a0

Browse files
FuYouJFuYouJ
committed
不依赖于user.dir环境变量,所以插件加载工厂便没有意义了
不依赖于user.dir环境变量,所以插件加载工厂便没有意义了
1 parent 03cfbfa commit af631a0

File tree

11 files changed

+269
-121
lines changed

11 files changed

+269
-121
lines changed

‎core/src/main/java/com/alibaba/datax/core/util/container/LoadUtil.java

+13-5
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public String value() {
4949
/**
5050
* jarLoader的缓冲
5151
*/
52-
private static Map<String, ClassLoader> jarLoaderCenter = new HashMap();
52+
private static Map<String, JarLoader> jarLoaderCenter = new HashMap();
5353

5454
/**
5555
* 设置pluginConfigs，方便后面插件来获取
@@ -167,7 +167,7 @@ private static synchronized Class<? extends AbstractPlugin> loadPluginClass(
167167
PluginType pluginType, String pluginName,
168168
ContainerType pluginRunType) {
169169
Configuration pluginConf = getPluginConf(pluginType, pluginName);
170-
ClassLoader jarLoader = LoadUtil.getJarLoader(pluginType, pluginName);
170+
JarLoader jarLoader = LoadUtil.getJarLoader(pluginType, pluginName);
171171
try {
172172
return (Class<? extends AbstractPlugin>) jarLoader
173173
.loadClass(pluginConf.getString("class") + "$"
@@ -177,14 +177,22 @@ private static synchronized Class<? extends AbstractPlugin> loadPluginClass(
177177
}
178178
}
179179

180-
public static synchronized ClassLoader getJarLoader(PluginType pluginType,
180+
public static synchronized JarLoader getJarLoader(PluginType pluginType,
181181
String pluginName) {
182182
Configuration pluginConf = getPluginConf(pluginType, pluginName);
183183

184-
ClassLoader jarLoader = jarLoaderCenter.get(generatePluginKey(pluginType,
184+
JarLoader jarLoader = jarLoaderCenter.get(generatePluginKey(pluginType,
185185
pluginName));
186186
if (null == jarLoader) {
187-
jarLoader = PluginLoaderFactory.create(pluginConf,pluginType,pluginName);
187+
String pluginPath = pluginConf.getString("path");
188+
if (StringUtils.isBlank(pluginPath)) {
189+
throw DataXException.asDataXException(
190+
FrameworkErrorCode.RUNTIME_ERROR,
191+
String.format(
192+
"%s插件[%s]路径非法!",
193+
pluginType, pluginName));
194+
}
195+
jarLoader = new JarLoader(new String[]{pluginPath});
188196
jarLoaderCenter.put(generatePluginKey(pluginType, pluginName),
189197
jarLoader);
190198
}

‎datax-example/doc/README.md

+66-12
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,82 @@
2121

2222
### 实现原理
2323

24-
- 不修改原有的ConfigParer,使用新的ExampleConfigParser,仅用于example模块。
25-
- 提供新的PluginLoader 插件加载器，可以从程序运行目录获取插件，与JarLoader各司其职。
24+
- 不修改原有的ConfigParer,使用新的ExampleConfigParser,仅用于example模块。他不依赖datax.home,而是依赖ide编译后的target目录
25+
- 将ide的target目录作为每个插件的目录类加载目录。
2626

2727
![img](img/img02.png)
2828

29-
### 如何使用
29+
### 如何使用
30+
1.修改插件的pom文件，做如下改动。以streamreader为例。<br/>
31+
改动前
32+
```xml
33+
<build>
34+
<plugins>
35+
<!-- compiler plugin -->
36+
<plugin>
37+
<artifactId>maven-compiler-plugin</artifactId>
38+
<configuration>
39+
<source>${jdk-version}</source>
40+
<target>${jdk-version}</target>
41+
<encoding>${project-sourceEncoding}</encoding>
42+
</configuration>
43+
</plugin>
44+
</plugins>
45+
</build>
46+
```
47+
改动后
48+
```xml
49+
<build>
50+
<resources>
51+
<!--将resource目录也输出到target-->
52+
<resource>
53+
<directory>src/main/resources</directory>
54+
<includes>
55+
<include>**/*.*</include>
56+
</includes>
57+
<filtering>true</filtering>
58+
</resource>
59+
</resources>
60+
<plugins>
61+
<!-- compiler plugin -->
62+
<plugin>
63+
<artifactId>maven-compiler-plugin</artifactId>
64+
<configuration>
65+
<source>${jdk-version}</source>
66+
<target>${jdk-version}</target>
67+
<encoding>${project-sourceEncoding}</encoding>
68+
</configuration>
69+
</plugin>
70+
</plugins>
71+
</build>
72+
```
73+
2.在datax-example模块引入你需要的插件，默认只引入了streamreader、writer
74+
75+
3.打开datax-example的Main class
3076

3177
```java
3278
public class Main {
79+
80+
/**
81+
* 注意！
82+
* 1.在example模块pom文件添加你依赖的的调试插件，
83+
* 你可以直接打开本模块的pom文件,参考是如何引入streamreader，streamwriter
84+
* 2. 在此处指定你的job文件
85+
*/
3386
public static void main(String[] args) {
34-
//1. 在 datax-example pom文件中加入测试插件模块的依赖，默认导入了streamreader/writer
35-
//2. 在此处指定你的测试文件路径
36-
String path = "/job/stream2stream.json";
3787

38-
Configuration configuration = ExampleConfigParser.parse(
39-
PathUtil.getAbsolutePathFromClassPath(path)
40-
);
88+
String classPathJobPath = "/job/stream2stream.json";
89+
String absJobPath = PathUtil.getAbsolutePathFromClassPath(classPathJobPath);
90+
startExample(absJobPath);
91+
}
92+
93+
public static void startExample(String jobPath) {
94+
95+
Configuration configuration = ExampleConfigParser.parse(jobPath);
4196

4297
Engine engine = new Engine();
4398
engine.start(configuration);
4499
}
100+
45101
}
46-
```
47-
### 注意
48-
此模块不参与打包
102+
```

‎datax-example/doc/img/img02.png

4.62 KB
Loading

‎datax-example/pom.xml

+22-2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
<artifactId>datax-core</artifactId>
2929
<version>0.0.1-SNAPSHOT</version>
3030
</dependency>
31+
<!-- 默认引入streamwriter,streamreader-->
3132
<dependency>
3233
<groupId>com.alibaba.datax</groupId>
3334
<artifactId>streamwriter</artifactId>
@@ -49,12 +50,31 @@
4950
<build>
5051
<resources>
5152
<resource>
52-
<directory>src/main/java</directory>
53+
<directory>src/main/resources</directory>
5354
<includes>
54-
<include>**/*.properties</include>
55+
<include>**/*.*</include>
5556
</includes>
57+
<filtering>true</filtering>
58+
</resource>
59+
<resource>
60+
<directory>src/main/resources</directory>
61+
<includes>
62+
<include>**/*.*</include>
63+
</includes>
64+
<filtering>true</filtering>
5665
</resource>
5766
</resources>
67+
<plugins>
68+
<!-- compiler plugin -->
69+
<plugin>
70+
<artifactId>maven-compiler-plugin</artifactId>
71+
<configuration>
72+
<source>${jdk-version}</source>
73+
<target>${jdk-version}</target>
74+
<encoding>${project-sourceEncoding}</encoding>
75+
</configuration>
76+
</plugin>
77+
</plugins>
5878
</build>
5979

6080
</project>

‎datax-example/src/main/java/com/alibaba/datax/example/Main.java

+15-5
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,25 @@
1010
* @author fuyouj
1111
*/
1212
public class Main {
13+
14+
/**
15+
* 1.在example模块pom文件添加你依赖的的调试插件，
16+
* 你可以直接打开本模块的pom文件,参考是如何引入streamreader，streamwriter
17+
* 2. 在此处指定你的job文件
18+
*/
1319
public static void main(String[] args) {
14-
//在此处指定你的测试文件路径
15-
String path = "/job/stream2stream.json";
1620

17-
Configuration configuration = ExampleConfigParser.parse(
18-
PathUtil.getAbsolutePathFromClassPath(path)
19-
);
21+
String classPathJobPath = "/job/stream2stream.json";
22+
String absJobPath = PathUtil.getAbsolutePathFromClassPath(classPathJobPath);
23+
startExample(absJobPath);
24+
}
25+
26+
public static void startExample(String jobPath) {
27+
28+
Configuration configuration = ExampleConfigParser.parse(jobPath);
2029

2130
Engine engine = new Engine();
2231
engine.start(configuration);
2332
}
33+
2434
}

‎datax-example/src/main/java/com/alibaba/datax/example/util/ExampleConfigParser.java

+58-13
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77
import com.alibaba.datax.core.util.container.CoreConstant;
88

99
import java.io.File;
10+
import java.io.IOException;
1011
import java.net.URL;
1112
import java.nio.file.Paths;
12-
import java.util.HashMap;
13-
import java.util.Map;
13+
import java.util.*;
1414

1515
/**
1616
* @author fuyouj
@@ -44,9 +44,15 @@ private static Configuration parsePluginsConfig(Map<String, String> pluginTypeMa
4444

4545
Configuration configuration = Configuration.newDefault();
4646

47-
String workingDirectory = System.getProperty("user.dir");
48-
File file = new File(workingDirectory);
49-
scanPlugin(configuration, file.listFiles(), pluginTypeMap);
47+
//最初打算通过user.dir获取工作目录来扫描插件，
48+
//但是user.dir在不同有一些不确定性，所以废弃了这个选择
49+
50+
for (File basePackage : runtimeBasePackages()) {
51+
if (pluginTypeMap.isEmpty()) {
52+
break;
53+
}
54+
scanPluginByPackage(basePackage, configuration, basePackage.listFiles(), pluginTypeMap);
55+
}
5056
if (!pluginTypeMap.isEmpty()) {
5157
throw DataXException.asDataXException(FrameworkErrorCode.PLUGIN_INIT_ERROR,
5258
"load plugin failed，未完成指定插件加载:"
@@ -55,7 +61,42 @@ private static Configuration parsePluginsConfig(Map<String, String> pluginTypeMa
5561
return configuration;
5662
}
5763

58-
private static void scanPlugin(Configuration configuration, File[] files, Map<String, String> pluginTypeMap) {
64+
/**
65+
* 通过classLoader获取程序编译的输出目录
66+
* @return File[/datax-example/target/classes,xxReader/target/classes,xxWriter/target/classes]
67+
*/
68+
private static File[] runtimeBasePackages() {
69+
List<File> basePackages = new ArrayList<>();
70+
ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
71+
Enumeration<URL> resources = null;
72+
try {
73+
resources = classLoader.getResources("");
74+
} catch (IOException e) {
75+
throw DataXException.asDataXException(e.getMessage());
76+
}
77+
78+
while (resources.hasMoreElements()) {
79+
URL resource = resources.nextElement();
80+
File file = new File(resource.getFile());
81+
if (file.isDirectory()) {
82+
basePackages.add(file);
83+
}
84+
}
85+
86+
return basePackages.toArray(new File[0]);
87+
}
88+
89+
/**
90+
*
91+
* @param packageFile 编译出来的target/classes根目录 便于找到插件时设置插件的URL目录，设置根目录是最保险的方式
92+
* @param configuration pluginConfig
93+
* @param files 待扫描文件
94+
* @param needPluginTypeMap 需要的插件
95+
*/
96+
private static void scanPluginByPackage(File packageFile,
97+
Configuration configuration,
98+
File[] files,
99+
Map<String, String> needPluginTypeMap) {
59100
if (files == null) {
60101
return;
61102
}
@@ -64,22 +105,26 @@ private static void scanPlugin(Configuration configuration, File[] files, Map<St
64105
Configuration pluginDesc = Configuration.from(file);
65106
String descPluginName = pluginDesc.getString("name", "");
66107

67-
if (pluginTypeMap.containsKey(descPluginName)) {
108+
if (needPluginTypeMap.containsKey(descPluginName)) {
68109

69-
String type = pluginTypeMap.get(descPluginName);
70-
configuration.merge(parseOnePlugin(type, descPluginName, pluginDesc), false);
71-
pluginTypeMap.remove(descPluginName);
110+
String type = needPluginTypeMap.get(descPluginName);
111+
configuration.merge(parseOnePlugin(packageFile.getAbsolutePath(),type, descPluginName, pluginDesc), false);
112+
needPluginTypeMap.remove(descPluginName);
72113

73114
}
74115
} else {
75-
scanPlugin(configuration, file.listFiles(), pluginTypeMap);
116+
scanPluginByPackage(packageFile,configuration, file.listFiles(), needPluginTypeMap);
76117
}
77118
}
78119
}
79120

80-
private static Configuration parseOnePlugin(String pluginType, String pluginName, Configuration pluginDesc) {
81121

82-
pluginDesc.set("loadType","pluginLoader");
122+
private static Configuration parseOnePlugin(String packagePath,
123+
String pluginType,
124+
String pluginName,
125+
Configuration pluginDesc) {
126+
//设置path 兼容jarLoader的加载方式URLClassLoader
127+
pluginDesc.set("path", packagePath);
83128
Configuration pluginConfInJob = Configuration.newDefault();
84129
pluginConfInJob.set(
85130
String.format("plugin.%s.%s", pluginType, pluginName),

‎datax-example/src/test/java/com/alibaba/datax/example/DataXExampleTest.java

+3-11
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,16 @@
11
package com.alibaba.datax.example;
22

33

4-
import com.alibaba.datax.common.util.Configuration;
5-
import com.alibaba.datax.core.Engine;
6-
import com.alibaba.datax.example.util.ExampleConfigParser;
74
import com.alibaba.datax.example.util.PathUtil;
85
import org.junit.Test;
96

107

11-
public class DataXExampleTest extends ExampleTestTemplate {
8+
public class DataXExampleTest {
129

1310
@Test
1411
public void testStreamReader2StreamWriter() {
15-
1612
String path = "/job/stream2stream.json";
17-
Configuration testConfiguration = ExampleConfigParser.parse(
18-
PathUtil.getAbsolutePathFromClassPath(path)
19-
);
20-
Engine engine = new Engine();
21-
engine.start(testConfiguration);
13+
String jobPath = PathUtil.getAbsolutePathFromClassPath(path);
14+
Main.startExample(jobPath);
2215
}
23-
2416
}

‎datax-example/src/test/java/com/alibaba/datax/example/util/com/alibaba/datax/example/util/ExampleConfigParserTest.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.alibaba.datax.example.util.com.alibaba.datax.example.util;
22

33
import com.alibaba.datax.common.util.Configuration;
4-
import com.alibaba.datax.example.ExampleTestTemplate;
54
import com.alibaba.datax.example.util.ExampleConfigParser;
65
import com.alibaba.datax.example.util.PathUtil;
76
import org.junit.Assert;
@@ -10,7 +9,7 @@
109
import java.io.File;
1110

1211

13-
public class ExampleConfigParserTest extends ExampleTestTemplate {
12+
public class ExampleConfigParserTest {
1413

1514

1615
@Test

0 commit comments

Comments
 (0)