Skip to content

Commit a206dde

Browse files
ralf0131beiwei30
authored andcommitted
[Dubbo-3653] Enable etcd to support metadata center (apache#3943)
* Support etcd as metadata center * Fix key format error for consul/redis and UT failure * Fix UT failure * Fix UT failure * Fix UT failure
1 parent 74a491b commit a206dde

File tree

21 files changed

+452
-48
lines changed

21 files changed

+452
-48
lines changed

dubbo-all/pom.xml

+7
Original file line numberDiff line numberDiff line change
@@ -450,6 +450,13 @@
450450
<scope>compile</scope>
451451
<optional>true</optional>
452452
</dependency>
453+
<dependency>
454+
<groupId>org.apache.dubbo</groupId>
455+
<artifactId>dubbo-metadata-report-etcd</artifactId>
456+
<version>${project.version}</version>
457+
<scope>compile</scope>
458+
<optional>true</optional>
459+
</dependency>
453460

454461
<!-- Transitive dependencies -->
455462
<dependency>

dubbo-bom/pom.xml

+5
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,11 @@
357357
<artifactId>dubbo-metadata-report-consul</artifactId>
358358
<version>${project.version}</version>
359359
</dependency>
360+
<dependency>
361+
<groupId>org.apache.dubbo</groupId>
362+
<artifactId>dubbo-metadata-report-etcd</artifactId>
363+
<version>${project.version}</version>
364+
</dependency>
360365
<dependency>
361366
<groupId>org.apache.dubbo</groupId>
362367
<artifactId>dubbo-configcenter-api</artifactId>

dubbo-configcenter/dubbo-configcenter-etcd/src/test/java/org/apache/dubbo/configcenter/support/etcd/EtcdDynamicConfigurationTest.java

-1
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,6 @@ public String getValue() {
120120

121121
private void put(String key, String value) {
122122
try {
123-
124123
client.getKVClient().put(ByteSequence.from(key, UTF_8), ByteSequence.from(value, UTF_8)).get();
125124
} catch (Exception e) {
126125
System.out.println("Error put value to etcd.");

dubbo-distribution/pom.xml

+50
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,56 @@
185185
<artifactId>dubbo-registry-sofa</artifactId>
186186
<version>${project.version}</version>
187187
</dependency>
188+
<dependency>
189+
<groupId>org.apache.dubbo</groupId>
190+
<artifactId>dubbo-configcenter-api</artifactId>
191+
<version>${project.version}</version>
192+
</dependency>
193+
<dependency>
194+
<groupId>org.apache.dubbo</groupId>
195+
<artifactId>dubbo-configcenter-zookeeper</artifactId>
196+
<version>${project.version}</version>
197+
</dependency>
198+
<dependency>
199+
<groupId>org.apache.dubbo</groupId>
200+
<artifactId>dubbo-configcenter-apollo</artifactId>
201+
<version>${project.version}</version>
202+
</dependency>
203+
<dependency>
204+
<groupId>org.apache.dubbo</groupId>
205+
<artifactId>dubbo-configcenter-consul</artifactId>
206+
<version>${project.version}</version>
207+
</dependency>
208+
<dependency>
209+
<groupId>org.apache.dubbo</groupId>
210+
<artifactId>dubbo-configcenter-etcd</artifactId>
211+
<version>${project.version}</version>
212+
</dependency>
213+
<dependency>
214+
<groupId>org.apache.dubbo</groupId>
215+
<artifactId>dubbo-metadata-report-api</artifactId>
216+
<version>${project.version}</version>
217+
</dependency>
218+
<dependency>
219+
<groupId>org.apache.dubbo</groupId>
220+
<artifactId>dubbo-metadata-report-zookeeper</artifactId>
221+
<version>${project.version}</version>
222+
</dependency>
223+
<dependency>
224+
<groupId>org.apache.dubbo</groupId>
225+
<artifactId>dubbo-metadata-report-redis</artifactId>
226+
<version>${project.version}</version>
227+
</dependency>
228+
<dependency>
229+
<groupId>org.apache.dubbo</groupId>
230+
<artifactId>dubbo-metadata-report-consul</artifactId>
231+
<version>${project.version}</version>
232+
</dependency>
233+
<dependency>
234+
<groupId>org.apache.dubbo</groupId>
235+
<artifactId>dubbo-metadata-report-etcd</artifactId>
236+
<version>${project.version}</version>
237+
</dependency>
188238
<dependency>
189239
<groupId>org.apache.dubbo</groupId>
190240
<artifactId>dubbo-monitor-api</artifactId>

dubbo-metadata-report/dubbo-metadata-report-api/src/main/java/org/apache/dubbo/metadata/identifier/MetadataIdentifier.java

+4-2
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,10 @@
2323
* 2018/10/25
2424
*/
2525
public class MetadataIdentifier {
26+
2627
public static final String SEPARATOR = ":";
2728
final static String DEFAULT_PATH_TAG = "metadata";
29+
final static String META_DATA_STORE_TAG = ".metaData";
2830

2931
private String serviceInterface;
3032
private String version;
@@ -53,9 +55,9 @@ public MetadataIdentifier(URL url) {
5355

5456
public String getUniqueKey(KeyTypeEnum keyType) {
5557
if (keyType == KeyTypeEnum.PATH) {
56-
return getFilePathKey();
58+
return getFilePathKey() + Constants.PATH_SEPARATOR + DEFAULT_PATH_TAG;
5759
}
58-
return getIdentifierKey();
60+
return getIdentifierKey() + META_DATA_STORE_TAG;
5961
}
6062

6163
public String getIdentifierKey() {

dubbo-metadata-report/dubbo-metadata-report-api/src/main/java/org/apache/dubbo/metadata/store/MetadataReport.java

-3
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,6 @@
2626
*/
2727
public interface MetadataReport {
2828

29-
public static final String META_DATA_STORE_TAG = ".metaData";
30-
31-
3229
void storeProviderMetadata(MetadataIdentifier providerMetadataIdentifier, FullServiceDefinition serviceDefinition);
3330

3431
void storeConsumerMetadata(MetadataIdentifier consumerMetadataIdentifier, Map<String, String> serviceParameterMap);

dubbo-metadata-report/dubbo-metadata-report-api/src/main/java/org/apache/dubbo/metadata/support/AbstractMetadataReport.java

+3-12
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
*/
5757
public abstract class AbstractMetadataReport implements MetadataReport {
5858

59+
protected final static String DEFAULT_ROOT = "dubbo";
5960

6061
private static final int ONE_DAY_IN_MIll = 60 * 24 * 60 * 1000;
6162
private static final int FOUR_HOURS_IN_MIll = 60 * 4 * 60 * 1000;
@@ -216,12 +217,7 @@ public void storeProviderMetadata(MetadataIdentifier providerMetadataIdentifier,
216217
if (syncReport) {
217218
storeProviderMetadataTask(providerMetadataIdentifier, serviceDefinition);
218219
} else {
219-
reportCacheExecutor.execute(new Runnable() {
220-
@Override
221-
public void run() {
222-
storeProviderMetadataTask(providerMetadataIdentifier, serviceDefinition);
223-
}
224-
});
220+
reportCacheExecutor.execute(() -> storeProviderMetadataTask(providerMetadataIdentifier, serviceDefinition));
225221
}
226222
}
227223

@@ -249,12 +245,7 @@ public void storeConsumerMetadata(MetadataIdentifier consumerMetadataIdentifier,
249245
if (syncReport) {
250246
storeConsumerMetadataTask(consumerMetadataIdentifier, serviceParameterMap);
251247
} else {
252-
reportCacheExecutor.execute(new Runnable() {
253-
@Override
254-
public void run() {
255-
storeConsumerMetadataTask(consumerMetadataIdentifier, serviceParameterMap);
256-
}
257-
});
248+
reportCacheExecutor.execute(() -> storeConsumerMetadataTask(consumerMetadataIdentifier, serviceParameterMap));
258249
}
259250
}
260251

dubbo-metadata-report/dubbo-metadata-report-api/src/test/java/org/apache/dubbo/metadata/identifier/MetadataIdentifierTest.java

+11-3
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
import org.junit.jupiter.api.Assertions;
2222
import org.junit.jupiter.api.Test;
2323

24+
import static org.apache.dubbo.metadata.identifier.MetadataIdentifier.META_DATA_STORE_TAG;
25+
2426
/**
2527
* 2019/1/7
2628
*/
@@ -34,10 +36,16 @@ public void testGetUniqueKey() {
3436
String application = "vic.zk.md";
3537
MetadataIdentifier providerMetadataIdentifier = new MetadataIdentifier(interfaceName, version, group, Constants.PROVIDER_SIDE, application);
3638
System.out.println(providerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.PATH));
37-
Assertions.assertEquals(providerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.PATH), "metadata" + Constants.PATH_SEPARATOR + interfaceName + Constants.PATH_SEPARATOR + (version == null ? "" : (version + Constants.PATH_SEPARATOR))
38-
+ (group == null ? "" : (group + Constants.PATH_SEPARATOR)) + Constants.PROVIDER_SIDE + Constants.PATH_SEPARATOR + application);
39+
Assertions.assertEquals(providerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.PATH),
40+
"metadata" + Constants.PATH_SEPARATOR + interfaceName + Constants.PATH_SEPARATOR +
41+
(version == null ? "" : (version + Constants.PATH_SEPARATOR))
42+
+ (group == null ? "" : (group + Constants.PATH_SEPARATOR)) + Constants.PROVIDER_SIDE
43+
+ Constants.PATH_SEPARATOR + application + Constants.PATH_SEPARATOR + "metadata");
3944
System.out.println(providerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.UNIQUE_KEY));
4045
Assertions.assertEquals(providerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.UNIQUE_KEY),
41-
interfaceName + MetadataIdentifier.SEPARATOR + (version == null ? "" : version + MetadataIdentifier.SEPARATOR) + (group == null ? "" : group + MetadataIdentifier.SEPARATOR) + Constants.PROVIDER_SIDE + MetadataIdentifier.SEPARATOR + application);
46+
interfaceName + MetadataIdentifier.SEPARATOR +
47+
(version == null ? "" : version + MetadataIdentifier.SEPARATOR)
48+
+ (group == null ? "" : group + MetadataIdentifier.SEPARATOR)
49+
+ Constants.PROVIDER_SIDE + MetadataIdentifier.SEPARATOR + application + META_DATA_STORE_TAG);
4250
}
4351
}

dubbo-metadata-report/dubbo-metadata-report-api/src/test/java/org/apache/dubbo/metadata/store/test/JTestMetadataReport4Test.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ protected void doStoreProviderMetadata(MetadataIdentifier providerMetadataIdenti
5454

5555
@Override
5656
protected void doStoreConsumerMetadata(MetadataIdentifier consumerMetadataIdentifier, String serviceParameterString) {
57-
store.put(consumerMetadataIdentifier.getIdentifierKey(), serviceParameterString);
57+
store.put(consumerMetadataIdentifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.UNIQUE_KEY), serviceParameterString);
5858
}
5959

6060
public static String getProviderKey(URL url) {

dubbo-metadata-report/dubbo-metadata-report-consul/src/main/java/org/apache/dubbo/metadata/store/consul/ConsulMetadataReport.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ protected void doStoreConsumerMetadata(MetadataIdentifier consumerMetadataIdenti
5555

5656
private void storeMetadata(MetadataIdentifier identifier, String v) {
5757
try {
58-
client.setKVValue(identifier.getIdentifierKey() + META_DATA_STORE_TAG, v);
58+
client.setKVValue(identifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.UNIQUE_KEY), v);
5959
} catch (Throwable t) {
6060
logger.error("Failed to put " + identifier + " to consul " + v + ", cause: " + t.getMessage(), t);
6161
throw new RpcException("Failed to put " + identifier + " to consul " + v + ", cause: " + t.getMessage(), t);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0"
20+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
21+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
22+
<parent>
23+
<artifactId>dubbo-metadata-report</artifactId>
24+
<groupId>org.apache.dubbo</groupId>
25+
<version>2.7.2-SNAPSHOT</version>
26+
</parent>
27+
<modelVersion>4.0.0</modelVersion>
28+
29+
<artifactId>dubbo-metadata-report-etcd</artifactId>
30+
31+
<dependencies>
32+
<dependency>
33+
<groupId>org.apache.dubbo</groupId>
34+
<artifactId>dubbo-metadata-report-api</artifactId>
35+
<version>${project.parent.version}</version>
36+
</dependency>
37+
<dependency>
38+
<groupId>org.apache.dubbo</groupId>
39+
<artifactId>dubbo-remoting-etcd3</artifactId>
40+
<version>${project.parent.version}</version>
41+
</dependency>
42+
<dependency>
43+
<groupId>io.etcd</groupId>
44+
<artifactId>jetcd-launcher</artifactId>
45+
<scope>test</scope>
46+
</dependency>
47+
<dependency>
48+
<groupId>org.testcontainers</groupId>
49+
<artifactId>testcontainers</artifactId>
50+
<scope>test</scope>
51+
</dependency>
52+
</dependencies>
53+
</project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
/*
19+
* Licensed to the Apache Software Foundation (ASF) under one or more
20+
* contributor license agreements. See the NOTICE file distributed with
21+
* this work for additional information regarding copyright ownership.
22+
* The ASF licenses this file to You under the Apache License, Version 2.0
23+
* (the "License"); you may not use this file except in compliance with
24+
* the License. You may obtain a copy of the License at
25+
*
26+
* http://www.apache.org/licenses/LICENSE-2.0
27+
*
28+
* Unless required by applicable law or agreed to in writing, software
29+
* distributed under the License is distributed on an "AS IS" BASIS,
30+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
31+
* See the License for the specific language governing permissions and
32+
* limitations under the License.
33+
*/
34+
package org.apache.dubbo.metadata.store.etcd;
35+
36+
import org.apache.dubbo.common.Constants;
37+
import org.apache.dubbo.common.URL;
38+
import org.apache.dubbo.common.logger.Logger;
39+
import org.apache.dubbo.common.logger.LoggerFactory;
40+
import org.apache.dubbo.metadata.identifier.MetadataIdentifier;
41+
import org.apache.dubbo.metadata.support.AbstractMetadataReport;
42+
import org.apache.dubbo.remoting.etcd.jetcd.JEtcdClient;
43+
44+
/**
45+
* Report Metadata to Etcd
46+
*/
47+
public class EtcdMetadataReport extends AbstractMetadataReport {
48+
49+
private final static Logger logger = LoggerFactory.getLogger(EtcdMetadataReport.class);
50+
51+
private final String root;
52+
53+
/**
54+
* The etcd client
55+
*/
56+
private final JEtcdClient etcdClient;
57+
58+
public EtcdMetadataReport(URL url) {
59+
super(url);
60+
if (url.isAnyHost()) {
61+
throw new IllegalStateException("registry address == null");
62+
}
63+
String group = url.getParameter(Constants.GROUP_KEY, DEFAULT_ROOT);
64+
if (!group.startsWith(Constants.PATH_SEPARATOR)) {
65+
group = Constants.PATH_SEPARATOR + group;
66+
}
67+
this.root = group;
68+
etcdClient = new JEtcdClient(url);
69+
}
70+
71+
@Override
72+
protected void doStoreProviderMetadata(MetadataIdentifier providerMetadataIdentifier, String serviceDefinitions) {
73+
storeMetadata(providerMetadataIdentifier, serviceDefinitions);
74+
}
75+
76+
@Override
77+
protected void doStoreConsumerMetadata(MetadataIdentifier consumerMetadataIdentifier, String value) {
78+
storeMetadata(consumerMetadataIdentifier, value);
79+
}
80+
81+
private void storeMetadata(MetadataIdentifier identifier, String v) {
82+
String key = getNodeKey(identifier);
83+
if (!etcdClient.put(key, v)) {
84+
logger.error("Failed to put " + identifier + " to etcd, value: " + v);
85+
}
86+
}
87+
88+
String getNodeKey(MetadataIdentifier identifier) {
89+
return toRootDir() + identifier.getUniqueKey(MetadataIdentifier.KeyTypeEnum.PATH);
90+
}
91+
92+
String toRootDir() {
93+
if (root.equals(Constants.PATH_SEPARATOR)) {
94+
return root;
95+
}
96+
return root + Constants.PATH_SEPARATOR;
97+
}
98+
}

0 commit comments

Comments
 (0)