Skip to content

Commit

Permalink
导入优化 (alibaba#4775)
Browse files Browse the repository at this point in the history 
* import 优化

* 无法识别项目 提示优化

* revert

* revert

* 改回原来的格式
  • Loading branch information
@wjm0729
wjm0729 authored Jan 22, 2021
1 parent a67f6b5 commit 0545e85
Show file tree
Hide file tree
Showing 7 changed files with 91 additions and 25 deletions.
2 changes: 2 additions & 0 deletions config/src/main/java/com/alibaba/nacos/config/server/constant/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,4 +267,6 @@ public class Constants {
* Specifies that reads wait without timeout.
*/
public static final String EXTEND_NEED_READ_UNTIL_HAVE_DATA = "00--0-read-join-0--00";

public static final String CONFIG_EXPORT_ITEM_FILE_SEPARATOR = "/";
}
28 changes: 19 additions & 9 deletions config/src/main/java/com/alibaba/nacos/config/server/controller/ConfigController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.alibaba.nacos.common.model.RestResult;
import com.alibaba.nacos.common.model.RestResultUtils;
import com.alibaba.nacos.common.utils.MapUtils;
import com.alibaba.nacos.common.utils.NamespaceUtil;
import com.alibaba.nacos.config.server.auth.ConfigResourceParser;
import com.alibaba.nacos.config.server.constant.Constants;
import com.alibaba.nacos.config.server.controller.parameters.SameNamespaceCloneConfigBean;
Expand All @@ -44,7 +45,6 @@
import com.alibaba.nacos.config.server.utils.MD5Util;
import com.alibaba.nacos.config.server.utils.ParamUtils;
import com.alibaba.nacos.config.server.utils.RequestUtil;
import com.alibaba.nacos.common.utils.NamespaceUtil;
import com.alibaba.nacos.config.server.utils.TimeUtils;
import com.alibaba.nacos.config.server.utils.ZipUtils;
import com.alibaba.nacos.sys.utils.InetUtils;
Expand Down Expand Up @@ -492,7 +492,7 @@ public ResponseEntity<byte[]> exportConfig(@RequestParam(value = "dataId", requi
// Fixed use of "\r\n" here
.append(ci.getAppName()).append("\r\n");
}
String itemName = ci.getGroup() + "/" + ci.getDataId();
String itemName = ci.getGroup() + Constants.CONFIG_EXPORT_ITEM_FILE_SEPARATOR + ci.getDataId();
zipItemList.add(new ZipUtils.ZipItem(itemName, ci.getContent()));
}
if (metaData != null) {
Expand Down Expand Up @@ -538,13 +538,15 @@ public RestResult<Map<String, Object>> importAndPublishConfig(HttpServletRequest
}

List<ConfigAllInfo> configInfoList = null;
List<Map<String, String>> unrecognizedList = null;
try {
ZipUtils.UnZipResult unziped = ZipUtils.unzip(file.getBytes());
ZipUtils.ZipItem metaDataZipItem = unziped.getMetaDataItem();
Map<String, String> metaDataMap = new HashMap<>(16);
if (metaDataZipItem != null) {
String metaDataStr = metaDataZipItem.getItemData();
String[] metaDataArr = metaDataStr.split("\r\n");
// compatible all file separator
String metaDataStr = metaDataZipItem.getItemData().replaceAll("[\r\n]+", "|");
String[] metaDataArr = metaDataStr.split("\\|");
for (String metaDataItem : metaDataArr) {
String[] metaDataItemArr = metaDataItem.split("=");
if (metaDataItemArr.length != 2) {
Expand All @@ -557,11 +559,14 @@ public RestResult<Map<String, Object>> importAndPublishConfig(HttpServletRequest
List<ZipUtils.ZipItem> itemList = unziped.getZipItemList();
if (itemList != null && !itemList.isEmpty()) {
configInfoList = new ArrayList<>(itemList.size());
unrecognizedList = new ArrayList<>();
for (ZipUtils.ZipItem item : itemList) {
String[] groupAdnDataId = item.getItemName().split("/");
if (!item.getItemName().contains("/") || groupAdnDataId.length != 2) {
failedData.put("succCount", 0);
return RestResultUtils.buildResult(ResultCodeEnum.DATA_VALIDATION_FAILED, failedData);
String[] groupAdnDataId = item.getItemName().split(Constants.CONFIG_EXPORT_ITEM_FILE_SEPARATOR);
if (groupAdnDataId.length != 2) {
Map<String, String> unrecognizedItem = new HashMap<>(1);
unrecognizedItem.put("itemName", item.getItemName());
unrecognizedList.add(unrecognizedItem);
continue;
}
String group = groupAdnDataId[0];
String dataId = groupAdnDataId[1];
Expand Down Expand Up @@ -605,6 +610,11 @@ public RestResult<Map<String, Object>> importAndPublishConfig(HttpServletRequest
requestIpApp, time.getTime(), InetUtils.getSelfIP(),
ConfigTraceService.PERSISTENCE_EVENT_PUB, configInfo.getContent());
}
// unrecognizedCount
if (!unrecognizedList.isEmpty()) {
saveResult.put("unrecognizedCount", unrecognizedList.size());
saveResult.put("unrecognizedData", unrecognizedList);
}
return RestResultUtils.success("导入成功", saveResult);
}

Expand Down Expand Up @@ -632,7 +642,7 @@ public RestResult<Map<String, Object>> cloneConfig(HttpServletRequest request,
return RestResultUtils.buildResult(ResultCodeEnum.NO_SELECTED_CONFIG, failedData);
}
configBeansList.removeAll(Collections.singleton(null));

namespace = NamespaceUtil.processNamespaceParameter(namespace);
if (StringUtils.isNotBlank(namespace) && persistService.tenantInfoCountByTenantId(namespace) <= 0) {
failedData.put("succCount", 0);
Expand Down
9 changes: 5 additions & 4 deletions config/src/main/java/com/alibaba/nacos/config/server/utils/ZipUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,12 @@ public static UnZipResult unzip(byte[] source) {
while ((offset = zipIn.read(buffer)) != -1) {
out.write(buffer, 0, offset);
}
if (".meta.yml".equals(entry.getName())) {
metaDataItem = new ZipItem(entry.getName(), out.toString("UTF-8"));
} else {
itemList.add(new ZipItem(entry.getName(), out.toString("UTF-8")));
String entryName = entry.getName();
if (".meta.yml".equals(entryName)) {
metaDataItem = new ZipItem(entryName, out.toString("UTF-8"));
continue;
}
itemList.add(new ZipItem(entryName, out.toString("UTF-8")));
} catch (IOException e) {
LOGGER.error("unzip error", e);
}
Expand Down
7 changes: 5 additions & 2 deletions console-ui/src/locales/en-US.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,8 @@ const I18N_CONF = {
configurationManagement8: 'configuration management',
queryResults: 'Search Results: Found',
articleMeetRequirements: 'configuration items',
fuzzyd: 'Add wildcard \'*\' for fuzzy query',
fuzzyg: 'Add wildcard \'*\' for fuzzy query',
fuzzyd: "Add wildcard '*' for fuzzy query",
fuzzyg: "Add wildcard '*' for fuzzy query",
query: 'Search',
advancedQuery9: 'Advanced Query',
application0: 'Application:',
Expand Down Expand Up @@ -301,8 +301,10 @@ const I18N_CONF = {
samePreparation: 'Same preparation',
targetNamespace: 'Target namespace',
conflictConfig: 'Conflict-detected configuration items',
importSuccEntries: 'Successful entries: ',
failureEntries: 'Failure entries',
unprocessedEntries: 'Unprocessed entries',
unrecognizedEntries: 'Unrecognized entries',
skippedEntries: 'skipped entries',
exportSelected: 'Export selected configs',
clone: 'Clone',
Expand All @@ -311,6 +313,7 @@ const I18N_CONF = {
cloneSucc: 'The clone was successful',
cloneAbort: 'Clone abort',
cloneSuccBegin: 'The clone was successful,with ',
cloneSuccEntries: 'Successful entries: ',
cloneSuccEnd: 'configuration items cloned',
cloneFail: 'Clone failed',
getNamespaceFailed: 'get the namespace failed',
Expand Down
7 changes: 5 additions & 2 deletions console-ui/src/locales/zh-CN.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,8 +270,8 @@ const I18N_CONF = {
configurationManagement8: '配置管理',
queryResults: '查询结果：共查询到',
articleMeetRequirements: '条满足要求的配置。',
fuzzyd: '添加通配符\'*\'进行模糊查询',
fuzzyg: '添加通配符\'*\'进行模糊查询',
fuzzyd: "添加通配符'*'进行模糊查询",
fuzzyg: "添加通配符'*'进行模糊查询",
query: '查询',
advancedQuery9: '高级查询',
application0: '归属应用:',
Expand Down Expand Up @@ -299,8 +299,10 @@ const I18N_CONF = {
samePreparation: '相同配置',
targetNamespace: '目标空间',
conflictConfig: '检测到冲突的配置项',
importSuccEntries: '成功导入条目数: ',
failureEntries: '失败的条目',
unprocessedEntries: '未处理的条目',
unrecognizedEntries: '未识别的条目',
skippedEntries: '跳过的条目',
exportSelected: '导出选中的配置',
clone: '克隆',
Expand All @@ -309,6 +311,7 @@ const I18N_CONF = {
cloneSucc: '克隆成功',
cloneAbort: '克隆终止',
cloneSuccBegin: '克隆成功,克隆了',
cloneSuccEntries: '成功克隆条目数: ',
cloneSuccEnd: '项配置',
cloneFail: '克隆失败',
getNamespaceFailed: '获取命名空间失败',
Expand Down
33 changes: 28 additions & 5 deletions ...e-ui/src/pages/ConfigurationManagement/ConfigurationManagement/ConfigurationManagement.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1004,7 +1004,10 @@ class ConfigurationManagement extends React.Component {
const resultCode = ret.code;
if (resultCode === 200) {
confirm.hide();
if (ret.data.failData && ret.data.failData.length > 0) {
let failCount = ret.data.failData ? ret.data.failData.length : 0;
let skipCount = ret.data.skipData ? ret.data.skipData.length : 0;
let unrecognizedCount = ret.data.unrecognizedCount ? ret.data.unrecognizedCount : 0;
if (failCount > 0) {
Dialog.alert({
title: isImport ? locale.importAbort : locale.cloneAbort,
content: (
Expand All @@ -1014,7 +1017,7 @@ class ConfigurationManagement extends React.Component {
</h4>
<div style={{ marginTop: 20 }}>
<h5>
{locale.failureEntries}: {ret.data.failData.length}
{locale.failureEntries}: {failCount}
</h5>
<Table dataSource={ret.data.failData}>
<Table.Column title="Data Id" dataIndex="dataId" />
Expand All @@ -1023,30 +1026,50 @@ class ConfigurationManagement extends React.Component {
</div>
<div>
<h5>
{locale.unprocessedEntries}: {ret.data.skipData ? ret.data.skipData.length : 0}
{locale.unprocessedEntries}: {skipCount}
</h5>
<Table dataSource={ret.data.skipData}>
<Table.Column title="Data Id" dataIndex="dataId" />
<Table.Column title="Group" dataIndex="group" />
</Table>
</div>
<div>
<h5>
{locale.unrecognizedEntries}: {unrecognizedCount}
</h5>
<Table dataSource={ret.data.unrecognizedData}>
<Table.Column title="Item Name" dataIndex="itemName" />
</Table>
</div>
</div>
),
});
} else if (ret.data.skipCount && ret.data.skipCount > 0) {
} else if (skipCount > 0 || unrecognizedCount > 0) {
let message = `${isImport ? locale.importSuccEntries : locale.cloneSuccEntries}${
ret.data.succCount
}`;
Dialog.alert({
title: isImport ? locale.importSucc : locale.cloneSucc,
content: (
<div style={{ width: '500px' }}>
<h5>{message}</h5>
<div>
<h5>
{locale.skippedEntries}: {ret.data.skipData.length}
{locale.skippedEntries}: {skipCount}
</h5>
<Table dataSource={ret.data.skipData}>
<Table.Column title="Data Id" dataIndex="dataId" />
<Table.Column title="Group" dataIndex="group" />
</Table>
</div>
<div>
<h5>
{locale.unrecognizedEntries}: {unrecognizedCount}
</h5>
<Table dataSource={ret.data.unrecognizedData}>
<Table.Column title="Item Name" dataIndex="itemName" />
</Table>
</div>
</div>
),
});
Expand Down
30 changes: 27 additions & 3 deletions test/src/test/java/com/alibaba/nacos/test/config/ConfigExportAndImportAPI_CITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

/**
* @author klw
Expand Down Expand Up @@ -121,7 +122,7 @@ public void cleanup() throws Exception{
params.put("beta", "false");
result = agent.httpDelete(CONFIG_CONTROLLER_PATH + "/", null, params, agent.getEncode(), TIME_OUT);
Assert.assertEquals(HttpURLConnection.HTTP_OK, result.getCode());

params.put("dataId", "testHasAppname1.properties");
params.put("group", "EXPORT_IMPORT_TEST_GROUP");
params.put("beta", "false");
Expand Down Expand Up @@ -265,7 +266,9 @@ public void testImport(){
zipItemList.add(new ZipUtils.ZipItem("TEST_IMPORT/test1.yml", "test: test1"));
zipItemList.add(new ZipUtils.ZipItem("TEST_IMPORT/test2.txt", "test: test1"));
zipItemList.add(new ZipUtils.ZipItem("TEST_IMPORT/test3.properties", "test.test1.value=test"));
String metaDataStr = "TEST_IMPORT.test2~txt.app=testApp1\r\nTEST_IMPORT.test3~properties.app=testApp2";
zipItemList.add(new ZipUtils.ZipItem("TEST_IMPORT_2/test4.properties", "test.test4.value=test"));
zipItemList.add(new ZipUtils.ZipItem("TEST_IMPORT/SUB_GROUP/test5.properties", "test.test5.value=test"));
String metaDataStr = "TEST_IMPORT.test1~yml.app=testApp1\rTEST_IMPORT.test2~txt.app=testApp2\r\nTEST_IMPORT.test3~properties.app=testApp3\nTEST_IMPORT_2.test4~properties.app=testApp4";
zipItemList.add(new ZipUtils.ZipItem(".meta.yml", metaDataStr));
String importUrl = "?import=true&namespace=";
Map<String, String> importPrarm = new HashMap<>(1);
Expand All @@ -275,7 +278,15 @@ public void testImport(){
uploadByteFile.setFileBytes(ZipUtils.zip(zipItemList));
uploadByteFile.setMediaType("application/zip");
uploadByteFile.setPrarmName("file");
httpClient.post(SERVER_ADDR + CONFIG_CONTROLLER_PATH + importUrl, importPrarm, Collections.singletonList(uploadByteFile), null);
String importResult = httpClient.post(SERVER_ADDR + CONFIG_CONTROLLER_PATH + importUrl, importPrarm, Collections.singletonList(uploadByteFile), null);

// test unrecognizedData
JsonNode importResObj = JacksonUtils.toObj(importResult);
int unrecognizedCount = importResObj.get("data").get("unrecognizedCount").intValue();
Assert.assertEquals(1, unrecognizedCount);
JsonNode unrecognizedData = importResObj.get("data").get("unrecognizedData").get(0);
Assert.assertEquals("TEST_IMPORT/SUB_GROUP/test5.properties", unrecognizedData.get("itemName").textValue());

String getDataUrl = "?search=accurate&dataId=&group=TEST_IMPORT&appName=&config_tags=&pageNo=1&pageSize=10&tenant=&namespaceId=";
String queryResult = httpClient.get(SERVER_ADDR + CONFIG_CONTROLLER_PATH + getDataUrl, null);
JsonNode resultObj = JacksonUtils.toObj(queryResult);
Expand All @@ -288,13 +299,26 @@ public void testImport(){
}
switch (config.get("dataId").textValue()){
case "test1.yml":
Assert.assertEquals(config.get("appName").textValue(), "testApp1");
break;
case "test2.txt":
Assert.assertEquals(config.get("appName").textValue(), "testApp2");
break;
case "test3.properties":
Assert.assertEquals(config.get("appName").textValue(), "testApp3");
break;
default:
Assert.fail();
}
}

getDataUrl = "?search=accurate&dataId=&group=TEST_IMPORT_2&appName=&config_tags=&pageNo=1&pageSize=10&tenant=&namespaceId=";
queryResult = httpClient.get(SERVER_ADDR + CONFIG_CONTROLLER_PATH + getDataUrl, null);
resultObj = JacksonUtils.toObj(queryResult);
resultConfigs = resultObj.get("pageItems");
Assert.assertEquals(1, resultConfigs.size());
JsonNode jsonNode = resultConfigs.get(0);
Assert.assertEquals(jsonNode.get("appName").textValue(), "testApp4");
}

private Map<String, String> processMetaData(ZipUtils.ZipItem metaDataZipItem){
Expand Down

0 comments on commit 0545e85

Please sign in to comment.