diff --git a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java index dbf023dd1..95c897347 100644 --- a/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java +++ b/tis-console/src/main/java/com/qlangtech/tis/config/module/action/CollectionAction.java @@ -47,6 +47,7 @@ import com.qlangtech.tis.runtime.module.action.SchemaAction; import com.qlangtech.tis.runtime.module.action.SysInitializeAction; import com.qlangtech.tis.runtime.module.misc.IMessageHandler; +import com.qlangtech.tis.runtime.module.screen.IndexQuery; import com.qlangtech.tis.runtime.module.screen.ViewPojo; import com.qlangtech.tis.solrdao.ISchemaField; import com.qlangtech.tis.solrdao.ISchemaPluginContext; @@ -86,8 +87,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -//import com.qlangtech.tis.coredefine.biz.CoreNode; - /** * @author 百岁（baisui@qlangtech.com） * @date 2020-12-13 16:10 @@ -115,6 +114,7 @@ public class CollectionAction extends com.qlangtech.tis.runtime.module.action.Ad public static final String KEY_CORE_URL = "coreUrl"; public static final String KEY_IS_ACTIVE = "active"; public static final String KEY_REPLICS = "replics"; + public static final String KEY_COLS_META = "colMetas"; // private @@ -177,6 +177,27 @@ public void doGetIndexTopology(Context context) throws Exception { }).getSchemaParseResult(); biz.put(KEY_PK, parseResult.getUniqueKey()); + + + QueryResutStrategy queryStrategy = QueryIndexServlet.createQueryResutStrategy( + this.getAppDomain(), new IndexQuery.QueryRequestWrapper(getRequest(), context), getResponse(), getDaoContext()); + List nodes = queryStrategy.queryProcess(); + List sfields = IndexQuery.getSfields(this.getRequest(), queryStrategy, nodes); + JSONArray colsMeta = new JSONArray(); + JSONObject colmeta = null; + for (PSchemaField field : sfields) { + colmeta = new JSONObject(); + colmeta.put("name", field.getName()); + colmeta.put("typeName", field.getType().getJavaType().getSimpleName()); + colmeta.put("typeCode", field.getType().getJavaType().getTypeCode()); + colmeta.put("docval", field.isDocValue()); + colmeta.put("indexd", field.isIndexed()); + colmeta.put("stored", field.isStored()); + colmeta.put("dynamic", field.isDynamic()); + colsMeta.add(colmeta); + } + biz.put(KEY_COLS_META, colsMeta); + this.setBizResult(context, biz); } @@ -296,9 +317,6 @@ public void doCreate(Context context) throws Exception { } this.setBizResult(context, new Object()); Objects.requireNonNull(dsTable, "dsTable can not be null"); -// DataSourceFactoryPluginStore dsPluginStore = TIS.getDataBasePluginStore(new PostedDSProp(targetTable)); -// // 保存表 -// dsPluginStore.saveTable(targetTable, targetColMetas.targetColMetas); // 开始创建DF final String topologyName = indexName.param; diff --git a/tis-console/src/test/java/com/qlangtech/tis/BasicActionTestCase.java b/tis-console/src/test/java/com/qlangtech/tis/BasicActionTestCase.java index 80b239e01..d3a2d0a3a 100644 --- a/tis-console/src/test/java/com/qlangtech/tis/BasicActionTestCase.java +++ b/tis-console/src/test/java/com/qlangtech/tis/BasicActionTestCase.java @@ -167,11 +167,11 @@ protected DocCollection buildDocCollectionMock(boolean getSlicesMap, boolean fet EasyMock.expect(replica.getBaseUrl()).andReturn("baseurl").anyTimes(); } - EasyMock.expect(docCollection.getSlicesMap()).andReturn(sliceMap); + EasyMock.expect(docCollection.getSlicesMap()).andReturn(sliceMap).anyTimes(); EasyMock.expect(tisZkStateReader.fetchCollectionState(collectionName, null)) - .andReturn(docCollection); + .andReturn(docCollection).anyTimes(); } return docCollection; } diff --git a/tis-console/src/test/java/com/qlangtech/tis/config/module/action/TestCollectionAction.java b/tis-console/src/test/java/com/qlangtech/tis/config/module/action/TestCollectionAction.java index 7d291c746..c1d16c5b1 100644 --- a/tis-console/src/test/java/com/qlangtech/tis/config/module/action/TestCollectionAction.java +++ b/tis-console/src/test/java/com/qlangtech/tis/config/module/action/TestCollectionAction.java @@ -75,6 +75,8 @@ * @date 2020-12-16 10:39 */ public class TestCollectionAction extends BasicActionTestCase { + private static final String TEST_TABLE_EMPLOYEES_NAME = "employees"; + private static final String COLLECTION_NAME = TISCollectionUtils.NAME_PREFIX + TEST_TABLE_EMPLOYEES_NAME; static { @@ -85,6 +87,8 @@ public class TestCollectionAction extends BasicActionTestCase { return TestCollectionAction.class.getResourceAsStream("s4employees_create_success.json"); }); + HttpUtils.addMockApply(0, "/solr/" + COLLECTION_NAME + "/admin/file/?file=schema.xml", "s4employee_schema.xml", TestCollectionAction.class); + // stub trigger collection indexbuild HttpUtils.addMockApply(CoreAction.TRIGGER_FULL_BUILD_COLLECTION_PATH, () -> { return TestCollectionAction.class.getResourceAsStream("s4employees_trigger_index_build_success.json"); @@ -92,7 +96,6 @@ public class TestCollectionAction extends BasicActionTestCase { } - private static final String TEST_TABLE_EMPLOYEES_NAME = "employees"; // private static final String TEST_TABLE_EMPLOYEES_NAME = "singleEmployee"; @@ -114,7 +117,6 @@ public class TestCollectionAction extends BasicActionTestCase { tabCols.put(TEST_TABLE_DEPARTMENT_NAME, getBuildDepartmentCols()); } - private static final String COLLECTION_NAME = TISCollectionUtils.NAME_PREFIX + TEST_TABLE_EMPLOYEES_NAME; // public void testSend2RemoteServer() throws Exception { // this.clearUpDB(); @@ -180,6 +182,10 @@ public void testGetIndexTopology() throws Exception { JSONObject replic = replics.getJSONObject(0); assertEquals(replica_core_url, replic.getString(CollectionAction.KEY_CORE_URL)); assertTrue(replic.getBoolean(CollectionAction.KEY_IS_ACTIVE)); + + JSONArray colsMeta = topology.getJSONArray(CollectionAction.KEY_COLS_META); + assertNotNull(colsMeta); + assertEquals(9, colsMeta.size()); } public void testDeleteCollection() throws Exception { diff --git a/tis-console/src/test/resources/com/qlangtech/tis/config/module/action/s4employee_schema.xml b/tis-console/src/test/resources/com/qlangtech/tis/config/module/action/s4employee_schema.xml new file mode 100644 index 000000000..b5dac85c3 --- /dev/null +++ b/tis-console/src/test/resources/com/qlangtech/tis/config/module/action/s4employee_schema.xml @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + emp_no + emp_no + diff --git a/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrFieldsParser.java b/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrFieldsParser.java index dca6620c6..69a4501a6 100644 --- a/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrFieldsParser.java +++ b/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrFieldsParser.java @@ -52,6 +52,7 @@ import java.io.StringReader; import java.lang.reflect.Method; import java.net.URL; +import java.sql.Timestamp; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -518,24 +519,32 @@ public static SolrType parseFieldType(String name, String fieldType, boolean spl Type t = new Type(name); t.setSolrType(fieldType); type.setSolrType(t); - if (isTypeMatch(fieldType, "int")) { - type.setJavaType(Integer.class); - return type; - } else if (isTypeMatch(fieldType, "float")) { - type.setJavaType(Float.class); - return type; - } else if (isTypeMatch(fieldType, "double")) { - type.setJavaType(Double.class); - return type; - } else if (isTypeMatch(fieldType, "long")) { - type.setJavaType(Long.class); - return type; - } else if (isTypeMatch(fieldType, "short")) { - type.setJavaType(Short.class); - return type; - } - type.setJavaType(String.class); + type.setJavaType(SolrJavaType.parse(fieldType)); return type; +// if (isTypeMatch(fieldType, "int")) { +// type.setJavaType(Integer.class); +// return type; +// } else if (isTypeMatch(fieldType, "float")) { +// type.setJavaType(Float.class); +// return type; +// } else if (isTypeMatch(fieldType, "double")) { +// type.setJavaType(Double.class); +// return type; +// } else if (isTypeMatch(fieldType, "long")) { +// type.setJavaType(Long.class); +// return type; +// } else if (isTypeMatch(fieldType, "short")) { +// type.setJavaType(Short.class); +// return type; +// } else if (isTypeMatch(fieldType, "timestamp")) { +// type.setJavaType(Timestamp.class); +// return type; +// } else if (isTypeMatch(fieldType, "date")) { +// type.setJavaType(Date.class); +// return type; +// } + + } public static class Type { @@ -579,13 +588,13 @@ public String getPluginName() { return StringUtils.substringAfter(this.getSolrType(), SolrFieldsParser.KEY_PLUGIN + ":"); } - private Class javaType; + private SolrJavaType javaType; private Type solrType; private Method valueof; - public Class getJavaType() { + public SolrJavaType getJavaType() { return this.javaType; } @@ -593,16 +602,16 @@ public Object valueOf(Object val) throws Exception { return valueof.invoke(null, val); } - public void setJavaType(Class javaType) { - try { - if (javaType == String.class) { - valueof = javaType.getMethod("valueOf", Object.class); - } else { - valueof = javaType.getMethod("valueOf", String.class); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + public void setJavaType(SolrJavaType javaType) { +// try { +// if (javaType == String.class) { +// valueof = javaType.getMethod("valueOf", Object.class); +// } else { +// valueof = javaType.getMethod("valueOf", String.class); +// } +// } catch (Exception e) { +// throw new RuntimeException(e); +// } this.javaType = javaType; } @@ -619,9 +628,7 @@ public void setSolrType(Type solrType) { } } - private static boolean isTypeMatch(String fieldType, String matchLetter) { - return StringUtils.indexOfAny(fieldType, new String[]{matchLetter, StringUtils.capitalize(matchLetter)}) > -1; - } + public static class SchemaFields extends ArrayList { diff --git a/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrJavaType.java b/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrJavaType.java new file mode 100644 index 000000000..e97fb62ac --- /dev/null +++ b/tis-solrconfig-parser/src/main/java/com/qlangtech/tis/solrdao/SolrJavaType.java @@ -0,0 +1,75 @@ +/** + * Copyright (c) 2020 QingLang, Inc. + *

+ * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + *

+ * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + *

+ * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +package com.qlangtech.tis.solrdao; + +import org.apache.commons.lang.StringUtils; + +import java.sql.Timestamp; +import java.util.Date; + +/** + * @author 百岁（baisui@qlangtech.com） + * @date 2021-02-25 17:00 + */ +public enum SolrJavaType { + INT(Integer.class, 1), FLOAT(Float.class, 2), DOUBLE(Double.class, 3), LONG(Long.class, 4), TIMESTAMP(Timestamp.class, 4), DATE(Date.class, 5), STRING(String.class, 6); + + public static SolrJavaType parse(String fieldType) { + if (isTypeMatch(fieldType, "int")) { + return INT; + } else if (isTypeMatch(fieldType, "float")) { + return FLOAT; + } else if (isTypeMatch(fieldType, "double")) { + return DOUBLE; + } else if (isTypeMatch(fieldType, "long")) { + return LONG; + } else if (isTypeMatch(fieldType, "timestamp")) { + return TIMESTAMP; + } else if (isTypeMatch(fieldType, "date")) { + return DATE; + } + return STRING; + } + + private static boolean isTypeMatch(String fieldType, String matchLetter) { + return StringUtils.indexOfAny(fieldType, new String[]{matchLetter, StringUtils.capitalize(matchLetter)}) > -1; + } + + private final Class javaType; + private final int typeCode; + // private final Method valueof; + + private SolrJavaType(Class javaType, int typeCode) { + this.javaType = javaType; + this.typeCode = typeCode; +// try { +// if (javaType == String.class) { +// valueof = javaType.getMethod("valueOf", Object.class); +// } else { +// valueof = javaType.getMethod("valueOf", String.class); +// } +// } catch (Exception e) { +// throw new RuntimeException(e); +// } + } + + public int getTypeCode() { + return typeCode; + } + + public String getSimpleName() { + return StringUtils.lowerCase(javaType.getSimpleName()); + } +} diff --git a/tis-solrcore-extend/src/main/java/com/qlangtech/tis/solrextend/core/TISCoresLocator.java b/tis-solrcore-extend/src/main/java/com/qlangtech/tis/solrextend/core/TISCoresLocator.java index 024938ee8..64395e5ed 100644 --- a/tis-solrcore-extend/src/main/java/com/qlangtech/tis/solrextend/core/TISCoresLocator.java +++ b/tis-solrcore-extend/src/main/java/com/qlangtech/tis/solrextend/core/TISCoresLocator.java @@ -22,6 +22,8 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.CoreDescriptor; import org.apache.solr.core.CorePropertiesLocator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.nio.file.Path; import java.util.List; @@ -33,6 +35,8 @@ * @date 2021-02-05 07:14 */ public class TISCoresLocator extends CorePropertiesLocator { + private static final Logger logger = LoggerFactory.getLogger(TISCoresLocator.class); + public TISCoresLocator(Path coreDiscoveryRoot) { super(coreDiscoveryRoot); } @@ -49,6 +53,8 @@ public List discover(CoreContainer cc) { } ComponentMeta coreComponent = TIS.getCoreComponent(resources); coreComponent.synchronizePluginsFromRemoteRepository(); + logger.info("synchronizePluginsFromRemoteRepository {},load resource size {}" + , collections.stream().collect(Collectors.joining(",")), resources.size()); return cores; } finally { TIS.permitInitialize = true; diff --git a/tis-solrj-client/src/test/java/TestJDBC.java b/tis-solrj-client/src/test/java/TestJDBC.java index 3ca6615a0..40c28b649 100644 --- a/tis-solrj-client/src/test/java/TestJDBC.java +++ b/tis-solrj-client/src/test/java/TestJDBC.java @@ -35,6 +35,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +import java.net.URLDecoder; import java.sql.Connection; import java.sql.ResultSet; import java.sql.Statement; @@ -48,6 +49,9 @@ public class TestJDBC extends TestCase { public void testSQL() throws Exception { + + System.out.println( URLDecoder.decode("%7B!terms%20f%3Demp_no%7D23436%2C23446%2C23451%2C23487%2C23496%2C23517%2C23771")); + DriverImpl driver = new DriverImpl(); // &aggregationMode=map_reduce Connection conn = driver.connect("jdbc:solr://10.1.6.65:2181,10.1.6.67:2181,10.1.6.80:2181/tis/cloud?collection=search4totalpay");