Refactor Shadow DistSQL and adapt the new data structure of tableConf…

…iguration (apache#12623)

* Refactored shadow DistSQL

* Refactored Shadow DistSQL, added `checker` and `supporter`

* Refactored Shadow DistSQL, Modify the naming in test

* Adapt to Shadow's new data structure

* Add license.

* Revert local configuration files.

* Fix problems that occurred during compare》

* Modify the aggregateData() to mergeConfiguration().

* Adjust the check method

* Use string.format() to replace '+'

shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-distsql/shardingsphere-shadow-distsql-handler/src/main/java/org/apache/shardingsphere/shadow/distsql/handler/checker/ShadowRuleStatementChecker.java

@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.shadow.distsql.handler.checker;
+
+import org.apache.shardingsphere.infra.config.scope.SchemaRuleConfiguration;
+import org.apache.shardingsphere.infra.distsql.exception.DistSQLException;
+import org.apache.shardingsphere.infra.distsql.exception.resource.RequiredResourceMissedException;
+import org.apache.shardingsphere.infra.distsql.exception.rule.InvalidAlgorithmConfigurationException;
+import org.apache.shardingsphere.infra.distsql.exception.rule.RequiredRuleMissedException;
+import org.apache.shardingsphere.infra.metadata.ShardingSphereMetaData;
+import org.apache.shardingsphere.shadow.distsql.parser.segment.ShadowAlgorithmSegment;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Shadow rule statement checker.
+ */
+public class ShadowRuleStatementChecker {
+
+    public static final String SHADOW = "shadow";
+
+    /**
+     * Check if the configuration exists.
+     *
+     * @param schemaName schema name
+     * @param configuration configuration
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkConfigurationExist(final String schemaName, final SchemaRuleConfiguration configuration) throws DistSQLException {
+        DistSQLException.predictionThrow(null != configuration, new RequiredRuleMissedException(SHADOW, schemaName));
+    }
+
+    /**
+     * Check if resources exist in meta data.
+     *
+     * @param resources resource being checked
+     * @param metaData meta rules
+     * @param schemaName schema name
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkResourceExist(final Collection<String> resources, final ShardingSphereMetaData metaData, final String schemaName) throws DistSQLException {
+        Collection<String> notExistedResources = metaData.getResource().getNotExistedResources(resources);
+        DistSQLException.predictionThrow(notExistedResources.isEmpty(), new RequiredResourceMissedException(schemaName, notExistedResources));
+    }
+
+    /**
+     * Check the completeness of the algorithm.
+     *
+     * @param algorithmSegments algorithmSegments to be checked
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkAlgorithmCompleteness(final Collection<ShadowAlgorithmSegment> algorithmSegments) throws DistSQLException {
+        Set<ShadowAlgorithmSegment> incompleteAlgorithms = algorithmSegments.stream().filter(each -> !each.isComplete()).collect(Collectors.toSet());
+        DistSQLException.predictionThrow(incompleteAlgorithms.isEmpty(), new InvalidAlgorithmConfigurationException(SHADOW));
+    }
+
+    /**
+     * Check if the rules exist.
+     * @param requireRules require rules
+     * @param currentRules current rules
+     * @param thrower thrower
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkRulesExist(final Collection<String> requireRules, final Collection<String> currentRules, final Function<Set<String>, DistSQLException> thrower) throws DistSQLException {
+        ShadowRuleStatementChecker.checkAnyDifferent(requireRules, currentRules, thrower);
+    }
+
+    /**
+     * Check if the algorithms exist.
+     * @param requireAlgorithms require algorithms
+     * @param currentAlgorithms current algorithms
+     * @param thrower thrower
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkAlgorithmExist(final Collection<String> requireAlgorithms, final Collection<String> currentAlgorithms, 
+                                           final Function<Set<String>, DistSQLException> thrower) throws DistSQLException {
+        ShadowRuleStatementChecker.checkAnyDifferent(requireAlgorithms, currentAlgorithms, thrower);
+    }
+
+    /**
+     * Check for any duplicate data in the rules, and throw the specified exception.
+     * @param rules rules to be checked
+     * @param thrower exception thrower
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkAnyDuplicate(final Collection<String> rules, final Function<Set<String>, DistSQLException> thrower) throws DistSQLException {
+        Set<String> duplicateRequire = getDuplicate(rules);
+        DistSQLException.predictionThrow(duplicateRequire.isEmpty(), thrower.apply(duplicateRequire));
+    }
+
+    /**
+     * Check if there are duplicates in the rules, and throw the specified exception.
+     *
+     * @param requireRules rules to be checked
+     * @param currentRules rules to be checked
+     * @param thrower exception thrower
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkAnyDuplicate(final Collection<String> requireRules, final Collection<String> currentRules, final Function<Set<String>, DistSQLException> thrower) throws DistSQLException {
+        Set<String> identical = getIdentical(requireRules, currentRules);
+        DistSQLException.predictionThrow(identical.isEmpty(), thrower.apply(identical));
+    }
+
+    /**
+     * Check for any different data in the rules, and throw the specified exception.
+     *
+     * @param requireRules rules to be checked
+     * @param currentRules rules to be checked
+     * @param thrower exception thrower
+     * @throws DistSQLException DistSQL exception
+     */
+    public static void checkAnyDifferent(final Collection<String> requireRules, final Collection<String> currentRules, final Function<Set<String>, DistSQLException> thrower) throws DistSQLException {
+        Set<String> different = getDifferent(requireRules, currentRules);
+        DistSQLException.predictionThrow(different.isEmpty(), thrower.apply(different));
+    }
+
+    private static Set<String> getDuplicate(final Collection<String> require) {
+        return require.stream().collect(Collectors.groupingBy(each -> each, Collectors.counting())).entrySet().stream()
+                .filter(each -> each.getValue() > 1).map(Map.Entry::getKey).collect(Collectors.toSet());
+    }
+
+    private static Set<String> getDifferent(final Collection<String> require, final Collection<String> current) {
+        return require.stream().filter(each -> !current.contains(each)).collect(Collectors.toSet());
+    }
+
+    private static Set<String> getIdentical(final Collection<String> require, final Collection<String> current) {
+        return require.stream().filter(current::contains).collect(Collectors.toSet());
+    }
+}

shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-distsql/shardingsphere-shadow-distsql-handler/src/main/java/org/apache/shardingsphere/shadow/distsql/handler/converter/ShadowRuleStatementConverter.java

@@ -23,12 +23,14 @@
 import org.apache.shardingsphere.shadow.api.config.ShadowRuleConfiguration;
 import org.apache.shardingsphere.shadow.api.config.datasource.ShadowDataSourceConfiguration;
 import org.apache.shardingsphere.shadow.api.config.table.ShadowTableConfiguration;
+import org.apache.shardingsphere.shadow.distsql.handler.supporter.ShadowRuleStatementSupporter;
 import org.apache.shardingsphere.shadow.distsql.parser.segment.ShadowAlgorithmSegment;
 import org.apache.shardingsphere.shadow.distsql.parser.segment.ShadowRuleSegment;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.LinkedList;
+import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.stream.Collectors;
@@ -55,13 +57,17 @@ public static ShadowRuleConfiguration convert(final Collection<ShadowRuleSegment
     }
 
     private static Map<String, ShadowTableConfiguration> getTables(final Collection<ShadowRuleSegment> rules) {
-        return rules.stream().flatMap(each -> each.getShadowTableRules().entrySet().stream()).collect(Collectors.toMap(Entry::getKey, ShadowRuleStatementConverter::buildShadowTableConfiguration));
+        Map<String, ShadowTableConfiguration> result = new HashMap<>();
+        rules.forEach(each -> {
+            Map<String, ShadowTableConfiguration> currentRuleTableConfig = each.getShadowTableRules().entrySet().stream()
+                    .collect(Collectors.toMap(Entry::getKey, entry -> buildShadowTableConfiguration(each.getRuleName(), entry), ShadowRuleStatementSupporter::mergeConfiguration));
+            currentRuleTableConfig.forEach((key, value) -> result.merge(key, value, ShadowRuleStatementSupporter::mergeConfiguration));
+        });
+        return result;
     }
 
-    private static ShadowTableConfiguration buildShadowTableConfiguration(final Entry<String, Collection<ShadowAlgorithmSegment>> entry) {
-        // FIXME replace empty collection
-        Collection<String> dataSourceNames = new LinkedList<>();
-        return new ShadowTableConfiguration(dataSourceNames, entry.getValue().stream().map(ShadowAlgorithmSegment::getAlgorithmName).collect(Collectors.toList()));
+    private static ShadowTableConfiguration buildShadowTableConfiguration(final String ruleName, final Entry<String, Collection<ShadowAlgorithmSegment>> entry) {
+        return new ShadowTableConfiguration(new ArrayList<>(Collections.singletonList(ruleName)), entry.getValue().stream().map(ShadowAlgorithmSegment::getAlgorithmName).collect(Collectors.toList()));
     }
 
     private static Map<String, ShadowDataSourceConfiguration> getDataSource(final Collection<ShadowRuleSegment> rules) {

shardingsphere-features/shardingsphere-shadow/shardingsphere-shadow-distsql/shardingsphere-shadow-distsql-handler/src/main/java/org/apache/shardingsphere/shadow/distsql/handler/supporter/ShadowRuleStatementSupporter.java

@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.shardingsphere.shadow.distsql.handler.supporter;
+
+import org.apache.shardingsphere.shadow.api.config.ShadowRuleConfiguration;
+import org.apache.shardingsphere.shadow.api.config.table.ShadowTableConfiguration;
+import org.apache.shardingsphere.shadow.distsql.parser.segment.ShadowAlgorithmSegment;
+import org.apache.shardingsphere.shadow.distsql.parser.segment.ShadowRuleSegment;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * Shadow rule statement supporter.
+ */
+public final class ShadowRuleStatementSupporter {
+
+    /**
+     * Get rule names from the configuration.
+     *
+     * @param configuration configuration
+     * @return rule names
+     */
+    public static List<String> getRuleNames(final ShadowRuleConfiguration configuration) {
+        if (null == configuration) {
+            return Collections.emptyList();
+        }
+        return new ArrayList<>(configuration.getDataSources().keySet());
+    }
+
+    /**
+     * Get rule names from the rules.
+     *
+     * @param rules rules
+     * @return rule names
+     */
+    public static List<String> getRuleNames(final Collection<ShadowRuleSegment> rules) {
+        if (rules.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return rules.stream().map(ShadowRuleSegment::getRuleName).collect(Collectors.toList());
+    }
+
+    /**
+     * Get table names from the configuration.
+     *
+     * @param configuration configuration
+     * @return table names
+     */
+    public static List<String> getTableNames(final ShadowRuleConfiguration configuration) {
+        if (null == configuration) {
+            return Collections.emptyList();
+        }
+        return new ArrayList<>(configuration.getTables().keySet());
+    }
+
+    /**
+     * Get the table names from the rules.
+     *
+     * @param rules rules
+     * @return table names
+     */
+    public static List<String> getTableNames(final Collection<ShadowRuleSegment> rules) {
+        if (rules.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return rules.stream().flatMap(each -> each.getShadowTableRules().keySet().stream()).collect(Collectors.toList());
+    }
+
+    /**
+     * Get the resource names from the rules.
+     *
+     * @param rules rules
+     * @return resource names
+     */
+    public static List<String> getResourceNames(final Collection<ShadowRuleSegment> rules) {
+        if (rules.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return rules.stream().map(each -> Arrays.asList(each.getSource(), each.getShadow())).flatMap(Collection::stream).filter(Objects::nonNull).collect(Collectors.toList());
+    }
+
+    /**
+     * Get the algorithm names from the configuration.
+     *
+     * @param configuration configuration
+     * @return algorithm names
+     */
+    public static List<String> getAlgorithmNames(final ShadowRuleConfiguration configuration) {
+        if (null == configuration) {
+            return Collections.emptyList();
+        }
+        return new ArrayList<>(configuration.getShadowAlgorithms().keySet());
+    }
+
+    /**
+     * Get the algorithm names from the rules.
+     *
+     * @param rules configuration
+     * @return algorithm names
+     */
+    public static List<String> getAlgorithmNames(final Collection<ShadowRuleSegment> rules) {
+        if (rules.isEmpty()) {
+            return Collections.emptyList();
+        }
+        return rules.stream().flatMap(each -> each.getShadowTableRules().values().stream()).flatMap(Collection::stream).map(ShadowAlgorithmSegment::getAlgorithmName).collect(Collectors.toList());
+    }
+
+    /**
+     * Get the algorithm segments from the rules.
+     *
+     * @param rules configuration
+     * @return algorithm segments
+     */
+    public static List<ShadowAlgorithmSegment> getShadowAlgorithmSegment(final Collection<ShadowRuleSegment> rules) {
+        return rules.stream().flatMap(each -> each.getShadowTableRules().values().stream()).flatMap(Collection::stream).collect(Collectors.toList());
+    }
+
+    /**
+     * Merge configuration.
+     *
+     * @param existingConfiguration already existing configuration
+     * @param newConfiguration new shadow table configuration
+     * @return shadow table configuration
+     */
+    public static ShadowTableConfiguration mergeConfiguration(final ShadowTableConfiguration existingConfiguration, final ShadowTableConfiguration newConfiguration) {
+        existingConfiguration.getDataSourceNames().addAll(newConfiguration.getDataSourceNames());
+        existingConfiguration.getShadowAlgorithmNames().addAll(newConfiguration.getShadowAlgorithmNames());
+        return existingConfiguration;
+    }
+}