[SRE API] query cluster migration

diversepwmeasurement · Mar 19, 2024 · 7e547d8 · 7e547d8
1 parent b3a26f9
commit 7e547d8
Show file tree

Hide file tree

Showing 10 changed files with 187 additions and 3 deletions.
diff --git a/...sole/src/main/java/com/ctrip/xpipe/redis/console/controller/api/migrate/MigrationApi.java b/...sole/src/main/java/com/ctrip/xpipe/redis/console/controller/api/migrate/MigrationApi.java
@@ -2,11 +2,13 @@
 
 import com.ctrip.xpipe.api.migration.DcMapper;
 import com.ctrip.xpipe.redis.checker.controller.result.RetMessage;
+import com.ctrip.xpipe.redis.console.cache.DcCache;
 import com.ctrip.xpipe.redis.console.controller.AbstractConsoleController;
 import com.ctrip.xpipe.redis.console.controller.api.migrate.meta.*;
 import com.ctrip.xpipe.redis.console.migration.model.MigrationCluster;
 import com.ctrip.xpipe.redis.console.migration.model.MigrationEvent;
 import com.ctrip.xpipe.redis.console.migration.status.MigrationStatus;
+import com.ctrip.xpipe.redis.console.model.MigrationClusterTbl;
 import com.ctrip.xpipe.redis.console.service.migration.MigrationService;
 import com.ctrip.xpipe.redis.console.service.migration.exception.*;
 import com.ctrip.xpipe.redis.console.service.migration.impl.MigrationRequest;
@@ -17,9 +19,8 @@
 import org.springframework.http.MediaType;
 import org.springframework.web.bind.annotation.*;
 
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
+import java.util.concurrent.TimeUnit;
 
 /**
  * @author wenchao.meng
@@ -35,6 +36,9 @@ public class MigrationApi extends AbstractConsoleController {
     @Autowired
     private MigrationService migrationService;
 
+    @Autowired
+    private DcCache dcCache;
+
     @RequestMapping(value = "/checkandprepare", method = RequestMethod.POST, produces = {MediaType.APPLICATION_JSON_UTF8_VALUE})
     public CheckPrepareResponse checkAndPrepare(@RequestBody(required = true) CheckPrepareRequest checkMeta) {
 
@@ -181,6 +185,27 @@ public RollbackResponse rollback(@RequestBody(required = true) RollbackRequest r
         return rollbackResponse;
     }
 
+    @GetMapping(value = "history")
+    public Map<String, List<ClusterMigrationStatus>> getClusterMigrationHistory(@RequestBody MigrationHistoryReq req) {
+        logger.info("[history][{}-{}] {}", req.from, req.to, req.clusters);
+        long current = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
+        if (req.from < 0 || req.from >= current) return Collections.emptyMap();
+        if (null == req.clusters || req.clusters.isEmpty()) return Collections.emptyMap();
+        if (req.to < req.from) req.to = current;
+
+        List<MigrationClusterTbl> migrationClusterTbls = migrationService.fetchMigrationClusters(req.clusters,
+                TimeUnit.SECONDS.toMillis(req.from), TimeUnit.SECONDS.toMillis(req.to));
+        Map<String, List<ClusterMigrationStatus>> resp = new HashMap<>();
+        migrationClusterTbls.forEach(migrationClusterTbl -> {
+            String clusterName = migrationClusterTbl.getCluster().getClusterName();
+            if (!resp.containsKey(clusterName)) resp.put(clusterName, new ArrayList<>());
+            ClusterMigrationStatus clusterMigrationStatus = ClusterMigrationStatus.from(migrationClusterTbl, dcCache);
+            resp.get(clusterName).add(clusterMigrationStatus);
+        });
+
+        return resp;
+    }
+
     @RequestMapping(value = "/migration/system/health/status", method = RequestMethod.GET)
     public RetMessage getMigrationSystemHealthStatus() {
         logger.info("[getMigrationSystemHealthStatus]");

diff --git a/...ava/com/ctrip/xpipe/redis/console/controller/api/migrate/meta/ClusterMigrationStatus.java b/...ava/com/ctrip/xpipe/redis/console/controller/api/migrate/meta/ClusterMigrationStatus.java
@@ -0,0 +1,54 @@
+package com.ctrip.xpipe.redis.console.controller.api.migrate.meta;
+
+import com.ctrip.xpipe.redis.console.cache.DcCache;
+import com.ctrip.xpipe.redis.console.migration.status.MigrationStatus;
+import com.ctrip.xpipe.redis.console.model.DcTbl;
+import com.ctrip.xpipe.redis.console.model.MigrationClusterTbl;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author lishanglin
+ * date 2024/3/18
+ */
+public class ClusterMigrationStatus {
+
+    public Long startAt;
+
+    public Long endAt;
+
+    public String sourceDc;
+
+    public String destDc;
+
+    public String status;
+
+    public static ClusterMigrationStatus from(MigrationClusterTbl migrationClusterTbl, DcCache dcCache) {
+        ClusterMigrationStatus migrationStatus = new ClusterMigrationStatus();
+        migrationStatus.startAt = TimeUnit.MILLISECONDS.toSeconds(migrationClusterTbl.getStartTime().getTime());
+        if (null != migrationClusterTbl.getEndTime()) {
+            migrationStatus.endAt = TimeUnit.MILLISECONDS.toSeconds(migrationClusterTbl.getEndTime().getTime());
+        } else {
+            migrationStatus.endAt = null;
+        }
+
+        DcTbl srcDcTbl = dcCache.find(migrationClusterTbl.getSourceDcId());
+        DcTbl destDcTbl = dcCache.find(migrationClusterTbl.getDestinationDcId());
+        if (null != srcDcTbl) migrationStatus.sourceDc = srcDcTbl.getDcName();
+        if (null != destDcTbl) migrationStatus.destDc = destDcTbl.getDcName();
+        migrationStatus.status = MigrationStatus.valueOf(migrationClusterTbl.getStatus()).getType();
+
+        return migrationStatus;
+    }
+
+    @Override
+    public String toString() {
+        return "ClusterMigrationStatus{" +
+                "startAt=" + startAt +
+                ", endAt=" + endAt +
+                ", sourceDc='" + sourceDc + '\'' +
+                ", destDc='" + destDc + '\'' +
+                ", status='" + status + '\'' +
+                '}';
+    }
+}
diff --git a/...n/java/com/ctrip/xpipe/redis/console/controller/api/migrate/meta/MigrationHistoryReq.java b/...n/java/com/ctrip/xpipe/redis/console/controller/api/migrate/meta/MigrationHistoryReq.java
@@ -0,0 +1,17 @@
+package com.ctrip.xpipe.redis.console.controller.api.migrate.meta;
+
+import java.util.Set;
+
+/**
+ * @author lishanglin
+ * date 2024/3/18
+ */
+public class MigrationHistoryReq {
+
+    public long from;
+
+    public long to;
+
+    public Set<String> clusters;
+
+}
diff --git a/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/dao/MigrationClusterDao.java b/redis/redis-console/src/main/java/com/ctrip/xpipe/redis/console/dao/MigrationClusterDao.java
@@ -13,6 +13,7 @@
 import javax.annotation.PostConstruct;
 import java.util.Date;
 import java.util.List;
+import java.util.Set;
 
 /**
  * @author wenchao.meng
@@ -262,5 +263,15 @@ public List<MigrationClusterTbl> doQuery() throws DalException {
         });
     }
 
+    public List<MigrationClusterTbl> findMigrationClustersByNameAndTime(Set<String> clusters, Date from, Date to) {
+        return queryHandler.handleQuery(new DalQuery<List<MigrationClusterTbl>>() {
+            @Override
+            public List<MigrationClusterTbl> doQuery() throws DalException {
+                return migrationClusterTblDao.findMigrationClustersByNameAndTime(clusters, from, to,
+                        MigrationClusterTblEntity.READSET_MIGRATION_CLUSTER_STATUS);
+            }
+        });
+    }
+
 
 }
diff --git a/...nsole/src/main/java/com/ctrip/xpipe/redis/console/service/migration/MigrationService.java b/...nsole/src/main/java/com/ctrip/xpipe/redis/console/service/migration/MigrationService.java
@@ -90,4 +90,6 @@ public interface MigrationService {
     Set<String> getLatestMigrationOperators(int hours);
 
     List<MigrationClusterTbl> getLatestMigrationClusters(int seconds);
+
+    List<MigrationClusterTbl> fetchMigrationClusters(Set<String> clusters, long from, long to);
 }
diff --git a/.../main/java/com/ctrip/xpipe/redis/console/service/migration/impl/MigrationServiceImpl.java b/.../main/java/com/ctrip/xpipe/redis/console/service/migration/impl/MigrationServiceImpl.java
@@ -646,6 +646,11 @@ public List<MigrationClusterTbl> getLatestMigrationClusters(int seconds) {
         return migrationClusterDao.findLatestMigrationClusterWithClusterName(DateTimeUtils.getSecondBeforeDate(new Date(), seconds));
     }
 
+    @Override
+    public List<MigrationClusterTbl> fetchMigrationClusters(Set<String> clusters, long from, long to) {
+        return migrationClusterDao.findMigrationClustersByNameAndTime(clusters, new Date(from), new Date(to));
+    }
+
     private String clusterRelatedDcToString(List<DcTbl> clusterRelatedDc) {
         return StringUtil.join(",", (dcTbl) -> dcTbl.getDcName() , clusterRelatedDc);
     }

diff --git a/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml b/redis/redis-console/src/main/resources/META-INF/dal/jdbc/meta-dal.xml
@@ -1925,6 +1925,9 @@
 		<var name="offset" value-type="long" />
 		<var name="operator" value-type="String" />
 		<var name="event-ids" value-type="java.util.List&lt;Long&gt;" />
+		<var name="from" value-type="Date" />
+		<var name="to" value-type="Date" />
+		<var name="clusters" value-type="java.util.Set&lt;String&gt;" />
 
 		<member name="count" value-type="long" select-expr="COUNT(*) count" all="false"/>
 		<member name="migEventCnt" value-type="long" select-expr=" COUNT(distinct mct.migration_event_id) migEventCnt" all="false" />
@@ -1961,6 +1964,10 @@
 				<readset-ref name="NAME_AND_ORG" relation-name="cluster" />
 				<readset-ref name="FULL" relation-name="migration-event" />
 			</readset>
+			<readset name="MIGRATION_CLUSTER_STATUS">
+				<readset-ref name="FULL"/>
+				<readset-ref name="NAME" relation-name="cluster" />
+			</readset>
 
 		</readsets>
 
@@ -1977,6 +1984,23 @@
 				</statement>
 			</query>
 
+			<query name="find-migration-clusters-by-name-and-time" type="SELECT" multiple="true">
+				<param name="clusters"/>
+				<param name="from"/>
+				<param name="to"/>
+				<statement>
+					<![CDATA[
+					SELECT <FIELDS/>
+					FROM <TABLE name='cluster'/>, <TABLE/>
+					WHERE <FIELD name='deleted'/> = 0 AND ct.deleted = 0
+					AND <FIELD name='cluster-id'/> = ct.id
+					AND ct.cluster_name IN <IN>${clusters}</IN>
+					AND <FIELD name='start-time'/> >= ${from}
+					AND <FIELD name='start-time'/> <= ${to}
+					]]>
+				</statement>
+			</query>
+
 			<query name="find-latest-migration-clusters-with-cluster-name" type="SELECT" multiple="true">
 				<param name="data-change-last-time"/>
 				<statement>

diff --git a/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java b/redis/redis-console/src/test/java/com/ctrip/xpipe/redis/console/AllTests.java
@@ -14,6 +14,7 @@
 import com.ctrip.xpipe.redis.console.controller.api.data.meta.CheckPrepareRequestTest;
 import com.ctrip.xpipe.redis.console.controller.api.data.meta.ClusterCreateInfoTest;
 import com.ctrip.xpipe.redis.console.controller.api.data.meta.RedisInstanceInfoTest;
+import com.ctrip.xpipe.redis.console.controller.api.migrate.MigrationApiIntegrationTest;
 import com.ctrip.xpipe.redis.console.controller.api.migrate.MigrationApiTest;
 import com.ctrip.xpipe.redis.console.controller.config.ClusterCheckInterceptorTest;
 import com.ctrip.xpipe.redis.console.controller.consoleportal.RedisControllerTest;
@@ -202,6 +203,7 @@
         BeaconMetaServiceImplTest.class,
         BeaconMigrationServiceImplTest.class,
         MigrationApiTest.class,
+        MigrationApiIntegrationTest.class,
         ExclusiveThreadsForMigrationTest.class,
         XPipeHandlerMethodCommandTest.class,
 

diff --git a/...ava/com/ctrip/xpipe/redis/console/controller/api/migrate/MigrationApiIntegrationTest.java b/...ava/com/ctrip/xpipe/redis/console/controller/api/migrate/MigrationApiIntegrationTest.java
@@ -0,0 +1,43 @@
+package com.ctrip.xpipe.redis.console.controller.api.migrate;
+
+import com.ctrip.xpipe.redis.console.AbstractConsoleIntegrationTest;
+import com.ctrip.xpipe.redis.console.controller.api.migrate.meta.ClusterMigrationStatus;
+import com.ctrip.xpipe.redis.console.controller.api.migrate.meta.MigrationHistoryReq;
+import org.junit.Assert;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author lishanglin
+ * date 2024/3/18
+ */
+public class MigrationApiIntegrationTest extends AbstractConsoleIntegrationTest {
+
+    @Autowired
+    private MigrationApi migrationApi;
+
+    @Test
+    public void testGetClusterMigrationHistory() {
+        MigrationHistoryReq req = new MigrationHistoryReq();
+        req.from = 0;
+        req.to = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()) + 600;
+        req.clusters = Collections.singleton("cluster2");
+
+        Map<String, List<ClusterMigrationStatus>> resp = migrationApi.getClusterMigrationHistory(req);
+        logger.info("[testGetClusterMigrationHistory] {}", resp);
+        Assert.assertTrue(resp.containsKey("cluster2"));
+        Assert.assertEquals(1, resp.get("cluster2").size());
+        Assert.assertEquals("Processing", resp.get("cluster2").get(0).status);
+    }
+
+    protected String prepareDatas() throws IOException {
+        return prepareDatasFromFile("src/test/resources/migration-test.sql");
+    }
+
+}
diff --git a/...pe/redis/console/controller/consoleportal/migration/ExclusiveThreadsForMigrationTest.java b/...pe/redis/console/controller/consoleportal/migration/ExclusiveThreadsForMigrationTest.java
@@ -94,6 +94,7 @@ public UserInfoHolder userInfoHolder() {
 
     @Before
     public void beforeRestTemplateFactoryTest() throws Exception {
+        this.port = randomPort();
         restOperations = RestTemplateFactory.createCommonsHttpRestTemplate(2, 2, 1200, 60000);
 
         springApplicationStarter = new SpringApplicationStarter(port, 1, SlowController.class, TestDependency.class, MigrationApi4Beacon.class);