Adding test for node join

raft_test.go

@@ -23,10 +23,11 @@ func (m *MockFSM) Apply(log []byte) {
 // Return configurations optimized for in-memory
 func inmemConfig() *Config {
 	return &Config{
-		HeartbeatTimeout: 5 * time.Millisecond,
+		HeartbeatTimeout: 10 * time.Millisecond,
 		ElectionTimeout:  10 * time.Millisecond,
 		CommitTimeout:    time.Millisecond,
 		MaxAppendEntries: 16,
+		ShutdownOnRemove: true,
 	}
 }
 
@@ -63,7 +64,7 @@ func TestRaft_SingleNode(t *testing.T) {
 	}
 	defer raft.Shutdown()
 
-	time.Sleep(15 * time.Millisecond)
+	time.Sleep(conf.HeartbeatTimeout * 3)
 
 	// Should be leader
 	if s := raft.State(); s != Leader {
@@ -90,6 +91,14 @@ type cluster struct {
 	rafts  []*Raft
 }
 
+func (c *cluster) Merge(other *cluster) {
+	c.dirs = append(c.dirs, other.dirs...)
+	c.stores = append(c.stores, other.stores...)
+	c.fsms = append(c.fsms, other.fsms...)
+	c.trans = append(c.trans, other.trans...)
+	c.rafts = append(c.rafts, other.rafts...)
+}
+
 func (c *cluster) Close() {
 	for _, r := range c.rafts {
 		r.Shutdown()
@@ -399,3 +408,56 @@ func TestRaft_ApplyConcurrent(t *testing.T) {
 	// Check the FSMs
 	c.EnsureSame(t)
 }
+
+func TestRaft_JoinNode(t *testing.T) {
+	// Make a cluster
+	c := MakeCluster(2, t, nil)
+	defer c.Close()
+
+	// Apply a log to this cluster to ensure it is 'newer'
+	leader := c.Leader()
+	future := leader.Apply([]byte("first"), 0)
+	if err := future.Error(); err != nil {
+		t.Fatalf("err: %v", err)
+	} else {
+		log.Printf("[INFO] Applied log")
+	}
+
+	// Make a new cluster of 1
+	c1 := MakeCluster(1, t, nil)
+
+	// Merge clusters
+	c.Merge(c1)
+	c.FullyConnect()
+
+	// Wait until we have 2 leaders
+	limit := time.Now().Add(100 * time.Millisecond)
+	var leaders []*Raft
+	for time.Now().Before(limit) && len(leaders) != 2 {
+		time.Sleep(10 * time.Millisecond)
+		leaders = c.GetInState(Leader)
+	}
+	if len(leaders) != 2 {
+		t.Fatalf("expected two leader: %v", leaders)
+	}
+
+	// Join the new node in
+	future = leader.AddPeer(c1.rafts[0].localAddr)
+	if err := future.Error(); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// Wait until we have 2 followers
+	limit = time.Now().Add(100 * time.Millisecond)
+	var followers []*Raft
+	for time.Now().Before(limit) && len(followers) != 2 {
+		time.Sleep(10 * time.Millisecond)
+		followers = c.GetInState(Follower)
+	}
+	if len(followers) != 2 {
+		t.Fatalf("expected two followers: %v", followers)
+	}
+
+	// Check the FSMs
+	c.EnsureSame(t)
+}