swarm: network simulation for swarm tests (ethereum#769)

* cmd/swarm: minor cli flag text adjustments

* cmd/swarm, swarm/storage, swarm: fix  mingw on windows test issues

* cmd/swarm: support for smoke tests on the production swarm cluster

* cmd/swarm/swarm-smoke: simplify cluster logic as per suggestion

* changed colour of landing page

* landing page reacts to enter keypress

* swarm/api/http: sticky footer for swarm landing page using flex

* swarm/api/http: sticky footer for error pages and fix for multiple choices

* swarm: propagate ctx to internal apis (ethereum#754)

* swarm/simnet: add basic node/service functions

* swarm/netsim: add buckets for global state and kademlia health check

* swarm/netsim: Use sync.Map as bucket and provide cleanup function for...

* swarm, swarm/netsim: adjust SwarmNetworkTest

* swarm/netsim: fix tests

* swarm: added visualization option to sim net redesign

* swarm/netsim: support multiple services per node

* swarm/netsim: remove redundant return statement

* swarm/netsim: add comments

* swarm: shutdown HTTP in Simulation.Close

* swarm: sim HTTP server timeout

* swarm/netsim: add more simulation methods and peer events examples

* swarm/netsim: add WaitKademlia example

* swarm/netsim: fix comments

* swarm/netsim: terminate peer events goroutines on simulation done

* swarm, swarm/netsim: naming updates

* swarm/netsim: return not healthy kademlias on WaitTillHealthy

* swarm: fix WaitTillHealthy call in testSwarmNetwork

* swarm/netsim: allow bucket to have any type for a key

* swarm: Added snapshots to new netsim

* swarm/netsim: add more tests for bucket

* swarm/netsim: move http related things into separate files

* swarm/netsim: add AddNodeWithService option

* swarm/netsim: add more tests and Start* methods

* swarm/netsim: add peer events and kademlia tests

* swarm/netsim: fix some tests flakiness

* swarm/netsim: improve random nodes selection, fix TestStartStop* tests

* swarm/netsim: remove time measurement from TestClose to avoid flakiness

* swarm/netsim: builder pattern for netsim HTTP server (ethereum#773)

* swarm/netsim: add connect related tests

* swarm/netsim: add comment for TestPeerEvents

* swarm: rename netsim package to network/simulation

p2p/simulations/adapters/inproc.go

@@ -296,6 +296,13 @@ func (sn *SimNode) Stop() error {
 	return sn.node.Stop()
 }
 
+// Service returns a running service by name
+func (sn *SimNode) Service(name string) node.Service {
+	sn.lock.RLock()
+	defer sn.lock.RUnlock()
+	return sn.running[name]
+}
+
 // Services returns a copy of the underlying services
 func (sn *SimNode) Services() []node.Service {
 	sn.lock.RLock()
@@ -307,6 +314,17 @@ func (sn *SimNode) Services() []node.Service {
 	return services
 }
 
+// ServiceMap returns a map by names of the underlying services
+func (sn *SimNode) ServiceMap() map[string]node.Service {
+	sn.lock.RLock()
+	defer sn.lock.RUnlock()
+	services := make(map[string]node.Service, len(sn.running))
+	for name, service := range sn.running {
+		services[name] = service
+	}
+	return services
+}
+
 // Server returns the underlying p2p.Server
 func (sn *SimNode) Server() *p2p.Server {
 	return sn.node.Server()

swarm/network/simulation/bucket.go

@@ -0,0 +1,81 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package simulation
+
+import (
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+// BucketKey is the type that should be used for keys in simulation buckets.
+type BucketKey string
+
+// NodeItem returns an item set in ServiceFunc function for a particualar node.
+func (s *Simulation) NodeItem(id discover.NodeID, key interface{}) (value interface{}, ok bool) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.buckets[id]; !ok {
+		return nil, false
+	}
+	return s.buckets[id].Load(key)
+}
+
+// SetNodeItem sets a new item associated with the node with provided NodeID.
+// Buckets should be used to avoid managing separate simulation global state.
+func (s *Simulation) SetNodeItem(id discover.NodeID, key interface{}, value interface{}) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	s.buckets[id].Store(key, value)
+}
+
+// NodeItems returns a map of items from all nodes that are all set under the
+// same BucketKey.
+func (s *Simulation) NodesItems(key interface{}) (values map[discover.NodeID]interface{}) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	ids := s.NodeIDs()
+	values = make(map[discover.NodeID]interface{}, len(ids))
+	for _, id := range ids {
+		if _, ok := s.buckets[id]; !ok {
+			continue
+		}
+		if v, ok := s.buckets[id].Load(key); ok {
+			values[id] = v
+		}
+	}
+	return values
+}
+
+// UpNodesItems returns a map of items with the same BucketKey from all nodes that are up.
+func (s *Simulation) UpNodesItems(key interface{}) (values map[discover.NodeID]interface{}) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+
+	ids := s.UpNodeIDs()
+	values = make(map[discover.NodeID]interface{})
+	for _, id := range ids {
+		if _, ok := s.buckets[id]; !ok {
+			continue
+		}
+		if v, ok := s.buckets[id].Load(key); ok {
+			values[id] = v
+		}
+	}
+	return values
+}

swarm/network/simulation/bucket_test.go

@@ -0,0 +1,155 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package simulation
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+)
+
+// TestServiceBucket tests all bucket functionalities using subtests.
+// It constructs a simulation of two nodes by adding items to their buckets
+// in ServiceFunc constructor, then by SetNodeItem. Testing UpNodesItems
+// is done by stopping one node and validating availability of its items.
+func TestServiceBucket(t *testing.T) {
+	testKey := "Key"
+	testValue := "Value"
+
+	sim := New(map[string]ServiceFunc{
+		"noop": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+			b.Store(testKey, testValue+ctx.Config.ID.String())
+			return newNoopService(), nil, nil
+		},
+	})
+	defer sim.Close()
+
+	id1, err := sim.AddNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	id2, err := sim.AddNode()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("ServiceFunc bucket Store", func(t *testing.T) {
+		v, ok := sim.NodeItem(id1, testKey)
+		if !ok {
+			t.Fatal("bucket item not found")
+		}
+		s, ok := v.(string)
+		if !ok {
+			t.Fatal("bucket item value is not string")
+		}
+		if s != testValue+id1.String() {
+			t.Fatalf("expected %q, got %q", testValue+id1.String(), s)
+		}
+
+		v, ok = sim.NodeItem(id2, testKey)
+		if !ok {
+			t.Fatal("bucket item not found")
+		}
+		s, ok = v.(string)
+		if !ok {
+			t.Fatal("bucket item value is not string")
+		}
+		if s != testValue+id2.String() {
+			t.Fatalf("expected %q, got %q", testValue+id2.String(), s)
+		}
+	})
+
+	customKey := "anotherKey"
+	customValue := "anotherValue"
+
+	t.Run("SetNodeItem", func(t *testing.T) {
+		sim.SetNodeItem(id1, customKey, customValue)
+
+		v, ok := sim.NodeItem(id1, customKey)
+		if !ok {
+			t.Fatal("bucket item not found")
+		}
+		s, ok := v.(string)
+		if !ok {
+			t.Fatal("bucket item value is not string")
+		}
+		if s != customValue {
+			t.Fatalf("expected %q, got %q", customValue, s)
+		}
+
+		v, ok = sim.NodeItem(id2, customKey)
+		if ok {
+			t.Fatal("bucket item should not be found")
+		}
+	})
+
+	if err := sim.StopNode(id2); err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("UpNodesItems", func(t *testing.T) {
+		items := sim.UpNodesItems(testKey)
+
+		v, ok := items[id1]
+		if !ok {
+			t.Errorf("node 1 item not found")
+		}
+		s, ok := v.(string)
+		if !ok {
+			t.Fatal("node 1 item value is not string")
+		}
+		if s != testValue+id1.String() {
+			t.Fatalf("expected %q, got %q", testValue+id1.String(), s)
+		}
+
+		v, ok = items[id2]
+		if ok {
+			t.Errorf("node 2 item should not be found")
+		}
+	})
+
+	t.Run("NodeItems", func(t *testing.T) {
+		items := sim.NodesItems(testKey)
+
+		v, ok := items[id1]
+		if !ok {
+			t.Errorf("node 1 item not found")
+		}
+		s, ok := v.(string)
+		if !ok {
+			t.Fatal("node 1 item value is not string")
+		}
+		if s != testValue+id1.String() {
+			t.Fatalf("expected %q, got %q", testValue+id1.String(), s)
+		}
+
+		v, ok = items[id2]
+		if !ok {
+			t.Errorf("node 2 item not found")
+		}
+		s, ok = v.(string)
+		if !ok {
+			t.Fatal("node 1 item value is not string")
+		}
+		if s != testValue+id2.String() {
+			t.Fatalf("expected %q, got %q", testValue+id2.String(), s)
+		}
+	})
+}