Mixnet v1 (#569)

* base

* Remove mixnet client from libp2p network backend (#572)

* Mixnet v1: Remove all mixnet legacies: mixnet crate, mixnode binary, tests, and docker (#573)

* Mixnet v1: Skeleton (#570)

* Use QUIC for libp2p (#580)

* Add Poisson interval function for Mixnet (#575)

* Mixnet network backend skeleton (#586)

* Libp2p stream read/write (#587)

* Emitting packets from mixclient using libp2p stream (#588)

* Handle outputs from mixnode using libp2p stream/gossipsub (#589)

* Refactor poisson (#590)

* Mix client Poisson emission (#591)

* Mix node packet handling (#592)

* Mix Packet / Fragment logic (#593)

* Move FisherYates to `nomos-utils` (#594)

* Mixnet topology (#595)

* Mix client/node unit tests (#596)

* change multiaddr from tcp to udp with quic-v1 (#607)

---------

Co-authored-by: Al Liu <[email protected]>

.env.example

@@ -6,4 +6,4 @@ DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
 DOCKER_COMPOSE_ETCDCTL_API=3
 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
 DOCKER_COMPOSE_OVERLAY_NODES=$DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY
-DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000
+DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1

.env.testnet

@@ -6,4 +6,4 @@ DOCKER_COMPOSE_ETCDCTL_ENDPOINTS=etcd:2379
 DOCKER_COMPOSE_ETCDCTL_API=3
 DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY=1000000000000000000000000000000000000000000000000000000000000000
 DOCKER_COMPOSE_OVERLAY_NODES=1000000000000000000000000000000000000000000000000000000000000000
-DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/tcp/3000
+DOCKER_COMPOSE_NET_INITIAL_PEERS=/dns/bootstrap/udp/3000/quic-v1

Cargo.toml

@@ -19,15 +19,11 @@ members = [
     "nomos-cli",
     "nomos-utils",
     "nodes/nomos-node",
-    "nodes/mixnode",
+    "mixnet",
     "simulations",
     "consensus/carnot-engine",
     "consensus/cryptarchia-engine",
     "ledger/cryptarchia-ledger",
     "tests",
-    "mixnet/node",
-    "mixnet/client",
-    "mixnet/protocol",
-    "mixnet/topology",
 ]
 resolver = "2"

ci/Jenkinsfile.nightly.integration

@@ -53,10 +53,6 @@ pipeline {
                     /* To prevent rebuilding node for each test, tests are defined here */
                     def tests = ['ten_nodes_happy', 'two_nodes_happy', 'ten_nodes_one_down']
 
-                    if (FEATURE == 'libp2p') {
-                      tests.add('mixnet')
-                    }
-
                     def report = runBuildAndTestsForFeature(FEATURE, tests)
                     writeFile(file: "${WORKSPACE}/report.txt", text: report)
                   }

compose.static.yml

@@ -7,14 +7,10 @@ services:
       context: .
       dockerfile: testnet/Dockerfile
     ports:
-      - "3000:3000/tcp"
+      - "3000:3000/udp"
       - "18080:18080/tcp"
     volumes:
       - ./testnet:/etc/nomos
-    depends_on:
-      - mix-node-0
-      - mix-node-1
-      - mix-node-2
     environment:
       - BOOTSTRAP_NODE_KEY=${DOCKER_COMPOSE_BOOSTRAP_NET_NODE_KEY:-1000000000000000000000000000000000000000000000000000000000000000}
       - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
@@ -32,11 +28,8 @@ services:
     depends_on:
       - bootstrap
       - etcd
-      - mix-node-0
-      - mix-node-1
-      - mix-node-2
     ports:
-      - "3001:3000/tcp"
+      - "3001:3000/udp"
       - "18081:18080/tcp"
     environment:
       - LIBP2P_REPLICAS=3
@@ -45,7 +38,7 @@ services:
       - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
+      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
   libp2p-node-2:
@@ -58,11 +51,8 @@ services:
     depends_on:
       - bootstrap
       - etcd
-      - mix-node-0
-      - mix-node-1
-      - mix-node-2
     ports:
-      - "3002:3000/tcp"
+      - "3002:3000/udp"
       - "18082:18080/tcp"
     environment:
       - LIBP2P_REPLICAS=3
@@ -71,7 +61,7 @@ services:
       - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
+      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
   libp2p-node-3:
@@ -84,11 +74,8 @@ services:
     depends_on:
       - bootstrap
       - etcd
-      - mix-node-0
-      - mix-node-1
-      - mix-node-2
     ports:
-      - "3003:3000/tcp"
+      - "3003:3000/udp"
       - "18083:18080/tcp"
     environment:
       - LIBP2P_REPLICAS=3
@@ -97,48 +84,9 @@ services:
       - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
+      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  mix-node-0:
-    container_name: mix_node_0
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    ports:
-      - "7707:7777/tcp"
-      - "7708:7778/tcp"
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
-  mix-node-1:
-    container_name: mix_node_1
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    ports:
-      - "7717:7777/tcp"
-      - "7718:7778/tcp"
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
-  mix-node-2:
-    container_name: mix_node_2
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    ports:
-      - "7727:7777/tcp"
-      - "7728:7778/tcp"
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
   chatbot:
     container_name: chatbot
     build:

compose.yml

@@ -5,7 +5,7 @@ services:
       context: .
       dockerfile: testnet/Dockerfile
     ports:
-      - "3000:3000/tcp"
+      - "3000:3000/udp"
       - "18080:18080/tcp"
     volumes:
       - ./testnet:/etc/nomos
@@ -27,46 +27,16 @@ services:
     depends_on:
       - bootstrap
       - etcd
-      - mix-node-0
-      - mix-node-1
-      - mix-node-2
     environment:
       - LIBP2P_REPLICAS=${DOCKER_COMPOSE_LIBP2P_REPLICAS:-1}
       - ETCDCTL_ENDPOINTS=${DOCKER_COMPOSE_ETCDCTL_ENDPOINTS:-etcd:2379}
       - ETCDCTL_API=${DOCKER_COMPOSE_ETCDCTL_API:-3}
       - LIBP2P_NODE_MASK=${DOCKER_COMPOSE_LIBP2P_NODE_KEY_MASK:-2000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_NODES=${DOCKER_COMPOSE_OVERLAY_NODES:-1000000000000000000000000000000000000000000000000000000000000000}
       - OVERLAY_SUPER_MAJORITY_THRESHOLD=${DOCKER_COMPOSE_SUPER_MAJORITY_THRESHOLD:-1}
-      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/tcp/3000}
+      - NET_INITIAL_PEERS=${DOCKER_COMPOSE_NET_INITIAL_PEERS:-/dns/bootstrap/udp/3000/quic-v1}
     entrypoint: /etc/nomos/scripts/run_nomos_node.sh
 
-  mix-node-0:
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
-  mix-node-1:
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
-  mix-node-2:
-    build:
-      context: .
-      dockerfile: testnet/Dockerfile
-    volumes:
-      - ./testnet:/etc/nomos
-    entrypoint: /usr/bin/mixnode
-    command: /etc/nomos/mixnode_config.yaml
-
   etcd:
     image: quay.io/coreos/etcd:v3.4.15
     ports:

consensus/carnot-engine/Cargo.toml

@@ -15,7 +15,7 @@ rand = "0.8"
 rand_chacha = "0.3"
 thiserror = "1"
 fraction = { version = "0.13" }
-nomos-utils = { path = "../../nomos-utils", optional = true }
+nomos-utils = { path = "../../nomos-utils" }
 
 utoipa = { version = "4.0", optional = true }
 serde_json = { version = "1.0", optional = true }

consensus/carnot-engine/src/overlay/branch_overlay.rs

@@ -188,7 +188,9 @@ fn build_committee_from_nodes_with_size(
 
 #[cfg(test)]
 mod tests {
-    use crate::overlay::{FisherYatesShuffle, RoundRobin};
+    use nomos_utils::fisheryates::FisherYatesShuffle;
+
+    use crate::overlay::RoundRobin;
 
     use super::*;
     const ENTROPY: [u8; 32] = [0; 32];

consensus/carnot-engine/src/overlay/membership.rs

@@ -1,33 +1,7 @@
-// std
-
-// crates
-use rand::{Rng, SeedableRng};
-use rand_chacha::ChaCha20Rng;
-
 // internal
 use crate::overlay::CommitteeMembership;
 use crate::NodeId;
-
-#[derive(Copy, Clone, Debug, Eq, PartialEq)]
-pub struct FisherYatesShuffle {
-    entropy: [u8; 32],
-}
-
-impl FisherYatesShuffle {
-    pub fn new(entropy: [u8; 32]) -> Self {
-        Self { entropy }
-    }
-
-    pub fn shuffle<T: Clone>(elements: &mut [T], entropy: [u8; 32]) {
-        let mut rng = ChaCha20Rng::from_seed(entropy);
-        // Implementation of fisher yates shuffling
-        // https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle
-        for i in (1..elements.len()).rev() {
-            let j = rng.gen_range(0..=i);
-            elements.swap(i, j);
-        }
-    }
-}
+use nomos_utils::fisheryates::FisherYatesShuffle;
 
 impl CommitteeMembership for FisherYatesShuffle {
     fn reshape_committees(&self, nodes: &mut [NodeId]) {

consensus/carnot-engine/src/overlay/mod.rs

@@ -53,8 +53,10 @@ pub trait CommitteeMembership: Clone {
 
 #[cfg(test)]
 mod tests {
+    use nomos_utils::fisheryates::FisherYatesShuffle;
+
     use super::*;
-    use crate::overlay::{FisherYatesShuffle, RoundRobin};
+    use crate::overlay::RoundRobin;
 
     const ENTROPY: [u8; 32] = [0; 32];
 

consensus/carnot-engine/src/overlay/random_beacon.rs

@@ -1,6 +1,7 @@
-use crate::overlay::{CommitteeMembership, FisherYatesShuffle};
+use crate::overlay::CommitteeMembership;
 use crate::types::*;
 use bls_signatures::{PrivateKey, PublicKey, Serialize, Signature};
+use nomos_utils::fisheryates::FisherYatesShuffle;
 use rand::{seq::SliceRandom, SeedableRng};
 use serde::{Deserialize, Serialize as SerdeSerialize};
 use sha2::{Digest, Sha256};

consensus/carnot-engine/src/overlay/tree_overlay/overlay.rs

@@ -231,8 +231,9 @@ where
 
 #[cfg(test)]
 mod tests {
+    use nomos_utils::fisheryates::FisherYatesShuffle;
+
     use crate::overlay::leadership::RoundRobin;
-    use crate::overlay::membership::FisherYatesShuffle;
     use crate::Overlay;
 
     use super::*;

mixnet/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "mixnet"
+version = "0.0.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+rand = "0.8"
+rand_distr = "0.4"
+nomos-utils = { path = "../nomos-utils" }
+thiserror = "1.0.57"
+tokio = { version = "1.36.0", features = ["sync"] }
+serde = { version = "1.0.197", features = ["derive"] }
+sphinx-packet = "0.1.0"
+nym-sphinx-addressing = { package = "nym-sphinx-addressing", git = "https://github.com/nymtech/nym", tag = "v1.1.22" }
+tracing = "0.1.40"
+uuid = { version = "1.7.0", features = ["v4"] }
+futures = "0.3"
+
+[dev-dependencies]
+tokio = { version = "1.36.0", features = ["test-util"] }