[Feature] Improve (dis-)connect handling between peers

.ci/utils.sh

@@ -82,14 +82,18 @@ function check_heights() {
 
 # Function checking that nodes created logs on disk and they contain no errors.
 function check_logs() {
-  echo "Checking logs exist for all nodes..."
+  echo "Checking logs exist for all nodes and contain no errors..."
   local log_dir=$1
   local total_validators=$2
   local total_clients=$3
 
   local all_reached=true
   local highest_height=0
 
+  # The maximum number of warnings allow in each node's log file.
+  # Nodes may create some warnings at startup because they cannot connect to each other yet.
+  local max_warnings=10
+
   for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
     if [ ! -s "$log_dir/validator-${validator_index}.log" ]; then
       echo "❌ Test failed! Validator #${validator_index} did not create any logs in \"$log_dir\"."
@@ -102,6 +106,12 @@ function check_logs() {
       grep "ERROR" "$log_dir/validator-${validator_index}.log"
       return 1
     fi
+
+    num_warnings=$(grep -c "WARN" "$log_dir/validator-${validator_index}.log")
+    if (( num_warnings > max_warnings )); then
+      echo "❌ Test failed! Validator #${validator_index} logs contain more than ${max_warnings} warnings."
+      return 1
+    fi
   done
 
   for ((client_index = 0; client_index < total_clients; client_index++)); do
@@ -116,7 +126,12 @@ function check_logs() {
       grep "ERROR" "$log_dir/client-${client_index}.log"
       return 1
     fi
-
+
+    num_warnings=$(grep -c "WARN" "$log_dir/client-${client_index}.log")
+    if (( num_warnings > max_warnings )); then
+      echo "❌ Test failed! Client #${client_index} logs contain more than ${max_warnings} warnings."
+      return 1
+    fi
   done
 
   return 0

.circleci/config.yml

@@ -420,6 +420,10 @@ commands:
       - run:
           name: "Install snarkos"
           no_output_timeout: 20m
+          environment:
+            CONSENSUS_VERSION_HEIGHTS: "0,100"
+            EXPECTED_MAX_CONSENSUS_VERSION: "10"
+            WAIT_BETWEEN_UPGRADES: "60"
           command: |
             cargo install --locked --path . --features test_network
       - run:

node/bft/Cargo.toml

@@ -25,6 +25,7 @@ locktick = [
   "snarkos-node-bft-storage-service/locktick",
   "snarkos-node-sync/locktick",
   "snarkos-node-tcp/locktick",
+  "snarkos-node-metrics/locktick",
   "snarkos-utilities/locktick",
   "snarkvm/locktick"
 ]

node/bft/events/src/disconnect.rs

@@ -29,6 +29,14 @@ pub enum DisconnectReason {
     ProtocolViolation = 2,
     /// The peer's client is outdated, judging by its version.
     OutdatedClientVersion = 3,
+    /// The two validators are the same node.
+    SelfConnect = 4,
+    /// No untrusted external peers are allowed.
+    NoExternalPeersAllowed = 5,
+    /// Already connecting to the same node (through another TCP channel).
+    AlreadyConnecting = 6,
+    /// Already connected to the same node (through another TCP channel).
+    AlreadyConnected = 7,
     /// The disconnect reason is not known. This is used for when the peers sends a disconnect reason that is not known to us.
     UnknownReason = u8::MAX,
 }
@@ -40,6 +48,10 @@ impl std::fmt::Display for DisconnectReason {
             Self::NoReasonGiven => write!(f, "no reason given"),
             Self::ProtocolViolation => write!(f, "protocol violation"),
             Self::OutdatedClientVersion => write!(f, "outdated client version"),
+            Self::SelfConnect => write!(f, "self connect"),
+            Self::NoExternalPeersAllowed => write!(f, "no external peers allowed"),
+            Self::AlreadyConnecting => write!(f, "already connecting"),
+            Self::AlreadyConnected => write!(f, "already connected"),
             Self::UnknownReason => write!(f, "unknown"),
         }
     }
@@ -86,6 +98,10 @@ impl FromBytes for Disconnect {
             1 => DisconnectReason::NoReasonGiven,
             2 => DisconnectReason::ProtocolViolation,
             3 => DisconnectReason::OutdatedClientVersion,
+            4 => DisconnectReason::SelfConnect,
+            5 => DisconnectReason::NoExternalPeersAllowed,
+            6 => DisconnectReason::AlreadyConnecting,
+            7 => DisconnectReason::AlreadyConnected,
             val => {
                 warn!("received unknown disconnect reason (id={val})");
                 DisconnectReason::UnknownReason
@@ -111,6 +127,7 @@ mod tests {
             DisconnectReason::NoReasonGiven,
             DisconnectReason::InvalidChallengeResponse,
             DisconnectReason::OutdatedClientVersion,
+            DisconnectReason::SelfConnect,
         ];
 
         for reason in all_reasons.iter() {