[Release] Mainnet v4.9.0

.cargo/release-version

@@ -1 +1 @@
-v4.8.1
+v4.9.0

.ci/test_db_backup.sh

@@ -4,7 +4,8 @@
 . ./.ci/utils.sh
 
 # Change this to increase/decrease log verbosity
-log_verbosity=2
+log_verbosity=4
+max_validator_log_size_bytes=$((2 * 1024 * 1024))
 
 # Network parameters
 total_validators=4
@@ -16,7 +17,7 @@ checkpoint_height=3
 rollback_height=10
 num_checkpoints=0
 remaining_checkpoints=2
-max_warnings=40
+max_warnings=300
 
 # Create log directory
 init_log_dir
@@ -134,7 +135,7 @@ while (( total_wait < 600 )); do  # 10 minutes max
       done
 
       if (( remaining_checkpoints == 0 )); then
-        if check_logs "$log_dir" "$total_validators" 0 "$max_warnings"; then
+        if check_logs "$log_dir" "$total_validators" 0 "$max_warnings" "$max_validator_log_size_bytes"; then
           exit 0
         else
           exit 1
@@ -154,4 +155,3 @@ done
 # The main loop has expired by now
 log "❌ Test failed!"
 exit 1
-

.ci/test_devnet.sh

@@ -16,15 +16,18 @@ network_id=$3
 min_height=$4
 max_warnings=$5
 
-# The verobsity of snarkos nodes.
-NODE_VERBOSITY=3
+# The verbosity of snarkos nodes.
+NODE_VERBOSITY=4
+# Max logfile sizes (bytes) for regression checks.
+MAX_VALIDATOR_LOG_SIZE_BYTES=$((2 * 1024 * 1024))
+MAX_CLIENT_LOG_SIZE_BYTES=$((1 * 1024 * 1024))
 
 # Default values if not provided
 : "${total_validators:=4}"
 : "${total_clients:=4}" # need at least 4 clients, so each validator has at least one client connected to it.
 : "${network_id:=0}"
 : "${min_height:=60}" # To likely go past the 100 round garbage collection limit.
-: "${max_warnings:=40}"
+: "${max_warnings:=300}"
 
 # shellcheck source=SCRIPTDIR/utils.sh
 . ./.ci/utils.sh
@@ -437,7 +440,7 @@ fi
 
 # Ensure no errors are generated during the devnet run, as all nodes are
 # expected to operate without failures or interruptions.
-if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings"; then
+if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings" "$MAX_VALIDATOR_LOG_SIZE_BYTES" "$MAX_CLIENT_LOG_SIZE_BYTES"; then
   exit 0
 else
   exit 1

.ci/test_full_upgrade.sh

@@ -24,10 +24,13 @@ max_warnings=$4
 : "${total_validators:=4}"
 : "${total_clients:=2}"
 : "${network_id:=0}"
-: "${max_warnings:=40}"
+: "${max_warnings:=45}"
 
 # Node verbosity
-NODE_VERBOSITY=1
+NODE_VERBOSITY=4
+# Max logfile sizes (bytes) for regression checks.
+MAX_VALIDATOR_LOG_SIZE_BYTES=$((8 * 1024 * 1024))
+MAX_CLIENT_LOG_SIZE_BYTES=$((1 * 1024 * 1024))
 
 
 # How long to wait between upgrades (seconds); used for block-height window
@@ -381,7 +384,7 @@ done
 
 log "Upgrade test passed: network reached highest consensus version with release, all nodes upgraded to PR snarkos, and consensus version remained correct."
 
-if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings"; then
+if check_logs "$log_dir" "$total_validators" "$total_clients" "$max_warnings" "$MAX_VALIDATOR_LOG_SIZE_BYTES" "$MAX_CLIENT_LOG_SIZE_BYTES"; then
   exit 0
 else
   exit 1

.ci/test_partial_upgrade.sh

@@ -27,10 +27,12 @@ max_warnings=$3
 # Default values if not provided
 : "${total_validators:=4}"
 : "${network_id:=0}"
-: "${max_warnings:=40}"
+: "${max_warnings:=45}"
 
 # Node verbosity
-NODE_VERBOSITY=1
+NODE_VERBOSITY=4
+# Max logfile size (bytes) for regression checks.
+MAX_VALIDATOR_LOG_SIZE_BYTES=$((4 * 1024 * 1024))
 
 # How long to wait between upgrades (seconds); used for block-height window
 WAIT_BETWEEN_UPGRADES="${WAIT_BETWEEN_UPGRADES:-60}"
@@ -294,7 +296,7 @@ fi
 
 log "🎉 Test passed! Node synced to new consensus height ($new_consensus_height) after another node was upgraded."
 
-if check_logs "$log_dir" "$total_validators" 0 "$max_warnings"; then
+if check_logs "$log_dir" "$total_validators" 0 "$max_warnings" "$MAX_VALIDATOR_LOG_SIZE_BYTES"; then
   exit 0
 else
   exit 1

.ci/test_reset_minority.sh

@@ -25,14 +25,15 @@ max_warnings=$6
 # Default values if not provided
 : "${total_validators:=7}"
 : "${network_id:=0}"
-: "${reset_interval:=20}"
-: "${final_height:=100}"
+: "${reset_interval:=50}"
+: "${final_height:=250}"
 : "${num_resets:=3}"
-: "${max_warnings:=40}"
+: "${max_warnings:=4000}" # Allow lots of warnings as we're doing funky stuff in this test.
 
 minority=$(( (total_validators - 1) / 3 ))
 network_name=$(get_network_name "$network_id")
-verbosity=0
+verbosity=1
+max_validator_log_size_bytes=$((6 * 1024 * 1024))
 
 # The time that is used to determine the total timeout for the test
 max_wait_per_block=10
@@ -101,7 +102,7 @@ else
   exit 1
 fi
 
-if check_logs "$log_dir" "$total_validators" 0 "$max_warnings"; then
+if check_logs "$log_dir" "$total_validators" 0 "$max_warnings" "$max_validator_log_size_bytes"; then
   exit 0
 else
   exit 1

.ci/test_restart_majority.sh

@@ -18,25 +18,26 @@ init_log_dir
 total_validators=$1
 network_id=$2
 reset_interval=$3
-final_height=$4
-num_resets=$5
-max_warnings=$6
+num_resets=$4
+max_warnings=$5
 
 # Default values if not provided
 : "${total_validators:=7}"
 : "${network_id:=0}"
-: "${reset_interval:=10}"
-: "${final_height:=20}"
+: "${reset_interval:=20}"
 : "${num_resets:=3}"
-: "${max_warnings:=40}"
+: "${max_warnings:=4000}" # Allow lots of warnings as we're doing funky stuff in this test.
+
+# Blocks to advance after all restarts complete.
+final_height_advance=20
 
 max_faulty=$(( (total_validators - 1) / 3 ))
 # AleoBFT needs at least N-f for a quorum, not 2*f+1.
 majority=$((total_validators - max_faulty))
 network_name=$(get_network_name "$network_id")
 
-# Keep verbosity low as we are running many nodes.
-verbosity=0
+verbosity=1
+max_validator_log_size_bytes=$((3 * 1024 * 1024))
 
 # The time that is used to determine the total timeout for the test.
 # Set this higher than the interval for the minority test, as more nodes need to sync.
@@ -67,9 +68,6 @@ done
 
 wait_for_nodes "$total_validators" 0 "$network_name" 180
 
-# Wait longer if there are more blocks to reach.
-max_wait=$((final_height * max_wait_per_block))
-
 for iter in $(seq 1 "$num_resets"); do
   reset_height=$(( iter * reset_interval ));
 
@@ -97,14 +95,24 @@ for iter in $(seq 1 "$num_resets"); do
   done
 done
 
-if ! wait_for_heights 0 "$total_validators" "$final_height" "$network_name" $(( max_wait - $(elapsed_since "$start") )); then
+# Sleep for a few seconds to make sure the nodes are fully restarted and connected.
+sleep 5
+
+log_check_since=$(epoch_now)
+latest_height=$(get_block_height 0 "$network_name")
+final_height=$((latest_height + final_height_advance))
+max_wait=$((final_height_advance * max_wait_per_block))
+
+log "All restarts complete at height ${latest_height}. Waiting for final height ${final_height}..."
+
+if ! wait_for_heights 0 "$total_validators" "$final_height" "$network_name" "$max_wait"; then
   log "❌ Test failed! Not all nodes reached final height of $final_height within $max_wait seconds."
   exit 1
 fi
 
 log "SUCCESS! Network took $(elapsed_since "$start") seconds to reach final height of $final_height after $num_resets resets."
 
-if check_logs "$log_dir" "$total_validators" 0 "$max_warnings"; then
+if check_logs "$log_dir" "$total_validators" 0 "$max_warnings" "$max_validator_log_size_bytes" "" "$log_check_since"; then
   exit 0
 else
   exit 1

.ci/utils.sh

@@ -79,6 +79,48 @@ function elapsed_since() {
   echo $((SECONDS - start))
 }
 
+# Get the current Unix epoch timestamp in seconds.
+function epoch_now() {
+  date +%s
+}
+
+# Convert a Unix epoch timestamp to an ISO-8601 prefix used by snarkOS log lines.
+function epoch_to_iso() {
+  local epoch=$1
+  if date -u -d "@${epoch}" +%Y-%m-%dT%H:%M:%S >/dev/null 2>&1; then
+    date -u -d "@${epoch}" +%Y-%m-%dT%H:%M:%S
+  else
+    date -u -r "${epoch}" +%Y-%m-%dT%H:%M:%S
+  fi
+}
+
+# Return log lines at or after the given Unix epoch timestamp.
+# When since_epoch is empty, returns the full log file.
+function log_lines_since() {
+  local log_file=$1
+  local since_epoch=$2
+
+  if [ -z "$since_epoch" ]; then
+    cat "$log_file"
+    return 0
+  fi
+
+  local since_prefix
+  since_prefix=$(epoch_to_iso "$since_epoch")
+
+  awk -v since="$since_prefix" '
+    BEGIN { from = 0 }
+    /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}/ {
+      if (substr($1, 1, 19) >= since) {
+        from = 1
+      } else {
+        from = 0
+      }
+    }
+    from { print }
+  ' "$log_file"
+}
+
 # Determine network name based on network_id
 function get_network_name() {
   local network_id=$1
@@ -344,26 +386,46 @@ function check_logs() {
   # The maximum number of warnings allow in each node's log file.
   # Nodes may create some warnings at startup because they cannot connect to each other yet.
   local max_warnings=$4
-
+  # Optional max logfile size in bytes.
+  local max_validator_log_size_bytes=${5:-}
+  local max_client_log_size_bytes=${6:-}
+  # Optional Unix epoch timestamp; only log lines at or after this time are checked.
+  local since_epoch=${7:-}
+
+  if [ -n "$since_epoch" ]; then
+    log "Only checking log lines at or after $(epoch_to_iso "$since_epoch")"
+  fi
+
   local all_reached=true
   local highest_height=0
 
   # Don't use `seq` here as `total_validators` can be 0.
-  for ((validator_index = 0; validator_index < validator_index; validator_index++)); do
-    if [ ! -s "$log_dir/validator-${validator_index}.log" ]; then
+  for ((validator_index = 0; validator_index < total_validators; validator_index++)); do
+    validator_log="$log_dir/validator-${validator_index}.log"
+    if [ ! -s "$validator_log" ]; then
       log "❌ Test failed! Validator #${validator_index} did not create any logs in \"$log_dir\"."
       return 1
     fi
 
+    if [ -n "$max_validator_log_size_bytes" ]; then
+      validator_log_size_bytes=$(wc -c < "$validator_log")
+      if (( validator_log_size_bytes > max_validator_log_size_bytes )); then
+        log "❌ Test failed! Validator #${validator_index} logfile is too large (${validator_log_size_bytes}B > ${max_validator_log_size_bytes}B)."
+        return 1
+      fi
+    fi
+
+    validator_log_content=$(log_lines_since "$validator_log" "$since_epoch")
+
     #TODO(kaimast): remove the grep -v "already exists in the ledger" once spurious sync errors are gone.
-    if grep "ERROR" "$log_dir/validator-${validator_index}.log" | grep -qv "already exists in the ledger"; then
+    if echo "$validator_log_content" | grep "ERROR" | grep -qv "already exists in the ledger"; then
       log "❌ Test failed! Validator #${validator_index} logs contain errors."
       # Print the errors to the console.
-      grep "ERROR" "$log_dir/validator-${validator_index}.log" | grep -v "already exists in the ledger"
+      echo "$validator_log_content" | grep "ERROR" | grep -v "already exists in the ledger"
       return 1
     fi
 
-    num_warnings=$(grep -c "WARN" "$log_dir/validator-${validator_index}.log")
+    num_warnings=$(echo "$validator_log_content" | grep -c "WARN" || true)
     if (( num_warnings > max_warnings )); then
       echo "❌ Test failed! Validator #${validator_index} logs contain more than ${max_warnings} warnings."
       return 1
@@ -372,19 +434,30 @@ function check_logs() {
 
   # Don't use `seq` here as `total_clients` can be 0.
   for ((client_index = 0; client_index < total_clients; client_index++)); do
-    if [ ! -s "$log_dir/client-${client_index}.log" ]; then
+    client_log="$log_dir/client-${client_index}.log"
+    if [ ! -s "$client_log" ]; then
       log "❌ Test failed! Client #${client_index} did not create any logs in \"$log_dir\"."
       return 1
     fi
 
-    if grep "ERROR" "$log_dir/client-${client_index}.log" | grep -qv "already exists in the ledger"; then
+    if [ -n "$max_client_log_size_bytes" ]; then
+      client_log_size_bytes=$(wc -c < "$client_log")
+      if (( client_log_size_bytes > max_client_log_size_bytes )); then
+        log "❌ Test failed! Client #${client_index} logfile is too large (${client_log_size_bytes}B > ${max_client_log_size_bytes}B)."
+        return 1
+      fi
+    fi
+
+    client_log_content=$(log_lines_since "$client_log" "$since_epoch")
+
+    if echo "$client_log_content" | grep "ERROR" | grep -qv "already exists in the ledger"; then
       log "❌ Test failed! Client #${client_index} logs contain errors."
       # Print the errors to the console.
-      grep "ERROR" "$log_dir/client-${client_index}.log" | grep -v "already exists in the ledger"
+      echo "$client_log_content" | grep "ERROR" | grep -v "already exists in the ledger"
       return 1
     fi
 
-    num_warnings=$(grep -c "WARN" "$log_dir/client-${client_index}.log")
+    num_warnings=$(echo "$client_log_content" | grep -c "WARN" || true)
     if (( num_warnings > max_warnings )); then
       echo "❌ Test failed! Client #${client_index} logs contain more than ${max_warnings} warnings."
       return 1
@@ -957,4 +1030,3 @@ function get_block_height {
   echo "$result"
   return 0
 }
-