Skip to content

[Testing] Add deterministic ALSP spam-decision boundary tests #8628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
janezpodhostnik wants to merge 1 commit into
base: master
Choose a base branch
from
+170 −91
Open

[Testing] Add deterministic ALSP spam-decision boundary tests #8628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 43 additions & 31 deletions engine/common/synchronization/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,38 @@ func (e *Engine) sendRequests(participants flow.IdentifierList, ranges []chainsy
}
}

// shouldReportProbabilistically returns true iff the random draw `n` - sampled uniformly from
// [0, spamProbabilityMultiplier) - falls below the probability `prob` ∈ [0, 1] scaled by
// spamProbabilityMultiplier. Properties (covered by unit tests):
// - prob = 0 never reports;
// - prob = 1 always reports;
// - for any prob > 0, the draw n = 0 always reports.
func shouldReportProbabilistically(n uint32, prob float32) bool {
return float32(n) < prob*spamProbabilityMultiplier
}

// batchRequestMisbehaviorProbability returns the probability of reporting a batch request as
// misbehavior: the larger the batch and the base probability, the higher the result. It is
// calculated as `baseProb * (len(batchRequest.BlockIDs) + 1) / synccore.DefaultConfig().MaxSize`.
// Examples:
// - 10 block IDs with baseProb 0.01: 0.01 * 11 / 64 = 0.00171875 = 0.171875%
// - 1000 block IDs with baseProb 0.01: 0.01 * 1001 / 64 = 0.15640625 = 15.640625%
func batchRequestMisbehaviorProbability(baseProb float32, batchRequest *flow.BatchRequest) float32 {
return baseProb * (float32(len(batchRequest.BlockIDs)) + 1) / float32(synccore.DefaultConfig().MaxSize)
}

// rangeRequestMisbehaviorProbability returns the probability of reporting a range request as
// misbehavior: the larger the requested range and the base probability, the higher the result. It
// is calculated as `baseProb * (rangeRequest.ToHeight - rangeRequest.FromHeight + 1) / synccore.DefaultConfig().MaxSize`.
// CAUTION: only valid for rangeRequest.ToHeight ≥ rangeRequest.FromHeight (the caller must handle
// invalid ranges beforehand, as the height difference would underflow).
// Examples:
// - range of 10 blocks with baseProb 0.01: 0.01 * 11 / 64 = 0.00171875 = 0.171875%
// - range of 1000 blocks with baseProb 0.01: 0.01 * 1001 / 64 = 0.15640625 = 15.640625%
func rangeRequestMisbehaviorProbability(baseProb float32, rangeRequest *flow.RangeRequest) float32 {
return baseProb * (float32(rangeRequest.ToHeight-rangeRequest.FromHeight) + 1) / float32(synccore.DefaultConfig().MaxSize)
}

// validateBatchRequestForALSP checks if a batch request should be reported as a misbehavior and sends misbehavior report to ALSP.
// The misbehavior is due to either:
// 1. unambiguous malicious or incorrect behavior (0 block IDs) OR
Expand Down Expand Up @@ -498,21 +530,11 @@ func (e *Engine) validateBatchRequestForALSP(originID flow.Identifier, batchRequ
return nil
}

// to avoid creating a misbehavior report for every batch request received, use a probabilistic approach.
// The larger the batch request and base probability, the higher the probability of creating a misbehavior report.

// batchRequestProb is calculated as follows:
// batchRequestBaseProb * (len(batchRequest.BlockIDs) + 1) / synccore.DefaultConfig().MaxSize
// Example 1 (small batch of block IDs) if the batch request is for 10 blocks IDs and batchRequestBaseProb is 0.01, then the probability of
// creating a misbehavior report is:
// batchRequestBaseProb * (10+1) / synccore.DefaultConfig().MaxSize
// = 0.01 * 11 / 64 = 0.00171875 = 0.171875%
// Example 2 (large batch of block IDs) if the batch request is for 1000 block IDs and batchRequestBaseProb is 0.01, then the probability of
// creating a misbehavior report is:
// batchRequestBaseProb * (1000+1) / synccore.DefaultConfig().MaxSize
// = 0.01 * 1001 / 64 = 0.15640625 = 15.640625%
batchRequestProb := e.spamDetectionConfig.batchRequestBaseProb * (float32(len(batchRequest.BlockIDs)) + 1) / float32(synccore.DefaultConfig().MaxSize)
if float32(n) < batchRequestProb*spamProbabilityMultiplier {
// to avoid creating a misbehavior report for every batch request received, use a probabilistic approach:
// the larger the batch request and base probability, the higher the probability of creating a misbehavior report
// (see `batchRequestMisbehaviorProbability` for the exact formula)
batchRequestProb := batchRequestMisbehaviorProbability(e.spamDetectionConfig.batchRequestBaseProb, batchRequest)
if shouldReportProbabilistically(n, batchRequestProb) {
// create a misbehavior report
e.log.Debug().
Hex("origin_id", logging.ID(originID)).
Expand Down Expand Up @@ -573,21 +595,11 @@ func (e *Engine) validateRangeRequestForALSP(originID flow.Identifier, rangeRequ
return nil
}

// to avoid creating a misbehavior report for every range request received, use a probabilistic approach.
// The higher the range request and base probability, the higher the probability of creating a misbehavior report.

// rangeRequestProb is calculated as follows:
// rangeRequestBaseProb * ((rangeRequest.ToHeight-rangeRequest.FromHeight) + 1) / synccore.DefaultConfig().MaxSize
// Example 1 (small range) if the range request is for 10 blocks and rangeRequestBaseProb is 0.01, then the probability of
// creating a misbehavior report is:
// rangeRequestBaseProb * (10+1) / synccore.DefaultConfig().MaxSize
// = 0.01 * 11 / 64 = 0.00171875 = 0.171875%
// Example 2 (large range) if the range request is for 1000 blocks and rangeRequestBaseProb is 0.01, then the probability of
// creating a misbehavior report is:
// rangeRequestBaseProb * (1000+1) / synccore.DefaultConfig().MaxSize
// = 0.01 * 1001 / 64 = 0.15640625 = 15.640625%
rangeRequestProb := e.spamDetectionConfig.rangeRequestBaseProb * (float32(rangeRequest.ToHeight-rangeRequest.FromHeight) + 1) / float32(synccore.DefaultConfig().MaxSize)
if float32(n) < rangeRequestProb*spamProbabilityMultiplier {
// to avoid creating a misbehavior report for every range request received, use a probabilistic approach:
// the larger the requested range and base probability, the higher the probability of creating a misbehavior report
// (see `rangeRequestMisbehaviorProbability` for the exact formula)
rangeRequestProb := rangeRequestMisbehaviorProbability(e.spamDetectionConfig.rangeRequestBaseProb, rangeRequest)
if shouldReportProbabilistically(n, rangeRequestProb) {
// create a misbehavior report
e.log.Debug().
Hex("origin_id", logging.ID(originID)).
Expand Down Expand Up @@ -628,7 +640,7 @@ func (e *Engine) validateSyncRequestForALSP(originID flow.Identifier) error {

// to avoid creating a misbehavior report for every sync request received, use a probabilistic approach.
// Create a report with a probability of spamDetectionConfig.syncRequestProb
if float32(n) < e.spamDetectionConfig.syncRequestProb*spamProbabilityMultiplier {
if shouldReportProbabilistically(n, e.spamDetectionConfig.syncRequestProb) {

// create misbehavior report
e.log.Debug().
Expand Down
103 changes: 103 additions & 0 deletions engine/common/synchronization/engine_alsp_probability_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package synchronization

import (
"testing"

"github.com/stretchr/testify/require"

"github.com/onflow/flow-go/model/flow"
"github.com/onflow/flow-go/utils/unittest"
)

// TestShouldReportProbabilistically tests the exact decision boundary of the probabilistic spam
// reporting: the random draw n is sampled uniformly from [0, spamProbabilityMultiplier), and a
// report is created iff n < prob*spamProbabilityMultiplier. These deterministic boundary cases
// complement the statistical load tests in engine_spam_test.go, which retain only
// high-probability wiring smoke groups and cannot exercise small-probability behavior.
func TestShouldReportProbabilistically(t *testing.T) {
testCases := []struct {
name string
n uint32
prob float32
expected bool
}{
// prob = 0 must never report, for any draw
{"prob 0, smallest draw", 0, 0.0, false},
{"prob 0, largest draw", spamProbabilityMultiplier - 1, 0.0, false},

// prob = 1 must always report, for any draw
{"prob 1, smallest draw", 0, 1.0, true},
{"prob 1, largest draw", spamProbabilityMultiplier - 1, 1.0, true},

// any prob >= 1/spamProbabilityMultiplier must report for the draw n = 0:
// an implementation that never reports is detected by this case
{"smallest meaningful prob, smallest draw", 0, 1.0 / spamProbabilityMultiplier, true},
{"smallest meaningful prob, draw at boundary", 1, 1.0 / spamProbabilityMultiplier, false},

// exact boundary at prob = 0.5: draws 0..499 report, draws 500.. do not
{"prob 0.5, draw below boundary", 499, 0.5, true},
{"prob 0.5, draw at boundary", 500, 0.5, false},

// exact boundary at prob = 0.1: draws 0..99 report, draws 100.. do not
{"prob 0.1, draw below boundary", 99, 0.1, true},
{"prob 0.1, draw at boundary", 100, 0.1, false},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
require.Equal(t, tc.expected, shouldReportProbabilistically(tc.n, tc.prob))
})
}
}

// TestBatchRequestMisbehaviorProbability pins the scaling formula
// `baseProb * (len(blockIDs) + 1) / MaxSize` (MaxSize = 64) to the documented example values.
func TestBatchRequestMisbehaviorProbability(t *testing.T) {
testCases := []struct {
name string
baseProb float32
numIDs int
expected float32
}{
{"single block ID", 0.1, 1, 0.003125},
{"small batch", 0.1, 10, 0.0171875},
{"large batch", 0.1, 99, 0.15625},
{"small batch, small base prob", 0.01, 10, 0.00171875},
{"very large batch, small base prob", 0.01, 999, 0.15625},
{"zero base prob", 0.0, 1000, 0.0},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := &flow.BatchRequest{BlockIDs: unittest.IdentifierListFixture(tc.numIDs)}
require.Equal(t, tc.expected, batchRequestMisbehaviorProbability(tc.baseProb, req))
})
}
}

// TestRangeRequestMisbehaviorProbability pins the scaling formula
// `baseProb * (toHeight - fromHeight + 1) / MaxSize` (MaxSize = 64) to the documented example values.
func TestRangeRequestMisbehaviorProbability(t *testing.T) {
testCases := []struct {
name string
baseProb float32
fromHeight uint64
toHeight uint64
expected float32
}{
{"flat range (from == to)", 0.01, 1, 1, 0.00015625},
{"single block range", 0.1, 9, 10, 0.003125},
{"small range", 0.1, 1, 11, 0.0171875},
{"large range", 0.1, 1, 100, 0.15625},
{"small range, small base prob", 0.01, 1, 11, 0.00171875},
{"very large range, small base prob", 0.01, 1, 1000, 0.15625},
{"zero base prob", 0.0, 1, 1000, 0.0},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
req := &flow.RangeRequest{FromHeight: tc.fromHeight, ToHeight: tc.toHeight}
require.Equal(t, tc.expected, rangeRequestMisbehaviorProbability(tc.baseProb, req))
})
}
}
84 changes: 24 additions & 60 deletions engine/common/synchronization/engine_spam_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,29 +114,21 @@ func (ss *SyncSuite) TestLoad_Process_SyncRequest_HigherThanReceiver_OutsideTole

loadGroups := []loadGroup{}

// These load tests are wiring smoke tests: the exact decision boundary of the probabilistic
// reporting is unit-tested deterministically in `TestShouldReportProbabilistically`, so we only
// keep the groups with discriminating power here. The bounds are the exact quantiles of the
// Binomial(1000, p) distribution of the misbehavior report count, such that the probability of
// the count falling outside the bounds is at most 1e-9 per tail. This keeps the test meaningful
// while making failures of a correct implementation practically impossible (previous, tighter
// bounds caused flakiness; low-probability groups were removed because their lower bound of 0
// could not discriminate at all).

// expect to never get misbehavior report
loadGroups = append(loadGroups, loadGroup{0.0, 0, 0})

// The lower and upper bounds below are the exact quantiles of the Binomial(1000, p) distribution
// of the misbehavior report count, such that the probability of the count falling outside the
// bounds is at most 1e-9 per tail. This keeps the test meaningful while making failures of a
// correct implementation practically impossible (previous, tighter bounds caused flakiness).

// expect to get misbehavior report about 0.1% of the time (1 in 1000 requests)
loadGroups = append(loadGroups, loadGroup{0.001, 0, 11})

// expect to get misbehavior report about 1% of the time
loadGroups = append(loadGroups, loadGroup{0.01, 0, 34})

// expect to get misbehavior report about 10% of the time
loadGroups = append(loadGroups, loadGroup{0.1, 48, 161})

// expect to get misbehavior report about 50% of the time
loadGroups = append(loadGroups, loadGroup{0.5, 405, 595})

// expect to get misbehavior report about 90% of the time
loadGroups = append(loadGroups, loadGroup{0.9, 839, 952})

// reset misbehavior report counter for each subtest
misbehaviorsCounter := 0

Expand Down Expand Up @@ -227,36 +219,20 @@ func (ss *SyncSuite) TestLoad_Process_RangeRequest_SometimesReportSpam() {

loadGroups := []loadGroup{}

// The lower and upper bounds below are the exact quantiles of the Binomial(1000, p) distribution
// of the misbehavior report count (with p being the expected probability factor), such that the
// These load tests are wiring smoke tests: the exact decision boundary and the range-scaling
// formula are unit-tested deterministically in `TestShouldReportProbabilistically` and
// `TestRangeRequestMisbehaviorProbability`, so we only keep the groups with discriminating
// power here. The bounds are the exact quantiles of the Binomial(1000, p) distribution of the
// misbehavior report count (with p being the expected probability factor), such that the
// probability of the count falling outside the bounds is at most 1e-9 per tail. This keeps the
// test meaningful while making failures of a correct implementation practically impossible
// (previous, tighter bounds caused flakiness).

// using a very small range (1) with a 10% base probability factor, expect to almost never get misbehavior report, about 0.3% of the time (3 in 1000 requests)
// expected probability factor: 0.1 * ((10-9) + 1)/64 = 0.003125
loadGroups = append(loadGroups, loadGroup{0.1, 0, 18, 9, 10})

// using a small range (10) with a 10% base probability factor, expect to get misbehavior report about 1.7% of the time (17 in 1000 requests)
// expected probability factor: 0.1 * ((11-1) + 1)/64 = 0.0171875
loadGroups = append(loadGroups, loadGroup{0.1, 0, 47, 1, 11})
// (previous, tighter bounds caused flakiness; low-probability groups were removed because
// their lower bound of 0 could not discriminate at all).

// using a large range (99) with a 10% base probability factor, expect to get misbehavior report about 15% of the time (150 in 1000 requests)
// expected probability factor: 0.1 * ((100-1) + 1)/64 = 0.15625
// expected probability factor: 0.1 * (99 + 1)/64 = 0.15625
loadGroups = append(loadGroups, loadGroup{0.1, 92, 229, 1, 100})

// using a flat range (0) (from height == to height) with a 1% base probability factor, expect to almost never get a misbehavior report, about 0.16% of the time (2 in 1000 requests)
// expected probability factor: 0.01 * ((1-1) + 1)/64 = 0.0015625
loadGroups = append(loadGroups, loadGroup{0.01, 0, 14, 1, 1})

// using a small range (10) with a 1% base probability factor, expect to almost never get misbehavior report, about 0.17% of the time (2 in 1000 requests)
// expected probability factor: 0.01 * ((11-1) + 1)/64 = 0.00171875
loadGroups = append(loadGroups, loadGroup{0.01, 0, 14, 1, 11})

// using a very large range (999) with a 1% base probability factor, expect to get misbehavior report about 15% of the time (150 in 1000 requests)
// expected probability factor: 0.01 * ((1000-1) + 1)/64 = 0.15625
loadGroups = append(loadGroups, loadGroup{0.01, 92, 229, 1, 1000})

// ALWAYS REPORT SPAM FOR INVALID RANGE REQUESTS OR RANGE REQUESTS THAT ARE FAR OUTSIDE OF THE TOLERANCE

// using an inverted range (from height > to height) always results in a misbehavior report, no matter how small the range is or how small the base probability factor is
Expand Down Expand Up @@ -347,32 +323,20 @@ func (ss *SyncSuite) TestLoad_Process_BatchRequest_SometimesReportSpam() {

loadGroups := []loadGroup{}

// The lower and upper bounds below are the exact quantiles of the Binomial(1000, p) distribution
// of the misbehavior report count (with p being the expected probability factor), such that the
// These load tests are wiring smoke tests: the exact decision boundary and the batch-scaling
// formula are unit-tested deterministically in `TestShouldReportProbabilistically` and
// `TestBatchRequestMisbehaviorProbability`, so we only keep the groups with discriminating
// power here. The bounds are the exact quantiles of the Binomial(1000, p) distribution of the
// misbehavior report count (with p being the expected probability factor), such that the
// probability of the count falling outside the bounds is at most 1e-9 per tail. This keeps the
// test meaningful while making failures of a correct implementation practically impossible
// (previous, tighter bounds caused flakiness).

// using a very small batch request (1 block ID) with a 10% base probability factor, expect to almost never get misbehavior report, about 0.3% of the time (3 in 1000 requests)
// expected probability factor: 0.1 * (1 + 1)/64 = 0.003125
loadGroups = append(loadGroups, loadGroup{0.1, 0, 18, repeatedBlockIDs(1)})

// using a small batch request (10 block IDs) with a 10% base probability factor, expect to get misbehavior report about 1.7% of the time (17 in 1000 requests)
// expected probability factor: 0.1 * (10 + 1)/64 = 0.0171875
loadGroups = append(loadGroups, loadGroup{0.1, 0, 47, repeatedBlockIDs(10)})
// (previous, tighter bounds caused flakiness; low-probability groups were removed because
// their lower bound of 0 could not discriminate at all).

// using a large batch request (99 block IDs) with a 10% base probability factor, expect to get misbehavior report about 15% of the time (150 in 1000 requests)
// expected probability factor: 0.1 * (99 + 1)/64 = 0.15625
loadGroups = append(loadGroups, loadGroup{0.1, 92, 229, repeatedBlockIDs(99)})

// using a small batch request (10 block IDs) with a 1% base probability factor, expect to almost never get misbehavior report, about 0.17% of the time (2 in 1000 requests)
// expected probability factor: 0.01 * (10 + 1)/64 = 0.00171875
loadGroups = append(loadGroups, loadGroup{0.01, 0, 14, repeatedBlockIDs(10)})

// using a very large batch request (999 block IDs) with a 1% base probability factor, expect to get misbehavior report about 15% of the time (150 in 1000 requests)
// expected probability factor: 0.01 * (999 + 1)/64 = 0.15625
loadGroups = append(loadGroups, loadGroup{0.01, 92, 229, repeatedBlockIDs(999)})

// ALWAYS REPORT SPAM FOR INVALID BATCH REQUESTS OR BATCH REQUESTS THAT ARE FAR OUTSIDE OF THE TOLERANCE

// using an empty batch request (0 block IDs) always results in a misbehavior report, no matter how small the base probability factor is
Expand Down