diff --git a/cmd/lotus-shed/finality.go b/cmd/lotus-shed/finality.go new file mode 100644 index 00000000000..37672096b85 --- /dev/null +++ b/cmd/lotus-shed/finality.go @@ -0,0 +1,228 @@ +package main + +import ( + "bufio" + "fmt" + "math" + "os" + "strconv" + + skellampmf "github.com/rvagg/go-skellam-pmf" + "github.com/urfave/cli/v2" + "golang.org/x/exp/constraints" + + "github.com/filecoin-project/lotus/build" +) + +var finalityCmd = &cli.Command{ + Name: "finality-calculator", + Description: "Calculate the finality probability of at a tipset", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "repo", + Value: "~/.lotus", + }, + &cli.StringFlag{ + Name: "input", + }, + &cli.IntFlag{ + Name: "target", + Usage: "target epoch for which finality is calculated", + }, + }, + ArgsUsage: "[inputFile]", + Action: func(cctx *cli.Context) error { + input := cctx.Args().Get(0) + file, err := os.Open(input) + if err != nil { + return err + } + defer func() { _ = file.Close() }() + + var chain []int + scanner := bufio.NewScanner(file) + for scanner.Scan() { + num, err := strconv.Atoi(scanner.Text()) + if err != nil { + return err + } + chain = append(chain, num) + } + + if err := scanner.Err(); err != nil { + return err + } + + blocksPerEpoch := 5.0 // Expected number of blocks per epoch + byzantineFraction := 0.3 // Upper bound on the fraction of malicious nodes in the network + currentEpoch := len(chain) - 1 // Current epoch (end of history) + // targetEpoch := currentEpoch - 30 // Target epoch for which finality is calculated + targetEpoch := cctx.Int("target") + + finality := FinalityCalcValidator(chain, blocksPerEpoch, byzantineFraction, currentEpoch, targetEpoch) + + _, _ = fmt.Fprintf(cctx.App.Writer, "Finality=%v @ %d for chain len=%d\n", finality, targetEpoch, currentEpoch) + + return nil + }, +} + +// FinalityCalcValidator computes the probability that a previous blockchain tipset gets replaced. +// +// Based on https://github.com/consensus-shipyard/ec-finality-calculator +func FinalityCalcValidator(chain []int, blocksPerEpoch float64, byzantineFraction float64, currentEpoch int, targetEpoch int) float64 { + // Threshold at which the probability of an event is considered negligible + const negligibleThreshold = 1e-25 + + maxKL := 400 // Max k for which to calculate Pr(L=k) + maxKB := (currentEpoch - targetEpoch) * int(blocksPerEpoch) // Max k for which to calculate Pr(B=k) + maxKM := 400 // Max k for which to calculate Pr(M=k) + maxIM := 100 // Maximum number of epochs for the calculation (after which the pr become negligible) + + rateMaliciousBlocks := blocksPerEpoch * byzantineFraction // upper bound + rateHonestBlocks := blocksPerEpoch - rateMaliciousBlocks // lower bound + + // Compute L + prL := make([]float64, maxKL+1) + + for k := 0; k <= maxKL; k++ { + sumExpectedAdversarialBlocksI := 0.0 + sumChainBlocksI := 0 + + for i := targetEpoch; i > currentEpoch-int(build.Finality); i-- { + sumExpectedAdversarialBlocksI += rateMaliciousBlocks + sumChainBlocksI += chain[i-1] + // Poisson(k=k, lambda=sum(f*e)) + prLi := poissonProb(sumExpectedAdversarialBlocksI, float64(k+sumChainBlocksI)) + prL[k] = math.Max(prL[k], prLi) + + } + if k > 1 && prL[k] < negligibleThreshold && prL[k] < prL[k-1] { + maxKL = k + prL = prL[:k+1] + break + } + } + + // As the adversarial lead is never negative, the missing probability is added to k=0 + prL[0] += 1 - sum(prL) + + // Compute B + prB := make([]float64, maxKB+1) + + // Calculate Pr(B=k) for each value of k + for k := 0; k <= maxKB; k++ { + prB[k] = poissonProb(float64(currentEpoch-targetEpoch)*rateMaliciousBlocks, float64(k)) + + // Break if prB[k] becomes negligible + if k > 1 && prB[k] < negligibleThreshold && prB[k] < prB[k-1] { + maxKB = k + prB = prB[:k+1] + break + } + } + + // Compute M + prHgt0 := 1 - poissonProb(rateHonestBlocks, 0) + + expZ := 0.0 + for k := 0; k < int(4*blocksPerEpoch); k++ { + pmf := poissonProb(rateMaliciousBlocks, float64(k)) + expZ += ((rateHonestBlocks + float64(k)) / math.Pow(2, float64(k))) * pmf + } + + ratePublicChain := prHgt0 * expZ + + prM := make([]float64, maxKM+1) + for k := 0; k <= maxKM; k++ { + for i := maxIM; i > 0; i-- { + probMI := skellampmf.SkellamPMF(k, float64(i)*rateMaliciousBlocks, float64(i)*ratePublicChain) + + // Break if probMI becomes negligible + if probMI < negligibleThreshold && probMI < prM[k] { + break + } + prM[k] = math.Max(prM[k], probMI) + } + + // Break if prM[k] becomes negligible + if k > 1 && prM[k] < negligibleThreshold && prM[k] < prM[k-1] { + maxKM = k + prM = prM[:k+1] + break + } + } + + prM[0] += 1 - sum(prM) + + // Compute error probability upper bound + cumsumL := cumsum(prL) + cumsumB := cumsum(prB) + cumsumM := cumsum(prM) + + k := sum(chain[targetEpoch:currentEpoch]) + + sumLgeK := cumsumL[len(cumsumL)-1] + if k > 0 { + sumLgeK -= cumsumL[min(k-1, maxKL)] + } + + doubleSum := 0.0 + + for l := 0; l < k; l++ { + sumBgeKminL := cumsumB[len(cumsumB)-1] + if k-l-1 > 0 { + sumBgeKminL -= cumsumB[min(k-l-1, maxKB)] + } + doubleSum += prL[min(l, maxKL)] * sumBgeKminL + + for b := 0; b < k-l; b++ { + sumMgeKminLminB := cumsumM[len(cumsumM)-1] + if k-l-b-1 > 0 { + sumMgeKminLminB -= cumsumM[min(k-l-b-1, maxKM)] + } + doubleSum += prL[min(l, maxKL)] * prB[min(b, maxKB)] * sumMgeKminLminB + } + } + + prError := sumLgeK + doubleSum + + return math.Min(prError, 1.0) +} + +func poissonProb(lambda float64, x float64) float64 { + return math.Exp(poissonLogProb(lambda, x)) +} + +func poissonLogProb(lambda float64, x float64) float64 { + if x < 0 || math.Floor(x) != x { + return math.Inf(-1) + } + lg, _ := math.Lgamma(math.Floor(x) + 1) + return x*math.Log(lambda) - lambda - lg +} + +func sum[T constraints.Integer | constraints.Float](s []T) T { + var total T + for _, v := range s { + total += v + } + return total +} + +func cumsum(arr []float64) []float64 { + cumsums := make([]float64, len(arr)) + cumSum := 0.0 + for i, value := range arr { + cumSum += value + cumsums[i] = cumSum + } + return cumsums +} + +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/cmd/lotus-shed/main.go b/cmd/lotus-shed/main.go index 911da346e97..8244e9d2db1 100644 --- a/cmd/lotus-shed/main.go +++ b/cmd/lotus-shed/main.go @@ -91,6 +91,7 @@ func main() { mismatchesCmd, blockCmd, adlCmd, + finalityCmd, } app := &cli.App{ diff --git a/go.mod b/go.mod index 08d462377d7..8456002af5b 100644 --- a/go.mod +++ b/go.mod @@ -126,6 +126,7 @@ require ( github.com/puzpuzpuz/xsync/v2 v2.4.0 github.com/raulk/clock v1.1.0 github.com/raulk/go-watchdog v1.3.0 + github.com/rvagg/go-skellam-pmf v0.0.1 github.com/samber/lo v1.39.0 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -149,6 +150,7 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.23.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 golang.org/x/net v0.25.0 golang.org/x/sync v0.7.0 golang.org/x/sys v0.20.0 @@ -320,7 +322,6 @@ require ( go.uber.org/dig v1.17.1 // indirect go.uber.org/mock v0.4.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.15.0 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index d1d53543052..5690e428f80 100644 --- a/go.sum +++ b/go.sum @@ -1178,6 +1178,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rvagg/go-skellam-pmf v0.0.1 h1:li3G0jioT+8bRUseNP+h0WRVv/CI+9s2q1qwNvEE994= +github.com/rvagg/go-skellam-pmf v0.0.1/go.mod h1:/xSBO272x+iW1BjHnPL6p2O7kCpVTh7QK9hZcXE/Kqk= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=