diff --git a/src/Nethermind/Benchmarks.sln b/src/Nethermind/Benchmarks.sln index 33a5ba018d41..d06c6ca9cadc 100644 --- a/src/Nethermind/Benchmarks.sln +++ b/src/Nethermind/Benchmarks.sln @@ -314,4 +314,7 @@ Global GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {14335C6A-B827-4069-8D11-9DC9398E51F3} EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {1710193C-E73E-42A4-AE70-B556B496C2C5} + EndGlobalSection EndGlobal diff --git a/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullExporter.cs b/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullExporter.cs new file mode 100644 index 000000000000..65124f10a939 --- /dev/null +++ b/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullExporter.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; + +namespace Imapp.Benchmark.Runner +{ + public class BenchmarkNullExporter : IExporter + { + public string Name => "ImappNullExporter"; + + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + return Enumerable.Empty(); + } + + public void ExportToLog(Summary summary, ILogger logger) + { + } + } +} diff --git a/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullLogger.cs b/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullLogger.cs new file mode 100644 index 000000000000..fa71b4d805d1 --- /dev/null +++ b/src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullLogger.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using BenchmarkDotNet.Loggers; + +namespace Imapp.Benchmark.Runner +{ + public class BenchmarkNullLogger : ILogger + { + public string Id => "ImappNullLogger"; + + public int Priority => 1; + + public void Flush() + { + + } + + public void Write(LogKind logKind, string text) + { + } + + public void WriteLine() + { + } + + public void WriteLine(LogKind logKind, string text) + { + + } + } +} diff --git a/src/Nethermind/Imapp.Benchmark.Runner/EvmByteCodeBenchmark.cs b/src/Nethermind/Imapp.Benchmark.Runner/EvmByteCodeBenchmark.cs new file mode 100644 index 000000000000..9f19fe6e389d --- /dev/null +++ b/src/Nethermind/Imapp.Benchmark.Runner/EvmByteCodeBenchmark.cs @@ -0,0 +1,88 @@ +using System; +using BenchmarkDotNet.Attributes; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Specs; +using Nethermind.Evm.Tracing; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie.Pruning; +using Nethermind.Evm; +using Nethermind.Evm.Benchmark; + +namespace Imapp.Benchmark.Runner +{ + [MaxWarmupCount(10)] + [MaxIterationCount(40)] + [MemoryDiagnoser] + public class EvmByteCodeBenchmark + { + public static byte[] ByteCode { get; set; } + + private IReleaseSpec _spec = MainnetSpecProvider.Instance.GetSpec(MainnetSpecProvider.IstanbulBlockNumber); + private ITxTracer _txTracer = NullTxTracer.Instance; + private ExecutionEnvironment _environment; + private IVirtualMachine _virtualMachine; + private BlockHeader _header = new BlockHeader(Keccak.Zero, Keccak.Zero, Address.Zero, UInt256.One, MainnetSpecProvider.IstanbulBlockNumber, Int64.MaxValue, UInt256.One, Bytes.Empty); + private IBlockhashProvider _blockhashProvider = new TestBlockhashProvider(); + private EvmState _evmState; + private StateProvider _stateProvider; + private StorageProvider _storageProvider; + private WorldState _worldState; + + [GlobalSetup] + public void GlobalSetup() + { + ByteCode = Bytes.FromHexString(Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE") ?? string.Empty); + //Console.WriteLine($"Running benchmark for bytecode {ByteCode?.ToHexString()}"); + + TrieStore trieStore = new(new MemDb(), new OneLoggerLogManager(NullLogger.Instance)); + IKeyValueStore codeDb = new MemDb(); + + _stateProvider = new StateProvider(trieStore, codeDb, new OneLoggerLogManager(NullLogger.Instance)); + _stateProvider.CreateAccount(Address.Zero, 1000.Ether()); + _stateProvider.Commit(_spec); + + _storageProvider = new StorageProvider(trieStore, _stateProvider, new OneLoggerLogManager(NullLogger.Instance)); + + _worldState = new WorldState(_stateProvider, _storageProvider); + + _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, LimboLogs.Instance); + + _environment = new ExecutionEnvironment + { + ExecutingAccount = Address.Zero, + CodeSource = Address.Zero, + Caller = Address.Zero, + CodeInfo = new CodeInfo(ByteCode), + Value = 0, + TransferValue = 0, + TxExecutionContext = new TxExecutionContext(_header, Address.Zero, 0) + }; + } + + [IterationSetup] + public void Setup() + { + _evmState = new EvmState(long.MaxValue, _environment, ExecutionType.Transaction, true, _worldState.TakeSnapshot(), false); + } + + [Benchmark] + public void ExecuteCode() + { + _virtualMachine.Run(_evmState, _worldState, _txTracer); + } + + [IterationCleanup] + public void Cleanup() + { + _stateProvider.Reset(); + _storageProvider.Reset(); + } + } +} diff --git a/src/Nethermind/Imapp.Benchmark.Runner/Imapp.Benchmark.Runner.csproj b/src/Nethermind/Imapp.Benchmark.Runner/Imapp.Benchmark.Runner.csproj new file mode 100644 index 000000000000..4a4b2e6e5a1f --- /dev/null +++ b/src/Nethermind/Imapp.Benchmark.Runner/Imapp.Benchmark.Runner.csproj @@ -0,0 +1,22 @@ + + + + Exe + net6.0 + + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Imapp.Benchmark.Runner/Program.cs b/src/Nethermind/Imapp.Benchmark.Runner/Program.cs new file mode 100644 index 000000000000..861df018a439 --- /dev/null +++ b/src/Nethermind/Imapp.Benchmark.Runner/Program.cs @@ -0,0 +1,62 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; +using System.CommandLine; + +namespace Imapp.Benchmark.Runner +{ + class Program + { + static int Main(string bytecode, int sampleSize = 1, bool printCSV = false) + { + var config = new ManualConfig() + //.WithOptions(ConfigOptions.DisableOptimizationsValidator) + .WithOptions(ConfigOptions.DisableLogFile) + .AddExporter(new BenchmarkNullExporter()); + + if (printCSV) + { + config = config.AddLogger(new BenchmarkNullLogger()); + } + else + { + config = config.AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default); + } + + if (String.IsNullOrEmpty(bytecode)) + { + throw new Exception("Bytecode cannot be empty"); + } + + for (int i = 1; i <= sampleSize; ++i) + { + Environment.SetEnvironmentVariable("NETH.BENCHMARK.BYTECODE", "00" + bytecode); + //Console.WriteLine("Code: " + Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE")); + var r1 = BenchmarkRunner.Run(config); + + Environment.SetEnvironmentVariable("NETH.BENCHMARK.BYTECODE", bytecode); + //Console.WriteLine("Code: " + Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE")); + var r2 = BenchmarkRunner.Run(config); + + OutputOverheadResults(i, r1, r2); + } + + return 0; + } + + private static void OutputOverheadResults(int sampleId, BenchmarkDotNet.Reports.Summary rEmpty, BenchmarkDotNet.Reports.Summary rActual) + { + var reportEmpty = rEmpty.Reports[0]; + var reportActual = rActual.Reports[0]; + var overheadTime = reportEmpty.ResultStatistics.Mean; + + var loopExecutionTime = reportActual.ResultStatistics.Mean - reportEmpty.ResultStatistics.Mean; + var totalTime = reportActual.ResultStatistics.Mean; + + var memAllocPerOp = reportActual.GcStats.GetBytesAllocatedPerOperation(reportActual.BenchmarkCase); + + Console.WriteLine($"{sampleId},{reportActual.ResultStatistics.N},{overheadTime},{loopExecutionTime},{totalTime},{reportActual.ResultStatistics.StandardDeviation},{reportActual.GcStats.TotalOperations},{memAllocPerOp}"); + } + } +} diff --git a/src/Nethermind/Imapp.Measurement.Runner/EvmByteCodeBenchmark.cs b/src/Nethermind/Imapp.Measurement.Runner/EvmByteCodeBenchmark.cs new file mode 100644 index 000000000000..8ad7e95d8ab1 --- /dev/null +++ b/src/Nethermind/Imapp.Measurement.Runner/EvmByteCodeBenchmark.cs @@ -0,0 +1,81 @@ +using System; +using BenchmarkDotNet.Attributes; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Evm.CodeAnalysis; +using Nethermind.Specs; +using Nethermind.Evm.Tracing; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.State; +using Nethermind.Trie.Pruning; +using Nethermind.Evm; +using Nethermind.Evm.Benchmark; + +namespace Imapp.Measurement.Runner +{ + public class EvmByteCodeBenchmark + { + public static byte[] ByteCode { get; set; } + + private IReleaseSpec _spec = MainnetSpecProvider.Instance.GetSpec(MainnetSpecProvider.IstanbulBlockNumber); + private ITxTracer _txTracer = NullTxTracer.Instance; + private ExecutionEnvironment _environment; + private IVirtualMachine _virtualMachine; + private BlockHeader _header = new BlockHeader(Keccak.Zero, Keccak.Zero, Address.Zero, UInt256.One, MainnetSpecProvider.IstanbulBlockNumber, Int64.MaxValue, UInt256.One, Bytes.Empty); + private IBlockhashProvider _blockhashProvider = new TestBlockhashProvider(); + private EvmState _evmState; + private StateProvider _stateProvider; + private StorageProvider _storageProvider; + private WorldState _worldState; + + public void GlobalSetup(string bytecode) + { + ByteCode = Bytes.FromHexString(bytecode); + //Console.WriteLine($"Running benchmark for bytecode {ByteCode?.ToHexString()}"); + + TrieStore trieStore = new(new MemDb(), new OneLoggerLogManager(NullLogger.Instance)); + IKeyValueStore codeDb = new MemDb(); + + _stateProvider = new StateProvider(trieStore, codeDb, new OneLoggerLogManager(NullLogger.Instance)); + _stateProvider.CreateAccount(Address.Zero, 1000.Ether()); + _stateProvider.Commit(_spec); + + _storageProvider = new StorageProvider(trieStore, _stateProvider, new OneLoggerLogManager(NullLogger.Instance)); + + _worldState = new WorldState(_stateProvider, _storageProvider); + + _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, LimboLogs.Instance); + + _environment = new ExecutionEnvironment + { + ExecutingAccount = Address.Zero, + CodeSource = Address.Zero, + Caller = Address.Zero, + CodeInfo = new CodeInfo(ByteCode), + Value = 0, + TransferValue = 0, + TxExecutionContext = new TxExecutionContext(_header, Address.Zero, 0) + }; + } + + public void Setup() + { + _evmState = new EvmState(long.MaxValue, _environment, ExecutionType.Transaction, true, _worldState.TakeSnapshot(), false); + } + + public void ExecuteCode() + { + _virtualMachine.Run(_evmState, _worldState, _txTracer); + } + + public void Cleanup() + { + _stateProvider.Reset(); + _storageProvider.Reset(); + } + } +} diff --git a/src/Nethermind/Imapp.Measurement.Runner/Imapp.Measurement.Runner.csproj b/src/Nethermind/Imapp.Measurement.Runner/Imapp.Measurement.Runner.csproj new file mode 100644 index 000000000000..4123fde07a8e --- /dev/null +++ b/src/Nethermind/Imapp.Measurement.Runner/Imapp.Measurement.Runner.csproj @@ -0,0 +1,22 @@ + + + + Exe + net6.0 + + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Imapp.Measurement.Runner/InstrumentationLogger.cs b/src/Nethermind/Imapp.Measurement.Runner/InstrumentationLogger.cs new file mode 100644 index 000000000000..2f3ee5eeb2b4 --- /dev/null +++ b/src/Nethermind/Imapp.Measurement.Runner/InstrumentationLogger.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Imapp.Measurement.Runner +{ + public class InstrumentationLogger + { + private List items = new List(); + + public void Add(InstrumentationLoggerItem item) + { + if (item.End < item.Start) throw new Exception("Invalid measurement"); + items.Add(item); + } + + public void PrintResults() + { + foreach (var item in items) + { + var ticksLength = item.End - item.Start; + + Console.WriteLine($"{item.SampleId},{TicksToNs(ticksLength)}"); + } + } + + private double TicksToNs(long ticks) + { + double ns = 1000000000.0 * (double)ticks / Stopwatch.Frequency; + return ns; + } + } + + public struct InstrumentationLoggerItem + { + public int SampleId { get; set; } + public long Start { get; set; } + public long End { get; set; } + } +} diff --git a/src/Nethermind/Imapp.Measurement.Runner/Program.cs b/src/Nethermind/Imapp.Measurement.Runner/Program.cs new file mode 100644 index 000000000000..4c6345a9a6e8 --- /dev/null +++ b/src/Nethermind/Imapp.Measurement.Runner/Program.cs @@ -0,0 +1,50 @@ +using System; +using System.Diagnostics; +using System.CommandLine; + +namespace Imapp.Measurement.Runner +{ + class Program + { + static int Main(string bytecode, int sampleSize = 1, bool printCSV = false) + { + if (String.IsNullOrEmpty(bytecode)) + { + throw new Exception("Bytecode cannot be empty"); + } + + var runner = new EvmByteCodeBenchmark(); + runner.GlobalSetup(bytecode); + + //warmup + for (int i = 0; i < 5; ++i) + { + runner.Setup(); + runner.ExecuteCode(); + runner.Cleanup(); + } + + var sw = new Stopwatch(); + sw.Start(); + var logger = new InstrumentationLogger(); + + for (int i = 1; i <= sampleSize; ++i) + { + var loggerItem = new InstrumentationLoggerItem() { SampleId = i }; + runner.Setup(); + + loggerItem.Start = sw.ElapsedTicks; + runner.ExecuteCode(); + loggerItem.End = sw.ElapsedTicks; + + runner.Cleanup(); + logger.Add(loggerItem); + } + sw.Stop(); + + logger.PrintResults(); + + return 0; + } + } +}