Imapp benchmark

src/Nethermind/Benchmarks.sln

@@ -314,4 +314,7 @@ Global
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {14335C6A-B827-4069-8D11-9DC9398E51F3}
 	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {1710193C-E73E-42A4-AE70-B556B496C2C5}
+	EndGlobalSection
 EndGlobal

src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullExporter.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Exporters;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Reports;
+
+namespace Imapp.Benchmark.Runner
+{
+    public class BenchmarkNullExporter : IExporter
+    {
+        public string Name => "ImappNullExporter";
+
+        public IEnumerable<string> ExportToFiles(Summary summary, ILogger consoleLogger)
+        {
+            return Enumerable.Empty<string>();
+        }
+
+        public void ExportToLog(Summary summary, ILogger logger)
+        {
+        }
+    }
+}

src/Nethermind/Imapp.Benchmark.Runner/BenchmarkNullLogger.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using BenchmarkDotNet.Loggers;
+
+namespace Imapp.Benchmark.Runner
+{
+    public class BenchmarkNullLogger : ILogger
+    {
+        public string Id => "ImappNullLogger";
+
+        public int Priority => 1;
+
+        public void Flush()
+        {
+
+        }
+
+        public void Write(LogKind logKind, string text)
+        {
+        }
+
+        public void WriteLine()
+        {
+        }
+
+        public void WriteLine(LogKind logKind, string text)
+        {
+
+        }
+    }
+}

src/Nethermind/Imapp.Benchmark.Runner/EvmByteCodeBenchmark.cs

@@ -0,0 +1,88 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
+using Nethermind.Core.Specs;
+using Nethermind.Db;
+using Nethermind.Evm.CodeAnalysis;
+using Nethermind.Specs;
+using Nethermind.Evm.Tracing;
+using Nethermind.Int256;
+using Nethermind.Logging;
+using Nethermind.State;
+using Nethermind.Trie.Pruning;
+using Nethermind.Evm;
+using Nethermind.Evm.Benchmark;
+
+namespace Imapp.Benchmark.Runner
+{
+    [MaxWarmupCount(10)]
+    [MaxIterationCount(40)]
+    [MemoryDiagnoser]
+    public class EvmByteCodeBenchmark
+    {
+        public static byte[] ByteCode { get; set; }
+
+        private IReleaseSpec _spec = MainnetSpecProvider.Instance.GetSpec(MainnetSpecProvider.IstanbulBlockNumber);
+        private ITxTracer _txTracer = NullTxTracer.Instance;
+        private ExecutionEnvironment _environment;
+        private IVirtualMachine _virtualMachine;
+        private BlockHeader _header = new BlockHeader(Keccak.Zero, Keccak.Zero, Address.Zero, UInt256.One, MainnetSpecProvider.IstanbulBlockNumber, Int64.MaxValue, UInt256.One, Bytes.Empty);
+        private IBlockhashProvider _blockhashProvider = new TestBlockhashProvider();
+        private EvmState _evmState;
+        private StateProvider _stateProvider;
+        private StorageProvider _storageProvider;
+        private WorldState _worldState;
+
+        [GlobalSetup]
+        public void GlobalSetup()
+        {
+            ByteCode = Bytes.FromHexString(Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE") ?? string.Empty);
+            //Console.WriteLine($"Running benchmark for bytecode {ByteCode?.ToHexString()}");
+
+            TrieStore trieStore = new(new MemDb(), new OneLoggerLogManager(NullLogger.Instance));
+            IKeyValueStore codeDb = new MemDb();
+
+            _stateProvider = new StateProvider(trieStore, codeDb, new OneLoggerLogManager(NullLogger.Instance));
+            _stateProvider.CreateAccount(Address.Zero, 1000.Ether());
+            _stateProvider.Commit(_spec);
+
+            _storageProvider = new StorageProvider(trieStore, _stateProvider, new OneLoggerLogManager(NullLogger.Instance));
+
+            _worldState = new WorldState(_stateProvider, _storageProvider);
+
+            _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, LimboLogs.Instance);
+
+            _environment = new ExecutionEnvironment
+            {
+                ExecutingAccount = Address.Zero,
+                CodeSource = Address.Zero,
+                Caller = Address.Zero,
+                CodeInfo = new CodeInfo(ByteCode),
+                Value = 0,
+                TransferValue = 0,
+                TxExecutionContext = new TxExecutionContext(_header, Address.Zero, 0)
+            };
+        }
+
+        [IterationSetup]
+        public void Setup()
+        {
+            _evmState = new EvmState(long.MaxValue, _environment, ExecutionType.Transaction, true, _worldState.TakeSnapshot(), false);
+        }
+
+        [Benchmark]
+        public void ExecuteCode()
+        {
+            _virtualMachine.Run(_evmState, _worldState, _txTracer);
+        }
+
+        [IterationCleanup]
+        public void Cleanup()
+        {
+            _stateProvider.Reset();
+            _storageProvider.Reset();
+        }
+    }
+}

src/Nethermind/Imapp.Benchmark.Runner/Imapp.Benchmark.Runner.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+    <PackageReference Include="System.CommandLine.DragonFruit" Version="0.4.0-alpha.22272.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Nethermind.Benchmark\Nethermind.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.EthereumTests.Benchmark\Nethermind.EthereumTests.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Evm.Benchmark\Nethermind.Evm.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.JsonRpc.Benchmark\Nethermind.JsonRpc.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Network.Benchmark\Nethermind.Network.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Precompiles.Benchmark\Nethermind.Precompiles.Benchmark.csproj" />
+  </ItemGroup>
+
+</Project>

src/Nethermind/Imapp.Benchmark.Runner/Program.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Linq;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Running;
+using System.CommandLine;
+
+namespace Imapp.Benchmark.Runner
+{
+    class Program
+    {
+        static int Main(string bytecode, int sampleSize = 1, bool printCSV = false)
+        {
+            var config = new ManualConfig()
+                //.WithOptions(ConfigOptions.DisableOptimizationsValidator)
+                .WithOptions(ConfigOptions.DisableLogFile)
+                .AddExporter(new BenchmarkNullExporter());
+
+            if (printCSV)
+            {
+                config = config.AddLogger(new BenchmarkNullLogger());
+            }
+            else
+            {
+                config = config.AddLogger(BenchmarkDotNet.Loggers.ConsoleLogger.Default);
+            }
+
+            if (String.IsNullOrEmpty(bytecode))
+            {
+                throw new Exception("Bytecode cannot be empty");
+            }
+
+            for (int i = 1; i <= sampleSize; ++i)
+            {
+                Environment.SetEnvironmentVariable("NETH.BENCHMARK.BYTECODE", "00" + bytecode);
+                //Console.WriteLine("Code: " + Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE"));
+                var r1 = BenchmarkRunner.Run<EvmByteCodeBenchmark>(config);
+
+                Environment.SetEnvironmentVariable("NETH.BENCHMARK.BYTECODE", bytecode);
+                //Console.WriteLine("Code: " + Environment.GetEnvironmentVariable("NETH.BENCHMARK.BYTECODE"));
+                var r2 = BenchmarkRunner.Run<EvmByteCodeBenchmark>(config);
+
+                OutputOverheadResults(i, r1, r2);
+            }
+
+            return 0;
+        }
+
+        private static void OutputOverheadResults(int sampleId, BenchmarkDotNet.Reports.Summary rEmpty, BenchmarkDotNet.Reports.Summary rActual)
+        {
+            var reportEmpty = rEmpty.Reports[0];
+            var reportActual = rActual.Reports[0];
+            var overheadTime = reportEmpty.ResultStatistics.Mean;
+
+            var loopExecutionTime = reportActual.ResultStatistics.Mean - reportEmpty.ResultStatistics.Mean;
+            var totalTime = reportActual.ResultStatistics.Mean;
+
+            var memAllocPerOp = reportActual.GcStats.GetBytesAllocatedPerOperation(reportActual.BenchmarkCase);
+
+            Console.WriteLine($"{sampleId},{reportActual.ResultStatistics.N},{overheadTime},{loopExecutionTime},{totalTime},{reportActual.ResultStatistics.StandardDeviation},{reportActual.GcStats.TotalOperations},{memAllocPerOp}");
+        }
+    }
+}

src/Nethermind/Imapp.Measurement.Runner/EvmByteCodeBenchmark.cs

@@ -0,0 +1,81 @@
+using System;
+using BenchmarkDotNet.Attributes;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using Nethermind.Core.Extensions;
+using Nethermind.Core.Specs;
+using Nethermind.Db;
+using Nethermind.Evm.CodeAnalysis;
+using Nethermind.Specs;
+using Nethermind.Evm.Tracing;
+using Nethermind.Int256;
+using Nethermind.Logging;
+using Nethermind.State;
+using Nethermind.Trie.Pruning;
+using Nethermind.Evm;
+using Nethermind.Evm.Benchmark;
+
+namespace Imapp.Measurement.Runner
+{
+    public class EvmByteCodeBenchmark
+    {
+        public static byte[] ByteCode { get; set; }
+
+        private IReleaseSpec _spec = MainnetSpecProvider.Instance.GetSpec(MainnetSpecProvider.IstanbulBlockNumber);
+        private ITxTracer _txTracer = NullTxTracer.Instance;
+        private ExecutionEnvironment _environment;
+        private IVirtualMachine _virtualMachine;
+        private BlockHeader _header = new BlockHeader(Keccak.Zero, Keccak.Zero, Address.Zero, UInt256.One, MainnetSpecProvider.IstanbulBlockNumber, Int64.MaxValue, UInt256.One, Bytes.Empty);
+        private IBlockhashProvider _blockhashProvider = new TestBlockhashProvider();
+        private EvmState _evmState;
+        private StateProvider _stateProvider;
+        private StorageProvider _storageProvider;
+        private WorldState _worldState;
+
+        public void GlobalSetup(string bytecode)
+        {
+            ByteCode = Bytes.FromHexString(bytecode);
+            //Console.WriteLine($"Running benchmark for bytecode {ByteCode?.ToHexString()}");
+
+            TrieStore trieStore = new(new MemDb(), new OneLoggerLogManager(NullLogger.Instance));
+            IKeyValueStore codeDb = new MemDb();
+
+            _stateProvider = new StateProvider(trieStore, codeDb, new OneLoggerLogManager(NullLogger.Instance));
+            _stateProvider.CreateAccount(Address.Zero, 1000.Ether());
+            _stateProvider.Commit(_spec);
+
+            _storageProvider = new StorageProvider(trieStore, _stateProvider, new OneLoggerLogManager(NullLogger.Instance));
+
+            _worldState = new WorldState(_stateProvider, _storageProvider);
+
+            _virtualMachine = new VirtualMachine(_blockhashProvider, MainnetSpecProvider.Instance, LimboLogs.Instance);
+
+            _environment = new ExecutionEnvironment
+            {
+                ExecutingAccount = Address.Zero,
+                CodeSource = Address.Zero,
+                Caller = Address.Zero,
+                CodeInfo = new CodeInfo(ByteCode),
+                Value = 0,
+                TransferValue = 0,
+                TxExecutionContext = new TxExecutionContext(_header, Address.Zero, 0)
+            };
+        }
+
+        public void Setup()
+        {
+            _evmState = new EvmState(long.MaxValue, _environment, ExecutionType.Transaction, true, _worldState.TakeSnapshot(), false);
+        }
+
+        public void ExecuteCode()
+        {
+            _virtualMachine.Run(_evmState, _worldState, _txTracer);
+        }
+
+        public void Cleanup()
+        {
+            _stateProvider.Reset();
+            _storageProvider.Reset();
+        }
+    }
+}

src/Nethermind/Imapp.Measurement.Runner/Imapp.Measurement.Runner.csproj

@@ -0,0 +1,22 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.13.1" />
+    <PackageReference Include="System.CommandLine.DragonFruit" Version="0.4.0-alpha.22272.1" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Nethermind.Benchmark\Nethermind.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.EthereumTests.Benchmark\Nethermind.EthereumTests.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Evm.Benchmark\Nethermind.Evm.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.JsonRpc.Benchmark\Nethermind.JsonRpc.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Network.Benchmark\Nethermind.Network.Benchmark.csproj" />
+    <ProjectReference Include="..\Nethermind.Precompiles.Benchmark\Nethermind.Precompiles.Benchmark.csproj" />
+  </ItemGroup>
+
+</Project>

src/Nethermind/Imapp.Measurement.Runner/InstrumentationLogger.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Imapp.Measurement.Runner
+{
+    public class InstrumentationLogger
+    {
+        private List<InstrumentationLoggerItem> items = new List<InstrumentationLoggerItem>();
+
+        public void Add(InstrumentationLoggerItem item)
+        {
+            if (item.End < item.Start) throw new Exception("Invalid measurement");
+            items.Add(item);
+        }
+
+        public void PrintResults()
+        {
+            foreach (var item in items)
+            {
+                var ticksLength = item.End - item.Start;
+
+                Console.WriteLine($"{item.SampleId},{TicksToNs(ticksLength)}");
+            }
+        }
+
+        private double TicksToNs(long ticks)
+        {
+            double ns = 1000000000.0 * (double)ticks / Stopwatch.Frequency;
+            return ns;
+        }
+    }
+
+    public struct InstrumentationLoggerItem
+    {
+        public int SampleId { get; set; }
+        public long Start { get; set; }
+        public long End { get; set; }
+    }
+}