Add benchmark measurements

.gitmodules

@@ -1,6 +1,14 @@
 [submodule "src/instrumentation_measurement/evmone"]
 	path = src/instrumentation_measurement/evmone
 	url = ../evmone.git
+[submodule "src/instrumentation_measurement/geth_benchmark"]
+	path = src/instrumentation_measurement/geth_benchmark
+	url = https://github.com/imapp-pl/go-ethereum
+	branch = imapp_benchmark
+[submodule "src/instrumentation_measurement/nethermind_benchmark"]
+	path = src/instrumentation_measurement/nethermind_benchmark
+	url = https://github.com/imapp-pl/nethermind.git
+	branch = imapp_benchmark
 [submodule "src/instrumentation_measurement/go-ethereum"]
 	path = src/instrumentation_measurement/go-ethereum
 	url = git@github.com:imapp-pl/go-ethereum.git

Dockerfile.nethermind

@@ -0,0 +1,24 @@
+FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS build
+
+WORKDIR /srv/app
+RUN git clone --single-branch --branch imapp_benchmark https://github.com/imapp-pl/nethermind.git
+
+WORKDIR /srv/app/nethermind
+RUN git submodule update --init src/Dirichlet src/int256 src/Math.Gmp.Native
+
+WORKDIR /srv/app/nethermind/src/Nethermind
+RUN dotnet build -c Release Benchmarks.sln
+
+
+FROM python:3.8 AS python
+
+COPY ./src/program_generator /srv/app/src/program_generator
+WORKDIR /srv/app/
+RUN pip install -r src/program_generator/requirements.txt
+
+COPY ./src/instrumentation_measurement/measurements.py /srv/app/src/instrumentation_measurement/measurements.py
+COPY --from=build /srv/app/nethermind/src/Nethermind/Imapp.Benchmark.Runner/bin /srv/app/src/instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Benchmark.Runner/bin
+COPY --from=build /srv/app/nethermind/src/Nethermind/Imapp.Measurement.Runner/bin /srv/app/src/instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Measurement.Runner/bin
+
+WORKDIR /srv/app/
+

src/instrumentation_measurement/README.md

@@ -28,3 +28,97 @@ sudo docker run --rm --privileged --security-opt seccomp:unconfined \
 ```
 
 For other EVMs use respective `Dockerfile`s and use the `--evm` flag on the `measure` command, e.g. `measure --evm openethereum`
+
+
+# Test environment setup
+## Go Etherum Benchmark
+Compile benchmark program
+```
+cd geth_benchmark\tests\imapp_benchmark
+go build
+```
+## Nethermind
+Requirements:
+- .Net Core 6.0
+
+Make sure that all submodules are fetched:
+```
+cd nethermind_benchmark
+git submodule update --recursive --remote --init
+```
+
+Compile benchmark program:
+```
+cd nethermind_benchmark\src
+dotnet build -c Release .\Benchmarks.sln
+```
+
+# Benchmark methodology
+
+## Tools
+The goal of benchmarking mode is to use well known libraries that track performance in reliable and precise manner. They tend to produce reproducible results and help to avoid common pitfalls while measuring execution time. In our approach we use the following tools:
+- Go Ethereum: [Go Testing](https://pkg.go.dev/testing#Benchmark) package
+- Nethermind: [DotNetBenchmark](https://benchmarkdotnet.org/articles/overview.html)
+
+These tools were selected as industry standards for respective languages. They all minimize influence and variability of: caching, warmups, memory allocation, garbage collection, process management, external programs impact and clock measurements.
+
+## Results explanation
+For each bytecode, benchmark is executed twice. The first time bytecode is prefixed with opcode 00 (STOP). This causes the program to terminate immediately on the first loop. The second time bytecode is executed as normal. This method is to assess the engine overhead for each program execution. Please see below for the description of what consists of overhead for each engine.
+The benchmark results contain:
+- iterations_count: How many times the benchmark library executed the program internally
+- engine_overhead_time_ns: Estimated time of engine overhead
+- execution_loop_time_ns: The actual loop over opcodes in the bytecode
+- total_time_ns: The two values above summed up
+- mem_allocs_count: Number of memory operations
+- mem_allocs_bytes: Total bytes allocated
+
+## Go Ethereum execution overhead analysis
+The certain 'preparation' steps are executed with every bytecode. They are performed no matter how long or complicated the bytecode is. This tend to be constant, so the longer program takes, the more negligible it becomes.
+
+Prepare environment and sender account (~7%)
+```go
+var (
+	address = common.BytesToAddress([]byte("contract"))
+	vmenv   = NewEnv(cfg)
+	sender  = vm.AccountRef(cfg.Origin)
+)
+```
+
+Get rule set London, Berlin, etc (~43%) (Note: this seems an obvious candidate for caching. Further analysis has to take place.)
+```go
+rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil)
+```
+
+Create a state object. If a state object with the address already exists the balance is carried over to the new account (~16%)
+```go
+cfg.State.CreateAccount(address)
+```
+
+Set the execution code (~23%)
+```
+cfg.State.SetCode(address, code)
+```
+
+Take snapshot (~9%)
+```go
+snapshot := evm.StateDB.Snapshot()
+```
+
+## Nethermind execution overhead analysis
+(Estimated times to follow)
+
+
+Get rule set London, Berlin, etc
+```csharp
+_specProvider.GetSpec(state.Env.TxExecutionContext.Header.Number)
+```
+
+Initiate stack
+```
+vmState.InitStacks();
+```
+
+Take snapshot
+```
+_worldState.TakeSnapshot()
+```

src/instrumentation_measurement/measurements.py

@@ -64,7 +64,7 @@ def __init__(self):
     reader = csv.DictReader(sys.stdin, delimiter=',', quotechar='"')
     self._programs = [self._program_from_csv_row(row) for row in reader]
 
-  def measure(self, sampleSize, mode="all", evm="geth", nSamples=1):
+  def measure(self, sampleSize=1, mode="all", evm="geth", nSamples=1):
     """
     Main entrypoint of the CLI tool.
 
@@ -81,21 +81,23 @@ def measure(self, sampleSize, mode="all", evm="geth", nSamples=1):
     openethereum = "openethereum"
     openethereum_ewasm = "openethereum_ewasm"
     evmone = "evmone"
+    nethermind = "nethermind"
 
     measure_total = "total"
     measure_all = "all"
     trace_opcodes = "trace"
+    benchmark_mode = "benchmark"
 
     if not self._check_clocksource():
       print("clocksource should be tsc, found something different. See docker_timer.md somewhere in the docs")
       return
 
-    if evm not in {geth, openethereum, evmone, openethereum_ewasm}:
-      print("Wrong evm parameter. Allowed are: {}, {}, {}, {}".format(geth, openethereum, evmone, openethereum_ewasm))
+    if evm not in {geth, openethereum, evmone, openethereum_ewasm, nethermind}:
+      print("Wrong evm parameter. Allowed are: {}, {}, {}, {}".format(geth, openethereum, evmone, openethereum_ewasm, nethermind))
       return
 
-    if mode not in {measure_total, measure_all, trace_opcodes}:
-        print("Invalid measurement mode. Allowed options: {}, {}, {}".format(measure_total, measure_all, trace_opcodes))
+    if mode not in {measure_total, measure_all, trace_opcodes, benchmark_mode}:
+        print("Invalid measurement mode. Allowed options: {}, {}, {}".format(measure_total, measure_all, trace_opcodes, benchmark_mode))
         return
     elif mode == measure_total:
         header = "program_id,sample_id,run_id,measure_total_time_ns,measure_total_timer_time_ns"
@@ -108,21 +110,34 @@ def measure(self, sampleSize, mode="all", evm="geth", nSamples=1):
         for i in range(MAX_OPCODE_ARGS):
             elem = ",arg_{}".format(i)
             header += elem
-
+        print(header)
+    elif mode == benchmark_mode:
+        if evm == geth:
+            header = "program_id,sample_id,run_id,iterations_count,engine_overhead_time_ns,execution_loop_time_ns,total_time_ns,mem_allocs_count,mem_allocs_bytes"
+        elif evm == nethermind:
+            header = "program_id,sample_id,run_id,iterations_count,engine_overhead_time_ns,execution_loop_time_ns,total_time_ns,std_dev_time_ns,mem_allocs_count,mem_allocs_bytes"
         print(header)
 
 
     for program in self._programs:
       for sample_id in range(nSamples):
         instrumenter_result = None
         if evm == geth:
-          instrumenter_result = self.run_geth(mode, program, sampleSize)
+          if mode == benchmark_mode:
+            instrumenter_result = self.run_geth_benchmark(program, sampleSize)
+          else:
+            instrumenter_result = self.run_geth(mode, program, sampleSize)
         elif evm == openethereum:
           instrumenter_result = self.run_openethereum(mode, program, sampleSize)
         elif evm == openethereum_ewasm:
           instrumenter_result = self.run_openethereum_wasm(program, sampleSize)
         elif evm == evmone:
           instrumenter_result = self.run_evmone(mode, program, sampleSize)
+        elif evm == nethermind:
+          if mode == benchmark_mode:
+            instrumenter_result = self.run_nethermind_benchmark(program, sampleSize)
+          else:
+            instrumenter_result = self.run_nethermind(program, sampleSize)
 
         if mode == trace_opcodes:
             instrumenter_result = self.sanitize_tracer_result(instrumenter_result)
@@ -145,6 +160,41 @@ def run_geth(self, mode, program, sampleSize):
 
     return instrumenter_result
 
+  def run_geth_benchmark(self, program, sampleSize):
+    geth_benchmark = ['./instrumentation_measurement/geth_benchmark/tests/imapp_benchmark/imapp_benchmark']
+
+    # alternative just-in-time compilation (could run 50% slower)
+    # geth_benchmark = ['go', 'run', './instrumentation_measurement/geth_benchmark/tests/imapp_benchmark/imapp_bench.go']
+
+    args = ['--sampleSize', '{}'.format(sampleSize)]
+    bytecode_arg = ['--bytecode', program.bytecode]
+    invocation = geth_benchmark + args + bytecode_arg
+    result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True)
+    assert result.returncode == 0
+    # strip the final newline
+    instrumenter_result = result.stdout.split('\n')[:-1]
+    return instrumenter_result
+
+  def run_nethermind(self, program, sampleSize):
+    geth_benchmark = ['./instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Measurement.Runner/bin/Release/net6.0/Imapp.Measurement.Runner']
+    args = ['--bytecode', program.bytecode, '--print-csv', '--sample-size={}'.format(sampleSize)]
+    invocation = geth_benchmark + args
+    result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True)
+    assert result.returncode == 0
+    # strip the final newline
+    instrumenter_result = result.stdout.split('\n')[:-1]
+    return instrumenter_result
+
+  def run_nethermind_benchmark(self, program, sampleSize):
+    geth_benchmark = ['./instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Benchmark.Runner/bin/Release/net6.0/Imapp.Benchmark.Runner']
+    args = ['--bytecode', program.bytecode, '--print-csv', '--sample-size={}'.format(sampleSize)]
+    invocation = geth_benchmark + args
+    result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True)
+    assert result.returncode == 0
+    # strip the final newline
+    instrumenter_result = result.stdout.split('\n')[:-1]
+    return instrumenter_result
+
   def run_openethereum(self, mode, program, sampleSize):
     openethereum_build_path = './instrumentation_measurement/openethereum/target/release/'
     openethereum_main = [openethereum_build_path + 'openethereum-evm']