diff --git a/.gitmodules b/.gitmodules index 5a1641e..d580bd8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,14 @@ [submodule "src/instrumentation_measurement/evmone"] path = src/instrumentation_measurement/evmone url = ../evmone.git +[submodule "src/instrumentation_measurement/geth_benchmark"] + path = src/instrumentation_measurement/geth_benchmark + url = https://github.com/imapp-pl/go-ethereum + branch = imapp_benchmark +[submodule "src/instrumentation_measurement/nethermind_benchmark"] + path = src/instrumentation_measurement/nethermind_benchmark + url = https://github.com/imapp-pl/nethermind.git + branch = imapp_benchmark [submodule "src/instrumentation_measurement/go-ethereum"] path = src/instrumentation_measurement/go-ethereum url = git@github.com:imapp-pl/go-ethereum.git diff --git a/Dockerfile.nethermind b/Dockerfile.nethermind new file mode 100644 index 0000000..129efe9 --- /dev/null +++ b/Dockerfile.nethermind @@ -0,0 +1,24 @@ +FROM mcr.microsoft.com/dotnet/sdk:6.0-alpine AS build + +WORKDIR /srv/app +RUN git clone --single-branch --branch imapp_benchmark https://github.com/imapp-pl/nethermind.git + +WORKDIR /srv/app/nethermind +RUN git submodule update --init src/Dirichlet src/int256 src/Math.Gmp.Native + +WORKDIR /srv/app/nethermind/src/Nethermind +RUN dotnet build -c Release Benchmarks.sln + + +FROM python:3.8 AS python + +COPY ./src/program_generator /srv/app/src/program_generator +WORKDIR /srv/app/ +RUN pip install -r src/program_generator/requirements.txt + +COPY ./src/instrumentation_measurement/measurements.py /srv/app/src/instrumentation_measurement/measurements.py +COPY --from=build /srv/app/nethermind/src/Nethermind/Imapp.Benchmark.Runner/bin /srv/app/src/instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Benchmark.Runner/bin +COPY --from=build /srv/app/nethermind/src/Nethermind/Imapp.Measurement.Runner/bin /srv/app/src/instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Measurement.Runner/bin + +WORKDIR /srv/app/ + diff --git a/src/instrumentation_measurement/README.md b/src/instrumentation_measurement/README.md index 9e95ee1..9926fb2 100644 --- a/src/instrumentation_measurement/README.md +++ b/src/instrumentation_measurement/README.md @@ -28,3 +28,97 @@ sudo docker run --rm --privileged --security-opt seccomp:unconfined \ ``` For other EVMs use respective `Dockerfile`s and use the `--evm` flag on the `measure` command, e.g. `measure --evm openethereum` + + +# Test environment setup +## Go Etherum Benchmark +Compile benchmark program +``` +cd geth_benchmark\tests\imapp_benchmark +go build +``` +## Nethermind +Requirements: +- .Net Core 6.0 + +Make sure that all submodules are fetched: +``` +cd nethermind_benchmark +git submodule update --recursive --remote --init +``` + +Compile benchmark program: +``` +cd nethermind_benchmark\src +dotnet build -c Release .\Benchmarks.sln +``` + +# Benchmark methodology + +## Tools +The goal of benchmarking mode is to use well known libraries that track performance in reliable and precise manner. They tend to produce reproducible results and help to avoid common pitfalls while measuring execution time. In our approach we use the following tools: +- Go Ethereum: [Go Testing](https://pkg.go.dev/testing#Benchmark) package +- Nethermind: [DotNetBenchmark](https://benchmarkdotnet.org/articles/overview.html) + +These tools were selected as industry standards for respective languages. They all minimize influence and variability of: caching, warmups, memory allocation, garbage collection, process management, external programs impact and clock measurements. + +## Results explanation +For each bytecode, benchmark is executed twice. The first time bytecode is prefixed with opcode 00 (STOP). This causes the program to terminate immediately on the first loop. The second time bytecode is executed as normal. This method is to assess the engine overhead for each program execution. Please see below for the description of what consists of overhead for each engine. +The benchmark results contain: +- iterations_count: How many times the benchmark library executed the program internally +- engine_overhead_time_ns: Estimated time of engine overhead +- execution_loop_time_ns: The actual loop over opcodes in the bytecode +- total_time_ns: The two values above summed up +- mem_allocs_count: Number of memory operations +- mem_allocs_bytes: Total bytes allocated + +## Go Ethereum execution overhead analysis +The certain 'preparation' steps are executed with every bytecode. They are performed no matter how long or complicated the bytecode is. This tend to be constant, so the longer program takes, the more negligible it becomes. + +Prepare environment and sender account (~7%) +```go +var ( + address = common.BytesToAddress([]byte("contract")) + vmenv = NewEnv(cfg) + sender = vm.AccountRef(cfg.Origin) +) +``` + +Get rule set London, Berlin, etc (~43%) (Note: this seems an obvious candidate for caching. Further analysis has to take place.) +```go +rules := cfg.ChainConfig.Rules(vmenv.Context.BlockNumber, vmenv.Context.Random != nil) +``` + +Create a state object. If a state object with the address already exists the balance is carried over to the new account (~16%) +```go +cfg.State.CreateAccount(address) +``` + +Set the execution code (~23%) +``` +cfg.State.SetCode(address, code) +``` + +Take snapshot (~9%) +```go +snapshot := evm.StateDB.Snapshot() +``` + +## Nethermind execution overhead analysis +(Estimated times to follow) + + +Get rule set London, Berlin, etc +```csharp +_specProvider.GetSpec(state.Env.TxExecutionContext.Header.Number) +``` + +Initiate stack +``` +vmState.InitStacks(); +``` + +Take snapshot +``` +_worldState.TakeSnapshot() +``` \ No newline at end of file diff --git a/src/instrumentation_measurement/geth_benchmark b/src/instrumentation_measurement/geth_benchmark new file mode 160000 index 0000000..506e841 --- /dev/null +++ b/src/instrumentation_measurement/geth_benchmark @@ -0,0 +1 @@ +Subproject commit 506e841026f2608d0b1e0662f9f0e14f61bb101a diff --git a/src/instrumentation_measurement/measurements.py b/src/instrumentation_measurement/measurements.py index 03afe68..64c877e 100644 --- a/src/instrumentation_measurement/measurements.py +++ b/src/instrumentation_measurement/measurements.py @@ -64,7 +64,7 @@ def __init__(self): reader = csv.DictReader(sys.stdin, delimiter=',', quotechar='"') self._programs = [self._program_from_csv_row(row) for row in reader] - def measure(self, sampleSize, mode="all", evm="geth", nSamples=1): + def measure(self, sampleSize=1, mode="all", evm="geth", nSamples=1): """ Main entrypoint of the CLI tool. @@ -81,21 +81,23 @@ def measure(self, sampleSize, mode="all", evm="geth", nSamples=1): openethereum = "openethereum" openethereum_ewasm = "openethereum_ewasm" evmone = "evmone" + nethermind = "nethermind" measure_total = "total" measure_all = "all" trace_opcodes = "trace" + benchmark_mode = "benchmark" if not self._check_clocksource(): print("clocksource should be tsc, found something different. See docker_timer.md somewhere in the docs") return - if evm not in {geth, openethereum, evmone, openethereum_ewasm}: - print("Wrong evm parameter. Allowed are: {}, {}, {}, {}".format(geth, openethereum, evmone, openethereum_ewasm)) + if evm not in {geth, openethereum, evmone, openethereum_ewasm, nethermind}: + print("Wrong evm parameter. Allowed are: {}, {}, {}, {}".format(geth, openethereum, evmone, openethereum_ewasm, nethermind)) return - if mode not in {measure_total, measure_all, trace_opcodes}: - print("Invalid measurement mode. Allowed options: {}, {}, {}".format(measure_total, measure_all, trace_opcodes)) + if mode not in {measure_total, measure_all, trace_opcodes, benchmark_mode}: + print("Invalid measurement mode. Allowed options: {}, {}, {}".format(measure_total, measure_all, trace_opcodes, benchmark_mode)) return elif mode == measure_total: header = "program_id,sample_id,run_id,measure_total_time_ns,measure_total_timer_time_ns" @@ -108,7 +110,12 @@ def measure(self, sampleSize, mode="all", evm="geth", nSamples=1): for i in range(MAX_OPCODE_ARGS): elem = ",arg_{}".format(i) header += elem - + print(header) + elif mode == benchmark_mode: + if evm == geth: + header = "program_id,sample_id,run_id,iterations_count,engine_overhead_time_ns,execution_loop_time_ns,total_time_ns,mem_allocs_count,mem_allocs_bytes" + elif evm == nethermind: + header = "program_id,sample_id,run_id,iterations_count,engine_overhead_time_ns,execution_loop_time_ns,total_time_ns,std_dev_time_ns,mem_allocs_count,mem_allocs_bytes" print(header) @@ -116,13 +123,21 @@ def measure(self, sampleSize, mode="all", evm="geth", nSamples=1): for sample_id in range(nSamples): instrumenter_result = None if evm == geth: - instrumenter_result = self.run_geth(mode, program, sampleSize) + if mode == benchmark_mode: + instrumenter_result = self.run_geth_benchmark(program, sampleSize) + else: + instrumenter_result = self.run_geth(mode, program, sampleSize) elif evm == openethereum: instrumenter_result = self.run_openethereum(mode, program, sampleSize) elif evm == openethereum_ewasm: instrumenter_result = self.run_openethereum_wasm(program, sampleSize) elif evm == evmone: instrumenter_result = self.run_evmone(mode, program, sampleSize) + elif evm == nethermind: + if mode == benchmark_mode: + instrumenter_result = self.run_nethermind_benchmark(program, sampleSize) + else: + instrumenter_result = self.run_nethermind(program, sampleSize) if mode == trace_opcodes: instrumenter_result = self.sanitize_tracer_result(instrumenter_result) @@ -145,6 +160,41 @@ def run_geth(self, mode, program, sampleSize): return instrumenter_result + def run_geth_benchmark(self, program, sampleSize): + geth_benchmark = ['./instrumentation_measurement/geth_benchmark/tests/imapp_benchmark/imapp_benchmark'] + + # alternative just-in-time compilation (could run 50% slower) + # geth_benchmark = ['go', 'run', './instrumentation_measurement/geth_benchmark/tests/imapp_benchmark/imapp_bench.go'] + + args = ['--sampleSize', '{}'.format(sampleSize)] + bytecode_arg = ['--bytecode', program.bytecode] + invocation = geth_benchmark + args + bytecode_arg + result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True) + assert result.returncode == 0 + # strip the final newline + instrumenter_result = result.stdout.split('\n')[:-1] + return instrumenter_result + + def run_nethermind(self, program, sampleSize): + geth_benchmark = ['./instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Measurement.Runner/bin/Release/net6.0/Imapp.Measurement.Runner'] + args = ['--bytecode', program.bytecode, '--print-csv', '--sample-size={}'.format(sampleSize)] + invocation = geth_benchmark + args + result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True) + assert result.returncode == 0 + # strip the final newline + instrumenter_result = result.stdout.split('\n')[:-1] + return instrumenter_result + + def run_nethermind_benchmark(self, program, sampleSize): + geth_benchmark = ['./instrumentation_measurement/nethermind_benchmark/src/Nethermind/Imapp.Benchmark.Runner/bin/Release/net6.0/Imapp.Benchmark.Runner'] + args = ['--bytecode', program.bytecode, '--print-csv', '--sample-size={}'.format(sampleSize)] + invocation = geth_benchmark + args + result = subprocess.run(invocation, stdout=subprocess.PIPE, universal_newlines=True) + assert result.returncode == 0 + # strip the final newline + instrumenter_result = result.stdout.split('\n')[:-1] + return instrumenter_result + def run_openethereum(self, mode, program, sampleSize): openethereum_build_path = './instrumentation_measurement/openethereum/target/release/' openethereum_main = [openethereum_build_path + 'openethereum-evm'] diff --git a/src/instrumentation_measurement/nethermind_benchmark b/src/instrumentation_measurement/nethermind_benchmark new file mode 160000 index 0000000..d27fcea --- /dev/null +++ b/src/instrumentation_measurement/nethermind_benchmark @@ -0,0 +1 @@ +Subproject commit d27fcea4e70f7a45211834dc2df6240b06aa480b