From 6d172792fdbf1d4a0e45c73dae46ac994588facc Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Thu, 30 Jul 2026 10:20:50 +0800 Subject: [PATCH 1/2] add pprof to ledger service --- admin/README.md | 14 +++++++++ cmd/ledger/admin.go | 61 +++++++++++++++++++++++++++++++++++-- cmd/ledger/main.go | 74 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/admin/README.md b/admin/README.md index 25632dbf1dd..fc7da9a9ee5 100644 --- a/admin/README.md +++ b/admin/README.md @@ -91,6 +91,20 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{" curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}' ``` +### Ledger service profiler +The standalone ledger service (`cmd/ledger`) exposes the same profiler configuration through its admin server (default port `9003`). It also registers the `net/http/pprof` endpoints on the admin server for on-demand profiling. + +``` +# Enable the auto-profiler +curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-enabled": true}}' + +# Trigger a profile run +curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}' + +# Get a heap profile via pprof +curl -o heap.prof localhost:9003/debug/pprof/heap +``` + ### Set a stop height ``` curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "stop-at-height", "data": { "height": 1111, "crash": false }}' diff --git a/cmd/ledger/admin.go b/cmd/ledger/admin.go index 607d9968b7a..2a4d1c66b68 100644 --- a/cmd/ledger/admin.go +++ b/cmd/ledger/admin.go @@ -4,9 +4,12 @@ import ( "encoding/json" "fmt" "net/http" + "net/http/pprof" "github.com/rs/zerolog" "go.uber.org/atomic" + + "github.com/onflow/flow-go/module/updatable_configs" ) // adminRequest represents the JSON request body for admin commands. @@ -29,19 +32,29 @@ type adminResponse struct { type adminHandler struct { logger zerolog.Logger triggerCheckpoint *atomic.Bool + configManager *updatable_configs.Manager commands []string } // newAdminHandler creates a new admin HTTP handler. -func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool) http.Handler { +func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool, configManager *updatable_configs.Manager) http.Handler { h := &adminHandler{ logger: logger.With().Str("component", "admin").Logger(), triggerCheckpoint: triggerCheckpoint, - commands: []string{"ping", "list-commands", "trigger-checkpoint"}, + configManager: configManager, + commands: []string{"ping", "list-commands", "trigger-checkpoint", "get-config", "set-config"}, } mux := http.NewServeMux() mux.HandleFunc("/admin/run_command", h.handleCommand) + + // Register pprof handlers for profiling (CPU, heap, goroutine, etc.) + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + return mux } @@ -78,6 +91,50 @@ func (h *adminHandler) handleCommand(w http.ResponseWriter, r *http.Request) { result = "checkpoint already triggered" } + case "get-config": + var configName string + if err := json.Unmarshal(req.Data, &configName); err != nil { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("get-config data must be a string config name: %v", err)) + return + } + field, ok := h.configManager.GetField(configName) + if !ok { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown config field: %s", configName)) + return + } + result = field.Get() + + case "set-config": + var data map[string]any + if err := json.Unmarshal(req.Data, &data); err != nil { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("set-config data must be a JSON object: %v", err)) + return + } + if len(data) != 1 { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("set-config data must have exactly one entry, got %d", len(data))) + return + } + var configName string + var configValue any + for k, v := range data { + configName = k + configValue = v + } + field, ok := h.configManager.GetField(configName) + if !ok { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown config field: %s", configName)) + return + } + oldValue := field.Get() + if err := field.Set(configValue); err != nil { + h.writeError(w, http.StatusBadRequest, fmt.Sprintf("failed to set config %s: %v", configName, err)) + return + } + result = map[string]any{ + "oldValue": oldValue, + "newValue": configValue, + } + default: h.writeError(w, http.StatusBadRequest, fmt.Sprintf("unknown command: %s", req.CommandName)) return diff --git a/cmd/ledger/main.go b/cmd/ledger/main.go index 0dd48ca6b98..d7c9c1e1887 100644 --- a/cmd/ledger/main.go +++ b/cmd/ledger/main.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "runtime" "strings" "syscall" "time" @@ -23,6 +24,8 @@ import ( "github.com/onflow/flow-go/ledger/remote" "github.com/onflow/flow-go/module/irrecoverable" "github.com/onflow/flow-go/module/metrics" + "github.com/onflow/flow-go/module/profiler" + "github.com/onflow/flow-go/module/updatable_configs" ) var ( @@ -37,6 +40,12 @@ var ( logLevel = flag.String("loglevel", "info", "Log level (panic, fatal, error, warn, info, debug)") maxRequestSize = flag.Uint("max-request-size", 1<<30, "Maximum request message size in bytes (default: 1 GiB)") maxResponseSize = flag.Uint("max-response-size", 1<<30, "Maximum response message size in bytes (default: 1 GiB)") + + profilerEnabled = flag.Bool("profiler-enabled", false, "Whether to enable the auto-profiler") + profilerDir = flag.String("profiler-dir", "profiler", "Directory to create auto-profiler profiles") + profilerInterval = flag.Duration("profiler-interval", 15*time.Minute, "Interval between auto-profiler runs") + profilerDuration = flag.Duration("profiler-duration", 10*time.Second, "Duration of each auto-profiler run") + profileUploaderEnabled = flag.Bool("profile-uploader-enabled", false, "Whether to upload profiles to a remote uploader (disabled for ledger service)") ) func main() { @@ -60,6 +69,68 @@ func main() { Str("service", "ledger"). Logger() + // Initialize updatable config manager and auto-profiler. + // The profiler is configured via admin get-config/set-config commands. + configManager := updatable_configs.NewManager() + profilerConfig := profiler.ProfilerConfig{ + Enabled: *profilerEnabled, + UploaderEnabled: false, // ledger service does not support remote profile upload + Dir: *profilerDir, + Interval: *profilerInterval, + Duration: *profilerDuration, + } + if *profileUploaderEnabled { + logger.Warn().Msg("profile-uploader-enabled is not supported by the ledger service, ignoring") + } + + autoProfiler, err := profiler.New(logger, &profiler.NoopUploader{}, profilerConfig) + if err != nil { + logger.Fatal().Err(err).Msg("failed to create auto-profiler") + } + + err = configManager.RegisterBoolConfig("profiler-enabled", autoProfiler.Enabled, autoProfiler.SetEnabled) + if err != nil { + logger.Fatal().Err(err).Msg("failed to register profiler-enabled config") + } + err = configManager.RegisterDurationConfig( + "profiler-trigger", + func() time.Duration { return profilerConfig.Duration }, + func(d time.Duration) error { return autoProfiler.TriggerRun(d) }, + ) + if err != nil { + logger.Fatal().Err(err).Msg("failed to register profiler-trigger config") + } + err = configManager.RegisterUintConfig( + "profiler-set-mem-profile-rate", + func() uint { return uint(runtime.MemProfileRate) }, + func(r uint) error { runtime.MemProfileRate = int(r); return nil }, + ) + if err != nil { + logger.Fatal().Err(err).Msg("failed to register profiler-set-mem-profile-rate config") + } + currentBlockRate := new(uint) + err = configManager.RegisterUintConfig( + "profiler-set-block-profile-rate", + func() uint { return *currentBlockRate }, + func(r uint) error { currentBlockRate = &r; runtime.SetBlockProfileRate(int(r)); return nil }, + ) + if err != nil { + logger.Fatal().Err(err).Msg("failed to register profiler-set-block-profile-rate config") + } + err = configManager.RegisterUintConfig( + "profiler-set-mutex-profile-fraction", + func() uint { return uint(runtime.SetMutexProfileFraction(-1)) }, + func(r uint) error { _ = runtime.SetMutexProfileFraction(int(r)); return nil }, + ) + if err != nil { + logger.Fatal().Err(err).Msg("failed to register profiler-set-mutex-profile-fraction config") + } + + go func() { + <-autoProfiler.Ready() + logger.Info().Bool("enabled", autoProfiler.Enabled()).Msg("auto-profiler ready") + }() + // Validate that at least one address is provided if *ledgerServiceTCP == "" && *ledgerServiceSocket == "" { logger.Fatal().Msg("at least one of --ledger-service-tcp or --ledger-service-socket must be provided") @@ -72,6 +143,7 @@ func main() { Str("admin_addr", *adminAddr). Uint("metrics_port", *metricsPort). Int("mtrie_cache_size", *mtrieCacheSize). + Bool("profiler_enabled", *profilerEnabled). Msg("starting ledger service") // Create trigger for manual checkpointing (used by admin command) @@ -229,7 +301,7 @@ func main() { // This is a lightweight HTTP-only server (no gRPC proxy layer) var adminServer *http.Server if *adminAddr != "" { - adminHandler := newAdminHandler(logger, triggerCheckpointOnNextSegmentFinish) + adminHandler := newAdminHandler(logger, triggerCheckpointOnNextSegmentFinish, configManager) adminServer = &http.Server{ Addr: *adminAddr, Handler: adminHandler, From aa3e283c3701b81e4c7517848cd6a7972891c111 Mon Sep 17 00:00:00 2001 From: "Leo Zhang (zhangchiqing)" Date: Fri, 31 Jul 2026 04:48:58 +0800 Subject: [PATCH 2/2] fix review comments --- admin/README.md | 9 ++++--- cmd/ledger/README.md | 24 +++++++++++++++++ cmd/ledger/admin.go | 40 ++++++++++++++++++++++++----- cmd/ledger/main.go | 61 +++++++++++++++++++++++++++----------------- 4 files changed, 101 insertions(+), 33 deletions(-) diff --git a/admin/README.md b/admin/README.md index fc7da9a9ee5..639b4432ef0 100644 --- a/admin/README.md +++ b/admin/README.md @@ -92,16 +92,19 @@ curl localhost:9002/admin/run_command -H 'Content-Type: application/json' -d '{" ``` ### Ledger service profiler -The standalone ledger service (`cmd/ledger`) exposes the same profiler configuration through its admin server (default port `9003`). It also registers the `net/http/pprof` endpoints on the admin server for on-demand profiling. +The standalone ledger service (`cmd/ledger`) can expose the same profiler configuration through its admin server. The admin server is disabled by default because `--admin-addr` defaults to an empty value; start the service with `--admin-addr=127.0.0.1:9003` to enable it. The `net/http/pprof` endpoints are registered on the admin server for on-demand profiling and are restricted to loopback addresses. + +```bash +# Start the ledger service with the admin server enabled +./ledger --triedir=/path/to/trie --admin-addr=127.0.0.1:9003 -``` # Enable the auto-profiler curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-enabled": true}}' # Trigger a profile run curl localhost:9003/admin/run_command -H 'Content-Type: application/json' -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}' -# Get a heap profile via pprof +# Get a heap profile via pprof (only accessible from loopback) curl -o heap.prof localhost:9003/debug/pprof/heap ``` diff --git a/cmd/ledger/README.md b/cmd/ledger/README.md index b71e7573c14..2fe187499ba 100644 --- a/cmd/ledger/README.md +++ b/cmd/ledger/README.md @@ -59,6 +59,10 @@ go build -o flow-ledger-service ./cmd/ledger - `-max-request-size`: Maximum request message size in bytes (default: 1 GiB) - `-max-response-size`: Maximum response message size in bytes (default: 1 GiB) - `-loglevel`: Log level (panic, fatal, error, warn, info, debug) (default: info) +- `-profiler-enabled`: Whether to enable the auto-profiler (default: false) +- `-profiler-dir`: Directory to create auto-profiler profiles (default: `profiler`) +- `-profiler-interval`: Interval between auto-profiler runs (default: 15m) +- `-profiler-duration`: Duration of each auto-profiler run (default: 10s) ## Admin Commands @@ -69,6 +73,8 @@ When `-admin-addr` is provided, the service exposes an HTTP admin API for managi - `trigger-checkpoint`: Triggers a checkpoint to be created as soon as the current WAL segment file is finished writing. This is useful for manually creating checkpoints without waiting for the automatic checkpoint distance. - `ping`: Simple health check command to verify the admin server is responsive. - `list-commands`: Lists all available admin commands. +- `get-config`: Returns the current value of a registered runtime config. +- `set-config`: Updates the value of a registered runtime config. **Examples:** ```bash @@ -86,6 +92,24 @@ curl -X POST http://localhost:9003/admin/run_command \ curl -X POST http://localhost:9003/admin/run_command \ -H "Content-Type: application/json" \ -d '{"commandName": "list-commands", "data": {}}' + +# Get the current auto-profiler enabled state +curl -X POST http://localhost:9003/admin/run_command \ + -H "Content-Type: application/json" \ + -d '{"commandName": "get-config", "data": "profiler-enabled"}' + +# Enable the auto-profiler +curl -X POST http://localhost:9003/admin/run_command \ + -H "Content-Type: application/json" \ + -d '{"commandName": "set-config", "data": {"profiler-enabled": true}}' + +# Trigger a profile run manually +curl -X POST http://localhost:9003/admin/run_command \ + -H "Content-Type: application/json" \ + -d '{"commandName": "set-config", "data": {"profiler-trigger": "1m"}}' + +# Get a heap profile via pprof (only accessible from loopback) +curl -o heap.prof http://localhost:9003/debug/pprof/heap ``` **Note:** When running an execution node with a remote ledger service (using `--ledger-service-addr`), the `trigger-checkpoint` command on the execution node is disabled. You must use the ledger service's admin endpoint to trigger checkpoints. diff --git a/cmd/ledger/admin.go b/cmd/ledger/admin.go index 2a4d1c66b68..bff71ef8636 100644 --- a/cmd/ledger/admin.go +++ b/cmd/ledger/admin.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "net" "net/http" "net/http/pprof" @@ -36,6 +37,25 @@ type adminHandler struct { commands []string } +// requireLoopback returns an http.HandlerFunc that only serves requests originating +// from the loopback interface. It is used to prevent sensitive profiling endpoints +// from being exposed when the admin server is bound to a publicly reachable address. +func requireLoopback(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + ip := net.ParseIP(host) + if ip == nil || !ip.IsLoopback() { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + next(w, r) + } +} + // newAdminHandler creates a new admin HTTP handler. func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool, configManager *updatable_configs.Manager) http.Handler { h := &adminHandler{ @@ -48,12 +68,14 @@ func newAdminHandler(logger zerolog.Logger, triggerCheckpoint *atomic.Bool, conf mux := http.NewServeMux() mux.HandleFunc("/admin/run_command", h.handleCommand) - // Register pprof handlers for profiling (CPU, heap, goroutine, etc.) - mux.HandleFunc("/debug/pprof/", pprof.Index) - mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) - mux.HandleFunc("/debug/pprof/profile", pprof.Profile) - mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) - mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + // Register pprof handlers for profiling (CPU, heap, goroutine, etc.). + // These endpoints are restricted to loopback to avoid exposing profiling + // data when the admin server is bound to a public address. + mux.HandleFunc("/debug/pprof/", requireLoopback(pprof.Index)) + mux.HandleFunc("/debug/pprof/cmdline", requireLoopback(pprof.Cmdline)) + mux.HandleFunc("/debug/pprof/profile", requireLoopback(pprof.Profile)) + mux.HandleFunc("/debug/pprof/symbol", requireLoopback(pprof.Symbol)) + mux.HandleFunc("/debug/pprof/trace", requireLoopback(pprof.Trace)) return mux } @@ -127,7 +149,11 @@ func (h *adminHandler) handleCommand(w http.ResponseWriter, r *http.Request) { } oldValue := field.Get() if err := field.Set(configValue); err != nil { - h.writeError(w, http.StatusBadRequest, fmt.Sprintf("failed to set config %s: %v", configName, err)) + status := http.StatusInternalServerError + if updatable_configs.IsValidationError(err) { + status = http.StatusBadRequest + } + h.writeError(w, status, fmt.Sprintf("failed to set config %s: %v", configName, err)) return } result = map[string]any{ diff --git a/cmd/ledger/main.go b/cmd/ledger/main.go index d7c9c1e1887..aed177dae0e 100644 --- a/cmd/ledger/main.go +++ b/cmd/ledger/main.go @@ -11,6 +11,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "syscall" "time" @@ -41,11 +42,10 @@ var ( maxRequestSize = flag.Uint("max-request-size", 1<<30, "Maximum request message size in bytes (default: 1 GiB)") maxResponseSize = flag.Uint("max-response-size", 1<<30, "Maximum response message size in bytes (default: 1 GiB)") - profilerEnabled = flag.Bool("profiler-enabled", false, "Whether to enable the auto-profiler") - profilerDir = flag.String("profiler-dir", "profiler", "Directory to create auto-profiler profiles") - profilerInterval = flag.Duration("profiler-interval", 15*time.Minute, "Interval between auto-profiler runs") - profilerDuration = flag.Duration("profiler-duration", 10*time.Second, "Duration of each auto-profiler run") - profileUploaderEnabled = flag.Bool("profile-uploader-enabled", false, "Whether to upload profiles to a remote uploader (disabled for ledger service)") + profilerEnabled = flag.Bool("profiler-enabled", false, "Whether to enable the auto-profiler") + profilerDir = flag.String("profiler-dir", "profiler", "Directory to create auto-profiler profiles") + profilerInterval = flag.Duration("profiler-interval", 15*time.Minute, "Interval between auto-profiler runs") + profilerDuration = flag.Duration("profiler-duration", 10*time.Second, "Duration of each auto-profiler run") ) func main() { @@ -69,6 +69,15 @@ func main() { Str("service", "ledger"). Logger() + if *profilerInterval <= 0 { + logger.Fatal().Dur("profiler_interval", *profilerInterval).Msg("profiler-interval must be positive") + } + + // Validate that at least one address is provided + if *ledgerServiceTCP == "" && *ledgerServiceSocket == "" { + logger.Fatal().Msg("at least one of --ledger-service-tcp or --ledger-service-socket must be provided") + } + // Initialize updatable config manager and auto-profiler. // The profiler is configured via admin get-config/set-config commands. configManager := updatable_configs.NewManager() @@ -79,9 +88,6 @@ func main() { Interval: *profilerInterval, Duration: *profilerDuration, } - if *profileUploaderEnabled { - logger.Warn().Msg("profile-uploader-enabled is not supported by the ledger service, ignoring") - } autoProfiler, err := profiler.New(logger, &profiler.NoopUploader{}, profilerConfig) if err != nil { @@ -108,11 +114,22 @@ func main() { if err != nil { logger.Fatal().Err(err).Msg("failed to register profiler-set-mem-profile-rate config") } - currentBlockRate := new(uint) + var currentBlockRateMu sync.Mutex + var currentBlockRate uint err = configManager.RegisterUintConfig( "profiler-set-block-profile-rate", - func() uint { return *currentBlockRate }, - func(r uint) error { currentBlockRate = &r; runtime.SetBlockProfileRate(int(r)); return nil }, + func() uint { + currentBlockRateMu.Lock() + defer currentBlockRateMu.Unlock() + return currentBlockRate + }, + func(r uint) error { + currentBlockRateMu.Lock() + defer currentBlockRateMu.Unlock() + runtime.SetBlockProfileRate(int(r)) + currentBlockRate = r + return nil + }, ) if err != nil { logger.Fatal().Err(err).Msg("failed to register profiler-set-block-profile-rate config") @@ -126,16 +143,6 @@ func main() { logger.Fatal().Err(err).Msg("failed to register profiler-set-mutex-profile-fraction config") } - go func() { - <-autoProfiler.Ready() - logger.Info().Bool("enabled", autoProfiler.Enabled()).Msg("auto-profiler ready") - }() - - // Validate that at least one address is provided - if *ledgerServiceTCP == "" && *ledgerServiceSocket == "" { - logger.Fatal().Msg("at least one of --ledger-service-tcp or --ledger-service-socket must be provided") - } - logger.Info(). Str("triedir", *triedir). Str("ledger_service_tcp", *ledgerServiceTCP). @@ -303,8 +310,12 @@ func main() { if *adminAddr != "" { adminHandler := newAdminHandler(logger, triggerCheckpointOnNextSegmentFinish, configManager) adminServer = &http.Server{ - Addr: *adminAddr, - Handler: adminHandler, + Addr: *adminAddr, + Handler: adminHandler, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 10 * time.Second, + IdleTimeout: 60 * time.Second, + WriteTimeout: 2 * time.Minute, } go func() { @@ -370,6 +381,10 @@ func main() { logger.Info().Msg("metrics server stopped") } + logger.Info().Msg("shutting down auto-profiler...") + <-autoProfiler.Done() + logger.Info().Msg("auto-profiler stopped") + logger.Info().Msg("waiting for ledger to stop...") <-ledgerStorage.Done()