Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## unreleased

* [FEATURE] Add a lifecycle `/-/clear` endpoint to clear registered StatsD time series without restarting the exporter ([#714](https://github.com/prometheus/statsd_exporter/pull/714))

## 0.30.0 / 2026-05-28
* [CHANGE] Remove the Dockerfile `HEALTHCHECK` from published container images ([#671](https://github.com/prometheus/statsd_exporter/pull/671))
* [ENHANCEMENT] Add a distroless container image variant ([#703](https://github.com/prometheus/statsd_exporter/pull/703))
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/king

## Lifecycle API

The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload or quit the exporter
by sending a `PUT` or `POST` request to the `/-/reload` or `/-/quit` endpoints.
The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload, quit, or clear dynamically registered StatsD metric series
by sending a `PUT` or `POST` request to the `/-/reload`, `/-/quit`, or `/-/clear` endpoints.

## Relay

Expand Down
21 changes: 21 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package main

import (
"bufio"
"context"
"fmt"
"log/slog"
"net"
Expand Down Expand Up @@ -205,6 +206,25 @@ func reloadConfig(fileName string, mapper *mapper.MetricMapper, logger *slog.Log
}
}

type metricsClearer interface {
ClearMetrics(context.Context) (int, error)
}

func clearMetricsHandler(clearer metricsClearer, logger *slog.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut || r.Method == http.MethodPost {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets return 405 Method Not Allowed when the method is not POST|PUT.

Allow: PUT, POST, and test both supported methods and at least one unsupported method.

cleared, err := clearer.ClearMetrics(r.Context())
if err != nil {
logger.Error("Failed to clear metrics", "error", err)
http.Error(w, err.Error(), http.StatusServiceUnavailable)
return
}
logger.Info("Received lifecycle api clear", "metrics", cleared)
fmt.Fprintf(w, "Cleared %d metric series", cleared)
}
}
}

func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string, logger *slog.Logger) error {
f, err := os.Create(dumpFilename)
if err != nil {
Expand Down Expand Up @@ -528,6 +548,7 @@ func main() {
quitChan <- struct{}{}
}
})
mux.HandleFunc("/-/clear", clearMetricsHandler(exporter, logger))
}

mux.HandleFunc("/-/healthy", func(w http.ResponseWriter, r *http.Request) {
Expand Down
70 changes: 70 additions & 0 deletions main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// Copyright 2026 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"

"github.com/prometheus/common/promslog"
)

type fakeMetricsClearer struct {
cleared int
called int
err error
}

func (f *fakeMetricsClearer) ClearMetrics(_ context.Context) (int, error) {
f.called++
return f.cleared, f.err
}

func TestClearMetricsHandler(t *testing.T) {
clearer := &fakeMetricsClearer{cleared: 3}
handler := clearMetricsHandler(clearer, promslog.NewNopLogger())

request := httptest.NewRequest(http.MethodPost, "/-/clear", nil)
response := httptest.NewRecorder()

handler.ServeHTTP(response, request)

if clearer.called != 1 {
t.Fatalf("expected clearer to be called once, got %d", clearer.called)
}
if body := response.Body.String(); !strings.Contains(body, "Cleared 3 metric series") {
t.Fatalf("unexpected response body: %q", body)
}
}

func TestClearMetricsHandlerError(t *testing.T) {
clearer := &fakeMetricsClearer{err: errors.New("clear failed")}
handler := clearMetricsHandler(clearer, promslog.NewNopLogger())

request := httptest.NewRequest(http.MethodPost, "/-/clear", nil)
response := httptest.NewRecorder()

handler.ServeHTTP(response, request)

if response.Code != http.StatusServiceUnavailable {
t.Fatalf("expected status %d, got %d", http.StatusServiceUnavailable, response.Code)
}
if body := response.Body.String(); !strings.Contains(body, "clear failed") {
t.Fatalf("unexpected response body: %q", body)
}
}
90 changes: 89 additions & 1 deletion pkg/exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,11 @@
package exporter

import (
"context"
"errors"
"log/slog"
"os"
"sync"
"time"

"github.com/prometheus/client_golang/prometheus"
Expand All @@ -39,6 +42,26 @@ type Registry interface {
RemoveStaleMetrics()
}

var (
// ErrClearMetricsUnsupported indicates that the configured registry cannot clear metrics.
ErrClearMetricsUnsupported = errors.New("registry does not support clearing metrics")
// ErrExporterNotRunning indicates that the exporter event loop is not running.
ErrExporterNotRunning = errors.New("exporter listener is not running")
)

type clearableRegistry interface {
ClearMetrics() int

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding ClearMetrics() int to the exported exporter.Registry interface is source-incompatible for downstream users that provide custom registry implementations or mocks. Because this repository is also consumed as a library, this change can cause immediate compile failures on upgrade even when runtime behavior is otherwise correct.

Suggestion: Consider keeping the exported Registry contract unchanged and introducing an internal optional capability interface (for example, type clearableRegistry interface { ClearMetrics() int }) behind a type assertion where clearing is needed. This preserves compatibility for existing library consumers.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. I restored the exported Registry interface so downstream custom implementations and mocks do not need to add ClearMetrics.

Clearing is now handled through an internal optional clearableRegistry capability interface, with ErrClearMetricsUnsupported returned if the configured registry does not support it. I also added a test registry without ClearMetrics to cover this compatibility case.

}

type clearMetricsRequest struct {
result chan clearMetricsResult
}

type clearMetricsResult struct {
cleared int
err error
}

type Exporter struct {
Mapper *mapper.MetricMapper
Registry Registry
Expand All @@ -49,21 +72,40 @@ type Exporter struct {
EventStats *prometheus.CounterVec
ConflictingEventStats *prometheus.CounterVec
MetricsCount *prometheus.GaugeVec
clearMetrics chan clearMetricsRequest
listenStateMtx sync.RWMutex
listening bool
listenDone chan struct{}
}

// Listen handles all events sent to the given channel sequentially. It
// terminates when the channel is closed.
func (b *Exporter) Listen(e <-chan event.Events) {
removeStaleMetricsTicker := clock.NewTicker(time.Second)
defer removeStaleMetricsTicker.Stop()

b.listenStateMtx.Lock()
b.listening = true
b.listenDone = make(chan struct{})
done := b.listenDone
b.listenStateMtx.Unlock()

defer func() {
b.listenStateMtx.Lock()
b.listening = false
close(done)
b.listenStateMtx.Unlock()
}()

for {
select {
case <-removeStaleMetricsTicker.C:
b.Registry.RemoveStaleMetrics()
case request := <-b.clearMetrics:
request.result <- b.clearRegistryMetrics()
case events, ok := <-e:
if !ok {
b.Logger.Debug("Channel is closed. Break out of Exporter.Listener.")
removeStaleMetricsTicker.Stop()
return
}
for _, event := range events {
Expand All @@ -73,6 +115,51 @@ func (b *Exporter) Listen(e <-chan event.Events) {
}
}

// ClearMetrics clears all dynamically registered StatsD time series.
func (b *Exporter) ClearMetrics(ctx context.Context) (int, error) {
if ctx == nil {
ctx = context.Background()
}

done, ok := b.listenState()
if !ok {
return 0, ErrExporterNotRunning
}

request := clearMetricsRequest{
result: make(chan clearMetricsResult, 1),
}

select {
case b.clearMetrics <- request:
case <-done:
return 0, ErrExporterNotRunning
case <-ctx.Done():
return 0, ctx.Err()
}

select {
case result := <-request.result:
return result.cleared, result.err
case <-ctx.Done():
return 0, ctx.Err()
}
}

func (b *Exporter) listenState() (<-chan struct{}, bool) {
b.listenStateMtx.RLock()
defer b.listenStateMtx.RUnlock()
return b.listenDone, b.listening
}

func (b *Exporter) clearRegistryMetrics() clearMetricsResult {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should instrument this, for large metric sets, we could block for a while and cause the whole exporter to get "stuck" here. Maybe lets add at least a doc block saying that this function may cause "hangs" when working with larger sets.

clearable, ok := b.Registry.(clearableRegistry)
if !ok {
return clearMetricsResult{err: ErrClearMetricsUnsupported}
}
return clearMetricsResult{cleared: clearable.ClearMetrics()}
}

// handleEvent processes a single Event according to the configured mapping.
func (b *Exporter) handleEvent(thisEvent event.Event) {
mapping, labels, present := b.Mapper.GetMapping(thisEvent.MetricName(), thisEvent.MetricType())
Expand Down Expand Up @@ -207,5 +294,6 @@ func NewExporter(reg prometheus.Registerer, mapper *mapper.MetricMapper, logger
EventStats: eventStats,
ConflictingEventStats: conflictingEventStats,
MetricsCount: metricsCount,
clearMetrics: make(chan clearMetricsRequest),
}
}
Loading