Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
## unreleased

* [FEATURE] Add a lifecycle `/-/clear` endpoint to clear registered StatsD time series without restarting the exporter ([#714](https://github.com/prometheus/statsd_exporter/pull/714))

## 0.30.0 / 2026-05-28
* [CHANGE] Remove the Dockerfile `HEALTHCHECK` from published container images ([#671](https://github.com/prometheus/statsd_exporter/pull/671))
* [ENHANCEMENT] Add a distroless container image variant ([#703](https://github.com/prometheus/statsd_exporter/pull/703))
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,8 @@ NOTE: Version 0.7.0 switched to the [kingpin](https://github.com/alecthomas/king

## Lifecycle API

The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload or quit the exporter
by sending a `PUT` or `POST` request to the `/-/reload` or `/-/quit` endpoints.
The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload, quit, or clear the exporter

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload, quit, or clear the exporter
The `statsd_exporter` has an optional lifecycle API (disabled by default) that can be used to reload, quit, or clear dynamically registered StatsD metric series

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Applied, thanks.

by sending a `PUT` or `POST` request to the `/-/reload`, `/-/quit`, or `/-/clear` endpoints.

## Relay

Expand Down
15 changes: 15 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,20 @@ func reloadConfig(fileName string, mapper *mapper.MetricMapper, logger *slog.Log
}
}

type metricsClearer interface {
ClearMetrics() int
}

func clearMetricsHandler(clearer metricsClearer, logger *slog.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPut || r.Method == http.MethodPost {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets return 405 Method Not Allowed when the method is not POST|PUT.

Allow: PUT, POST, and test both supported methods and at least one unsupported method.

cleared := clearer.ClearMetrics()
logger.Info("Received lifecycle api clear", "metrics", cleared)
fmt.Fprintf(w, "Cleared %d metric series", cleared)
}
}
}

func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string, logger *slog.Logger) error {
f, err := os.Create(dumpFilename)
if err != nil {
Expand Down Expand Up @@ -528,6 +542,7 @@ func main() {
quitChan <- struct{}{}
}
})
mux.HandleFunc("/-/clear", clearMetricsHandler(exporter, logger))
}

mux.HandleFunc("/-/healthy", func(w http.ResponseWriter, r *http.Request) {
Expand Down
50 changes: 50 additions & 0 deletions main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Copyright 2026 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
"net/http"
"net/http/httptest"
"strings"
"testing"

"github.com/prometheus/common/promslog"
)

type fakeMetricsClearer struct {
cleared int
called int
}

func (f *fakeMetricsClearer) ClearMetrics() int {
f.called++
return f.cleared
}

func TestClearMetricsHandler(t *testing.T) {
clearer := &fakeMetricsClearer{cleared: 3}
handler := clearMetricsHandler(clearer, promslog.NewNopLogger())

request := httptest.NewRequest(http.MethodPost, "/-/clear", nil)
response := httptest.NewRecorder()

handler.ServeHTTP(response, request)

if clearer.called != 1 {
t.Fatalf("expected clearer to be called once, got %d", clearer.called)
}
if body := response.Body.String(); !strings.Contains(body, "Cleared 3 metric series") {
t.Fatalf("unexpected response body: %q", body)
}
}
12 changes: 12 additions & 0 deletions pkg/exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ type Registry interface {
GetHistogram(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping, metricsCount *prometheus.GaugeVec) (prometheus.Observer, error)
GetSummary(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping, metricsCount *prometheus.GaugeVec) (prometheus.Observer, error)
RemoveStaleMetrics()
ClearMetrics() int

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Adding ClearMetrics() int to the exported exporter.Registry interface is source-incompatible for downstream users that provide custom registry implementations or mocks. Because this repository is also consumed as a library, this change can cause immediate compile failures on upgrade even when runtime behavior is otherwise correct.

Suggestion: Consider keeping the exported Registry contract unchanged and introducing an internal optional capability interface (for example, type clearableRegistry interface { ClearMetrics() int }) behind a type assertion where clearing is needed. This preserves compatibility for existing library consumers.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done. I restored the exported Registry interface so downstream custom implementations and mocks do not need to add ClearMetrics.

Clearing is now handled through an internal optional clearableRegistry capability interface, with ErrClearMetricsUnsupported returned if the configured registry does not support it. I also added a test registry without ClearMetrics to cover this compatibility case.

}

type Exporter struct {
Expand All @@ -49,6 +50,7 @@ type Exporter struct {
EventStats *prometheus.CounterVec
ConflictingEventStats *prometheus.CounterVec
MetricsCount *prometheus.GaugeVec
clearMetrics chan chan int
}

// Listen handles all events sent to the given channel sequentially. It
Expand All @@ -60,6 +62,8 @@ func (b *Exporter) Listen(e <-chan event.Events) {
select {
case <-removeStaleMetricsTicker.C:
b.Registry.RemoveStaleMetrics()
case cleared := <-b.clearMetrics:
cleared <- b.Registry.ClearMetrics()
case events, ok := <-e:
if !ok {
b.Logger.Debug("Channel is closed. Break out of Exporter.Listener.")
Expand All @@ -73,6 +77,13 @@ func (b *Exporter) Listen(e <-chan event.Events) {
}
}

// ClearMetrics clears all dynamically registered StatsD time series.
func (b *Exporter) ClearMetrics() int {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This synchronously sends on an unbuffered channel and waits for a reply that is only processed inside Listen(). If called before Listen() starts, after it exits, or while it is stuck, callers can block indefinitely. This is especially risky now that ClearMetrics is public and callable by library users outside the binary's lifecycle flow.

Suggestion: Consider guarding send/receive with lifecycle state. A robust pattern is a done channel closed when Listen exits, then using select in ClearMetrics() for both send and receive paths (including case <-done:) so calls fail fast instead of hanging indefinitely.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, good catch. I updated ClearMetrics to take a context.Context and return (int, error) so callers do not block indefinitely when the exporter listener is not running.

It now tracks the Listen lifecycle, returns ErrExporterNotRunning before/after the event loop, and still serializes successful clear requests through the exporter loop. The HTTP handler now returns 503 if clearing cannot be performed.

cleared := make(chan int)
b.clearMetrics <- cleared
return <-cleared
}

// handleEvent processes a single Event according to the configured mapping.
func (b *Exporter) handleEvent(thisEvent event.Event) {
mapping, labels, present := b.Mapper.GetMapping(thisEvent.MetricName(), thisEvent.MetricType())
Expand Down Expand Up @@ -207,5 +218,6 @@ func NewExporter(reg prometheus.Registerer, mapper *mapper.MetricMapper, logger
EventStats: eventStats,
ConflictingEventStats: conflictingEventStats,
MetricsCount: metricsCount,
clearMetrics: make(chan chan int),
}
}
68 changes: 68 additions & 0 deletions pkg/exporter/exporter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1168,6 +1168,74 @@ mappings:
}
}

func TestClearMetrics(t *testing.T) {
clockInstance := clock.ClockInstance
clock.ClockInstance = nil
defer func() {
clock.ClockInstance = clockInstance
}()

reg := prometheus.NewRegistry()
testMapper := mapper.MetricMapper{}
ex := NewExporter(reg, &testMapper, promslog.NewNopLogger(), eventsActions, eventsUnmapped, errorEventStats, eventStats, conflictingEventStats, metricsCount)
events := make(chan event.Events)
done := make(chan struct{})
go func() {
ex.Listen(events)
close(done)
}()
defer func() {
close(events)
<-done
}()

events <- event.Events{
&event.GaugeEvent{
GMetricName: "clearable_gauge",
GValue: 200,
},
}
events <- event.Events{}

metrics, err := reg.Gather()
if err != nil {
t.Fatal("Gather should not fail")
}
gaugeValue := getFloat64(metrics, "clearable_gauge", prometheus.Labels{})
if gaugeValue == nil || *gaugeValue != 200 {
t.Fatalf("Gauge `clearable_gauge` should be gathered with value 200, got %v", gaugeValue)
}

if cleared := ex.ClearMetrics(); cleared != 1 {
t.Fatalf("Expected to clear 1 metric series, cleared %d", cleared)
}

metrics, err = reg.Gather()
if err != nil {
t.Fatal("Gather should not fail")
}
if gaugeValue = getFloat64(metrics, "clearable_gauge", prometheus.Labels{}); gaugeValue != nil {
t.Fatalf("Gauge `clearable_gauge` should be cleared, got %v", *gaugeValue)
}

events <- event.Events{
&event.GaugeEvent{
GMetricName: "clearable_gauge",
GValue: 42,
},
}
events <- event.Events{}

metrics, err = reg.Gather()
if err != nil {
t.Fatal("Gather should not fail")
}
gaugeValue = getFloat64(metrics, "clearable_gauge", prometheus.Labels{})
if gaugeValue == nil || *gaugeValue != 42 {
t.Fatalf("Gauge `clearable_gauge` should be gathered again with value 42, got %v", gaugeValue)
}
}

func TestHashLabelNames(t *testing.T) {
r := registry.NewRegistry(prometheus.DefaultRegisterer, nil)
// Validate value hash changes and name has doesn't when just the value changes.
Expand Down
25 changes: 22 additions & 3 deletions pkg/registry/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -387,14 +387,33 @@ func (r *Registry) RemoveStaleMetrics() {
continue
}
if rm.LastRegisteredAt.Add(rm.TTL).Before(now) {
metric.Vectors[rm.VecKey].Holder.Delete(rm.Labels)
metric.Vectors[rm.VecKey].RefCount--
delete(metric.Metrics, hash)
r.removeMetric(metric, hash, rm)
}
}
}
}

// ClearMetrics deletes all registered time series from the registry.
func (r *Registry) ClearMetrics() int {
removed := 0
for _, metric := range r.Metrics {
for hash, rm := range metric.Metrics {
r.removeMetric(metric, hash, rm)
removed++
}
}
return removed
}

func (r *Registry) removeMetric(metric metrics.Metric, hash metrics.ValueHash, rm *metrics.RegisteredMetric) {
vector := metric.Vectors[rm.VecKey]
vector.Holder.Delete(rm.Labels)
if vector.RefCount > 0 {
vector.RefCount--
}
delete(metric.Metrics, hash)
}

// Calculates a hash of both the label names and values.
func (r *Registry) HashLabels(labels prometheus.Labels) (metrics.LabelHash, []string) {
r.Hasher.Reset()
Expand Down