-
Notifications
You must be signed in to change notification settings - Fork 259
Add lifecycle endpoint to clear metrics #714
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -205,6 +205,20 @@ func reloadConfig(fileName string, mapper *mapper.MetricMapper, logger *slog.Log | |
| } | ||
| } | ||
|
|
||
| type metricsClearer interface { | ||
| ClearMetrics() int | ||
| } | ||
|
|
||
| func clearMetricsHandler(clearer metricsClearer, logger *slog.Logger) http.HandlerFunc { | ||
| return func(w http.ResponseWriter, r *http.Request) { | ||
| if r.Method == http.MethodPut || r.Method == http.MethodPost { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lets return 405 Method Not Allowed when the method is not POST|PUT. Allow: PUT, POST, and test both supported methods and at least one unsupported method. |
||
| cleared := clearer.ClearMetrics() | ||
| logger.Info("Received lifecycle api clear", "metrics", cleared) | ||
| fmt.Fprintf(w, "Cleared %d metric series", cleared) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func dumpFSM(mapper *mapper.MetricMapper, dumpFilename string, logger *slog.Logger) error { | ||
| f, err := os.Create(dumpFilename) | ||
| if err != nil { | ||
|
|
@@ -528,6 +542,7 @@ func main() { | |
| quitChan <- struct{}{} | ||
| } | ||
| }) | ||
| mux.HandleFunc("/-/clear", clearMetricsHandler(exporter, logger)) | ||
| } | ||
|
|
||
| mux.HandleFunc("/-/healthy", func(w http.ResponseWriter, r *http.Request) { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,50 @@ | ||
| // Copyright 2026 The Prometheus Authors | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "net/http" | ||
| "net/http/httptest" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/prometheus/common/promslog" | ||
| ) | ||
|
|
||
| type fakeMetricsClearer struct { | ||
| cleared int | ||
| called int | ||
| } | ||
|
|
||
| func (f *fakeMetricsClearer) ClearMetrics() int { | ||
| f.called++ | ||
| return f.cleared | ||
| } | ||
|
|
||
| func TestClearMetricsHandler(t *testing.T) { | ||
| clearer := &fakeMetricsClearer{cleared: 3} | ||
| handler := clearMetricsHandler(clearer, promslog.NewNopLogger()) | ||
|
|
||
| request := httptest.NewRequest(http.MethodPost, "/-/clear", nil) | ||
| response := httptest.NewRecorder() | ||
|
|
||
| handler.ServeHTTP(response, request) | ||
|
|
||
| if clearer.called != 1 { | ||
| t.Fatalf("expected clearer to be called once, got %d", clearer.called) | ||
| } | ||
| if body := response.Body.String(); !strings.Contains(body, "Cleared 3 metric series") { | ||
| t.Fatalf("unexpected response body: %q", body) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -37,6 +37,7 @@ type Registry interface { | |
| GetHistogram(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping, metricsCount *prometheus.GaugeVec) (prometheus.Observer, error) | ||
| GetSummary(metricName string, labels prometheus.Labels, help string, mapping *mapper.MetricMapping, metricsCount *prometheus.GaugeVec) (prometheus.Observer, error) | ||
| RemoveStaleMetrics() | ||
| ClearMetrics() int | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Adding Suggestion: Consider keeping the exported
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. I restored the exported Clearing is now handled through an internal optional |
||
| } | ||
|
|
||
| type Exporter struct { | ||
|
|
@@ -49,6 +50,7 @@ type Exporter struct { | |
| EventStats *prometheus.CounterVec | ||
| ConflictingEventStats *prometheus.CounterVec | ||
| MetricsCount *prometheus.GaugeVec | ||
| clearMetrics chan chan int | ||
| } | ||
|
|
||
| // Listen handles all events sent to the given channel sequentially. It | ||
|
|
@@ -60,6 +62,8 @@ func (b *Exporter) Listen(e <-chan event.Events) { | |
| select { | ||
| case <-removeStaleMetricsTicker.C: | ||
| b.Registry.RemoveStaleMetrics() | ||
| case cleared := <-b.clearMetrics: | ||
| cleared <- b.Registry.ClearMetrics() | ||
| case events, ok := <-e: | ||
| if !ok { | ||
| b.Logger.Debug("Channel is closed. Break out of Exporter.Listener.") | ||
|
|
@@ -73,6 +77,13 @@ func (b *Exporter) Listen(e <-chan event.Events) { | |
| } | ||
| } | ||
|
|
||
| // ClearMetrics clears all dynamically registered StatsD time series. | ||
| func (b *Exporter) ClearMetrics() int { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This synchronously sends on an unbuffered channel and waits for a reply that is only processed inside Suggestion: Consider guarding send/receive with lifecycle state. A robust pattern is a
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks, good catch. I updated It now tracks the |
||
| cleared := make(chan int) | ||
| b.clearMetrics <- cleared | ||
| return <-cleared | ||
| } | ||
|
|
||
| // handleEvent processes a single Event according to the configured mapping. | ||
| func (b *Exporter) handleEvent(thisEvent event.Event) { | ||
| mapping, labels, present := b.Mapper.GetMapping(thisEvent.MetricName(), thisEvent.MetricType()) | ||
|
|
@@ -207,5 +218,6 @@ func NewExporter(reg prometheus.Registerer, mapper *mapper.MetricMapper, logger | |
| EventStats: eventStats, | ||
| ConflictingEventStats: conflictingEventStats, | ||
| MetricsCount: metricsCount, | ||
| clearMetrics: make(chan chan int), | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Applied, thanks.