cmd/stdiscosrv: Metric for returned retry-after

cmd/stdiscosrv/apisrv.go

@@ -212,7 +212,9 @@ func (s *apiSrv) handleGET(w http.ResponseWriter, req *http.Request) {
 			s.db.put(key, rec)
 		}
 
-		w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
+		afterS := notFoundRetryAfterSeconds(int(misses))
+		retryAfterHistogram.Observe(float64(afterS))
+		w.Header().Set("Retry-After", strconv.Itoa(afterS))
 		http.Error(w, "Not Found", http.StatusNotFound)
 		return
 	}
@@ -492,13 +494,13 @@ func errorRetryAfterString() string {
 	return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
 }
 
-func notFoundRetryAfterString(misses int) string {
+func notFoundRetryAfterSeconds(misses int) int {
 	retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
 	if retryAfterS > notFoundRetryMaxSeconds {
 		retryAfterS = notFoundRetryMaxSeconds
 	}
 	retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
-	return strconv.Itoa(retryAfterS)
+	return retryAfterS
 }
 
 func reannounceAfterString() string {

cmd/stdiscosrv/stats.go

@@ -90,6 +90,14 @@ var (
 			Help:       "Latency of database operations.",
 			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
 		}, []string{"operation"})
+
+	retryAfterHistogram = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: "syncthing",
+		Subsystem: "discovery",
+		Name:      "retry_after_seconds",
+		Help:      "Retry-After header value in seconds.",
+		Buckets:   prometheus.ExponentialBuckets(60, 2, 7), // 60, 120, 240, 480, 960, 1920, 3840
+	})
 )
 
 const (
@@ -108,7 +116,8 @@ func init() {
 		lookupRequestsTotal, announceRequestsTotal,
 		replicationSendsTotal, replicationRecvsTotal,
 		databaseKeys, databaseStatisticsSeconds,
-		databaseOperations, databaseOperationSeconds)
+		databaseOperations, databaseOperationSeconds,
+		retryAfterHistogram)
 
 	processCollectorOpts := collectors.ProcessCollectorOpts{
 		Namespace: "syncthing_discovery",
@@ -120,5 +129,4 @@ func init() {
 	prometheus.MustRegister(
 		collectors.NewProcessCollector(processCollectorOpts),
 	)
-
 }