Update broker metrics to count matches, denials, and timeouts

Our metrics were undercounting client polls by missing the case where
clients are matched with a snowflake but receive a timeout before the
snowflake responds with its answer. This change adds a new metric,
called client-snowflake-timeout-count, to the 24 hour broker stats and a
new "timeout" status label for prometheus metrics.
This commit is contained in:
Cecylia Bocovich 2025-03-06 13:45:07 -05:00
parent 583178f4f2
commit 57dc276e48
No known key found for this signature in database
GPG key ID: 009DE379FD9B7B90
4 changed files with 27 additions and 10 deletions

View file

@ -55,6 +55,7 @@ type Metrics struct {
clientRestrictedDeniedCount map[messages.RendezvousMethod]uint
clientUnrestrictedDeniedCount map[messages.RendezvousMethod]uint
clientProxyMatchCount map[messages.RendezvousMethod]uint
clientProxyTimeoutCount map[messages.RendezvousMethod]uint
rendezvousCountryStats map[messages.RendezvousMethod]map[string]int
@ -147,7 +148,7 @@ func (m *Metrics) UpdateCountryStats(addr string, proxyType string, natType stri
}
}
func (m *Metrics) UpdateRendezvousStats(addr string, rendezvousMethod messages.RendezvousMethod, natType string, matched bool) {
func (m *Metrics) UpdateRendezvousStats(addr string, rendezvousMethod messages.RendezvousMethod, natType, status string) {
ip := net.ParseIP(addr)
country := "??"
if m.geoipdb != nil {
@ -157,18 +158,20 @@ func (m *Metrics) UpdateRendezvousStats(addr string, rendezvousMethod messages.R
}
}
var status string
if !matched {
switch status {
case "denied":
m.clientDeniedCount[rendezvousMethod]++
if natType == NATUnrestricted {
m.clientUnrestrictedDeniedCount[rendezvousMethod]++
} else {
m.clientRestrictedDeniedCount[rendezvousMethod]++
}
status = "denied"
} else {
status = "matched"
case "matched":
m.clientProxyMatchCount[rendezvousMethod]++
case "timeout":
m.clientProxyTimeoutCount[rendezvousMethod]++
default:
log.Printf("Unknown rendezvous status: %s", status)
}
m.rendezvousCountryStats[rendezvousMethod][country]++
m.promMetrics.ClientPollTotal.With(prometheus.Labels{
@ -216,6 +219,7 @@ func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
m.clientRestrictedDeniedCount = make(map[messages.RendezvousMethod]uint)
m.clientUnrestrictedDeniedCount = make(map[messages.RendezvousMethod]uint)
m.clientProxyMatchCount = make(map[messages.RendezvousMethod]uint)
m.clientProxyTimeoutCount = make(map[messages.RendezvousMethod]uint)
m.rendezvousCountryStats = make(map[messages.RendezvousMethod]map[string]int)
for _, rendezvousMethod := range rendezvoudMethodList {
@ -275,6 +279,7 @@ func (m *Metrics) printMetrics() {
m.logger.Println("client-restricted-denied-count", binCount(sumMapValues(&m.clientRestrictedDeniedCount)))
m.logger.Println("client-unrestricted-denied-count", binCount(sumMapValues(&m.clientUnrestrictedDeniedCount)))
m.logger.Println("client-snowflake-match-count", binCount(sumMapValues(&m.clientProxyMatchCount)))
m.logger.Println("client-snowflake-timeout-count", binCount(sumMapValues(&m.clientProxyTimeoutCount)))
for _, rendezvousMethod := range rendezvoudMethodList {
m.logger.Printf("client-%s-count %d\n", rendezvousMethod, binCount(
@ -299,6 +304,7 @@ func (m *Metrics) zeroMetrics() {
m.proxyPollWithRelayURLExtension = 0
m.proxyPollWithoutRelayURLExtension = 0
m.clientProxyMatchCount = make(map[messages.RendezvousMethod]uint)
m.clientProxyTimeoutCount = make(map[messages.RendezvousMethod]uint)
m.rendezvousCountryStats = make(map[messages.RendezvousMethod]map[string]int)
for _, rendezvousMethod := range rendezvoudMethodList {