snowflake/broker/metrics.go

/*
We export metrics in the format specified in our broker spec:
https://gitweb.torproject.org/pluggable-transports/snowflake.git/tree/doc/broker-spec.txt
*/

package main

import (
	"fmt"
	"log"
	"net"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"gitlab.torproject.org/tpo/anti-censorship/geoip"
	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/safeprom"
	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
)

const (
	prometheusNamespace = "snowflake"
	metricsResolution   = 60 * 60 * 24 * time.Second //86400 seconds
)

type Metrics struct {
	logger  *log.Logger
	geoipdb *geoip.Geoip

	ips      *sync.Map // proxy IP addresses we've seen before
	counters *sync.Map // counters for ip-based metrics

	// counters for country-based metrics
	proxies         *sync.Map // ip-based counts of proxy country codes
	clientHTTPPolls *sync.Map // poll-based counts of client HTTP rendezvous
	clientAMPPolls  *sync.Map // poll-based counts of client AMP cache rendezvous
	clientSQSPolls  *sync.Map // poll-based counts of client SQS rendezvous

	promMetrics *PromMetrics
}

func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
	m := new(Metrics)

	m.logger = metricsLogger
	m.promMetrics = initPrometheus()
	m.ips = new(sync.Map)
	m.counters = new(sync.Map)
	m.proxies = new(sync.Map)
	m.clientHTTPPolls = new(sync.Map)
	m.clientAMPPolls = new(sync.Map)
	m.clientSQSPolls = new(sync.Map)

	// Write to log file every day with updated metrics
	go m.logMetrics()

	return m, nil
}

func incrementMapCounter(counters *sync.Map, key string) {
	start := uint64(1)
	val, loaded := counters.LoadOrStore(key, &start)
	if loaded {
		ptr := val.(*uint64)
		atomic.AddUint64(ptr, 1)
	}
}

func (m *Metrics) IncrementCounter(key string) {
	incrementMapCounter(m.counters, key)
}

func (m *Metrics) UpdateProxyStats(addr string, proxyType string, natType string) {

	// perform geolocation of IP address
	ip := net.ParseIP(addr)
	if m.geoipdb == nil {
		return
	}
	country, ok := m.geoipdb.GetCountryByAddr(ip)
	if !ok {
		country = "??"
	}

	// check whether we've seen this proxy ip before
	if _, loaded := m.ips.LoadOrStore(addr, true); !loaded {
		m.IncrementCounter("proxy-total")
		incrementMapCounter(m.proxies, country)
		m.promMetrics.ProxyTotal.With(prometheus.Labels{
			"nat":  natType,
			"type": proxyType,
			"cc":   country,
		}).Inc()
	}

	// update unique IP proxy NAT metrics
	key := fmt.Sprintf("%s-%s", addr, natType)
	if _, loaded := m.ips.LoadOrStore(key, true); !loaded {
		switch natType {
		case NATRestricted:
			m.IncrementCounter("proxy-nat-restricted")
		case NATUnrestricted:
			m.IncrementCounter("proxy-nat-unrestricted")
		default:
			m.IncrementCounter("proxy-nat-unknown")
		}
	}
	// update unique IP proxy type metrics
	key = fmt.Sprintf("%s-%s", addr, proxyType)
	if _, loaded := m.ips.LoadOrStore(key, true); !loaded {
		switch proxyType {
		case "standalone":
			m.IncrementCounter("proxy-standalone")
		case "badge":
			m.IncrementCounter("proxy-badge")
		case "iptproxy":
			m.IncrementCounter("proxy-iptproxy")
		case "webext":
			m.IncrementCounter("proxy-webext")
		}
	}
}

func (m *Metrics) UpdateClientStats(addr string, rendezvousMethod messages.RendezvousMethod, natType, status string) {
	ip := net.ParseIP(addr)
	country := "??"
	if m.geoipdb != nil {
		country_by_addr, ok := m.geoipdb.GetCountryByAddr(ip)
		if ok {
			country = country_by_addr
		}
	}

	switch status {
	case "denied":
		m.IncrementCounter("client-denied")
		if natType == NATUnrestricted {
			m.IncrementCounter("client-unrestricted-denied")
		} else {
			m.IncrementCounter("client-restricted-denied")
		}
	case "matched":
		m.IncrementCounter("client-match")
	case "timeout":
		m.IncrementCounter("client-timeout")
	default:
		log.Printf("Unknown rendezvous status: %s", status)
	}

	switch rendezvousMethod {
	case messages.RendezvousHttp:
		m.IncrementCounter("client-http")
		incrementMapCounter(m.clientHTTPPolls, country)
	case messages.RendezvousAmpCache:
		m.IncrementCounter("client-amp")
		incrementMapCounter(m.clientAMPPolls, country)
	case messages.RendezvousSqs:
		m.IncrementCounter("client-sqs")
		incrementMapCounter(m.clientSQSPolls, country)
	}
	m.promMetrics.ClientPollTotal.With(prometheus.Labels{
		"nat":               natType,
		"status":            status,
		"rendezvous_method": string(rendezvousMethod),
		"cc":                country,
	}).Inc()
}

// Types to facilitate sorting in formatAndClearCountryStats.
type record struct {
	cc    string
	count uint64
}
type records []record

// Implementation of sort.Interface for records. The ordering is lexicographic:
// first by count (descending), then by cc (ascending).
func (r records) Len() int      { return len(r) }
func (r records) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r records) Less(i, j int) bool {
	return r[i].count > r[j].count || (r[i].count == r[j].count && r[i].cc < r[j].cc)
}

// formatAndClearCountryStats takes a map from country codes to counts, and
// returns a formatted string of comma-separated CC=COUNT. Entries are sorted by
// count from largest to smallest. When counts are equal, entries are sorted by
// country code in ascending order.
//
// formatAndClearCountryStats has the side effect of deleting all entries in m.
func formatAndClearCountryStats(m *sync.Map, binned bool) string {
	// Extract entries from the map into a slice of records, binning counts
	// if asked to.
	rs := records{}
	m.Range(func(cc, countPtr any) bool {
		count := *countPtr.(*uint64)
		if binned {
			count = binCount(count)
		}
		rs = append(rs, record{cc: cc.(string), count: count})
		m.Delete(cc)
		return true
	})
	// Sort the records.
	sort.Sort(rs)
	// Format and concatenate.
	var output strings.Builder
	for i, r := range rs {
		if i != 0 {
			output.WriteString(",")
		}
		fmt.Fprintf(&output, "%s=%d", r.cc, r.count)
	}
	return output.String()
}

func (m *Metrics) LoadGeoipDatabases(geoipDB string, geoip6DB string) error {

	// Load geoip databases
	var err error
	log.Println("Loading geoip databases")
	m.geoipdb, err = geoip.New(geoipDB, geoip6DB)
	return err
}

// Logs metrics in intervals specified by metricsResolution
func (m *Metrics) logMetrics() {
	heartbeat := time.Tick(metricsResolution)
	for range heartbeat {
		m.printMetrics()
	}
}

func (m *Metrics) loadAndZero(key string) uint64 {
	count, loaded := m.counters.LoadAndDelete(key)
	if !loaded {
		count = new(uint64)
	}
	ptr := count.(*uint64)
	return *ptr
}

func (m *Metrics) printMetrics() {
	m.logger.Println(
		"snowflake-stats-end",
		time.Now().UTC().Format("2006-01-02 15:04:05"),
		fmt.Sprintf("(%d s)", int(metricsResolution.Seconds())),
	)
	m.logger.Println("snowflake-ips", formatAndClearCountryStats(m.proxies, false))
	m.logger.Printf("snowflake-ips-iptproxy %d\n", m.loadAndZero("proxy-iptproxy"))
	m.logger.Printf("snowflake-ips-standalone %d\n", m.loadAndZero("proxy-standalone"))
	m.logger.Printf("snowflake-ips-webext %d\n", m.loadAndZero("proxy-webext"))
	m.logger.Printf("snowflake-ips-badge %d\n", m.loadAndZero("proxy-badge"))
	m.logger.Println("snowflake-ips-total", m.loadAndZero("proxy-total"))
	m.logger.Println("snowflake-idle-count", binCount(m.loadAndZero("proxy-idle")))
	m.logger.Println("snowflake-proxy-poll-with-relay-url-count", binCount(m.loadAndZero("proxy-poll-with-relay-url")))
	m.logger.Println("snowflake-proxy-poll-without-relay-url-count", binCount(m.loadAndZero("proxy-poll-without-relay-url")))
	m.logger.Println("snowflake-proxy-rejected-for-relay-url-count", binCount(m.loadAndZero("proxy-poll-rejected-relay-url")))

	m.logger.Println("client-denied-count", binCount(m.loadAndZero("client-denied")))
	m.logger.Println("client-restricted-denied-count", binCount(m.loadAndZero("client-restricted-denied")))
	m.logger.Println("client-unrestricted-denied-count", binCount(m.loadAndZero("client-unrestricted-denied")))
	m.logger.Println("client-snowflake-match-count", binCount(m.loadAndZero("client-match")))
	m.logger.Println("client-snowflake-timeout-count", binCount(m.loadAndZero("client-timeout")))

	m.logger.Printf("client-http-count %d\n", binCount(m.loadAndZero("client-http")))
	m.logger.Printf("client-http-ips %s\n", formatAndClearCountryStats(m.clientHTTPPolls, true))
	m.logger.Printf("client-ampcache-count %d\n", binCount(m.loadAndZero("client-amp")))
	m.logger.Printf("client-ampcache-ips %s\n", formatAndClearCountryStats(m.clientAMPPolls, true))
	m.logger.Printf("client-sqs-count %d\n", binCount(m.loadAndZero("client-sqs")))
	m.logger.Printf("client-sqs-ips %s\n", formatAndClearCountryStats(m.clientSQSPolls, true))

	m.logger.Println("snowflake-ips-nat-restricted", m.loadAndZero("proxy-nat-restricted"))
	m.logger.Println("snowflake-ips-nat-unrestricted", m.loadAndZero("proxy-nat-unrestricted"))
	m.logger.Println("snowflake-ips-nat-unknown", m.loadAndZero("proxy-nat-unknown"))

	m.ips.Clear()
}

// binCount rounds count up to the next multiple of 8. Returns 0 on integer
// overflow.
func binCount(count uint64) uint64 {
	return (count + 7) / 8 * 8
}

type PromMetrics struct {
	registry         *prometheus.Registry
	ProxyTotal       *prometheus.CounterVec
	ProxyPollTotal   *safeprom.CounterVec
	ClientPollTotal  *safeprom.CounterVec
	ProxyAnswerTotal *safeprom.CounterVec
	AvailableProxies *prometheus.GaugeVec

	ProxyPollWithRelayURLExtensionTotal    *safeprom.CounterVec
	ProxyPollWithoutRelayURLExtensionTotal *safeprom.CounterVec

	ProxyPollRejectedForRelayURLExtensionTotal *safeprom.CounterVec
}

// Initialize metrics for prometheus exporter
func initPrometheus() *PromMetrics {
	promMetrics := &PromMetrics{}

	promMetrics.registry = prometheus.NewRegistry()

	promMetrics.ProxyTotal = prometheus.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "proxy_total",
			Help:      "The number of unique snowflake IPs",
		},
		[]string{"type", "nat", "cc"},
	)

	promMetrics.AvailableProxies = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Namespace: prometheusNamespace,
			Name:      "available_proxies",
			Help:      "The number of currently available snowflake proxies",
		},
		[]string{"type", "nat"},
	)

	promMetrics.ProxyPollTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_proxy_poll_total",
			Help:      "The number of snowflake proxy polls, rounded up to a multiple of 8",
		},
		[]string{"nat", "type", "status"},
	)

	promMetrics.ProxyAnswerTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_proxy_answer_total",
			Help:      "The number of snowflake proxy answers, rounded up to a multiple of 8",
		},
		[]string{"type", "status"},
	)

	promMetrics.ProxyPollWithRelayURLExtensionTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_proxy_poll_with_relay_url_extension_total",
			Help:      "The number of snowflake proxy polls with Relay URL Extension, rounded up to a multiple of 8",
		},
		[]string{"nat", "type"},
	)

	promMetrics.ProxyPollWithoutRelayURLExtensionTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_proxy_poll_without_relay_url_extension_total",
			Help:      "The number of snowflake proxy polls without Relay URL Extension, rounded up to a multiple of 8",
		},
		[]string{"nat", "type"},
	)

	promMetrics.ProxyPollRejectedForRelayURLExtensionTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_proxy_poll_rejected_relay_url_extension_total",
			Help:      "The number of snowflake proxy polls rejected by Relay URL Extension, rounded up to a multiple of 8",
		},
		[]string{"nat", "type"},
	)

	promMetrics.ClientPollTotal = safeprom.NewCounterVec(
		prometheus.CounterOpts{
			Namespace: prometheusNamespace,
			Name:      "rounded_client_poll_total",
			Help:      "The number of snowflake client polls, rounded up to a multiple of 8",
		},
		[]string{"nat", "status", "cc", "rendezvous_method"},
	)

	// We need to register our metrics so they can be exported.
	promMetrics.registry.MustRegister(
		promMetrics.ClientPollTotal, promMetrics.ProxyPollTotal,
		promMetrics.ProxyTotal, promMetrics.ProxyAnswerTotal, promMetrics.AvailableProxies,
		promMetrics.ProxyPollWithRelayURLExtensionTotal,
		promMetrics.ProxyPollWithoutRelayURLExtensionTotal,
		promMetrics.ProxyPollRejectedForRelayURLExtensionTotal,
	)

	return promMetrics
}