Merge remote-tracking branch 'origin/mr/258'

broker/amp.go

@@ -7,6 +7,7 @@ import (
 
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/amp"
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
+	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"
 )
 
 // ampClientOffers is the AMP-speaking endpoint for client poll messages,
@@ -35,7 +36,7 @@ func ampClientOffers(i *IPC, w http.ResponseWriter, r *http.Request) {
 	if err == nil {
 		arg := messages.Arg{
 			Body:             encPollReq,
-			RemoteAddr:       "",
+			RemoteAddr:       util.GetClientIp(r),
 			RendezvousMethod: messages.RendezvousAmpCache,
 		}
 		err = i.ClientOffers(arg, &response)

broker/http.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
+	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"
 )
 
 const (
@@ -102,7 +103,7 @@ func proxyPolls(i *IPC, w http.ResponseWriter, r *http.Request) {
 
 	arg := messages.Arg{
 		Body:       body,
-		RemoteAddr: r.RemoteAddr,
+		RemoteAddr: util.GetClientIp(r),
 	}
 
 	var response []byte
@@ -167,7 +168,7 @@ func clientOffers(i *IPC, w http.ResponseWriter, r *http.Request) {
 
 	arg := messages.Arg{
 		Body:             body,
-		RemoteAddr:       "",
+		RemoteAddr:       util.GetClientIp(r),
 		RendezvousMethod: messages.RendezvousHttp,
 	}
 
@@ -227,7 +228,7 @@ func proxyAnswers(i *IPC, w http.ResponseWriter, r *http.Request) {
 
 	arg := messages.Arg{
 		Body:       body,
-		RemoteAddr: "",
+		RemoteAddr: util.GetClientIp(r),
 	}
 
 	var response []byte

broker/ipc.go

@@ -5,7 +5,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"log"
-	"net"
 	"time"
 
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/bridgefingerprint"
@@ -101,7 +100,7 @@ func (i *IPC) ProxyPolls(arg messages.Arg, response *[]byte) error {
 	}
 
 	// Log geoip stats
-	remoteIP, _, err := net.SplitHostPort(arg.RemoteAddr)
+	remoteIP := arg.RemoteAddr
 	if err != nil {
 		log.Println("Warning: cannot process proxy IP: ", err.Error())
 	} else {
@@ -196,13 +195,7 @@ func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte) error {
 		snowflake.offerChannel <- offer
 	} else {
 		i.ctx.metrics.lock.Lock()
-		i.ctx.metrics.clientDeniedCount[arg.RendezvousMethod]++
+		i.ctx.metrics.UpdateRendezvousStats(arg.RemoteAddr, arg.RendezvousMethod, offer.natType, false)
-		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "denied", "rendezvous_method": string(arg.RendezvousMethod)}).Inc()
-		if offer.natType == NATUnrestricted {
-			i.ctx.metrics.clientUnrestrictedDeniedCount[arg.RendezvousMethod]++
-		} else {
-			i.ctx.metrics.clientRestrictedDeniedCount[arg.RendezvousMethod]++
-		}
 		i.ctx.metrics.lock.Unlock()
 		resp := &messages.ClientPollResponse{Error: messages.StrNoProxies}
 		return sendClientResponse(resp, response)
@@ -212,8 +205,7 @@ func (i *IPC) ClientOffers(arg messages.Arg, response *[]byte) error {
 	select {
 	case answer := <-snowflake.answerChannel:
 		i.ctx.metrics.lock.Lock()
-		i.ctx.metrics.clientProxyMatchCount[arg.RendezvousMethod]++
+		i.ctx.metrics.UpdateRendezvousStats(arg.RemoteAddr, arg.RendezvousMethod, offer.natType, true)
-		i.ctx.metrics.promMetrics.ClientPollTotal.With(prometheus.Labels{"nat": offer.natType, "status": "matched", "rendezvous_method": string(arg.RendezvousMethod)}).Inc()
 		i.ctx.metrics.lock.Unlock()
 		resp := &messages.ClientPollResponse{Answer: answer}
 		err = sendClientResponse(resp, response)

broker/metrics.go

@@ -24,6 +24,12 @@ const (
 	metricsResolution   = 60 * 60 * 24 * time.Second //86400 seconds
 )
 
+var rendezvoudMethodList = [...]messages.RendezvousMethod{
+	messages.RendezvousHttp,
+	messages.RendezvousAmpCache,
+	messages.RendezvousSqs,
+}
+
 type CountryStats struct {
 	// map[proxyType][address]bool
 	proxies map[string]map[string]bool
@@ -49,6 +55,8 @@ type Metrics struct {
 	clientUnrestrictedDeniedCount map[messages.RendezvousMethod]uint
 	clientProxyMatchCount         map[messages.RendezvousMethod]uint
 
+	rendezvousCountryStats map[messages.RendezvousMethod]map[string]int
+
 	proxyPollWithRelayURLExtension         uint
 	proxyPollWithoutRelayURLExtension      uint
 	proxyPollRejectedWithRelayURLExtension uint
@@ -96,7 +104,6 @@ func (s CountryStats) Display() string {
 }
 
 func (m *Metrics) UpdateCountryStats(addr string, proxyType string, natType string) {
-
 	var country string
 	var ok bool
 
@@ -137,7 +144,59 @@ func (m *Metrics) UpdateCountryStats(addr string, proxyType string, natType stri
 	default:
 		m.countryStats.natUnknown[addr] = true
 	}
+}
 
+func (m *Metrics) UpdateRendezvousStats(addr string, rendezvousMethod messages.RendezvousMethod, natType string, matched bool) {
+	ip := net.ParseIP(addr)
+	country := "??"
+	if m.geoipdb != nil {
+		country_by_addr, ok := m.geoipdb.GetCountryByAddr(ip)
+		if ok {
+			country = country_by_addr
+		}
+	}
+
+	var status string
+	if !matched {
+		m.clientDeniedCount[rendezvousMethod]++
+		if natType == NATUnrestricted {
+			m.clientUnrestrictedDeniedCount[rendezvousMethod]++
+		} else {
+			m.clientRestrictedDeniedCount[rendezvousMethod]++
+		}
+		status = "denied"
+	} else {
+		status = "matched"
+		m.clientProxyMatchCount[rendezvousMethod]++
+	}
+	m.rendezvousCountryStats[rendezvousMethod][country]++
+	m.promMetrics.ClientPollTotal.With(prometheus.Labels{
+		"nat":               natType,
+		"status":            status,
+		"rendezvous_method": string(rendezvousMethod),
+		"cc":                country,
+	}).Inc()
+}
+
+func (m *Metrics) DisplayRendezvousStatsByCountry(rendezvoudMethod messages.RendezvousMethod) string {
+	output := ""
+
+	// Use the records struct to sort our counts map by value.
+	rs := records{}
+	for cc, count := range m.rendezvousCountryStats[rendezvoudMethod] {
+		rs = append(rs, record{cc: cc, count: count})
+	}
+	sort.Sort(sort.Reverse(rs))
+	for _, r := range rs {
+		output += fmt.Sprintf("%s=%d,", r.cc, binCount(uint(r.count)))
+	}
+
+	// cut off trailing ","
+	if len(output) > 0 {
+		return output[:len(output)-1]
+	}
+
+	return output
 }
 
 func (m *Metrics) LoadGeoipDatabases(geoipDB string, geoip6DB string) error {
@@ -157,6 +216,11 @@ func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
 	m.clientUnrestrictedDeniedCount = make(map[messages.RendezvousMethod]uint)
 	m.clientProxyMatchCount = make(map[messages.RendezvousMethod]uint)
 
+	m.rendezvousCountryStats = make(map[messages.RendezvousMethod]map[string]int)
+	for _, rendezvousMethod := range rendezvoudMethodList {
+		m.rendezvousCountryStats[rendezvousMethod] = make(map[string]int)
+	}
+
 	m.countryStats = CountryStats{
 		counts:          make(map[string]int),
 		proxies:         make(map[string]map[string]bool),
@@ -211,14 +275,11 @@ func (m *Metrics) printMetrics() {
 	m.logger.Println("client-unrestricted-denied-count", binCount(sumMapValues(&m.clientUnrestrictedDeniedCount)))
 	m.logger.Println("client-snowflake-match-count", binCount(sumMapValues(&m.clientProxyMatchCount)))
 
-	for _, rendezvousMethod := range [3]messages.RendezvousMethod{
+	for _, rendezvousMethod := range rendezvoudMethodList {
-		messages.RendezvousHttp,
-		messages.RendezvousAmpCache,
-		messages.RendezvousSqs,
-	} {
 		m.logger.Printf("client-%s-count %d\n", rendezvousMethod, binCount(
 			m.clientDeniedCount[rendezvousMethod]+m.clientProxyMatchCount[rendezvousMethod],
 		))
+		m.logger.Printf("client-%s-ips %s\n", rendezvousMethod, m.DisplayRendezvousStatsByCountry(rendezvousMethod))
 	}
 
 	m.logger.Println("snowflake-ips-nat-restricted", len(m.countryStats.natRestricted))
@@ -237,6 +298,12 @@ func (m *Metrics) zeroMetrics() {
 	m.proxyPollWithRelayURLExtension = 0
 	m.proxyPollWithoutRelayURLExtension = 0
 	m.clientProxyMatchCount = make(map[messages.RendezvousMethod]uint)
+
+	m.rendezvousCountryStats = make(map[messages.RendezvousMethod]map[string]int)
+	for _, rendezvousMethod := range rendezvoudMethodList {
+		m.rendezvousCountryStats[rendezvousMethod] = make(map[string]int)
+	}
+
 	m.countryStats.counts = make(map[string]int)
 	for pType := range m.countryStats.proxies {
 		m.countryStats.proxies[pType] = make(map[string]bool)
@@ -339,7 +406,7 @@ func initPrometheus() *PromMetrics {
 			Name:      "rounded_client_poll_total",
 			Help:      "The number of snowflake client polls, rounded up to a multiple of 8",
 		},
-		[]string{"nat", "status", "rendezvous_method"},
+		[]string{"nat", "status", "cc", "rendezvous_method"},
 	)
 
 	// We need to register our metrics so they can be exported.

broker/snowflake-broker_test.go

@@ -157,8 +157,11 @@ client-restricted-denied-count 8
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 8
+client-http-ips ??=8
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -184,8 +187,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 8
 client-http-count 8
+client-http-ips ??=8
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -260,8 +266,11 @@ client-restricted-denied-count 8
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 8
+client-http-ips ??=8
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -287,8 +296,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 8
 client-http-count 8
+client-http-ips ??=8
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -340,8 +352,11 @@ client-restricted-denied-count 8
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 0
+client-http-ips 
 client-ampcache-count 8
+client-ampcache-ips ??=8
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -369,8 +384,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 8
 client-http-count 0
+client-http-ips 
 client-ampcache-count 8
+client-ampcache-ips ??=8
 client-sqs-count 0
+client-sqs-ips 
 `)
 			})
 
@@ -728,8 +746,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 0
+client-http-ips 
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 snowflake-ips-nat-restricted 0
 snowflake-ips-nat-unrestricted 0
 snowflake-ips-nat-unknown 1
@@ -742,6 +763,7 @@ snowflake-ips-nat-unknown 1
 			data, err := createClientOffer(sdp, NATUnknown, "")
 			So(err, ShouldBeNil)
 			r, err := http.NewRequest("POST", "snowflake.broker/client", data)
+			r.RemoteAddr = "129.97.208.23:8888" //CA geoip
 			So(err, ShouldBeNil)
 
 			clientOffers(i, w, r)
@@ -752,9 +774,11 @@ client-restricted-denied-count 8
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 8
+client-http-ips CA=8
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
-`)
+client-sqs-ips `)
 
 			// Test reset
 			buf.Reset()
@@ -774,8 +798,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 0
 client-http-count 0
+client-http-ips 
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 0
+client-sqs-ips 
 snowflake-ips-nat-restricted 0
 snowflake-ips-nat-unrestricted 0
 snowflake-ips-nat-unknown 0

broker/sqs.go

@@ -12,6 +12,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/sqs/types"
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
 	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/sqsclient"
+	"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"
 )
 
 const (
@@ -144,9 +145,27 @@ func (r *sqsHandler) handleMessage(context context.Context, message *types.Messa
 	answerSQSURL := res.QueueUrl
 
 	encPollReq = []byte(*message.Body)
+
+	// Get best guess Client IP for geolocating
+	remoteAddr := ""
+	req, err := messages.DecodeClientPollRequest(encPollReq)
+	if err != nil {
+		log.Printf("SQSHandler: error encounted when decoding client poll request %s: %v\n", *clientID, err)
+	} else {
+		sdp, err := util.DeserializeSessionDescription(req.Offer)
+		if err != nil {
+			log.Printf("SQSHandler: error encounted when deserializing session desc %s: %v\n", *clientID, err)
+		} else {
+			candidateAddrs := util.GetCandidateAddrs(sdp.SDP)
+			if len(candidateAddrs) > 0 {
+				remoteAddr = candidateAddrs[0].String()
+			}
+		}
+	}
+
 	arg := messages.Arg{
 		Body:             encPollReq,
-		RemoteAddr:       "",
+		RemoteAddr:       remoteAddr,
 		RendezvousMethod: messages.RendezvousSqs,
 	}
 	err = r.IPC.ClientOffers(arg, &response)

broker/sqs_test.go

@@ -195,8 +195,11 @@ client-restricted-denied-count 0
 client-unrestricted-denied-count 0
 client-snowflake-match-count 8
 client-http-count 0
+client-http-ips 
 client-ampcache-count 0
+client-ampcache-ips 
 client-sqs-count 8
+client-sqs-ips ??=8
 `)
 								wg.Done()
 							}

common/util/util.go

@@ -3,11 +3,16 @@ package util
 import (
 	"encoding/json"
 	"errors"
+	"log"
 	"net"
+	"net/http"
+	"slices"
+	"sort"
 
 	"github.com/pion/ice/v2"
 	"github.com/pion/sdp/v3"
 	"github.com/pion/webrtc/v3"
+	"github.com/realclientip/realclientip-go"
 )
 
 func SerializeSessionDescription(desc *webrtc.SessionDescription) (string, error) {
@@ -97,3 +102,66 @@ func StripLocalAddresses(str string) string {
 	}
 	return string(bts)
 }
+
+// Attempts to retrieve the client IP of where the HTTP request originating.
+// There is no standard way to do this since the original client IP can be included in a number of different headers,
+// depending on the proxies and load balancers between the client and the server. We attempt to check as many of these
+// headers as possible to determine a "best guess" of the client IP
+// Using this as a reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Forwarded
+func GetClientIp(req *http.Request) string {
+	// We check the "Fowarded" header first, followed by the "X-Forwarded-For" header, and then use the "RemoteAddr" as
+	// a last resort. We use the leftmost address since it is the closest one to the client.
+	strat := realclientip.NewChainStrategy(
+		realclientip.Must(realclientip.NewLeftmostNonPrivateStrategy("Forwarded")),
+		realclientip.Must(realclientip.NewLeftmostNonPrivateStrategy("X-Forwarded-For")),
+		realclientip.RemoteAddrStrategy{},
+	)
+	clientIp := strat.ClientIP(req.Header, req.RemoteAddr)
+	return clientIp
+}
+
+// Returns a list of IP addresses of ICE candidates, roughly in descending order for accuracy for geolocation
+func GetCandidateAddrs(sdpStr string) []net.IP {
+	var desc sdp.SessionDescription
+	err := desc.Unmarshal([]byte(sdpStr))
+	if err != nil {
+		log.Printf("GetCandidateAddrs: failed to unmarshal SDP: %v\n", err)
+		return []net.IP{}
+	}
+
+	iceCandidates := make([]ice.Candidate, 0)
+
+	for _, m := range desc.MediaDescriptions {
+		for _, a := range m.Attributes {
+			if a.IsICECandidate() {
+				c, err := ice.UnmarshalCandidate(a.Value)
+				if err == nil {
+					iceCandidates = append(iceCandidates, c)
+				}
+			}
+		}
+	}
+
+	// ICE candidates are first sorted in asecending order of priority, to match convention of providing a custom Less
+	// function to sort
+	sort.Slice(iceCandidates, func(i, j int) bool {
+		if iceCandidates[i].Type() != iceCandidates[j].Type() {
+			// Sort by candidate type first, in the order specified in https://datatracker.ietf.org/doc/html/rfc8445#section-5.1.2.2
+			// Higher priority candidate types are more efficient, which likely means they are closer to the client
+			// itself, providing a more accurate result for geolocation
+			return ice.CandidateType(iceCandidates[i].Type().Preference()) < ice.CandidateType(iceCandidates[j].Type().Preference())
+		}
+		// Break ties with the ICE candidate's priority property
+		return iceCandidates[i].Priority() < iceCandidates[j].Priority()
+	})
+	slices.Reverse(iceCandidates)
+
+	sortedIpAddr := make([]net.IP, 0)
+	for _, c := range iceCandidates {
+		ip := net.ParseIP(c.Address())
+		if ip != nil {
+			sortedIpAddr = append(sortedIpAddr, ip)
+		}
+	}
+	return sortedIpAddr
+}

common/util/util_test.go

@@ -1,6 +1,8 @@
 package util
 
 import (
+	"net"
+	"net/http"
 	"testing"
 
 	. "github.com/smartystreets/goconvey/convey"
@@ -25,4 +27,49 @@ func TestUtil(t *testing.T) {
 
 		So(StripLocalAddresses(offer), ShouldEqual, offerStart+goodCandidate+offerEnd)
 	})
+
+	Convey("GetClientIp", t, func() {
+		// Should use Forwarded header
+		req1, _ := http.NewRequest("GET", "https://example.com", nil)
+		req1.Header.Add("X-Forwarded-For", "1.1.1.1, 2001:db8:cafe::99%eth0, 3.3.3.3, 192.168.1.1")
+		req1.Header.Add("Forwarded", `For=fe80::abcd;By=fe80::1234, Proto=https;For=::ffff:188.0.2.128, For="[2001:db8:cafe::17]:4848", For=fc00::1`)
+		req1.RemoteAddr = "192.168.1.2:8888"
+		So(GetClientIp(req1), ShouldEqual, "188.0.2.128")
+
+		// Should use X-Forwarded-For header
+		req2, _ := http.NewRequest("GET", "https://example.com", nil)
+		req2.Header.Add("X-Forwarded-For", "1.1.1.1, 2001:db8:cafe::99%eth0, 3.3.3.3, 192.168.1.1")
+		req2.RemoteAddr = "192.168.1.2:8888"
+		So(GetClientIp(req2), ShouldEqual, "1.1.1.1")
+
+		// Should use RemoteAddr
+		req3, _ := http.NewRequest("GET", "https://example.com", nil)
+		req3.RemoteAddr = "192.168.1.2:8888"
+		So(GetClientIp(req3), ShouldEqual, "192.168.1.2")
+
+		// Should return empty client IP
+		req4, _ := http.NewRequest("GET", "https://example.com", nil)
+		So(GetClientIp(req4), ShouldEqual, "")
+	})
+
+	Convey("GetCandidateAddrs", t, func() {
+		// Should prioritize type in the following order: https://datatracker.ietf.org/doc/html/rfc8445#section-5.1.2.2
+		// Break ties using priority value
+		const offerStart = "v=0\r\no=- 4358805017720277108 2 IN IP4 8.8.8.8\r\ns=-\r\nt=0 0\r\na=group:BUNDLE data\r\na=msid-semantic: WMS\r\nm=application 56688 DTLS/SCTP 5000\r\nc=IN IP4 8.8.8.8\r\n"
+		const offerEnd = "a=ice-ufrag:aMAZ\r\na=ice-pwd:jcHb08Jjgrazp2dzjdrvPPvV\r\na=ice-options:trickle\r\na=fingerprint:sha-256 C8:88:EE:B9:E7:02:2E:21:37:ED:7A:D1:EB:2B:A3:15:A2:3B:5B:1C:3D:D4:D5:1F:06:CF:52:40:03:F8:DD:66\r\na=setup:actpass\r\na=mid:data\r\na=sctpmap:5000 webrtc-datachannel 1024\r\n"
+
+		const sdp = offerStart + "a=candidate:3769337065 1 udp 2122260223 8.8.8.8 56688 typ prflx\r\n" +
+			"a=candidate:3769337065 1 udp 2122260223 129.97.124.13 56688 typ relay\r\n" +
+			"a=candidate:3769337065 1 udp 2122260223 129.97.124.14 56688 typ srflx\r\n" +
+			"a=candidate:3769337065 1 udp 2122260223 129.97.124.15 56688 typ host\r\n" +
+			"a=candidate:3769337065 1 udp 2122260224 129.97.124.16 56688 typ host\r\n" + offerEnd
+
+		So(GetCandidateAddrs(sdp), ShouldEqual, []net.IP{
+			net.ParseIP("129.97.124.16"),
+			net.ParseIP("129.97.124.15"),
+			net.ParseIP("8.8.8.8"),
+			net.ParseIP("129.97.124.14"),
+			net.ParseIP("129.97.124.13"),
+		})
+	})
 }

doc/broker-spec.txt

@@ -88,6 +88,14 @@ Metrics data from the Snowflake broker can be retrieved by sending an HTTP GET r
         A count of the number of times a client has requested a proxy using
         the HTTP rendezvous method from the broker, rounded up to the nearest 
         multiple of 8.
+    
+    "client-http-ips" [CC=NUM,CC=NUM,...,CC=NUM] NL
+        [At most once.]
+
+        List of mappings from two-letter country codes to the number of
+        times a client has requested a proxy using the HTTP rendezvous method, 
+        rounded up to the nearest multiple of 8.  Each country code only appears 
+        once.
 
     "client-ampcache-count" NUM NL
         [At most once.]
@@ -95,6 +103,14 @@ Metrics data from the Snowflake broker can be retrieved by sending an HTTP GET r
         A count of the number of times a client has requested a proxy using
         the ampcache rendezvous method from the broker, rounded up to the 
         nearest multiple of 8.
+    
+    "client-ampcache-ips" [CC=NUM,CC=NUM,...,CC=NUM] NL
+        [At most once.]
+
+        List of mappings from two-letter country codes to the number of
+        times a client has requested a proxy using the ampcache rendezvous 
+        method, rounded up to the nearest multiple of 8.  Each country code only 
+        appears once.
 
     "client-sqs-count" NUM NL
         [At most once.]
@@ -103,6 +119,14 @@ Metrics data from the Snowflake broker can be retrieved by sending an HTTP GET r
         the sqs rendezvous method from the broker, rounded up to the nearest 
         multiple of 8.
 
+    "client-sqs-ips" [CC=NUM,CC=NUM,...,CC=NUM] NL
+        [At most once.]
+
+        List of mappings from two-letter country codes to the number of
+        times a client has requested a proxy using the sqs rendezvous method, 
+        rounded up to the nearest multiple of 8.  Each country code only appears 
+        once.
+
     "snowflake-ips-nat-restricted" NUM NL
         [At most once.]
 

go.mod

@@ -72,6 +72,7 @@ require (
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/quic-go/quic-go v0.40.1 // indirect
+	github.com/realclientip/realclientip-go v1.0.0 // indirect
 	github.com/smarty/assertions v1.15.0 // indirect
 	github.com/templexxx/cpu v0.1.0 // indirect
 	github.com/templexxx/xorsimd v0.4.2 // indirect

go.sum

@@ -181,6 +181,8 @@ github.com/quic-go/quic-go v0.40.1 h1:X3AGzUNFs0jVuO3esAGnTfvdgvL4fq655WaOi1snv1
 github.com/quic-go/quic-go v0.40.1/go.mod h1:PeN7kuVJ4xZbxSv/4OX6S1USOX8MJvydwpTx31vx60c=
 github.com/refraction-networking/utls v1.6.3 h1:MFOfRN35sSx6K5AZNIoESsBuBxS2LCgRilRIdHb6fDc=
 github.com/refraction-networking/utls v1.6.3/go.mod h1:yil9+7qSl+gBwJqztoQseO6Pr3h62pQoY1lXiNR/FPs=
+github.com/realclientip/realclientip-go v1.0.0 h1:+yPxeC0mEaJzq1BfCt2h4BxlyrvIIBzR6suDc3BEF1U=
+github.com/realclientip/realclientip-go v1.0.0/go.mod h1:CXnUdVwFRcXFJIRb/dTYqbT7ud48+Pi2pFm80bxDmcI=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=