archives/snowflake
2
0
Fork 0
mirror of https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git synced 2025-10-13 11:11:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add proxy type to stats exported by broker

Browse source
This commit is contained in:
Cecylia Bocovich 2019-11-20 13:27:04 -05:00
parent 8ab81fc6cd
commit 981abffbd9
3 changed files with 111 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

2
broker/broker.go
View file

@ -173,7 +173,7 @@ func proxyPolls(ctx *BrokerContext, w http.ResponseWriter, r *http.Request) {
if err != nil {
log.Println("Error processing proxy IP: ", err.Error())
} else {
ctx.metrics.UpdateCountryStats(remoteIP)
ctx.metrics.UpdateCountryStats(remoteIP, ptype)
}
// Wait for a client to avail an offer to the snowflake, or timeout if nil.

73
broker/metrics.go
View file

@ -19,6 +19,24 @@ We export metrics in the following format:
A count of the total number of unique IP addresses of snowflake
proxies that have polled.
"snowflake-ips-standalone" NUM NL
[At most once.]
A count of the total number of unique IP addresses of snowflake
proxies of type "standalone" that have polled.
"snowflake-ips-badge" NUM NL
[At most once.]
A count of the total number of unique IP addresses of snowflake
proxies of type "badge" that have polled.
"snowflake-ips-webext" NUM NL
[At most once.]
A count of the total number of unique IP addresses of snowflake
proxies of type "webext" that have polled.
"snowflake-idle-count" NUM NL
[At most once.]
@ -58,8 +76,11 @@ var (
const metricsResolution = 60 * 60 * 24 * time.Second //86400 seconds
type CountryStats struct {
addrs map[string]bool
counts map[string]int
standalone map[string]bool
badge map[string]bool
webext map[string]bool
unknown map[string]bool
counts map[string]int
}
// Implements Observable
@ -89,13 +110,27 @@ func (s CountryStats) Display() string {
return output
}
func (m *Metrics) UpdateCountryStats(addr string) {
func (m *Metrics) UpdateCountryStats(addr string, ptype string) {
var country string
var ok bool
if m.countryStats.addrs[addr] {
return
if ptype == "standalone" {
if m.countryStats.standalone[addr] {
return
}
} else if ptype == "badge" {
if m.countryStats.badge[addr] {
return
}
} else if ptype == "webext" {
if m.countryStats.webext[addr] {
return
}
} else {
if m.countryStats.unknown[addr] {
return
}
}
ip := net.ParseIP(addr)
@ -118,7 +153,15 @@ func (m *Metrics) UpdateCountryStats(addr string) {
//update map of unique ips and counts
m.countryStats.counts[country]++
m.countryStats.addrs[addr] = true
if ptype == "standalone" {
m.countryStats.standalone[addr] = true
} else if ptype == "badge" {
m.countryStats.badge[addr] = true
} else if ptype == "webext" {
m.countryStats.webext[addr] = true
} else {
m.countryStats.unknown[addr] = true
}
}
@ -148,8 +191,11 @@ func NewMetrics(metricsLogger *log.Logger) (*Metrics, error) {
m := new(Metrics)
m.countryStats = CountryStats{
counts: make(map[string]int),
addrs: make(map[string]bool),
counts: make(map[string]int),
standalone: make(map[string]bool),
badge: make(map[string]bool),
webext: make(map[string]bool),
unknown: make(map[string]bool),
}
m.logger = metricsLogger
@ -172,7 +218,11 @@ func (m *Metrics) logMetrics() {
func (m *Metrics) printMetrics() {
m.logger.Println("snowflake-stats-end", time.Now().UTC().Format("2006-01-02 15:04:05"), fmt.Sprintf("(%d s)", int(metricsResolution.Seconds())))
m.logger.Println("snowflake-ips", m.countryStats.Display())
m.logger.Println("snowflake-ips-total", len(m.countryStats.addrs))
m.logger.Println("snowflake-ips-total", len(m.countryStats.standalone)+
len(m.countryStats.badge)+len(m.countryStats.webext)+len(m.countryStats.unknown))
m.logger.Println("snowflake-ips-standalone", len(m.countryStats.standalone))
m.logger.Println("snowflake-ips-badge", len(m.countryStats.badge))
m.logger.Println("snowflake-ips-webext", len(m.countryStats.webext))
m.logger.Println("snowflake-idle-count", binCount(m.proxyIdleCount))
m.logger.Println("client-denied-count", binCount(m.clientDeniedCount))
m.logger.Println("client-snowflake-match-count", binCount(m.clientProxyMatchCount))
@ -184,7 +234,10 @@ func (m *Metrics) zeroMetrics() {
m.clientDeniedCount = 0
m.clientProxyMatchCount = 0
m.countryStats.counts = make(map[string]int)
m.countryStats.addrs = make(map[string]bool)
m.countryStats.standalone = make(map[string]bool)
m.countryStats.badge = make(map[string]bool)
m.countryStats.webext = make(map[string]bool)
m.countryStats.unknown = make(map[string]bool)
}
// Rounds up a count to the nearest multiple of 8.

58
broker/snowflake-broker_test.go
View file

@ -387,7 +387,7 @@ func TestGeoip(t *testing.T) {
if err := ctx.metrics.LoadGeoipDatabases("invalid_filename", "invalid_filename6"); err != nil {
log.Printf("loading geo ip databases returned error: %v", err)
}
ctx.metrics.UpdateCountryStats("127.0.0.1")
ctx.metrics.UpdateCountryStats("127.0.0.1", "")
So(ctx.metrics.tablev4, ShouldEqual, nil)
})
@ -408,7 +408,6 @@ func TestMetrics(t *testing.T) {
w := httptest.NewRecorder()
data := bytes.NewReader([]byte("{\"Sid\":\"ymbcCMto7KHNGYlp\",\"Version\":\"1.0\"}"))
r, err := http.NewRequest("POST", "snowflake.broker/proxy", data)
r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
@ -419,8 +418,47 @@ func TestMetrics(t *testing.T) {
p.offerChannel <- nil
<-done
w = httptest.NewRecorder()
data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"standalone"}`))
r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
proxyPolls(ctx, w, r)
done <- true
}(ctx)
p = <-ctx.proxyPolls //manually unblock poll
p.offerChannel <- nil
<-done
w = httptest.NewRecorder()
data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"badge"}`))
r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
proxyPolls(ctx, w, r)
done <- true
}(ctx)
p = <-ctx.proxyPolls //manually unblock poll
p.offerChannel <- nil
<-done
w = httptest.NewRecorder()
data = bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0","Type":"webext"}`))
r, err = http.NewRequest("POST", "snowflake.broker/proxy", data)
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
proxyPolls(ctx, w, r)
done <- true
}(ctx)
p = <-ctx.proxyPolls //manually unblock poll
p.offerChannel <- nil
<-done
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=4\nsnowflake-ips-total 4\nsnowflake-ips-standalone 1\nsnowflake-ips-badge 1\nsnowflake-ips-webext 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
})
//Test addition of client failures
@ -433,13 +471,13 @@ func TestMetrics(t *testing.T) {
clientOffers(ctx, w, r)
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
// Test reset
buf.Reset()
ctx.metrics.zeroMetrics()
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
})
//Test addition of client matches
Convey("for client-proxy match", func() {
@ -460,7 +498,7 @@ func TestMetrics(t *testing.T) {
<-done
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 8\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 0\nclient-snowflake-match-count 8\n")
})
//Test rounding boundary
Convey("binning boundary", func() {
@ -479,12 +517,12 @@ func TestMetrics(t *testing.T) {
clientOffers(ctx, w, r)
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 8\nclient-snowflake-match-count 0\n")
clientOffers(ctx, w, r)
buf.Reset()
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-idle-count 0\nclient-denied-count 16\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips \nsnowflake-ips-total 0\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 0\nclient-denied-count 16\nclient-snowflake-match-count 0\n")
})
//Test unique ip
@ -492,7 +530,6 @@ func TestMetrics(t *testing.T) {
w := httptest.NewRecorder()
data := bytes.NewReader([]byte(`{"Sid":"ymbcCMto7KHNGYlp","Version":"1.0"}`))
r, err := http.NewRequest("POST", "snowflake.broker/proxy", data)
r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
So(err, ShouldBeNil)
go func(ctx *BrokerContext) {
@ -508,7 +545,6 @@ func TestMetrics(t *testing.T) {
if err != nil {
log.Printf("unable to get NewRequest with error: %v", err)
}
r.Header.Set("X-Session-ID", "test")
r.RemoteAddr = "129.97.208.23:8888" //CA geoip
go func(ctx *BrokerContext) {
proxyPolls(ctx, w, r)
@ -519,7 +555,7 @@ func TestMetrics(t *testing.T) {
<-done
ctx.metrics.printMetrics()
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
So(buf.String(), ShouldResemble, "snowflake-stats-end "+time.Now().UTC().Format("2006-01-02 15:04:05")+" (86400 s)\nsnowflake-ips CA=1\nsnowflake-ips-total 1\nsnowflake-ips-standalone 0\nsnowflake-ips-badge 0\nsnowflake-ips-webext 0\nsnowflake-idle-count 8\nclient-denied-count 0\nclient-snowflake-match-count 0\n")
})
})
}