From 26f7ee4b0620b5b64f3b7df6b139891a7b0170c8 Mon Sep 17 00:00:00 2001 From: onyinyang Date: Wed, 29 Jan 2025 10:59:31 -0500 Subject: [PATCH] Remove utls library from snowflake and Use ptuil/utls --- client/lib/rendezvous.go | 2 +- common/utls/client_hello_id.go | 46 ----- common/utls/client_hello_id_version.go | 24 --- common/utls/roundtripper.go | 265 ------------------------- common/utls/roundtripper_test.go | 183 ----------------- go.mod | 2 +- go.sum | 2 + 7 files changed, 4 insertions(+), 520 deletions(-) delete mode 100644 common/utls/client_hello_id.go delete mode 100644 common/utls/client_hello_id_version.go delete mode 100644 common/utls/roundtripper.go delete mode 100644 common/utls/roundtripper_test.go diff --git a/client/lib/rendezvous.go b/client/lib/rendezvous.go index bf1a48c..da06027 100644 --- a/client/lib/rendezvous.go +++ b/client/lib/rendezvous.go @@ -16,12 +16,12 @@ import ( "github.com/pion/webrtc/v4" utls "github.com/refraction-networking/utls" + utlsutil "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/utls" "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/certs" "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/event" "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages" "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/nat" "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util" - utlsutil "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/utls" ) const ( diff --git a/common/utls/client_hello_id.go b/common/utls/client_hello_id.go deleted file mode 100644 index c134ecf..0000000 --- a/common/utls/client_hello_id.go +++ /dev/null @@ -1,46 +0,0 @@ -package utls - -import ( - "errors" - utls "github.com/refraction-networking/utls" - "strings" -) - -// ported from https://github.com/max-b/snowflake/commit/9dded063cb74c6941a16ad90b9dd0e06e618e55e -var clientHelloIDMap = map[string]utls.ClientHelloID{ - // No HelloCustom: not useful for external configuration. - // No HelloRandomized: doesn't negotiate consistent ALPN. - "hellorandomizedalpn": utls.HelloRandomizedALPN, - "hellorandomizednoalpn": utls.HelloRandomizedNoALPN, - "hellofirefox_auto": utls.HelloFirefox_Auto, - "hellofirefox_55": utls.HelloFirefox_55, - "hellofirefox_56": utls.HelloFirefox_56, - "hellofirefox_63": utls.HelloFirefox_63, - "hellofirefox_65": utls.HelloFirefox_65, - "hellochrome_auto": utls.HelloChrome_Auto, - "hellochrome_58": utls.HelloChrome_58, - "hellochrome_62": utls.HelloChrome_62, - "hellochrome_70": utls.HelloChrome_70, - "hellochrome_72": utls.HelloChrome_72, - "helloios_auto": utls.HelloIOS_Auto, - "helloios_11_1": utls.HelloIOS_11_1, - "helloios_12_1": utls.HelloIOS_12_1, -} - -var errNameNotFound = errors.New("client hello name is unrecognized") - -func NameToUTLSID(name string) (utls.ClientHelloID, error) { - normalizedName := strings.ToLower(name) - if id, ok := clientHelloIDMap[normalizedName]; ok { - return id, nil - } - return utls.ClientHelloID{}, errNameNotFound -} - -func ListAllNames() []string { - var names []string - for k, _ := range clientHelloIDMap { - names = append(names, k) - } - return names -} diff --git a/common/utls/client_hello_id_version.go b/common/utls/client_hello_id_version.go deleted file mode 100644 index f17de21..0000000 --- a/common/utls/client_hello_id_version.go +++ /dev/null @@ -1,24 +0,0 @@ -package utls - -import ( - "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/version" - "strings" -) - -func generateVersionOutput() string { - var versionOutputBuilder strings.Builder - - versionOutputBuilder.WriteString(`Known utls-imitate values: -(empty) -`) - - for _, name := range ListAllNames() { - versionOutputBuilder.WriteString(name) - versionOutputBuilder.WriteRune('\n') - } - return versionOutputBuilder.String() -} - -func init() { - version.AddVersionDetail(generateVersionOutput()) -} diff --git a/common/utls/roundtripper.go b/common/utls/roundtripper.go deleted file mode 100644 index 10c9155..0000000 --- a/common/utls/roundtripper.go +++ /dev/null @@ -1,265 +0,0 @@ -package utls - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "golang.org/x/net/proxy" - "net" - "net/http" - "net/url" - "sync" - "time" - - utls "github.com/refraction-networking/utls" - "golang.org/x/net/http2" -) - -// Deprecated: use NewUTLSHTTPRoundTripperWithProxy instead -func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config, - backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper { - return NewUTLSHTTPRoundTripperWithProxy(clientHelloID, uTlsConfig, backdropTransport, removeSNI, nil) -} - -// NewUTLSHTTPRoundTripperWithProxy creates an instance of RoundTripper that dial to remote HTTPS endpoint with -// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint. -// clientHelloID is the clientHello that uTLS attempts to imitate -// uTlsConfig is the TLS Configuration template -// backdropTransport is the transport that will be used for non-https traffic -// removeSNI indicates not to send Server Name Indication Extension -// returns a RoundTripper: its behaviour is documented at -// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/merge_requests/76#note_2777161 -func NewUTLSHTTPRoundTripperWithProxy(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config, - backdropTransport http.RoundTripper, removeSNI bool, proxy *url.URL) http.RoundTripper { - rtImpl := &uTLSHTTPRoundTripperImpl{ - clientHelloID: clientHelloID, - config: uTlsConfig, - connectWithH1: map[string]bool{}, - backdropTransport: backdropTransport, - pendingConn: map[pendingConnKey]*unclaimedConnection{}, - removeSNI: removeSNI, - proxyAddr: proxy, - } - rtImpl.init() - return rtImpl -} - -type uTLSHTTPRoundTripperImpl struct { - clientHelloID utls.ClientHelloID - config *utls.Config - - accessConnectWithH1 sync.Mutex - connectWithH1 map[string]bool - - httpsH1Transport http.RoundTripper - httpsH2Transport http.RoundTripper - backdropTransport http.RoundTripper - - accessDialingConnection sync.Mutex - pendingConn map[pendingConnKey]*unclaimedConnection - - removeSNI bool - proxyAddr *url.URL -} - -type pendingConnKey struct { - isH2 bool - dest string -} - -var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN") -var errEAGAINTooMany = errors.New("incorrect ALPN negotiated") -var errExpired = errors.New("connection have expired") - -func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) { - if req.URL.Scheme != "https" { - return r.backdropTransport.RoundTrip(req) - } - for retryCount := 0; retryCount < 5; retryCount++ { - effectivePort := req.URL.Port() - if effectivePort == "" { - effectivePort = "443" - } - if r.getShouldConnectWithH1(fmt.Sprintf("%v:%v", req.URL.Hostname(), effectivePort)) { - resp, err := r.httpsH1Transport.RoundTrip(req) - if errors.Is(err, errEAGAIN) { - continue - } - return resp, err - } - resp, err := r.httpsH2Transport.RoundTrip(req) - if errors.Is(err, errEAGAIN) { - continue - } - return resp, err - } - return nil, errEAGAINTooMany -} - -func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool { - r.accessConnectWithH1.Lock() - defer r.accessConnectWithH1.Unlock() - if value, set := r.connectWithH1[domainName]; set { - return value - } - return false -} - -func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) { - r.accessConnectWithH1.Lock() - defer r.accessConnectWithH1.Unlock() - r.connectWithH1[domainName] = true -} - -func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) { - r.accessConnectWithH1.Lock() - defer r.accessConnectWithH1.Unlock() - r.connectWithH1[domainName] = false -} - -func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey { - return pendingConnKey{isH2: alpnIsH2, dest: dest} -} - -func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) { - connId := getPendingConnectionID(addr, alpnIsH2) - r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute) -} -func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn { - connId := getPendingConnectionID(addr, alpnIsH2) - if conn, ok := r.pendingConn[connId]; ok { - delete(r.pendingConn, connId) - if claimedConnection, err := conn.claimConnection(); err == nil { - return claimedConnection - } - } - return nil -} -func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) { - r.accessDialingConnection.Lock() - defer r.accessDialingConnection.Unlock() - - if r.getShouldConnectWithH1(addr) == expectedH2 { - return nil, errEAGAIN - } - - //Get a cached connection if possible to reduce preflight connection closed without sending data - if gconn := r.getConn(addr, expectedH2); gconn != nil { - return gconn, nil - } - - conn, err := r.dialTLS(ctx, addr) - if err != nil { - return nil, err - } - - protocol := conn.ConnectionState().NegotiatedProtocol - - protocolIsH2 := protocol == http2.NextProtoTLS - - if protocolIsH2 == expectedH2 { - return conn, err - } - - r.putConn(addr, protocolIsH2, conn) - - if protocolIsH2 { - r.clearShouldConnectWithH1(addr) - } else { - r.setShouldConnectWithH1(addr) - } - - return nil, errEAGAIN -} - -// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c -// by dcf1 -func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) { - config := r.config.Clone() - - host, _, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - config.ServerName = host - - systemDialer := &net.Dialer{} - - var dialer proxy.ContextDialer - dialer = systemDialer - - if r.proxyAddr != nil { - proxyDialer, err := proxy.FromURL(r.proxyAddr, systemDialer) - if err != nil { - return nil, err - } - dialer = proxyDialer.(proxy.ContextDialer) - } - - conn, err := dialer.DialContext(ctx, "tcp", addr) - if err != nil { - return nil, err - } - uconn := utls.UClient(conn, config, r.clientHelloID) - if (net.ParseIP(config.ServerName) != nil) || r.removeSNI { - err := uconn.RemoveSNIExtension() - if err != nil { - uconn.Close() - return nil, err - } - } - - err = uconn.Handshake() - if err != nil { - return nil, err - } - return uconn, nil -} - -func (r *uTLSHTTPRoundTripperImpl) init() { - r.httpsH2Transport = &http2.Transport{ - DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) { - return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true) - }, - } - r.httpsH1Transport = &http.Transport{ - DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) { - return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false) - }, - } -} - -func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection { - c := &unclaimedConnection{ - Conn: conn, - } - time.AfterFunc(expireTime, c.tick) - return c -} - -type unclaimedConnection struct { - net.Conn - claimed bool - access sync.Mutex -} - -func (c *unclaimedConnection) claimConnection() (net.Conn, error) { - c.access.Lock() - defer c.access.Unlock() - if !c.claimed { - c.claimed = true - return c.Conn, nil - } - return nil, errExpired -} - -func (c *unclaimedConnection) tick() { - c.access.Lock() - defer c.access.Unlock() - if !c.claimed { - c.claimed = true - c.Conn.Close() - c.Conn = nil - } -} diff --git a/common/utls/roundtripper_test.go b/common/utls/roundtripper_test.go deleted file mode 100644 index 905b78e..0000000 --- a/common/utls/roundtripper_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package utls - -import ( - "crypto/rsa" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "math/big" - "math/rand" - "net/http" - "os" - "testing" - "time" - - stdcontext "context" - - utls "github.com/refraction-networking/utls" - "golang.org/x/net/http2" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestRoundTripper(t *testing.T) { - runRoundTripperTest(t, "127.0.0.1:23802", "127.0.0.1:23801", "https://127.0.0.1:23802/", "https://127.0.0.1:23801/") -} - -func TestRoundTripperOnH1DefaultPort(t *testing.T) { - if os.Getuid() != 0 { - t.SkipNow() - } - runRoundTripperTest(t, "127.0.0.1:23802", "127.0.0.1:443", "https://127.0.0.1:23802/", "https://127.0.0.1/") -} - -func TestRoundTripperOnH2DefaultPort(t *testing.T) { - if os.Getuid() != 0 { - t.SkipNow() - } - runRoundTripperTest(t, "127.0.0.1:443", "127.0.0.1:23801", "https://127.0.0.1/", "https://127.0.0.1:23801/") -} - -func runRoundTripperTest(t *testing.T, h2listen, h1listen, h2addr, h1addr string) { - var selfSignedCert []byte - var selfSignedPrivateKey *rsa.PrivateKey - httpServerContext, cancel := stdcontext.WithCancel(stdcontext.Background()) - Convey("[Test]Set up http servers", t, func(c C) { - c.Convey("[Test]Generate Self-Signed Cert", func(c C) { - // Ported from https://gist.github.com/samuel/8b500ddd3f6118d052b5e6bc16bc4c09 - - // note that we use the insecure math/rand here because some platforms - // fail the test suite at build time in Debian, due to entropy starvation. - // since that's not a problem at test time, we do *not* use a secure - // mechanism for key generation. - // - // DO NOT REUSE THIS CODE IN PRODUCTION, IT IS DANGEROUS - insecureRandReader := rand.New(rand.NewSource(1337)) - priv, err := rsa.GenerateKey(insecureRandReader, 4096) - c.So(err, ShouldBeNil) - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: "Testing Certificate", - }, - NotBefore: time.Now(), - NotAfter: time.Now().Add(time.Hour * 24 * 180), - - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - } - derBytes, err := x509.CreateCertificate(insecureRandReader, &template, &template, priv.Public(), priv) - c.So(err, ShouldBeNil) - selfSignedPrivateKey = priv - selfSignedCert = derBytes - }) - c.Convey("[Test]Setup http2 server", func(c C) { - listener, err := tls.Listen("tcp", h2listen, &tls.Config{ - NextProtos: []string{http2.NextProtoTLS}, - Certificates: []tls.Certificate{ - tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey}, - }, - }) - c.So(err, ShouldBeNil) - s := http.Server{} - go s.Serve(listener) - go func() { - <-httpServerContext.Done() - s.Close() - }() - }) - c.Convey("[Test]Setup http1 server", func(c C) { - listener, err := tls.Listen("tcp", h1listen, &tls.Config{ - NextProtos: []string{"http/1.1"}, - Certificates: []tls.Certificate{ - tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey}, - }, - }) - c.So(err, ShouldBeNil) - s := http.Server{} - go s.Serve(listener) - go func() { - <-httpServerContext.Done() - s.Close() - }() - }) - }) - for _, v := range []struct { - id utls.ClientHelloID - name string - }{ - { - id: utls.HelloChrome_58, - name: "HelloChrome_58", - }, - { - id: utls.HelloChrome_62, - name: "HelloChrome_62", - }, - { - id: utls.HelloChrome_70, - name: "HelloChrome_70", - }, - { - id: utls.HelloChrome_72, - name: "HelloChrome_72", - }, - { - id: utls.HelloChrome_83, - name: "HelloChrome_83", - }, - { - id: utls.HelloFirefox_55, - name: "HelloFirefox_55", - }, - { - id: utls.HelloFirefox_55, - name: "HelloFirefox_55", - }, - { - id: utls.HelloFirefox_63, - name: "HelloFirefox_63", - }, - { - id: utls.HelloFirefox_65, - name: "HelloFirefox_65", - }, - { - id: utls.HelloIOS_11_1, - name: "HelloIOS_11_1", - }, - { - id: utls.HelloIOS_12_1, - name: "HelloIOS_12_1", - }, - } { - t.Run("Testing fingerprint for "+v.name, func(t *testing.T) { - rtter := NewUTLSHTTPRoundTripper(v.id, &utls.Config{ - InsecureSkipVerify: true, - }, http.DefaultTransport, false) - - for count := 0; count <= 10; count++ { - Convey("HTTP 1.1 Test", t, func(c C) { - { - req, err := http.NewRequest("GET", h2addr, nil) - So(err, ShouldBeNil) - _, err = rtter.RoundTrip(req) - So(err, ShouldBeNil) - } - }) - - Convey("HTTP 2 Test", t, func(c C) { - { - req, err := http.NewRequest("GET", h1addr, nil) - So(err, ShouldBeNil) - _, err = rtter.RoundTrip(req) - So(err, ShouldBeNil) - } - }) - } - }) - } - - cancel() -} diff --git a/go.mod b/go.mod index 3732d08..98dce00 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/xtaci/smux v1.5.33 gitlab.torproject.org/tpo/anti-censorship/geoip v0.0.0-20210928150955-7ce4b3d98d01 gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0 - gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027 + gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500 golang.org/x/crypto v0.32.0 golang.org/x/net v0.34.0 golang.org/x/sys v0.29.0 diff --git a/go.sum b/go.sum index 6ab4b45..cd1ca5b 100644 --- a/go.sum +++ b/go.sum @@ -176,6 +176,8 @@ gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0 h1 gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0/go.mod h1:70bhd4JKW/+1HLfm+TMrgHJsUHG4coelMWwiVEJ2gAg= gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027 h1:zATW8o41V5jE5rkznMl85TbtNqRPMdexGevpjsNxQH4= gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027/go.mod h1:n/u74vECtThx3cvWkCD7j7PRtMb9oBTq33m74g4hL+c= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500 h1:lt8iyIWtJGIF2uOiPtcbhqNOmnrVmi+x04jNwudqPBA= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500/go.mod h1:PK7EvweKeypdelDyh1m7N922aldSeCAG8n0lJ7RAXWQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=