Add utls roundtripper

common/utls/roundtripper.go

@@ -0,0 +1,191 @@
+package utls
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/http"
+	"sync"
+
+	utls "github.com/refraction-networking/utls"
+	"golang.org/x/net/http2"
+)
+
+func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config,
+	backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper {
+	rtImpl := &uTLSHTTPRoundTripperImpl{
+		clientHelloID:     clientHelloID,
+		config:            uTlsConfig,
+		connectWithH1:     map[string]bool{},
+		backdropTransport: backdropTransport,
+		pendingConn:       map[pendingConnKey]net.Conn{},
+		removeSNI:         removeSNI,
+	}
+	rtImpl.init()
+	return rtImpl
+}
+
+type uTLSHTTPRoundTripperImpl struct {
+	clientHelloID utls.ClientHelloID
+	config        *utls.Config
+
+	accessConnectWithH1 sync.Mutex
+	connectWithH1       map[string]bool
+
+	httpsH1Transport  http.RoundTripper
+	httpsH2Transport  http.RoundTripper
+	backdropTransport http.RoundTripper
+
+	accessDialingConnection sync.Mutex
+	pendingConn             map[pendingConnKey]net.Conn
+
+	removeSNI bool
+}
+
+type pendingConnKey struct {
+	isH2 bool
+	dest string
+}
+
+var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN")
+var errEAGAINTooMany = errors.New("incorrect ALPN negotiated")
+
+func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.URL.Scheme != "https" {
+		return r.backdropTransport.RoundTrip(req)
+	}
+	for retryCount := 0; retryCount < 5; retryCount++ {
+		if r.getShouldConnectWithH1(req.URL.Host) {
+			resp, err := r.httpsH1Transport.RoundTrip(req)
+			if errors.Is(err, errEAGAIN) {
+				continue
+			}
+			return resp, err
+		}
+		resp, err := r.httpsH2Transport.RoundTrip(req)
+		if errors.Is(err, errEAGAIN) {
+			continue
+		}
+		return resp, err
+	}
+	return nil, errEAGAINTooMany
+}
+
+func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool {
+	r.accessConnectWithH1.Lock()
+	defer r.accessConnectWithH1.Unlock()
+	if value, set := r.connectWithH1[domainName]; set {
+		return value
+	}
+	return false
+}
+
+func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) {
+	r.accessConnectWithH1.Lock()
+	defer r.accessConnectWithH1.Unlock()
+	r.connectWithH1[domainName] = true
+}
+
+func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) {
+	r.accessConnectWithH1.Lock()
+	defer r.accessConnectWithH1.Unlock()
+	r.connectWithH1[domainName] = false
+}
+
+func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey {
+	return pendingConnKey{isH2: alpnIsH2, dest: dest}
+}
+
+func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) {
+	connId := getPendingConnectionID(addr, alpnIsH2)
+	r.pendingConn[connId] = conn
+}
+func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn {
+	connId := getPendingConnectionID(addr, alpnIsH2)
+	if conn, ok := r.pendingConn[connId]; ok {
+		return conn
+	}
+	return nil
+}
+func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) {
+	r.accessDialingConnection.Lock()
+	defer r.accessDialingConnection.Unlock()
+
+	if r.getShouldConnectWithH1(addr) == expectedH2 {
+		return nil, errEAGAIN
+	}
+
+	//Get a cached connection if possible to reduce preflight connection closed without sending data
+	if gconn := r.getConn(addr, expectedH2); gconn != nil {
+		return gconn, nil
+	}
+
+	conn, err := r.dialTLS(ctx, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	protocol := conn.ConnectionState().NegotiatedProtocol
+
+	protocolIsH2 := protocol == http2.NextProtoTLS
+
+	if protocolIsH2 == expectedH2 {
+		return conn, err
+	}
+
+	r.putConn(addr, protocolIsH2, conn)
+
+	if protocolIsH2 {
+		r.clearShouldConnectWithH1(addr)
+	} else {
+		r.setShouldConnectWithH1(addr)
+	}
+
+	return nil, errEAGAIN
+}
+
+// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c
+// by dcf1
+func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) {
+	config := r.config.Clone()
+
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+	config.ServerName = host
+
+	dialer := &net.Dialer{}
+	conn, err := dialer.DialContext(ctx, "tcp", addr)
+	if err != nil {
+		return nil, err
+	}
+	uconn := utls.UClient(conn, config, r.clientHelloID)
+	if (net.ParseIP(config.ServerName) != nil) || r.removeSNI {
+		err := uconn.RemoveSNIExtension()
+		if err != nil {
+			uconn.Close()
+			return nil, err
+		}
+	}
+
+	err = uconn.Handshake()
+	if err != nil {
+		return nil, err
+	}
+	return uconn, nil
+}
+
+func (r *uTLSHTTPRoundTripperImpl) init() {
+	r.httpsH2Transport = &http2.Transport{
+		DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
+			return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true)
+		},
+	}
+	r.httpsH1Transport = &http.Transport{
+		DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
+			return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false)
+		},
+	}
+}

common/utls/roundtripper_test.go

@@ -0,0 +1,153 @@
+package utls
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	utls "github.com/refraction-networking/utls"
+	"golang.org/x/net/http2"
+	"math/big"
+	"net/http"
+	"testing"
+	"time"
+)
+
+import . "github.com/smartystreets/goconvey/convey"
+
+import stdcontext "context"
+
+func TestRoundTripper(t *testing.T) {
+	var selfSignedCert []byte
+	var selfSignedPrivateKey *rsa.PrivateKey
+	httpServerContext, cancel := stdcontext.WithCancel(stdcontext.Background())
+	Convey("[Test]Set up http servers", t, func(c C) {
+		c.Convey("[Test]Generate Self-Signed Cert", func(c C) {
+			// Ported from https://gist.github.com/samuel/8b500ddd3f6118d052b5e6bc16bc4c09
+			priv, err := rsa.GenerateKey(rand.Reader, 4096)
+			c.So(err, ShouldBeNil)
+			template := x509.Certificate{
+				SerialNumber: big.NewInt(1),
+				Subject: pkix.Name{
+					CommonName: "Testing Certificate",
+				},
+				NotBefore: time.Now(),
+				NotAfter:  time.Now().Add(time.Hour * 24 * 180),
+
+				KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+				ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+				BasicConstraintsValid: true,
+			}
+			derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, priv.Public(), priv)
+			c.So(err, ShouldBeNil)
+			selfSignedPrivateKey = priv
+			selfSignedCert = derBytes
+		})
+		c.Convey("[Test]Setup http2 server", func(c C) {
+			listener, err := tls.Listen("tcp", "127.0.0.1:23802", &tls.Config{
+				NextProtos: []string{http2.NextProtoTLS},
+				Certificates: []tls.Certificate{
+					tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey},
+				},
+			})
+			c.So(err, ShouldBeNil)
+			s := http.Server{}
+			go s.Serve(listener)
+			go func() {
+				<-httpServerContext.Done()
+				s.Close()
+			}()
+		})
+		c.Convey("[Test]Setup http1 server", func(c C) {
+			listener, err := tls.Listen("tcp", "127.0.0.1:23801", &tls.Config{
+				NextProtos: []string{"http/1.1"},
+				Certificates: []tls.Certificate{
+					tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey},
+				},
+			})
+			c.So(err, ShouldBeNil)
+			s := http.Server{}
+			go s.Serve(listener)
+			go func() {
+				<-httpServerContext.Done()
+				s.Close()
+			}()
+		})
+	})
+	for _, v := range []struct {
+		id   utls.ClientHelloID
+		name string
+	}{
+		{
+			id:   utls.HelloChrome_58,
+			name: "HelloChrome_58",
+		},
+		{
+			id:   utls.HelloChrome_62,
+			name: "HelloChrome_62",
+		},
+		{
+			id:   utls.HelloChrome_70,
+			name: "HelloChrome_70",
+		},
+		{
+			id:   utls.HelloChrome_72,
+			name: "HelloChrome_72",
+		},
+		{
+			id:   utls.HelloChrome_83,
+			name: "HelloChrome_83",
+		},
+		{
+			id:   utls.HelloFirefox_55,
+			name: "HelloFirefox_55",
+		},
+		{
+			id:   utls.HelloFirefox_55,
+			name: "HelloFirefox_55",
+		},
+		{
+			id:   utls.HelloFirefox_63,
+			name: "HelloFirefox_63",
+		},
+		{
+			id:   utls.HelloFirefox_65,
+			name: "HelloFirefox_65",
+		},
+		{
+			id:   utls.HelloIOS_11_1,
+			name: "HelloIOS_11_1",
+		},
+		{
+			id:   utls.HelloIOS_12_1,
+			name: "HelloIOS_12_1",
+		},
+	} {
+		t.Run("Testing fingerprint for "+v.name, func(t *testing.T) {
+			rtter := NewUTLSHTTPRoundTripper(v.id, &utls.Config{
+				InsecureSkipVerify: true,
+			}, http.DefaultTransport)
+
+			Convey("HTTP 1.1 Test", t, func(c C) {
+				{
+					req, err := http.NewRequest("GET", "https://127.0.0.1:23801/", nil)
+					So(err, ShouldBeNil)
+					_, err = rtter.RoundTrip(req)
+					So(err, ShouldBeNil)
+				}
+			})
+
+			Convey("HTTP 2 Test", t, func(c C) {
+				{
+					req, err := http.NewRequest("GET", "https://127.0.0.1:23802/", nil)
+					So(err, ShouldBeNil)
+					_, err = rtter.RoundTrip(req)
+					So(err, ShouldBeNil)
+				}
+			})
+		})
+	}
+
+	cancel()
+}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/pion/webrtc/v3 v3.0.15
 	github.com/prometheus/client_golang v1.10.0
 	github.com/prometheus/client_model v0.2.0
+	github.com/refraction-networking/utls v1.0.0 // indirect
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/xtaci/kcp-go/v5 v5.6.1

go.sum

@@ -302,6 +302,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/refraction-networking/utls v1.0.0 h1:6XQHSjDmeBCF9sPq8p2zMVGq7Ud3rTD2q88Fw8Tz1tA=
+github.com/refraction-networking/utls v1.0.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=