Remove utls library from snowflake and Use ptuil/utls

This commit is contained in:
onyinyang 2025-01-29 10:59:31 -05:00
parent 0dee9d68bd
commit 26f7ee4b06
No known key found for this signature in database
GPG key ID: 156A6435430C2036
7 changed files with 4 additions and 520 deletions

View file

@ -16,12 +16,12 @@ import (
"github.com/pion/webrtc/v4"
utls "github.com/refraction-networking/utls"
utlsutil "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil/utls"
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/certs"
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/event"
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/messages"
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/nat"
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/util"
utlsutil "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/utls"
)
const (

View file

@ -1,46 +0,0 @@
package utls
import (
"errors"
utls "github.com/refraction-networking/utls"
"strings"
)
// ported from https://github.com/max-b/snowflake/commit/9dded063cb74c6941a16ad90b9dd0e06e618e55e
var clientHelloIDMap = map[string]utls.ClientHelloID{
// No HelloCustom: not useful for external configuration.
// No HelloRandomized: doesn't negotiate consistent ALPN.
"hellorandomizedalpn": utls.HelloRandomizedALPN,
"hellorandomizednoalpn": utls.HelloRandomizedNoALPN,
"hellofirefox_auto": utls.HelloFirefox_Auto,
"hellofirefox_55": utls.HelloFirefox_55,
"hellofirefox_56": utls.HelloFirefox_56,
"hellofirefox_63": utls.HelloFirefox_63,
"hellofirefox_65": utls.HelloFirefox_65,
"hellochrome_auto": utls.HelloChrome_Auto,
"hellochrome_58": utls.HelloChrome_58,
"hellochrome_62": utls.HelloChrome_62,
"hellochrome_70": utls.HelloChrome_70,
"hellochrome_72": utls.HelloChrome_72,
"helloios_auto": utls.HelloIOS_Auto,
"helloios_11_1": utls.HelloIOS_11_1,
"helloios_12_1": utls.HelloIOS_12_1,
}
var errNameNotFound = errors.New("client hello name is unrecognized")
func NameToUTLSID(name string) (utls.ClientHelloID, error) {
normalizedName := strings.ToLower(name)
if id, ok := clientHelloIDMap[normalizedName]; ok {
return id, nil
}
return utls.ClientHelloID{}, errNameNotFound
}
func ListAllNames() []string {
var names []string
for k, _ := range clientHelloIDMap {
names = append(names, k)
}
return names
}

View file

@ -1,24 +0,0 @@
package utls
import (
"gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/common/version"
"strings"
)
func generateVersionOutput() string {
var versionOutputBuilder strings.Builder
versionOutputBuilder.WriteString(`Known utls-imitate values:
(empty)
`)
for _, name := range ListAllNames() {
versionOutputBuilder.WriteString(name)
versionOutputBuilder.WriteRune('\n')
}
return versionOutputBuilder.String()
}
func init() {
version.AddVersionDetail(generateVersionOutput())
}

View file

@ -1,265 +0,0 @@
package utls
import (
"context"
"crypto/tls"
"errors"
"fmt"
"golang.org/x/net/proxy"
"net"
"net/http"
"net/url"
"sync"
"time"
utls "github.com/refraction-networking/utls"
"golang.org/x/net/http2"
)
// Deprecated: use NewUTLSHTTPRoundTripperWithProxy instead
func NewUTLSHTTPRoundTripper(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config,
backdropTransport http.RoundTripper, removeSNI bool) http.RoundTripper {
return NewUTLSHTTPRoundTripperWithProxy(clientHelloID, uTlsConfig, backdropTransport, removeSNI, nil)
}
// NewUTLSHTTPRoundTripperWithProxy creates an instance of RoundTripper that dial to remote HTTPS endpoint with
// an alternative version of TLS implementation that attempts to imitate browsers' fingerprint.
// clientHelloID is the clientHello that uTLS attempts to imitate
// uTlsConfig is the TLS Configuration template
// backdropTransport is the transport that will be used for non-https traffic
// removeSNI indicates not to send Server Name Indication Extension
// returns a RoundTripper: its behaviour is documented at
// https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/merge_requests/76#note_2777161
func NewUTLSHTTPRoundTripperWithProxy(clientHelloID utls.ClientHelloID, uTlsConfig *utls.Config,
backdropTransport http.RoundTripper, removeSNI bool, proxy *url.URL) http.RoundTripper {
rtImpl := &uTLSHTTPRoundTripperImpl{
clientHelloID: clientHelloID,
config: uTlsConfig,
connectWithH1: map[string]bool{},
backdropTransport: backdropTransport,
pendingConn: map[pendingConnKey]*unclaimedConnection{},
removeSNI: removeSNI,
proxyAddr: proxy,
}
rtImpl.init()
return rtImpl
}
type uTLSHTTPRoundTripperImpl struct {
clientHelloID utls.ClientHelloID
config *utls.Config
accessConnectWithH1 sync.Mutex
connectWithH1 map[string]bool
httpsH1Transport http.RoundTripper
httpsH2Transport http.RoundTripper
backdropTransport http.RoundTripper
accessDialingConnection sync.Mutex
pendingConn map[pendingConnKey]*unclaimedConnection
removeSNI bool
proxyAddr *url.URL
}
type pendingConnKey struct {
isH2 bool
dest string
}
var errEAGAIN = errors.New("incorrect ALPN negotiated, try again with another ALPN")
var errEAGAINTooMany = errors.New("incorrect ALPN negotiated")
var errExpired = errors.New("connection have expired")
func (r *uTLSHTTPRoundTripperImpl) RoundTrip(req *http.Request) (*http.Response, error) {
if req.URL.Scheme != "https" {
return r.backdropTransport.RoundTrip(req)
}
for retryCount := 0; retryCount < 5; retryCount++ {
effectivePort := req.URL.Port()
if effectivePort == "" {
effectivePort = "443"
}
if r.getShouldConnectWithH1(fmt.Sprintf("%v:%v", req.URL.Hostname(), effectivePort)) {
resp, err := r.httpsH1Transport.RoundTrip(req)
if errors.Is(err, errEAGAIN) {
continue
}
return resp, err
}
resp, err := r.httpsH2Transport.RoundTrip(req)
if errors.Is(err, errEAGAIN) {
continue
}
return resp, err
}
return nil, errEAGAINTooMany
}
func (r *uTLSHTTPRoundTripperImpl) getShouldConnectWithH1(domainName string) bool {
r.accessConnectWithH1.Lock()
defer r.accessConnectWithH1.Unlock()
if value, set := r.connectWithH1[domainName]; set {
return value
}
return false
}
func (r *uTLSHTTPRoundTripperImpl) setShouldConnectWithH1(domainName string) {
r.accessConnectWithH1.Lock()
defer r.accessConnectWithH1.Unlock()
r.connectWithH1[domainName] = true
}
func (r *uTLSHTTPRoundTripperImpl) clearShouldConnectWithH1(domainName string) {
r.accessConnectWithH1.Lock()
defer r.accessConnectWithH1.Unlock()
r.connectWithH1[domainName] = false
}
func getPendingConnectionID(dest string, alpnIsH2 bool) pendingConnKey {
return pendingConnKey{isH2: alpnIsH2, dest: dest}
}
func (r *uTLSHTTPRoundTripperImpl) putConn(addr string, alpnIsH2 bool, conn net.Conn) {
connId := getPendingConnectionID(addr, alpnIsH2)
r.pendingConn[connId] = NewUnclaimedConnection(conn, time.Minute)
}
func (r *uTLSHTTPRoundTripperImpl) getConn(addr string, alpnIsH2 bool) net.Conn {
connId := getPendingConnectionID(addr, alpnIsH2)
if conn, ok := r.pendingConn[connId]; ok {
delete(r.pendingConn, connId)
if claimedConnection, err := conn.claimConnection(); err == nil {
return claimedConnection
}
}
return nil
}
func (r *uTLSHTTPRoundTripperImpl) dialOrGetTLSWithExpectedALPN(ctx context.Context, addr string, expectedH2 bool) (net.Conn, error) {
r.accessDialingConnection.Lock()
defer r.accessDialingConnection.Unlock()
if r.getShouldConnectWithH1(addr) == expectedH2 {
return nil, errEAGAIN
}
//Get a cached connection if possible to reduce preflight connection closed without sending data
if gconn := r.getConn(addr, expectedH2); gconn != nil {
return gconn, nil
}
conn, err := r.dialTLS(ctx, addr)
if err != nil {
return nil, err
}
protocol := conn.ConnectionState().NegotiatedProtocol
protocolIsH2 := protocol == http2.NextProtoTLS
if protocolIsH2 == expectedH2 {
return conn, err
}
r.putConn(addr, protocolIsH2, conn)
if protocolIsH2 {
r.clearShouldConnectWithH1(addr)
} else {
r.setShouldConnectWithH1(addr)
}
return nil, errEAGAIN
}
// based on https://repo.or.cz/dnstt.git/commitdiff/d92a791b6864901f9263f7d73d97cfd30ac53b09..98bdffa1706dfc041d1e99b86c47f29d72ad3a0c
// by dcf1
func (r *uTLSHTTPRoundTripperImpl) dialTLS(ctx context.Context, addr string) (*utls.UConn, error) {
config := r.config.Clone()
host, _, err := net.SplitHostPort(addr)
if err != nil {
return nil, err
}
config.ServerName = host
systemDialer := &net.Dialer{}
var dialer proxy.ContextDialer
dialer = systemDialer
if r.proxyAddr != nil {
proxyDialer, err := proxy.FromURL(r.proxyAddr, systemDialer)
if err != nil {
return nil, err
}
dialer = proxyDialer.(proxy.ContextDialer)
}
conn, err := dialer.DialContext(ctx, "tcp", addr)
if err != nil {
return nil, err
}
uconn := utls.UClient(conn, config, r.clientHelloID)
if (net.ParseIP(config.ServerName) != nil) || r.removeSNI {
err := uconn.RemoveSNIExtension()
if err != nil {
uconn.Close()
return nil, err
}
}
err = uconn.Handshake()
if err != nil {
return nil, err
}
return uconn, nil
}
func (r *uTLSHTTPRoundTripperImpl) init() {
r.httpsH2Transport = &http2.Transport{
DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
return r.dialOrGetTLSWithExpectedALPN(context.Background(), addr, true)
},
}
r.httpsH1Transport = &http.Transport{
DialTLSContext: func(ctx context.Context, network string, addr string) (net.Conn, error) {
return r.dialOrGetTLSWithExpectedALPN(ctx, addr, false)
},
}
}
func NewUnclaimedConnection(conn net.Conn, expireTime time.Duration) *unclaimedConnection {
c := &unclaimedConnection{
Conn: conn,
}
time.AfterFunc(expireTime, c.tick)
return c
}
type unclaimedConnection struct {
net.Conn
claimed bool
access sync.Mutex
}
func (c *unclaimedConnection) claimConnection() (net.Conn, error) {
c.access.Lock()
defer c.access.Unlock()
if !c.claimed {
c.claimed = true
return c.Conn, nil
}
return nil, errExpired
}
func (c *unclaimedConnection) tick() {
c.access.Lock()
defer c.access.Unlock()
if !c.claimed {
c.claimed = true
c.Conn.Close()
c.Conn = nil
}
}

View file

@ -1,183 +0,0 @@
package utls
import (
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"math/big"
"math/rand"
"net/http"
"os"
"testing"
"time"
stdcontext "context"
utls "github.com/refraction-networking/utls"
"golang.org/x/net/http2"
. "github.com/smartystreets/goconvey/convey"
)
func TestRoundTripper(t *testing.T) {
runRoundTripperTest(t, "127.0.0.1:23802", "127.0.0.1:23801", "https://127.0.0.1:23802/", "https://127.0.0.1:23801/")
}
func TestRoundTripperOnH1DefaultPort(t *testing.T) {
if os.Getuid() != 0 {
t.SkipNow()
}
runRoundTripperTest(t, "127.0.0.1:23802", "127.0.0.1:443", "https://127.0.0.1:23802/", "https://127.0.0.1/")
}
func TestRoundTripperOnH2DefaultPort(t *testing.T) {
if os.Getuid() != 0 {
t.SkipNow()
}
runRoundTripperTest(t, "127.0.0.1:443", "127.0.0.1:23801", "https://127.0.0.1/", "https://127.0.0.1:23801/")
}
func runRoundTripperTest(t *testing.T, h2listen, h1listen, h2addr, h1addr string) {
var selfSignedCert []byte
var selfSignedPrivateKey *rsa.PrivateKey
httpServerContext, cancel := stdcontext.WithCancel(stdcontext.Background())
Convey("[Test]Set up http servers", t, func(c C) {
c.Convey("[Test]Generate Self-Signed Cert", func(c C) {
// Ported from https://gist.github.com/samuel/8b500ddd3f6118d052b5e6bc16bc4c09
// note that we use the insecure math/rand here because some platforms
// fail the test suite at build time in Debian, due to entropy starvation.
// since that's not a problem at test time, we do *not* use a secure
// mechanism for key generation.
//
// DO NOT REUSE THIS CODE IN PRODUCTION, IT IS DANGEROUS
insecureRandReader := rand.New(rand.NewSource(1337))
priv, err := rsa.GenerateKey(insecureRandReader, 4096)
c.So(err, ShouldBeNil)
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
CommonName: "Testing Certificate",
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour * 24 * 180),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
}
derBytes, err := x509.CreateCertificate(insecureRandReader, &template, &template, priv.Public(), priv)
c.So(err, ShouldBeNil)
selfSignedPrivateKey = priv
selfSignedCert = derBytes
})
c.Convey("[Test]Setup http2 server", func(c C) {
listener, err := tls.Listen("tcp", h2listen, &tls.Config{
NextProtos: []string{http2.NextProtoTLS},
Certificates: []tls.Certificate{
tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey},
},
})
c.So(err, ShouldBeNil)
s := http.Server{}
go s.Serve(listener)
go func() {
<-httpServerContext.Done()
s.Close()
}()
})
c.Convey("[Test]Setup http1 server", func(c C) {
listener, err := tls.Listen("tcp", h1listen, &tls.Config{
NextProtos: []string{"http/1.1"},
Certificates: []tls.Certificate{
tls.Certificate{Certificate: [][]byte{selfSignedCert}, PrivateKey: selfSignedPrivateKey},
},
})
c.So(err, ShouldBeNil)
s := http.Server{}
go s.Serve(listener)
go func() {
<-httpServerContext.Done()
s.Close()
}()
})
})
for _, v := range []struct {
id utls.ClientHelloID
name string
}{
{
id: utls.HelloChrome_58,
name: "HelloChrome_58",
},
{
id: utls.HelloChrome_62,
name: "HelloChrome_62",
},
{
id: utls.HelloChrome_70,
name: "HelloChrome_70",
},
{
id: utls.HelloChrome_72,
name: "HelloChrome_72",
},
{
id: utls.HelloChrome_83,
name: "HelloChrome_83",
},
{
id: utls.HelloFirefox_55,
name: "HelloFirefox_55",
},
{
id: utls.HelloFirefox_55,
name: "HelloFirefox_55",
},
{
id: utls.HelloFirefox_63,
name: "HelloFirefox_63",
},
{
id: utls.HelloFirefox_65,
name: "HelloFirefox_65",
},
{
id: utls.HelloIOS_11_1,
name: "HelloIOS_11_1",
},
{
id: utls.HelloIOS_12_1,
name: "HelloIOS_12_1",
},
} {
t.Run("Testing fingerprint for "+v.name, func(t *testing.T) {
rtter := NewUTLSHTTPRoundTripper(v.id, &utls.Config{
InsecureSkipVerify: true,
}, http.DefaultTransport, false)
for count := 0; count <= 10; count++ {
Convey("HTTP 1.1 Test", t, func(c C) {
{
req, err := http.NewRequest("GET", h2addr, nil)
So(err, ShouldBeNil)
_, err = rtter.RoundTrip(req)
So(err, ShouldBeNil)
}
})
Convey("HTTP 2 Test", t, func(c C) {
{
req, err := http.NewRequest("GET", h1addr, nil)
So(err, ShouldBeNil)
_, err = rtter.RoundTrip(req)
So(err, ShouldBeNil)
}
})
}
})
}
cancel()
}

2
go.mod
View file

@ -25,7 +25,7 @@ require (
github.com/xtaci/smux v1.5.33
gitlab.torproject.org/tpo/anti-censorship/geoip v0.0.0-20210928150955-7ce4b3d98d01
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500
golang.org/x/crypto v0.32.0
golang.org/x/net v0.34.0
golang.org/x/sys v0.29.0

2
go.sum
View file

@ -176,6 +176,8 @@ gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0 h1
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.6.0/go.mod h1:70bhd4JKW/+1HLfm+TMrgHJsUHG4coelMWwiVEJ2gAg=
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027 h1:zATW8o41V5jE5rkznMl85TbtNqRPMdexGevpjsNxQH4=
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20240710081135-6c4d8ed41027/go.mod h1:n/u74vECtThx3cvWkCD7j7PRtMb9oBTq33m74g4hL+c=
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500 h1:lt8iyIWtJGIF2uOiPtcbhqNOmnrVmi+x04jNwudqPBA=
gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/ptutil v0.0.0-20250129175826-48a566259500/go.mod h1:PK7EvweKeypdelDyh1m7N922aldSeCAG8n0lJ7RAXWQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=