Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP towards removing oohttp and oocrypto #1674

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

WIP towards removing oohttp and oocrypto #1674

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions internal/experiment/echcheck/handshake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,21 @@

import (
"context"
"crypto/rand"
"crypto/tls"
"net"
"time"

"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/logx"
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
utls "gitlab.com/yawning/utls.git"
)

const echExtensionType uint16 = 0xfe0d

func connectAndHandshake(
ctx context.Context,
trace *measurexlite.Trace,
startTime time.Time,
address string, sni string, outerSni string,
logger model.Logger) (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
Expand All @@ -38,6 +36,7 @@
if outerSni == "" {
res = handshake(
ctx,
trace,
conn,
startTime,
address,
Expand All @@ -47,6 +46,7 @@
} else {
res = handshakeWithEch(
ctx,
trace,
conn,
startTime,
address,
Expand All @@ -63,24 +63,15 @@
return channel, nil
}

func handshake(ctx context.Context, conn net.Conn, zeroTime time.Time,
func handshake(ctx context.Context, trace *measurexlite.Trace, conn net.Conn, zeroTime time.Time,
address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
return handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{}, logger)
return doHandshake(ctx, trace, conn, zeroTime, address, sni, []byte{}, logger)
}

func handshakeWithEch(ctx context.Context, conn net.Conn, zeroTime time.Time,
func handshakeWithEch(ctx context.Context, trace *measurexlite.Trace, conn net.Conn, zeroTime time.Time,
address string, sni string, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
payload, err := generateGreaseExtension(rand.Reader)
if err != nil {
panic("failed to generate grease ECH: " + err.Error())
}

var utlsEchExtension utls.GenericExtension

utlsEchExtension.Id = echExtensionType
utlsEchExtension.Data = payload

hs := handshakeWithExtension(ctx, conn, zeroTime, address, sni, []utls.TLSExtension{&utlsEchExtension}, logger)
hs := doHandshake(ctx, trace, conn, zeroTime, address, sni, []byte("ECHCONFIG"), logger)
hs.ECHConfig = "GREASE"
hs.OuterServerName = sni
return hs
Expand All @@ -93,14 +84,18 @@
return ""
}

func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Time, address string, sni string,
extensions []utls.TLSExtension, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
func doHandshake(ctx context.Context, trace *measurexlite.Trace, conn net.Conn, zeroTime time.Time, address string, sni string,
echConfig []byte, logger model.Logger) *model.ArchivalTLSOrQUICHandshakeResult {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
tlsConfig := genTLSConfig(sni)
if len(echConfig) > 0 {
tlsConfig.EncryptedClientHelloConfigList = echConfig

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.22, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.15, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.20, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.18, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.19, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.16, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.21, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / build_with_specific_go_version (1.17, ubuntu-latest) 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / test 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / test 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)

Check failure on line 93 in internal/experiment/echcheck/handshake.go

View workflow job for this annotation

GitHub Actions / measure_coverage 

tlsConfig.EncryptedClientHelloConfigList undefined (type *tls.Config has no field or method EncryptedClientHelloConfigList)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO: figure out if it's possible to pass in here something equivalent to GREASE

}

handshakerConstructor := newHandshakerWithExtensions(extensions)
tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto)
tracedHandshaker := trace.NewTLSHandshakerStdlib(logger)

ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake%s", handshakeMaybePrintWithECH(len(extensions) > 0))
ol := logx.NewOperationLogger(logger, "echcheck: TLSHandshake with ECH")
start := time.Now()
maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig)
finish := time.Now()
Expand Down
6 changes: 3 additions & 3 deletions internal/experiment/echcheck/measure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,17 @@ func (m *Measurer) Run(
handshakes := []func() (chan model.ArchivalTLSOrQUICHandshakeResult, error){
// handshake with ECH disabled and SNI coming from the URL
func() (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
return connectAndHandshake(ctx, args.Measurement.MeasurementStartTimeSaved,
return connectAndHandshake(ctx, trace, args.Measurement.MeasurementStartTimeSaved,
address, parsed.Host, "", args.Session.Logger())
},
// handshake with ECH enabled and ClientHelloOuter SNI coming from the URL
func() (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
return connectAndHandshake(ctx, args.Measurement.MeasurementStartTimeSaved,
return connectAndHandshake(ctx, trace, args.Measurement.MeasurementStartTimeSaved,
address, parsed.Host, parsed.Host, args.Session.Logger())
},
// handshake with ECH enabled and hardcoded different ClientHelloOuter SNI
func() (chan model.ArchivalTLSOrQUICHandshakeResult, error) {
return connectAndHandshake(ctx, args.Measurement.MeasurementStartTimeSaved,
return connectAndHandshake(ctx, trace, args.Measurement.MeasurementStartTimeSaved,
address, parsed.Host, "cloudflare.com", args.Session.Logger())
},
}
Expand Down
49 changes: 0 additions & 49 deletions internal/experiment/echcheck/utls.go
View file
Open in desktop

This file was deleted.

14 changes: 0 additions & 14 deletions internal/experiment/tlsmiddlebox/tracing.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,8 @@ import (
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
utls "gitlab.com/yawning/utls.git"
)

// ClientIDs to map configurable inputs to uTLS fingerprints
// We use a non-zero index to map to each ClientID
var ClientIDs = map[int]*utls.ClientHelloID{
1: &utls.HelloGolang,
2: &utls.HelloChrome_Auto,
3: &utls.HelloFirefox_Auto,
4: &utls.HelloIOS_Auto,
}

// TLSTrace performs tracing using control and target SNI
func (m *Measurer) TLSTrace(ctx context.Context, index int64, zeroTime time.Time, logger model.Logger,
address string, targetSNI string, trace *CompleteTrace) {
Expand Down Expand Up @@ -93,10 +83,6 @@ func (m *Measurer) handshakeWithTTL(ctx context.Context, index int64, zeroTime t
// 3. Perform the handshake and extract the SO_ERROR value (if any)
// Note: we switch to a uTLS Handshaker if the configured ClientID is non-zero
thx := trace.NewTLSHandshakerStdlib(logger)
clientId := m.config.clientid()
if clientId > 0 {
thx = trace.NewTLSHandshakerUTLS(logger, ClientIDs[clientId])
}
_, err = thx.Handshake(ctx, conn, genTLSConfig(sni))
ol.Stop(err)
soErr := extractSoError(conn)
Expand Down
19 changes: 0 additions & 19 deletions internal/measurexlite/utls.go
View file
Open in desktop

This file was deleted.

31 changes: 0 additions & 31 deletions internal/measurexlite/utls_test.go
View file
Open in desktop

This file was deleted.

15 changes: 1 addition & 14 deletions internal/model/netx.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@ import (
"syscall"
"time"

oohttp "github.com/ooni/oohttp"
"github.com/quic-go/quic-go"
utls "gitlab.com/yawning/utls.git"
)

// DNSResponse is a parsed DNS response ready for further processing.
Expand Down Expand Up @@ -225,14 +223,6 @@ type MeasuringNetwork interface {
// that is using the go standard library to manage TLS.
NewTLSHandshakerStdlib(logger DebugLogger) TLSHandshaker

// NewTLSHandshakerUTLS creates a new TLS handshaker using
// gitlab.com/yawning/utls for TLS that implements error wrapping.
//
// The id is the address of something like utls.HelloFirefox_55.
//
// Passing a nil `id` will make this function panic.
NewTLSHandshakerUTLS(logger DebugLogger, id *utls.ClientHelloID) TLSHandshaker

// NewUDPListener creates a new UDPListener with error wrapping.
NewUDPListener() UDPListener
}
Expand Down Expand Up @@ -314,10 +304,7 @@ type Resolver interface {
// kind of TLSConn we're able to use both the standard library
// and gitlab.com/yawning/utls.git to perform TLS operations. Note
// that the stdlib's tls.Conn implements this interface.
type TLSConn = oohttp.TLSConn

// Ensures that a [*tls.Conn] implements the [TLSConn] interface.
var _ TLSConn = &tls.Conn{}
type TLSConn = *tls.Conn

// TLSDialer is a Dialer dialing TLS connections.
type TLSDialer interface {
Expand Down
18 changes: 9 additions & 9 deletions internal/netxlite/httpfactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ package netxlite

import (
"crypto/tls"
"net/http"
"net/url"

oohttp "github.com/ooni/oohttp"
"github.com/ooni/probe-cli/v3/internal/model"
)

// HTTPTransportOption is an initialization option for [NewHTTPTransport].
type HTTPTransportOption func(txp *oohttp.Transport)
type HTTPTransportOption func(txp *http.Transport)

// NewHTTPTransport is the high-level factory to create a [model.HTTPTransport] using
// github.com/ooni/oohttp as the HTTP library with HTTP/1.1 and HTTP2 support.
Expand Down Expand Up @@ -42,7 +42,7 @@ type HTTPTransportOption func(txp *oohttp.Transport)
func NewHTTPTransportWithOptions(logger model.Logger,
dialer model.Dialer, tlsDialer model.TLSDialer, options ...HTTPTransportOption) model.HTTPTransport {
// Using oohttp to support any TLS library.
txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone()
txp := http.DefaultTransport.(*http.Transport).Clone()

// This wrapping ensures that we always have a timeout when we
// are using HTTP; see https://github.com/ooni/probe/issues/1609.
Expand All @@ -63,7 +63,7 @@ func NewHTTPTransportWithOptions(logger model.Logger,

// Return a fully wrapped HTTP transport
return WrapHTTPTransport(logger, &httpTransportConnectionsCloser{
HTTPTransport: &httpTransportStdlib{&oohttp.StdlibTransport{Transport: txp}},
HTTPTransport: &httpTransportStdlib{StdlibTransport: txp},
Dialer: dialer,
TLSDialer: tlsDialer,
})
Expand All @@ -72,8 +72,8 @@ func NewHTTPTransportWithOptions(logger model.Logger,
// HTTPTransportOptionProxyURL configures the transport to use the given proxyURL
// or disables proxying (already the default) if the proxyURL is nil.
func HTTPTransportOptionProxyURL(proxyURL *url.URL) HTTPTransportOption {
return func(txp *oohttp.Transport) {
txp.Proxy = func(r *oohttp.Request) (*url.URL, error) {
return func(txp *http.Transport) {
txp.Proxy = func(r *http.Request) (*url.URL, error) {
// "If Proxy is nil or returns a nil *URL, no proxy is used."
return proxyURL, nil
}
Expand All @@ -83,15 +83,15 @@ func HTTPTransportOptionProxyURL(proxyURL *url.URL) HTTPTransportOption {
// HTTPTransportOptionMaxConnsPerHost configures the .MaxConnPerHosts field, which
// otherwise uses the default set in github.com/ooni/oohttp.
func HTTPTransportOptionMaxConnsPerHost(value int) HTTPTransportOption {
return func(txp *oohttp.Transport) {
return func(txp *http.Transport) {
txp.MaxConnsPerHost = value
}
}

// HTTPTransportOptionDisableCompression configures the .DisableCompression field, which
// otherwise is set to true, so that this code is ready for measuring out of the box.
func HTTPTransportOptionDisableCompression(value bool) HTTPTransportOption {
return func(txp *oohttp.Transport) {
return func(txp *http.Transport) {
txp.DisableCompression = value
}
}
Expand All @@ -104,7 +104,7 @@ func HTTPTransportOptionDisableCompression(value bool) HTTPTransportOption {
// this limitation. Future releases MIGHT use a different technique and, as such,
// we MAY remove this option when we don't need it anymore.
func HTTPTransportOptionTLSClientConfig(config *tls.Config) HTTPTransportOption {
return func(txp *oohttp.Transport) {
return func(txp *http.Transport) {
txp.TLSClientConfig = config
}
}
Loading
Loading