From 6a78f5e5a6701b046f080e67eebfd2f7cd9acbea Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Sep 2023 17:44:34 +0200 Subject: [PATCH] feat(enginenetx): extend beacons policy for THs (#1318) Provided that we get correct IP addresses, which is a big IF, we can try to avoid using offending SNIs when using the THs. Part of https://github.com/ooni/probe/issues/2531 --- internal/enginenetx/beaconspolicy.go | 74 ++++++++++++++++++++--- internal/enginenetx/beaconspolicy_test.go | 57 +++++++++++++++-- 2 files changed, 115 insertions(+), 16 deletions(-) diff --git a/internal/enginenetx/beaconspolicy.go b/internal/enginenetx/beaconspolicy.go index a2a1fbcf4a..ab1994a9d9 100644 --- a/internal/enginenetx/beaconspolicy.go +++ b/internal/enginenetx/beaconspolicy.go @@ -35,7 +35,7 @@ func (p *beaconsPolicy) LookupTactics(ctx context.Context, domain, port string) // emit beacons related tactics first which are empty if there are // no beacons for the givend domain and port - for tx := range p.tacticsForDomain(domain, port) { + for tx := range p.beaconsTacticsForDomain(domain, port) { tx.InitialDelay = happyEyeballsDelay(index) index += 1 out <- tx @@ -43,7 +43,10 @@ func (p *beaconsPolicy) LookupTactics(ctx context.Context, domain, port string) // now fallback to get more tactics (typically here the fallback // uses the DNS and obtains some extra tactics) - for tx := range p.Fallback.LookupTactics(ctx, domain, port) { + // + // we wrap whatever the underlying policy returns us with some + // extra logic for better communicating with test helpers + for tx := range p.maybeRewriteTestHelpersTactics(p.Fallback.LookupTactics(ctx, domain, port)) { tx.InitialDelay = happyEyeballsDelay(index) index += 1 out <- tx @@ -53,7 +56,55 @@ func (p *beaconsPolicy) LookupTactics(ctx context.Context, domain, port string) return out } -func (p *beaconsPolicy) tacticsForDomain(domain, port string) <-chan *httpsDialerTactic { +var beaconsPolicyTestHelpersDomains = []string{ + "0.th.ooni.org", + "1.th.ooni.org", + "2.th.ooni.org", + "3.th.ooni.org", + "d33d1gs9kpq1c5.cloudfront.net", +} + +// TODO(bassosimone): this would be slices.Contains when we'll use go1.21 +func beaconsPolicySlicesContains(slice []string, value string) bool { + for _, entry := range slice { + if value == entry { + return true + } + } + return false +} + +func (p *beaconsPolicy) maybeRewriteTestHelpersTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + defer close(out) // tell the parent when we're done + + for tactic := range input { + // When we're not connecting to a TH, pass the policy down the chain unmodified + if !beaconsPolicySlicesContains(beaconsPolicyTestHelpersDomains, tactic.VerifyHostname) { + out <- tactic + continue + } + + // This is the case where we're connecting to a test helper. Let's try + // to produce policies hiding the SNI to censoring middleboxes. + for _, sni := range p.beaconsDomainsInRandomOrder() { + out <- &httpsDialerTactic{ + Address: tactic.Address, + InitialDelay: 0, + Port: tactic.Port, + SNI: sni, + VerifyHostname: tactic.VerifyHostname, + } + } + } + }() + + return out +} + +func (p *beaconsPolicy) beaconsTacticsForDomain(domain, port string) <-chan *httpsDialerTactic { out := make(chan *httpsDialerTactic) go func() { @@ -64,14 +115,8 @@ func (p *beaconsPolicy) tacticsForDomain(domain, port string) <-chan *httpsDiale return } - snis := p.beaconsDomains() - r := rand.New(rand.NewSource(time.Now().UnixNano())) - r.Shuffle(len(snis), func(i, j int) { - snis[i], snis[j] = snis[j], snis[i] - }) - for _, ipAddr := range p.beaconsAddrs() { - for _, sni := range snis { + for _, sni := range p.beaconsDomainsInRandomOrder() { out <- &httpsDialerTactic{ Address: ipAddr, InitialDelay: 0, @@ -86,6 +131,15 @@ func (p *beaconsPolicy) tacticsForDomain(domain, port string) <-chan *httpsDiale return out } +func (p *beaconsPolicy) beaconsDomainsInRandomOrder() (out []string) { + out = p.beaconsDomains() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Shuffle(len(out), func(i, j int) { + out[i], out[j] = out[j], out[i] + }) + return +} + func (p *beaconsPolicy) beaconsAddrs() (out []string) { return append( out, diff --git a/internal/enginenetx/beaconspolicy_test.go b/internal/enginenetx/beaconspolicy_test.go index fadd041a55..f371e5bcc2 100644 --- a/internal/enginenetx/beaconspolicy_test.go +++ b/internal/enginenetx/beaconspolicy_test.go @@ -12,7 +12,7 @@ import ( func TestBeaconsPolicy(t *testing.T) { t.Run("for domains for which we don't have beacons and DNS failure", func(t *testing.T) { expected := errors.New("mocked error") - policy := &beaconsPolicy{ + p := &beaconsPolicy{ Fallback: &dnsPolicy{ Logger: model.DiscardLogger, Resolver: &mocks.Resolver{ @@ -24,7 +24,7 @@ func TestBeaconsPolicy(t *testing.T) { } ctx := context.Background() - tactics := policy.LookupTactics(ctx, "www.example.com", "443") + tactics := p.LookupTactics(ctx, "www.example.com", "443") var count int for range tactics { @@ -37,7 +37,7 @@ func TestBeaconsPolicy(t *testing.T) { }) t.Run("for domains for which we don't have beacons and DNS success", func(t *testing.T) { - policy := &beaconsPolicy{ + p := &beaconsPolicy{ Fallback: &dnsPolicy{ Logger: model.DiscardLogger, Resolver: &mocks.Resolver{ @@ -49,7 +49,7 @@ func TestBeaconsPolicy(t *testing.T) { } ctx := context.Background() - tactics := policy.LookupTactics(ctx, "www.example.com", "443") + tactics := p.LookupTactics(ctx, "www.example.com", "443") var count int for tactic := range tactics { @@ -78,7 +78,7 @@ func TestBeaconsPolicy(t *testing.T) { t.Run("for the api.ooni.io domain", func(t *testing.T) { expected := errors.New("mocked error") - policy := &beaconsPolicy{ + p := &beaconsPolicy{ Fallback: &dnsPolicy{ Logger: model.DiscardLogger, Resolver: &mocks.Resolver{ @@ -90,7 +90,7 @@ func TestBeaconsPolicy(t *testing.T) { } ctx := context.Background() - tactics := policy.LookupTactics(ctx, "api.ooni.io", "443") + tactics := p.LookupTactics(ctx, "api.ooni.io", "443") var count int for tactic := range tactics { @@ -116,4 +116,49 @@ func TestBeaconsPolicy(t *testing.T) { t.Fatal("expected to see at least one tactic") } }) + + t.Run("for test helper domains", func(t *testing.T) { + for _, domain := range beaconsPolicyTestHelpersDomains { + t.Run(domain, func(t *testing.T) { + expectedAddrs := []string{"164.92.180.7"} + + p := &beaconsPolicy{ + Fallback: &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return expectedAddrs, nil + }, + }, + }, + } + + ctx := context.Background() + index := 0 + for tactics := range p.LookupTactics(ctx, domain, "443") { + + if tactics.Address != "164.92.180.7" { + t.Fatal("unexpected .Address") + } + + if tactics.InitialDelay != happyEyeballsDelay(index) { + t.Fatal("unexpected .InitialDelay") + } + index++ + + if tactics.Port != "443" { + t.Fatal("unexpected .Port") + } + + if tactics.SNI == domain { + t.Fatal("unexpected .Domain") + } + + if tactics.VerifyHostname != domain { + t.Fatal("unexpected .VerifyHostname") + } + } + }) + } + }) }