fix(webconnectivitylte): add classic and tcptls_experiment tags (ooni…

…#1502)

Closes ooni/probe#2673

Part of ooni/probe#2666

internal/experiment/webconnectivitylte/cleartextflow.go

@@ -8,7 +8,6 @@ package webconnectivitylte
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -32,6 +31,9 @@ type CleartextFlow struct {
 	// Address is the MANDATORY address to connect to.
 	Address string
 
+	// Classic is true if this address was discovered using getaddrinfo.
+	Classic bool
+
 	// DNSCache is the MANDATORY DNS cache.
 	DNSCache *DNSCache
 
@@ -106,8 +108,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error {
 	}
 
 	// create trace
-	trace := measurexlite.NewTrace(index, t.ZeroTime, fmt.Sprintf("depth=%d", t.Depth),
-		fmt.Sprintf("fetch_body=%v", t.PrioSelector != nil))
+	trace := measurexlite.NewTrace(index, t.ZeroTime, generateTagsForEndpoints(t.Depth, t.PrioSelector, t.Classic)...)
 
 	// start measuring throttling
 	sampler := throttling.NewSampler(trace)

internal/experiment/webconnectivitylte/dnsresolvers.go

@@ -223,7 +223,7 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- []
 	index := t.IDGenerator.NewIDForGetaddrinfo()
 
 	// create trace
-	trace := measurexlite.NewTrace(index, t.ZeroTime, fmt.Sprintf("depth=%d", t.Depth))
+	trace := measurexlite.NewTrace(index, t.ZeroTime, "classic", fmt.Sprintf("depth=%d", t.Depth))
 
 	// start the operation logger
 	ol := logx.NewOperationLogger(
@@ -394,6 +394,7 @@ func (t *DNSResolvers) startCleartextFlows(
 		MaybeDelayCleartextFlows(index) // allow specific callers to space flows apart
 		task := &CleartextFlow{
 			Address:                 net.JoinHostPort(addr.Addr, port),
+			Classic:                 addr.Flags&DNSAddrFlagSystemResolver != 0,
 			Depth:                   t.Depth,
 			DNSCache:                t.DNSCache,
 			DNSOverHTTPSURLProvider: t.DNSOverHTTPSURLProvider,
@@ -444,6 +445,7 @@ func (t *DNSResolvers) startSecureFlows(
 		MaybeDelaySecureFlows(index) // allow specific callers to space flows apart
 		task := &SecureFlow{
 			Address:                 net.JoinHostPort(addr.Addr, port),
+			Classic:                 addr.Flags&DNSAddrFlagSystemResolver != 0,
 			Depth:                   t.Depth,
 			DNSCache:                t.DNSCache,
 			DNSOverHTTPSURLProvider: t.DNSOverHTTPSURLProvider,

internal/experiment/webconnectivitylte/secureflow.go

@@ -9,7 +9,6 @@ package webconnectivitylte
 import (
 	"context"
 	"crypto/tls"
-	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -33,6 +32,9 @@ type SecureFlow struct {
 	// Address is the MANDATORY address to connect to.
 	Address string
 
+	// Classic is true if this address was discovered using getaddrinfo.
+	Classic bool
+
 	// DNSCache is the MANDATORY DNS cache.
 	DNSCache *DNSCache
 
@@ -113,8 +115,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error {
 	}
 
 	// create trace
-	trace := measurexlite.NewTrace(index, t.ZeroTime, fmt.Sprintf("depth=%d", t.Depth),
-		fmt.Sprintf("fetch_body=%v", t.PrioSelector != nil))
+	trace := measurexlite.NewTrace(index, t.ZeroTime, generateTagsForEndpoints(t.Depth, t.PrioSelector, t.Classic)...)
 
 	// start measuring throttling
 	sampler := throttling.NewSampler(trace)

internal/experiment/webconnectivitylte/tags.go

@@ -0,0 +1,31 @@
+package webconnectivitylte
+
+import "fmt"
+
+// generateTagsForEndpoints generates the tags for the endpoints.
+func generateTagsForEndpoints(depth int64, ps *prioritySelector, classic bool) (output []string) {
+	// The classic flag marks all observations using IP addresses
+	// fetched using the system resolver. Strictly speaking classic
+	// means that these measurements derive from the resolver that
+	// we consider primary, and for us it it the system one.
+	if classic {
+		output = append(output, "classic")
+	}
+
+	// The depth=0|1|... tag indicates the current redirect depth.
+	//
+	// When the depth is zero, we also include the tcptls_experiment tag
+	// for backwards compatibility with Web Connectivity v0.4.
+	if depth < 1 {
+		output = append(output, "tcptls_experiment")
+	}
+	output = append(output, fmt.Sprintf("depth=%d", depth))
+
+	// The fetch_body=true|false tag allows to distinguish between observations
+	// with the objective of fetching the body and extra observations. For example,
+	// for http:// requests we perform TLS handshakes for the purpose of checking
+	// whether IP addresses are valid without fetching the body.
+	output = append(output, fmt.Sprintf("fetch_body=%v", ps != nil))
+
+	return output
+}

internal/minipipeline/classic.go

@@ -14,6 +14,8 @@ package minipipeline
 //
 // The result should approximate what v0.4 would have measured.
 func ClassicFilter(input *WebObservationsContainer) (output *WebObservationsContainer) {
+	// TODO(bassosimone): now that there's a "classic" tag it would probably
+	// be simpler to just always use the "classic" tag to extract.
 	output = &WebObservationsContainer{
 		DNSLookupFailures:  []*WebObservation{},
 		DNSLookupSuccesses: []*WebObservation{},

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithExpiredCertificate/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -148,6 +149,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -166,6 +169,8 @@
         "server_name": "expired.badssl.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithConsistentDNS/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -148,6 +149,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -166,6 +169,8 @@
         "server_name": "untrusted-root.badssl.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithUnknownAuthorityWithInconsistentDNS/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -215,6 +216,7 @@
         },
         "t": 0,
         "tags": [
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -231,6 +233,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -245,6 +249,7 @@
         },
         "t": 0,
         "tags": [
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -263,6 +268,8 @@
         "server_name": "www.example.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -280,6 +287,7 @@
         "server_name": "www.example.com",
         "t": 0,
         "tags": [
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],

internal/minipipeline/testdata/webconnectivity/generated/badSSLWithWrongServerName/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -148,6 +149,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -166,6 +169,8 @@
         "server_name": "wrong.host.badssl.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],

internal/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTP/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -249,6 +250,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -265,6 +268,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -279,6 +284,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=false"
         ],
@@ -297,6 +304,8 @@
         "server_name": "www.cloudflare-cache.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=false"
         ],

internal/minipipeline/testdata/webconnectivity/generated/cloudflareCAPTCHAWithHTTPS/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -250,6 +251,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -266,6 +269,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -284,6 +289,8 @@
         "server_name": "www.cloudflare-cache.com",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],

internal/minipipeline/testdata/webconnectivity/generated/controlFailureWithSuccessfulHTTPSWebsite/measurement.json

@@ -94,6 +94,7 @@
         "resolver_address": "",
         "t": 0,
         "tags": [
+          "classic",
           "depth=0"
         ],
         "transaction_id": 10001
@@ -215,6 +216,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -231,6 +234,8 @@
         },
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],
@@ -249,6 +254,8 @@
         "server_name": "www.example.org",
         "t": 0,
         "tags": [
+          "classic",
+          "tcptls_experiment",
           "depth=0",
           "fetch_body=true"
         ],