Pull request 285: AG-23599 use hostsfile vol.2

Merge in GO/dnsproxy from AG-23599-use-hostsfile-vol.2 to master

Squashed commit of the following:

commit b8fd8d5
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 10 19:21:33 2023 +0300

    upstream: fix nil deref

commit 11ec4fe
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 10 18:54:45 2023 +0300

    bootstrap: imp docs

commit 93aa3b1
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 10 17:30:15 2023 +0300

    bootstrap: imp doc

commit 9779260
Merge: 1240ee1 ea0f2b1
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 10 16:43:54 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile-vol.2

commit 1240ee1
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Nov 9 17:37:58 2023 +0300

    upstream: remove sort from upstream resolver

commit f22fe9b
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Nov 9 15:37:01 2023 +0300

    all: upd go

commit 45c5215
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Nov 9 15:34:22 2023 +0300

    upstream: remove empty impl

commit 8582ed9
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Nov 8 13:00:14 2023 +0300

    upstream: imp cognit

commit 35ce9dc
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Nov 8 12:47:25 2023 +0300

    all: imp code, fmt

commit 1823da5
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Nov 8 12:33:35 2023 +0300

    all: imp docs, restore behavior

commit aeda179
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Nov 7 15:23:17 2023 +0300

    main: fix import

commit fa37464
Merge: eb12aec 254e5a8
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Nov 7 14:33:18 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile-vol.2

commit eb12aec
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 3 19:44:15 2023 +0300

    upstream: fix nil deref

commit e897536
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Fri Nov 3 18:38:00 2023 +0300

    all: imp code, docs

commit 60e7099
Merge: 0e6e9ad 8d20902
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Tue Oct 31 17:16:14 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile-vol.2

commit 0e6e9ad
Merge: b10f900 b4abccf
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Oct 11 17:09:08 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile-vol.2

commit b10f900
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Wed Oct 11 16:30:18 2023 +0300

    upstream: imp returning types & docs

commit a5f5077
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Oct 5 18:52:20 2023 +0300

    proxy: fix merged test

commit a153556
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Oct 5 18:50:27 2023 +0300

    netutil: doc more, fix type check

commit 718aee8
Merge: 34750a6 60d2174
Author: Eugene Burkov <E.Burkov@AdGuard.COM>
Date:   Thu Oct 5 16:07:10 2023 +0300

    Merge branch 'master' into AG-23599-use-hostsfile-vol.2

... and 8 more commits

internal/bootstrap/bootstrap.go

@@ -21,12 +21,12 @@ import (
 // specified at initialization and ignores the addr.
 type DialHandler func(ctx context.Context, network, addr string) (conn net.Conn, err error)
 
-// ResolveDialContext returns a DialHandler that uses addresses resolved from
-// u using resolvers.  u must not be nil.
+// ResolveDialContext returns a DialHandler that uses addresses resolved from u
+// using resolver.  u must not be nil.
 func ResolveDialContext(
 	u *url.URL,
 	timeout time.Duration,
-	resolvers []Resolver,
+	resolver Resolver,
 	preferIPv6 bool,
 ) (h DialHandler, err error) {
 	defer func() { err = errors.Annotate(err, "dialing %q: %w", u.Host) }()
@@ -38,14 +38,18 @@ func ResolveDialContext(
 		return nil, err
 	}
 
+	if resolver == nil {
+		return nil, fmt.Errorf("resolver is nil: %w", ErrNoResolvers)
+	}
+
 	ctx := context.Background()
 	if timeout > 0 {
 		var cancel func()
 		ctx, cancel = context.WithTimeout(ctx, timeout)
 		defer cancel()
 	}
 
-	ips, err := LookupParallel(ctx, resolvers, host)
+	ips, err := resolver.LookupNetIP(ctx, "ip", host)
 	if err != nil {
 		return nil, fmt.Errorf("resolving hostname: %w", err)
 	}

internal/bootstrap/bootstrap_test.go

@@ -98,7 +98,7 @@ func TestResolveDialContext(t *testing.T) {
 			dialContext, err := bootstrap.ResolveDialContext(
 				&url.URL{Host: netutil.JoinHostPort(hostname, port)},
 				testTimeout,
-				[]bootstrap.Resolver{r},
+				bootstrap.ParallelResolver{r},
 				tc.preferIPv6,
 			)
 			require.NoError(t, err)
@@ -130,7 +130,7 @@ func TestResolveDialContext(t *testing.T) {
 		dialContext, err := bootstrap.ResolveDialContext(
 			&url.URL{Host: netutil.JoinHostPort(hostname, port)},
 			testTimeout,
-			[]bootstrap.Resolver{r},
+			bootstrap.ParallelResolver{r},
 			false,
 		)
 		require.NoError(t, err)

internal/bootstrap/resolver.go

@@ -13,7 +13,8 @@ import (
 // Resolver resolves the hostnames to IP addresses.
 type Resolver interface {
 	// LookupNetIP looks up the IP addresses for the given host.  network must
-	// be one of "ip", "ip4" or "ip6".
+	// be one of "ip", "ip4" or "ip6".  The response may be empty even if err is
+	// nil.
 	LookupNetIP(ctx context.Context, network, host string) (addrs []netip.Addr, err error)
 }
 
@@ -23,67 +24,71 @@ var _ Resolver = &net.Resolver{}
 // ErrNoResolvers is returned when zero resolvers specified.
 const ErrNoResolvers errors.Error = "no resolvers specified"
 
-// LookupParallel performs lookup for IP address of host with all resolvers
-// concurrently.
-func LookupParallel(
+// ParallelResolver is a slice of resolvers that are queried concurrently.  The
+// first successful response is returned.
+type ParallelResolver []Resolver
+
+// type check
+var _ Resolver = ParallelResolver(nil)
+
+// LookupNetIP implements the [Resolver] interface for ParallelResolver.
+func (r ParallelResolver) LookupNetIP(
 	ctx context.Context,
-	resolvers []Resolver,
+	network string,
 	host string,
 ) (addrs []netip.Addr, err error) {
-	resolversNum := len(resolvers)
+	resolversNum := len(r)
 	switch resolversNum {
 	case 0:
 		return nil, ErrNoResolvers
 	case 1:
-		return lookup(ctx, resolvers[0], host)
+		return lookup(ctx, r[0], network, host)
 	default:
 		// Go on.
 	}
 
 	// Size of channel must accommodate results of lookups from all resolvers,
 	// sending into channel will be block otherwise.
-	ch := make(chan *lookupResult, resolversNum)
-	for _, res := range resolvers {
-		go lookupAsync(ctx, res, host, ch)
+	ch := make(chan any, resolversNum)
+	for _, rslv := range r {
+		go lookupAsync(ctx, rslv, network, host, ch)
 	}
 
 	var errs []error
-	for range resolvers {
-		result := <-ch
-		if result.err == nil {
-			return result.addrs, nil
+	for range r {
+		switch result := <-ch; result := result.(type) {
+		case error:
+			errs = append(errs, result)
+		case []netip.Addr:
+			return result, nil
 		}
-
-		errs = append(errs, result.err)
 	}
 
-	// TODO(e.burkov):  Use [errors.Join] in Go 1.20.
-	return nil, errors.List("all resolvers failed", errs...)
+	return nil, errors.Join(errs...)
 }
 
-// lookupResult is a structure that represents the result of a lookup.
-type lookupResult struct {
-	err   error
-	addrs []netip.Addr
-}
-
-// lookupAsync tries to lookup for ip of host with r and sends the result into
-// resCh.  It's inteneded to be used as a goroutine.
-func lookupAsync(ctx context.Context, r Resolver, host string, resCh chan<- *lookupResult) {
+// lookupAsync performs a lookup for ip of host with r and sends the result into
+// resCh.  It is intended to be used as a goroutine.
+func lookupAsync(ctx context.Context, r Resolver, network, host string, resCh chan<- any) {
 	defer log.OnPanic("parallel lookup")
 
-	addrs, err := lookup(ctx, r, host)
-	resCh <- &lookupResult{
-		err:   err,
-		addrs: addrs,
+	addrs, err := lookup(ctx, r, network, host)
+	if err != nil {
+		resCh <- err
+	} else {
+		resCh <- addrs
 	}
 }
 
 // lookup tries to lookup ip of host with r.
-func lookup(ctx context.Context, r Resolver, host string) (addrs []netip.Addr, err error) {
+//
+// TODO(e.burkov):  Get rid of this function?  It only wraps the actual lookup
+// with dubious logging.
+func lookup(ctx context.Context, r Resolver, network, host string) (addrs []netip.Addr, err error) {
 	start := time.Now()
-	addrs, err = r.LookupNetIP(ctx, "ip", host)
+	addrs, err = r.LookupNetIP(ctx, network, host)
 	elapsed := time.Since(start)
+
 	if err != nil {
 		log.Debug("parallel lookup: lookup for %s failed in %s: %s", host, elapsed, err)
 	} else {

internal/bootstrap/resolver_test.go

@@ -2,8 +2,8 @@ package bootstrap_test
 
 import (
 	"context"
-	"fmt"
 	"net/netip"
+	"strings"
 	"testing"
 
 	"github.com/AdguardTeam/dnsproxy/internal/bootstrap"
@@ -31,7 +31,7 @@ func TestLookupParallel(t *testing.T) {
 	const hostname = "host.name"
 
 	t.Run("no_resolvers", func(t *testing.T) {
-		addrs, err := bootstrap.LookupParallel(context.Background(), nil, "")
+		addrs, err := bootstrap.ParallelResolver(nil).LookupNetIP(context.Background(), "ip", "")
 		assert.ErrorIs(t, err, bootstrap.ErrNoResolvers)
 		assert.Nil(t, addrs)
 	})
@@ -49,9 +49,9 @@ func TestLookupParallel(t *testing.T) {
 	}
 
 	t.Run("one_resolver", func(t *testing.T) {
-		addrs, err := bootstrap.LookupParallel(
+		addrs, err := bootstrap.ParallelResolver{immediate}.LookupNetIP(
 			context.Background(),
-			[]bootstrap.Resolver{immediate},
+			"ip",
 			hostname,
 		)
 		require.NoError(t, err)
@@ -72,9 +72,9 @@ func TestLookupParallel(t *testing.T) {
 			},
 		}
 
-		addrs, err := bootstrap.LookupParallel(
+		addrs, err := bootstrap.ParallelResolver{immediate, delayed}.LookupNetIP(
 			context.Background(),
-			[]bootstrap.Resolver{immediate, delayed},
+			"ip",
 			hostname,
 		)
 		require.NoError(t, err)
@@ -85,17 +85,18 @@ func TestLookupParallel(t *testing.T) {
 
 	t.Run("all_errors", func(t *testing.T) {
 		err := assert.AnError
-		wantErrMsg := fmt.Sprintf("all resolvers failed: 3 errors: %[1]q, %[1]q, %[1]q", err)
+		errStr := err.Error()
+		wantErrMsg := strings.Join([]string{errStr, errStr, errStr}, "\n")
 
 		r := &testResolver{
 			onLookupNetIP: func(_ context.Context, network, host string) ([]netip.Addr, error) {
 				return nil, assert.AnError
 			},
 		}
 
-		addrs, err := bootstrap.LookupParallel(
+		addrs, err := bootstrap.ParallelResolver{r, r, r}.LookupNetIP(
 			context.Background(),
-			[]bootstrap.Resolver{r, r, r},
+			"ip",
 			hostname,
 		)
 		testutil.AssertErrorMsg(t, wantErrMsg, err)

internal/netutil/hosts.go

@@ -44,6 +44,9 @@ type (
 // It must be initialized with [NewHosts].
 //
 // TODO(e.burkov):  Think of storing only slices.
+//
+// TODO(e.burkov):  Move to netutil/hostsfile in module golibs as a default
+// implementation of some storage interface.
 type Hosts struct {
 	// names maps each address to its names in original case and in original
 	// adding order without duplicates.
@@ -54,9 +57,6 @@ type Hosts struct {
 	addrs map[string]*addrsSet
 }
 
-// type check
-var _ hostsfile.HandleSet = (*Hosts)(nil)
-
 // NewHosts parses hosts files from r and returns a new Hosts set.  readers are
 // optional, the error is only returned in case of parsing error.
 func NewHosts(readers ...io.Reader) (h *Hosts, err error) {

internal/netutil/netutil.go

@@ -8,6 +8,7 @@ package netutil
 import (
 	"net"
 	"net/netip"
+	"strings"
 
 	glnetutil "github.com/AdguardTeam/golibs/netutil"
 	"golang.org/x/exp/slices"
@@ -93,3 +94,26 @@ func SortNetIPAddrs(addrs []netip.Addr, preferIPv6 bool) {
 		return 1
 	})
 }
+
+// ParseSubnet parses s either as a CIDR prefix itself, or as an IP address,
+// returning the corresponding single-IP CIDR prefix.
+//
+// TODO(e.burkov):  Move to golibs.
+func ParseSubnet(s string) (p netip.Prefix, err error) {
+	if strings.Contains(s, "/") {
+		p, err = netip.ParsePrefix(s)
+		if err != nil {
+			return netip.Prefix{}, err
+		}
+	} else {
+		var ip netip.Addr
+		ip, err = netip.ParseAddr(s)
+		if err != nil {
+			return netip.Prefix{}, err
+		}
+
+		p = netip.PrefixFrom(ip, ip.BitLen())
+	}
+
+	return p, nil
+}

internal/osutil/osutil.go

@@ -0,0 +1,14 @@
+// Package osutil contains utilities for functions requiring system calls and
+// other OS-specific APIs, except for network-related ones.
+package osutil
+
+import "io/fs"
+
+// RootDirFS returns the fs.FS rooted at the operating system's root.  On
+// Windows it returns the fs.FS rooted at the volume of the system directory
+// (usually, C:).
+//
+// TODO(e.burkov):  Move to golibs.
+func RootDirFS() (fsys fs.FS) {
+	return rootDirFS()
+}

internal/osutil/osutil_unix.go

@@ -0,0 +1,12 @@
+//go:build !windows
+
+package osutil
+
+import (
+	"io/fs"
+	"os"
+)
+
+func rootDirFS() (fsys fs.FS) {
+	return os.DirFS("/")
+}

internal/osutil/osutil_windows.go

@@ -0,0 +1,25 @@
+//go:build windows
+
+package osutil
+
+import (
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/AdguardTeam/golibs/log"
+	"golang.org/x/sys/windows"
+)
+
+func rootDirFS() (fsys fs.FS) {
+	// TODO(a.garipov): Use a better way if golang/go#44279 is ever resolved.
+	sysDir, err := windows.GetSystemDirectory()
+	if err != nil {
+		log.Error("aghos: getting root filesystem: %s; using C:", err)
+
+		// Assume that C: is the safe default.
+		return os.DirFS("C:")
+	}
+
+	return os.DirFS(filepath.VolumeName(sysDir))
+}

internal/tools/go.mod

@@ -8,27 +8,27 @@ require (
 	github.com/gordonklaus/ineffassign v0.0.0-20230610083614-0e73809eb601
 	github.com/kisielk/errcheck v1.6.3
 	github.com/kyoh86/looppointer v0.2.1
-	github.com/securego/gosec/v2 v2.16.0
+	github.com/securego/gosec/v2 v2.17.0
 	github.com/uudashr/gocognit v1.0.7
-	golang.org/x/tools v0.12.0
-	golang.org/x/vuln v1.0.0
-	honnef.co/go/tools v0.4.3
+	golang.org/x/tools v0.13.0
+	golang.org/x/vuln v1.0.1
+	honnef.co/go/tools v0.4.6
 	mvdan.cc/gofumpt v0.5.0
-	mvdan.cc/unparam v0.0.0-20230815095028-f7c6fb1088f0
+	mvdan.cc/unparam v0.0.0-20230917202934-3ee2d22f45fb
 )
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect
+	github.com/ccojocar/zxcvbn-go v1.0.1 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
 	github.com/gookit/color v1.5.4 // indirect
 	github.com/kyoh86/nolint v0.0.1 // indirect
-	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	golang.org/x/exp v0.0.0-20230307190834-24139beb5833 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20230811145659-89c5cff77bcb // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )