From 48c1814545eb94a61501930b8e7f473635eb1682 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 12 Mar 2021 16:41:58 -0500 Subject: [PATCH 01/29] crawler client v1 --- ctx_mutex.go | 28 - dht.go | 9 +- dht_net.go | 44 +- fullrt/dht.go | 1141 +++++++++++++++++ go.mod | 1 + go.sum | 4 + internal/config/quorum.go | 16 + internal/ctx_mutex.go | 28 + .../net/message_manager.go | 66 +- routing.go | 5 +- routing_options.go | 19 +- subscriber_notifee.go | 9 +- 12 files changed, 1272 insertions(+), 98 deletions(-) delete mode 100644 ctx_mutex.go create mode 100644 fullrt/dht.go create mode 100644 internal/config/quorum.go create mode 100644 internal/ctx_mutex.go rename message_manager.go => internal/net/message_manager.go (82%) diff --git a/ctx_mutex.go b/ctx_mutex.go deleted file mode 100644 index c28d89875..000000000 --- a/ctx_mutex.go +++ /dev/null @@ -1,28 +0,0 @@ -package dht - -import ( - "context" -) - -type ctxMutex chan struct{} - -func newCtxMutex() ctxMutex { - return make(ctxMutex, 1) -} - -func (m ctxMutex) Lock(ctx context.Context) error { - select { - case m <- struct{}{}: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - -func (m ctxMutex) Unlock() { - select { - case <-m: - default: - panic("not locked") - } -} diff --git a/dht.go b/dht.go index 7c89ab56a..0f24c5f34 100644 --- a/dht.go +++ b/dht.go @@ -16,6 +16,7 @@ import ( "github.com/libp2p/go-libp2p-core/routing" "github.com/libp2p/go-libp2p-kad-dht/internal" + "github.com/libp2p/go-libp2p-kad-dht/internal/net" "github.com/libp2p/go-libp2p-kad-dht/metrics" pb "github.com/libp2p/go-libp2p-kad-dht/pb" "github.com/libp2p/go-libp2p-kad-dht/providers" @@ -96,7 +97,7 @@ type IpfsDHT struct { proc goprocess.Process protoMessenger *pb.ProtocolMessenger - msgSender *messageSenderImpl + msgSender pb.MessageSender plk sync.Mutex @@ -188,11 +189,7 @@ func New(ctx context.Context, h host.Host, options ...Option) (*IpfsDHT, error) dht.disableFixLowPeers = cfg.disableFixLowPeers dht.Validator = cfg.validator - dht.msgSender = &messageSenderImpl{ - host: h, - strmap: make(map[peer.ID]*peerMessageSender), - protocols: dht.protocols, - } + dht.msgSender = net.NewMessageSenderImpl(h, dht.protocols) dht.protoMessenger, err = pb.NewProtocolMessenger(dht.msgSender, pb.WithValidator(dht.Validator)) if err != nil { return nil, err diff --git a/dht_net.go b/dht_net.go index 278216625..18f4bd733 100644 --- a/dht_net.go +++ b/dht_net.go @@ -1,15 +1,12 @@ package dht import ( - "bufio" - "fmt" "io" - "sync" "time" "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-msgio/protoio" + "github.com/libp2p/go-libp2p-kad-dht/internal/net" "github.com/libp2p/go-libp2p-kad-dht/metrics" pb "github.com/libp2p/go-libp2p-kad-dht/pb" @@ -19,45 +16,10 @@ import ( "go.uber.org/zap" ) -var dhtReadMessageTimeout = 10 * time.Second var dhtStreamIdleTimeout = 1 * time.Minute // ErrReadTimeout is an error that occurs when no message is read within the timeout period. -var ErrReadTimeout = fmt.Errorf("timed out reading response") - -// The Protobuf writer performs multiple small writes when writing a message. -// We need to buffer those writes, to make sure that we're not sending a new -// packet for every single write. -type bufferedDelimitedWriter struct { - *bufio.Writer - protoio.WriteCloser -} - -var writerPool = sync.Pool{ - New: func() interface{} { - w := bufio.NewWriter(nil) - return &bufferedDelimitedWriter{ - Writer: w, - WriteCloser: protoio.NewDelimitedWriter(w), - } - }, -} - -func writeMsg(w io.Writer, mes *pb.Message) error { - bw := writerPool.Get().(*bufferedDelimitedWriter) - bw.Reset(w) - err := bw.WriteMsg(mes) - if err == nil { - err = bw.Flush() - } - bw.Reset(nil) - writerPool.Put(bw) - return err -} - -func (w *bufferedDelimitedWriter) Flush() error { - return w.Writer.Flush() -} +var ErrReadTimeout = net.ErrReadTimeout // handleNewStream implements the network.StreamHandler func (dht *IpfsDHT) handleNewStream(s network.Stream) { @@ -180,7 +142,7 @@ func (dht *IpfsDHT) handleNewMessage(s network.Stream) bool { } // send out response msg - err = writeMsg(s, resp) + err = net.WriteMsg(s, resp) if err != nil { stats.Record(ctx, metrics.ReceivedMessageErrors.M(1)) if c := baseLogger.Check(zap.DebugLevel, "error writing response"); c != nil { diff --git a/fullrt/dht.go b/fullrt/dht.go new file mode 100644 index 000000000..c7b9a7403 --- /dev/null +++ b/fullrt/dht.go @@ -0,0 +1,1141 @@ +package fullrt + +import ( + "bytes" + "context" + "fmt" + "github.com/gogo/protobuf/proto" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + u "github.com/ipfs/go-ipfs-util" + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p-core/routing" + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p-kad-dht/crawler" + "github.com/libp2p/go-libp2p-kad-dht/internal" + "github.com/libp2p/go-libp2p-kad-dht/internal/net" + dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb" + "github.com/libp2p/go-libp2p-kad-dht/providers" + "github.com/libp2p/go-libp2p-kad-dht/qpeerset" + kb "github.com/libp2p/go-libp2p-kbucket" + record "github.com/libp2p/go-libp2p-record" + recpb "github.com/libp2p/go-libp2p-record/pb" + "github.com/multiformats/go-base32" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multihash" + "sync" + "sync/atomic" + "time" + + kaddht "github.com/libp2p/go-libp2p-kad-dht" + internalConfig "github.com/libp2p/go-libp2p-kad-dht/internal/config" + + "github.com/libp2p/go-libp2p-xor/kademlia" + kadkey "github.com/libp2p/go-libp2p-xor/key" + "github.com/libp2p/go-libp2p-xor/trie" +) + +var logger = logging.Logger("fullrtdht") + +type FullRT struct { + ctx context.Context + + enableValues, enableProviders bool + Validator record.Validator + ProviderManager *providers.ProviderManager + datastore ds.Datastore + h host.Host + + crawler *crawler.Crawler + protoMessenger *dht_pb.ProtocolMessenger + + filterFromTable dht.QueryFilterFunc + rtLk sync.RWMutex + rt *trie.Trie + + kMapLk sync.RWMutex + keyToPeerMap map[string]peer.ID + + peerAddrsLk sync.RWMutex + peerAddrs map[peer.ID][]multiaddr.Multiaddr + + bootstrapPeers []*peer.AddrInfo + + bucketSize int + + triggerRefresh chan struct{} +} + +func NewFullRT(ctx context.Context, h host.Host, validator record.Validator, ds ds.Batching) (*FullRT, error) { + ms := net.NewMessageSenderImpl(h, []protocol.ID{"/ipfs/kad/1.0.0"}) + protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(validator)) + if err != nil { + return nil, err + } + + pm, err := providers.NewProviderManager(ctx, h.ID(), ds) + if err != nil { + return nil, err + } + + c, err := crawler.New(h, crawler.WithParallelism(200)) + if err != nil { + return nil, err + } + + bsPeersNormal := dht.GetDefaultBootstrapPeerAddrInfos() + bsPeers := make([]*peer.AddrInfo, len(bsPeersNormal)) + for i, ai := range bsPeersNormal { + tmpai := ai + bsPeers[i] = &tmpai + } + + rt := &FullRT{ + ctx: ctx, + enableValues: true, + enableProviders: true, + Validator: validator, + ProviderManager: pm, + datastore: ds, + h: h, + crawler: c, + protoMessenger: protoMessenger, + filterFromTable: kaddht.PublicQueryFilter, + rt: trie.New(), + keyToPeerMap: make(map[string]peer.ID), + bucketSize: 20, + + peerAddrs: make(map[peer.ID][]multiaddr.Multiaddr), + bootstrapPeers: bsPeers, + + triggerRefresh: make(chan struct{}), + } + + go rt.runCrawler(ctx) + + return rt, nil +} + +type crawlVal struct { + addrs []multiaddr.Multiaddr + key kadkey.Key +} + +func (dht *FullRT) TriggerRefresh(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case dht.triggerRefresh <- struct{}{}: + return nil + } +} + +func (dht *FullRT) Stat() map[string]peer.ID { + newMap := make(map[string]peer.ID) + + dht.kMapLk.RLock() + for k, v := range dht.keyToPeerMap { + newMap[k] = v + } + dht.kMapLk.RUnlock() + return newMap +} + +func (dht *FullRT) runCrawler(ctx context.Context) { + t := time.NewTicker(time.Minute * 60) + + m := make(map[peer.ID]*crawlVal) + mxLk := sync.Mutex{} + + initialTrigger := make(chan struct{}, 1) + initialTrigger <- struct{}{} + + for { + select { + case <-t.C: + case <-initialTrigger: + case <-dht.triggerRefresh: + case <-ctx.Done(): + return + } + + var addrs []*peer.AddrInfo + dht.peerAddrsLk.Lock() + for k, v := range m { + addrs = append(addrs, &peer.AddrInfo{ID: k, Addrs: v.addrs}) + } + for _, ai := range dht.bootstrapPeers { + addrs = append(addrs, ai) + } + dht.peerAddrsLk.Unlock() + + start := time.Now() + dht.crawler.Run(ctx, addrs, + func(p peer.ID, rtPeers []*peer.AddrInfo) { + addrs := dht.h.Peerstore().Addrs(p) + mxLk.Lock() + defer mxLk.Unlock() + m[p] = &crawlVal{ + addrs: addrs, + } + }, + func(p peer.ID, err error) { + return + }) + dur := time.Since(start) + fmt.Printf("crawl took %v\n", dur) + start = time.Now() + + peerAddrs := make(map[peer.ID][]multiaddr.Multiaddr) + kPeerMap := make(map[string]peer.ID) + newRt := trie.New() + for k, v := range m { + v.key = kadkey.KbucketIDToKey(kb.ConvertPeerID(k)) + peerAddrs[k] = v.addrs + kPeerMap[string(v.key)] = k + newRt.Add(v.key) + } + + dht.peerAddrsLk.Lock() + dht.peerAddrs = peerAddrs + dht.peerAddrsLk.Unlock() + + dht.kMapLk.Lock() + dht.keyToPeerMap = kPeerMap + dht.kMapLk.Unlock() + + dht.rtLk.Lock() + dht.rt = newRt + dht.rtLk.Unlock() + + dur = time.Since(start) + fmt.Printf("processing crawl took %v\n", dur) + } +} + +func (dht *FullRT) Bootstrap(ctx context.Context) error { + return nil +} + +// CheckPeers return (success, total) +func (dht *FullRT) CheckPeers(ctx context.Context, peers ...peer.ID) (int, int) { + var peerAddrs chan interface{} + var total int + if len(peers) == 0 { + dht.peerAddrsLk.RLock() + total = len(dht.peerAddrs) + peerAddrs = make(chan interface{}, total) + for k, v := range dht.peerAddrs { + peerAddrs <- peer.AddrInfo{ + ID: k, + Addrs: v, + } + } + close(peerAddrs) + dht.peerAddrsLk.RUnlock() + } else { + total = len(peers) + peerAddrs = make(chan interface{}, total) + dht.peerAddrsLk.RLock() + for _, p := range peers { + peerAddrs <- peer.AddrInfo{ + ID: p, + Addrs: dht.peerAddrs[p], + } + } + close(peerAddrs) + dht.peerAddrsLk.RUnlock() + } + + var success uint64 + + workers(100, func(i interface{}) { + a := i.(peer.AddrInfo) + dialctx, dialcancel := context.WithTimeout(ctx, time.Second*3) + if err := dht.h.Connect(dialctx, a); err == nil { + atomic.AddUint64(&success, 1) + } + dialcancel() + }, peerAddrs) + return int(success), total +} + +func workers(numWorkers int, fn func(interface{}), inputs <-chan interface{}) { + jobs := make(chan interface{}) + defer close(jobs) + for i := 0; i < numWorkers; i++ { + go func() { + for j := range jobs { + fn(j) + } + }() + } + for i := range inputs { + jobs <- i + } +} + +func (dht *FullRT) GetClosestPeers(ctx context.Context, key string) ([]peer.ID, error) { + kbID := kb.ConvertKey(key) + kadKey := kadkey.KbucketIDToKey(kbID) + dht.rtLk.RLock() + closestKeys := kademlia.ClosestN(kadKey, dht.rt, dht.bucketSize) + dht.rtLk.RUnlock() + + peers := make([]peer.ID, 0, len(closestKeys)) + for _, k := range closestKeys { + dht.kMapLk.RLock() + p, ok := dht.keyToPeerMap[string(k)] + if !ok { + logger.Errorf("key not found in map") + } + dht.kMapLk.RUnlock() + peers = append(peers, p) + } + return peers, nil +} + +// PutValue adds value corresponding to given Key. +// This is the top level "Store" operation of the DHT +func (dht *FullRT) PutValue(ctx context.Context, key string, value []byte, opts ...routing.Option) (err error) { + if !dht.enableValues { + return routing.ErrNotSupported + } + + logger.Debugw("putting value", "key", internal.LoggableRecordKeyString(key)) + + // don't even allow local users to put bad values. + if err := dht.Validator.Validate(key, value); err != nil { + return err + } + + old, err := dht.getLocal(key) + if err != nil { + // Means something is wrong with the datastore. + return err + } + + // Check if we have an old value that's not the same as the new one. + if old != nil && !bytes.Equal(old.GetValue(), value) { + // Check to see if the new one is better. + i, err := dht.Validator.Select(key, [][]byte{value, old.GetValue()}) + if err != nil { + return err + } + if i != 0 { + return fmt.Errorf("can't replace a newer value with an older value") + } + } + + rec := record.MakePutRecord(key, value) + rec.TimeReceived = u.FormatRFC3339(time.Now()) + err = dht.putLocal(key, rec) + if err != nil { + return err + } + + peers, err := dht.GetClosestPeers(ctx, key) + if err != nil { + return err + } + + successes := putToMany(ctx, func(ctx context.Context, p peer.ID) error { + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.Value, + ID: p, + }) + err := dht.protoMessenger.PutValue(ctx, p, rec) + return err + }, peers, 0.3, time.Second*5) + + if successes == 0 { + return fmt.Errorf("failed to complete put") + } + + return nil +} + +// RecvdVal stores a value and the peer from which we got the value. +type RecvdVal struct { + Val []byte + From peer.ID +} + +// GetValue searches for the value corresponding to given Key. +func (dht *FullRT) GetValue(ctx context.Context, key string, opts ...routing.Option) (_ []byte, err error) { + if !dht.enableValues { + return nil, routing.ErrNotSupported + } + + // apply defaultQuorum if relevant + var cfg routing.Options + if err := cfg.Apply(opts...); err != nil { + return nil, err + } + opts = append(opts, kaddht.Quorum(internalConfig.GetQuorum(&cfg))) + + responses, err := dht.SearchValue(ctx, key, opts...) + if err != nil { + return nil, err + } + var best []byte + + for r := range responses { + best = r + } + + if ctx.Err() != nil { + return best, ctx.Err() + } + + if best == nil { + return nil, routing.ErrNotFound + } + logger.Debugf("GetValue %v %x", internal.LoggableRecordKeyString(key), best) + return best, nil +} + +// SearchValue searches for the value corresponding to given Key and streams the results. +func (dht *FullRT) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) { + if !dht.enableValues { + return nil, routing.ErrNotSupported + } + + var cfg routing.Options + if err := cfg.Apply(opts...); err != nil { + return nil, err + } + + responsesNeeded := 0 + if !cfg.Offline { + responsesNeeded = internalConfig.GetQuorum(&cfg) + } + + stopCh := make(chan struct{}) + valCh, lookupRes := dht.getValues(ctx, key, stopCh) + + out := make(chan []byte) + go func() { + defer close(out) + best, peersWithBest, aborted := dht.searchValueQuorum(ctx, key, valCh, stopCh, out, responsesNeeded) + if best == nil || aborted { + return + } + + updatePeers := make([]peer.ID, 0, dht.bucketSize) + select { + case l := <-lookupRes: + if l == nil { + return + } + + for _, p := range l.peers { + if _, ok := peersWithBest[p]; !ok { + updatePeers = append(updatePeers, p) + } + } + case <-ctx.Done(): + return + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + dht.updatePeerValues(ctx, key, best, updatePeers) + cancel() + }() + + return out, nil +} + +func (dht *FullRT) searchValueQuorum(ctx context.Context, key string, valCh <-chan RecvdVal, stopCh chan struct{}, + out chan<- []byte, nvals int) ([]byte, map[peer.ID]struct{}, bool) { + numResponses := 0 + return dht.processValues(ctx, key, valCh, + func(ctx context.Context, v RecvdVal, better bool) bool { + numResponses++ + if better { + select { + case out <- v.Val: + case <-ctx.Done(): + return false + } + } + + if nvals > 0 && numResponses > nvals { + close(stopCh) + return true + } + return false + }) +} + +// GetValues gets nvals values corresponding to the given key. +func (dht *FullRT) GetValues(ctx context.Context, key string, nvals int) (_ []RecvdVal, err error) { + if !dht.enableValues { + return nil, routing.ErrNotSupported + } + + queryCtx, cancel := context.WithCancel(ctx) + defer cancel() + valCh, _ := dht.getValues(queryCtx, key, nil) + + out := make([]RecvdVal, 0, nvals) + for val := range valCh { + out = append(out, val) + if len(out) == nvals { + cancel() + } + } + + return out, ctx.Err() +} + +func (dht *FullRT) processValues(ctx context.Context, key string, vals <-chan RecvdVal, + newVal func(ctx context.Context, v RecvdVal, better bool) bool) (best []byte, peersWithBest map[peer.ID]struct{}, aborted bool) { +loop: + for { + if aborted { + return + } + + select { + case v, ok := <-vals: + if !ok { + break loop + } + + // Select best value + if best != nil { + if bytes.Equal(best, v.Val) { + peersWithBest[v.From] = struct{}{} + aborted = newVal(ctx, v, false) + continue + } + sel, err := dht.Validator.Select(key, [][]byte{best, v.Val}) + if err != nil { + logger.Warnw("failed to select best value", "key", internal.LoggableRecordKeyString(key), "error", err) + continue + } + if sel != 1 { + aborted = newVal(ctx, v, false) + continue + } + } + peersWithBest = make(map[peer.ID]struct{}) + peersWithBest[v.From] = struct{}{} + best = v.Val + aborted = newVal(ctx, v, true) + case <-ctx.Done(): + return + } + } + + return +} + +func (dht *FullRT) updatePeerValues(ctx context.Context, key string, val []byte, peers []peer.ID) { + fixupRec := record.MakePutRecord(key, val) + for _, p := range peers { + go func(p peer.ID) { + //TODO: Is this possible? + if p == dht.h.ID() { + err := dht.putLocal(key, fixupRec) + if err != nil { + logger.Error("Error correcting local dht entry:", err) + } + return + } + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + err := dht.protoMessenger.PutValue(ctx, p, fixupRec) + if err != nil { + logger.Debug("Error correcting DHT entry: ", err) + } + }(p) + } +} + +type lookupWithFollowupResult struct { + peers []peer.ID // the top K not unreachable peers at the end of the query + state []qpeerset.PeerState // the peer states at the end of the query + + // indicates that neither the lookup nor the followup has been prematurely terminated by an external condition such + // as context cancellation or the stop function being called. + completed bool +} + +func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan struct{}) (<-chan RecvdVal, <-chan *lookupWithFollowupResult) { + valCh := make(chan RecvdVal, 1) + lookupResCh := make(chan *lookupWithFollowupResult, 1) + + logger.Debugw("finding value", "key", internal.LoggableRecordKeyString(key)) + + if rec, err := dht.getLocal(key); rec != nil && err == nil { + select { + case valCh <- RecvdVal{ + Val: rec.GetValue(), + From: dht.h.ID(), + }: + case <-ctx.Done(): + } + } + peers, err := dht.GetClosestPeers(ctx, key) + if err != nil { + lookupResCh <- &lookupWithFollowupResult{} + close(valCh) + close(lookupResCh) + return valCh, lookupResCh + } + + go func() { + defer close(valCh) + defer close(lookupResCh) + queryFn := func(ctx context.Context, p peer.ID) ([]*peer.AddrInfo, error) { + // For DHT query command + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.SendingQuery, + ID: p, + }) + + rec, peers, err := dht.protoMessenger.GetValue(ctx, p, key) + switch err { + case routing.ErrNotFound: + // in this case, they responded with nothing, + // still send a notification so listeners can know the + // request has completed 'successfully' + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + }) + return nil, err + default: + return nil, err + case nil, internal.ErrInvalidRecord: + // in either of these cases, we want to keep going + } + + // TODO: What should happen if the record is invalid? + // Pre-existing code counted it towards the quorum, but should it? + if rec != nil && rec.GetValue() != nil { + rv := RecvdVal{ + Val: rec.GetValue(), + From: p, + } + + select { + case valCh <- rv: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + // For DHT query command + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + Responses: peers, + }) + + return peers, err + } + + for i := range peers { + p := peers[i] + go func() { + queryFn(ctx, p) + }() + } + }() + return valCh, lookupResCh +} + +// Provider abstraction for indirect stores. +// Some DHTs store values directly, while an indirect store stores pointers to +// locations of the value, similarly to Coral and Mainline DHT. + +// Provide makes this node announce that it can provide a value for the given key +func (dht *FullRT) Provide(ctx context.Context, key cid.Cid, brdcst bool) (err error) { + if !dht.enableProviders { + return routing.ErrNotSupported + } else if !key.Defined() { + return fmt.Errorf("invalid cid: undefined") + } + keyMH := key.Hash() + logger.Debugw("providing", "cid", key, "mh", internal.LoggableProviderRecordBytes(keyMH)) + + // add self locally + dht.ProviderManager.AddProvider(ctx, keyMH, dht.h.ID()) + if !brdcst { + return nil + } + + closerCtx := ctx + if deadline, ok := ctx.Deadline(); ok { + now := time.Now() + timeout := deadline.Sub(now) + + if timeout < 0 { + // timed out + return context.DeadlineExceeded + } else if timeout < 10*time.Second { + // Reserve 10% for the final put. + deadline = deadline.Add(-timeout / 10) + } else { + // Otherwise, reserve a second (we'll already be + // connected so this should be fast). + deadline = deadline.Add(-time.Second) + } + var cancel context.CancelFunc + closerCtx, cancel = context.WithDeadline(ctx, deadline) + defer cancel() + } + + var exceededDeadline bool + peers, err := dht.GetClosestPeers(closerCtx, string(keyMH)) + switch err { + case context.DeadlineExceeded: + // If the _inner_ deadline has been exceeded but the _outer_ + // context is still fine, provide the value to the closest peers + // we managed to find, even if they're not the _actual_ closest peers. + if ctx.Err() != nil { + return ctx.Err() + } + exceededDeadline = true + case nil: + default: + return err + } + + successes := putToMany(ctx, func(ctx context.Context, p peer.ID) error { + err := dht.protoMessenger.PutProvider(ctx, p, keyMH, dht.h) + return err + }, peers, 0.3, time.Second*5) + + if exceededDeadline { + return context.DeadlineExceeded + } + + if successes == 0 { + return fmt.Errorf("could not complete provide") + } + + return ctx.Err() +} + +func putToMany(ctx context.Context, fn func(context.Context, peer.ID) error, peers []peer.ID, waitFrac float64, timeoutPerOp time.Duration) int { + putctx, cancel := context.WithCancel(ctx) + + waitAllCh := make(chan struct{}, len(peers)) + numSuccessfulToWaitFor := int(float64(len(peers)) * waitFrac) + waitSuccessCh := make(chan struct{}, numSuccessfulToWaitFor) + for _, p := range peers { + go func(p peer.ID) { + fnCtx, fnCancel := context.WithTimeout(putctx, timeoutPerOp) + defer fnCancel() + err := fn(fnCtx, p) + if err != nil { + logger.Debug(err) + } else { + waitSuccessCh <- struct{}{} + } + waitAllCh <- struct{}{} + }(p) + } + + numSuccess, numDone := 0, 0 + t := time.NewTimer(time.Hour) + for numDone != len(peers) { + select { + case <-waitAllCh: + numDone++ + case <-waitSuccessCh: + if numSuccess >= numSuccessfulToWaitFor { + t.Reset(time.Millisecond * 500) + } + numSuccess++ + numDone++ + case <-t.C: + cancel() + } + } + return numSuccess +} + +func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) error { + if !dht.enableProviders { + return routing.ErrNotSupported + } + + keysAsPeerIDs := make([]peer.ID, len(keys)) + for _, k := range keys { + keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) + } + sortedKeys := kb.SortClosestPeers(keysAsPeerIDs, kb.ID(make([]byte, 32))) + for _, k := range sortedKeys { + peers, err := dht.GetClosestPeers(ctx, string(k)) + if err != nil { + return err + } + sendMsgToSome(ctx, peers, func(ctx context.Context, p peer.ID) error { + return dht.protoMessenger.PutProvider(ctx, p, multihash.Multihash(k), dht.h) + }) + } + return nil +} + +func sendMsgToSome(ctx context.Context, peers []peer.ID, sendFn func(context.Context, peer.ID) error) { + groupSendCtx, cancel := context.WithCancel(ctx) + + waitAllCh := make(chan struct{}, len(peers)) + numSuccessfulToWaitFor := int(float64(len(peers)) * 0.3) + waitSuccessCh := make(chan struct{}, numSuccessfulToWaitFor) + for _, p := range peers { + go func(p peer.ID) { + err := sendFn(groupSendCtx, p) + if err != nil { + logger.Debug(err) + } else { + waitSuccessCh <- struct{}{} + } + waitAllCh <- struct{}{} + }(p) + } + + numSuccess, numDone := 0, 0 + t := time.NewTimer(time.Hour) + for numDone != len(peers) { + select { + case <-waitAllCh: + numDone++ + case <-waitSuccessCh: + if numSuccess >= numSuccessfulToWaitFor { + t.Reset(time.Millisecond * 500) + } + numSuccess++ + numDone++ + case <-t.C: + cancel() + } + } +} + +// FindProviders searches until the context expires. +func (dht *FullRT) FindProviders(ctx context.Context, c cid.Cid) ([]peer.AddrInfo, error) { + if !dht.enableProviders { + return nil, routing.ErrNotSupported + } else if !c.Defined() { + return nil, fmt.Errorf("invalid cid: undefined") + } + + var providers []peer.AddrInfo + for p := range dht.FindProvidersAsync(ctx, c, dht.bucketSize) { + providers = append(providers, p) + } + return providers, nil +} + +// FindProvidersAsync is the same thing as FindProviders, but returns a channel. +// Peers will be returned on the channel as soon as they are found, even before +// the search query completes. If count is zero then the query will run until it +// completes. Note: not reading from the returned channel may block the query +// from progressing. +func (dht *FullRT) FindProvidersAsync(ctx context.Context, key cid.Cid, count int) <-chan peer.AddrInfo { + if !dht.enableProviders || !key.Defined() { + peerOut := make(chan peer.AddrInfo) + close(peerOut) + return peerOut + } + + chSize := count + if count == 0 { + chSize = 1 + } + peerOut := make(chan peer.AddrInfo, chSize) + + keyMH := key.Hash() + + logger.Debugw("finding providers", "cid", key, "mh", internal.LoggableProviderRecordBytes(keyMH)) + go dht.findProvidersAsyncRoutine(ctx, keyMH, count, peerOut) + return peerOut +} + +func (dht *FullRT) findProvidersAsyncRoutine(ctx context.Context, key multihash.Multihash, count int, peerOut chan peer.AddrInfo) { + defer close(peerOut) + + findAll := count == 0 + var ps *peer.Set + if findAll { + ps = peer.NewSet() + } else { + ps = peer.NewLimitedSet(count) + } + + provs := dht.ProviderManager.GetProviders(ctx, key) + for _, p := range provs { + // NOTE: Assuming that this list of peers is unique + if ps.TryAdd(p) { + pi := dht.h.Peerstore().PeerInfo(p) + select { + case peerOut <- pi: + case <-ctx.Done(): + return + } + } + + // If we have enough peers locally, don't bother with remote RPC + // TODO: is this a DOS vector? + if !findAll && ps.Size() >= count { + return + } + } + + peers, err := dht.GetClosestPeers(ctx, string(key)) + if err != nil { + return + } + + queryctx, cancelquery := context.WithCancel(ctx) + defer cancelquery() + + for i := range peers { + p := peers[i] + go func() { + // For DHT query command + routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + Type: routing.SendingQuery, + ID: p, + }) + + provs, closest, err := dht.protoMessenger.GetProviders(queryctx, p, key) + if err != nil { + return + } + + logger.Debugf("%d provider entries", len(provs)) + + // Add unique providers from request, up to 'count' + for _, prov := range provs { + dht.maybeAddAddrs(prov.ID, prov.Addrs, peerstore.TempAddrTTL) + logger.Debugf("got provider: %s", prov) + if ps.TryAdd(prov.ID) { + logger.Debugf("using provider: %s", prov) + select { + case peerOut <- *prov: + case <-queryctx.Done(): + logger.Debug("context timed out sending more providers") + return + } + } + if !findAll && ps.Size() >= count { + logger.Debugf("got enough providers (%d/%d)", ps.Size(), count) + cancelquery() + return + } + } + + // Give closer peers back to the query to be queried + logger.Debugf("got closer peers: %d %s", len(closest), closest) + + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + Responses: closest, + }) + }() + } +} + +// FindPeer searches for a peer with given ID. +func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, err error) { + if err := id.Validate(); err != nil { + return peer.AddrInfo{}, err + } + + logger.Debugw("finding peer", "peer", id) + + // Check if were already connected to them + if pi := dht.FindLocal(id); pi.ID != "" { + return pi, nil + } + + peers, err := dht.GetClosestPeers(ctx, string(id)) + if err != nil { + return peer.AddrInfo{}, err + } + + queryctx, cancelquery := context.WithCancel(ctx) + defer cancelquery() + + addrsCh := make(chan *peer.AddrInfo, 1) + go func() { + addrsSoFar := make(map[multiaddr.Multiaddr]struct{}) + for { + select { + case ai, ok := <-addrsCh: + if !ok { + return + } + + newAddrs := make([]multiaddr.Multiaddr, 0) + for _, a := range ai.Addrs { + _, found := addrsSoFar[a] + if !found { + newAddrs = append(newAddrs, a) + addrsSoFar[a] = struct{}{} + } + } + + err := dht.h.Connect(ctx, peer.AddrInfo{ + ID: id, + Addrs: ai.Addrs, + }) + if err == nil { + cancelquery() + return + } + case <-ctx.Done(): + return + } + } + }() + + wg := sync.WaitGroup{} + wg.Add(len(peers)) + for i := range peers { + p := peers[i] + go func() { + defer wg.Done() + // For DHT query command + routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + Type: routing.SendingQuery, + ID: p, + }) + + peers, err := dht.protoMessenger.GetClosestPeers(queryctx, p, id) + if err != nil { + logger.Debugf("error getting closer peers: %s", err) + return + } + + // For DHT query command + routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + Responses: peers, + }) + + for _, a := range peers { + if a.ID == id { + select { + case addrsCh <- a: + case <-queryctx.Done(): + return + } + return + } + } + }() + } + + wg.Wait() + + // Return peer information if we tried to dial the peer during the query or we are (or recently were) connected + // to the peer. + connectedness := dht.h.Network().Connectedness(id) + if connectedness == network.Connected || connectedness == network.CanConnect { + return dht.h.Peerstore().PeerInfo(id), nil + } + + return peer.AddrInfo{}, routing.ErrNotFound +} + +var _ routing.Routing = (*FullRT)(nil) + +// getLocal attempts to retrieve the value from the datastore. +// +// returns nil, nil when either nothing is found or the value found doesn't properly validate. +// returns nil, some_error when there's a *datastore* error (i.e., something goes very wrong) +func (dht *FullRT) getLocal(key string) (*recpb.Record, error) { + logger.Debugw("finding value in datastore", "key", internal.LoggableRecordKeyString(key)) + + rec, err := dht.getRecordFromDatastore(mkDsKey(key)) + if err != nil { + logger.Warnw("get local failed", "key", internal.LoggableRecordKeyString(key), "error", err) + return nil, err + } + + // Double check the key. Can't hurt. + if rec != nil && string(rec.GetKey()) != key { + logger.Errorw("BUG: found a DHT record that didn't match it's key", "expected", internal.LoggableRecordKeyString(key), "got", rec.GetKey()) + return nil, nil + + } + return rec, nil +} + +// putLocal stores the key value pair in the datastore +func (dht *FullRT) putLocal(key string, rec *recpb.Record) error { + data, err := proto.Marshal(rec) + if err != nil { + logger.Warnw("failed to put marshal record for local put", "error", err, "key", internal.LoggableRecordKeyString(key)) + return err + } + + return dht.datastore.Put(mkDsKey(key), data) +} + +func mkDsKey(s string) ds.Key { + return ds.NewKey(base32.RawStdEncoding.EncodeToString([]byte(s))) +} + +// returns nil, nil when either nothing is found or the value found doesn't properly validate. +// returns nil, some_error when there's a *datastore* error (i.e., something goes very wrong) +func (dht *FullRT) getRecordFromDatastore(dskey ds.Key) (*recpb.Record, error) { + buf, err := dht.datastore.Get(dskey) + if err == ds.ErrNotFound { + return nil, nil + } + if err != nil { + logger.Errorw("error retrieving record from datastore", "key", dskey, "error", err) + return nil, err + } + rec := new(recpb.Record) + err = proto.Unmarshal(buf, rec) + if err != nil { + // Bad data in datastore, log it but don't return an error, we'll just overwrite it + logger.Errorw("failed to unmarshal record from datastore", "key", dskey, "error", err) + return nil, nil + } + + err = dht.Validator.Validate(string(rec.GetKey()), rec.GetValue()) + if err != nil { + // Invalid record in datastore, probably expired but don't return an error, + // we'll just overwrite it + logger.Debugw("local record verify failed", "key", rec.GetKey(), "error", err) + return nil, nil + } + + return rec, nil +} + +// FindLocal looks for a peer with a given ID connected to this dht and returns the peer and the table it was found in. +func (dht *FullRT) FindLocal(id peer.ID) peer.AddrInfo { + switch dht.h.Network().Connectedness(id) { + case network.Connected, network.CanConnect: + return dht.h.Peerstore().PeerInfo(id) + default: + return peer.AddrInfo{} + } +} + +func (dht *FullRT) maybeAddAddrs(p peer.ID, addrs []multiaddr.Multiaddr, ttl time.Duration) { + // Don't add addresses for self or our connected peers. We have better ones. + if p == dht.h.ID() || dht.h.Network().Connectedness(p) == network.Connected { + return + } + dht.h.Peerstore().AddAddrs(p, addrs, ttl) +} diff --git a/go.mod b/go.mod index 2ba606e08..bf25c800b 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.2.3 github.com/libp2p/go-libp2p-swarm v0.4.0 github.com/libp2p/go-libp2p-testing v0.4.0 + github.com/libp2p/go-libp2p-xor v0.0.0-20200501025846-71e284145d58 github.com/libp2p/go-msgio v0.0.6 github.com/libp2p/go-netroute v0.1.6 github.com/multiformats/go-base32 v0.0.3 diff --git a/go.sum b/go.sum index 298e40e2a..b54217d82 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,7 @@ github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfx github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0 h1:Qfl+e5+lfDgwdrXdu4YNCWyEo3fWuP+WgN9mN0iWviQ= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= github.com/libp2p/go-libp2p-kbucket v0.4.7/go.mod h1:XyVo99AfQH0foSf176k4jY1xUJ2+jUJIZCSDm7r2YKk= github.com/libp2p/go-libp2p-loggables v0.1.0 h1:h3w8QFfCt2UJl/0/NW4K829HX/0S4KD31PQ7m8UXXO8= @@ -312,6 +313,8 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSo github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= github.com/libp2p/go-libp2p-transport-upgrader v0.4.0 h1:xwj4h3hJdBrxqMOyMUjwscjoVst0AASTsKtZiTChoHI= github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s= +github.com/libp2p/go-libp2p-xor v0.0.0-20200501025846-71e284145d58 h1:GcTNu27BMpOTtMnQqun03+kbtHA1qTxJ/J8cZRRYu2k= +github.com/libp2p/go-libp2p-xor v0.0.0-20200501025846-71e284145d58/go.mod h1:AYjOiqJIdcmI4SXE2ouKQuFrUbE5myv8txWaB2pl4TI= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= @@ -510,6 +513,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= diff --git a/internal/config/quorum.go b/internal/config/quorum.go new file mode 100644 index 000000000..ce5fba2a8 --- /dev/null +++ b/internal/config/quorum.go @@ -0,0 +1,16 @@ +package config + +import "github.com/libp2p/go-libp2p-core/routing" + +type QuorumOptionKey struct{} + +const defaultQuorum = 0 + +// GetQuorum defaults to 0 if no option is found +func GetQuorum(opts *routing.Options) int { + responsesNeeded, ok := opts.Other[QuorumOptionKey{}].(int) + if !ok { + responsesNeeded = defaultQuorum + } + return responsesNeeded +} diff --git a/internal/ctx_mutex.go b/internal/ctx_mutex.go new file mode 100644 index 000000000..4e923f6e0 --- /dev/null +++ b/internal/ctx_mutex.go @@ -0,0 +1,28 @@ +package internal + +import ( + "context" +) + +type CtxMutex chan struct{} + +func NewCtxMutex() CtxMutex { + return make(CtxMutex, 1) +} + +func (m CtxMutex) Lock(ctx context.Context) error { + select { + case m <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m CtxMutex) Unlock() { + select { + case <-m: + default: + panic("not locked") + } +} diff --git a/message_manager.go b/internal/net/message_manager.go similarity index 82% rename from message_manager.go rename to internal/net/message_manager.go index 8cc3e22e3..d808803e5 100644 --- a/message_manager.go +++ b/internal/net/message_manager.go @@ -1,8 +1,14 @@ -package dht +package net import ( + "bufio" "context" "fmt" + logging "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-kad-dht/internal" + pb "github.com/libp2p/go-libp2p-kad-dht/pb" + "github.com/libp2p/go-msgio/protoio" + "io" "sync" "time" @@ -12,13 +18,19 @@ import ( "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-kad-dht/metrics" - pb "github.com/libp2p/go-libp2p-kad-dht/pb" "github.com/libp2p/go-msgio" "go.opencensus.io/stats" "go.opencensus.io/tag" ) +var dhtReadMessageTimeout = 10 * time.Second + +// ErrReadTimeout is an error that occurs when no message is read within the timeout period. +var ErrReadTimeout = fmt.Errorf("timed out reading response") + +var logger = logging.Logger("dht") + // messageSenderImpl is responsible for sending requests and messages to peers efficiently, including reuse of streams. // It also tracks metrics for sent requests and messages. type messageSenderImpl struct { @@ -28,7 +40,15 @@ type messageSenderImpl struct { protocols []protocol.ID } -func (m *messageSenderImpl) streamDisconnect(ctx context.Context, p peer.ID) { +func NewMessageSenderImpl(h host.Host, protos []protocol.ID) pb.MessageSender { + return &messageSenderImpl{ + host: h, + strmap: make(map[peer.ID]*peerMessageSender), + protocols: protos, + } +} + +func (m *messageSenderImpl) StreamDisconnect(ctx context.Context, p peer.ID) { m.smlk.Lock() defer m.smlk.Unlock() ms, ok := m.strmap[p] @@ -120,7 +140,7 @@ func (m *messageSenderImpl) messageSenderForPeer(ctx context.Context, p peer.ID) m.smlk.Unlock() return ms, nil } - ms = &peerMessageSender{p: p, m: m, lk: newCtxMutex()} + ms = &peerMessageSender{p: p, m: m, lk: internal.NewCtxMutex()} m.strmap[p] = ms m.smlk.Unlock() @@ -149,7 +169,7 @@ func (m *messageSenderImpl) messageSenderForPeer(ctx context.Context, p peer.ID) type peerMessageSender struct { s network.Stream r msgio.ReadCloser - lk ctxMutex + lk internal.CtxMutex p peer.ID m *messageSenderImpl @@ -297,7 +317,7 @@ func (ms *peerMessageSender) SendRequest(ctx context.Context, pmes *pb.Message) } func (ms *peerMessageSender) writeMsg(pmes *pb.Message) error { - return writeMsg(ms.s, pmes) + return WriteMsg(ms.s, pmes) } func (ms *peerMessageSender) ctxReadMsg(ctx context.Context, mes *pb.Message) error { @@ -325,3 +345,37 @@ func (ms *peerMessageSender) ctxReadMsg(ctx context.Context, mes *pb.Message) er return ErrReadTimeout } } + +// The Protobuf writer performs multiple small writes when writing a message. +// We need to buffer those writes, to make sure that we're not sending a new +// packet for every single write. +type bufferedDelimitedWriter struct { + *bufio.Writer + protoio.WriteCloser +} + +var writerPool = sync.Pool{ + New: func() interface{} { + w := bufio.NewWriter(nil) + return &bufferedDelimitedWriter{ + Writer: w, + WriteCloser: protoio.NewDelimitedWriter(w), + } + }, +} + +func WriteMsg(w io.Writer, mes *pb.Message) error { + bw := writerPool.Get().(*bufferedDelimitedWriter) + bw.Reset(w) + err := bw.WriteMsg(mes) + if err == nil { + err = bw.Flush() + } + bw.Reset(nil) + writerPool.Put(bw) + return err +} + +func (w *bufferedDelimitedWriter) Flush() error { + return w.Writer.Flush() +} diff --git a/routing.go b/routing.go index 6d31e0c5c..9204b367d 100644 --- a/routing.go +++ b/routing.go @@ -15,6 +15,7 @@ import ( "github.com/ipfs/go-cid" u "github.com/ipfs/go-ipfs-util" "github.com/libp2p/go-libp2p-kad-dht/internal" + internalConfig "github.com/libp2p/go-libp2p-kad-dht/internal/config" "github.com/libp2p/go-libp2p-kad-dht/qpeerset" kb "github.com/libp2p/go-libp2p-kbucket" record "github.com/libp2p/go-libp2p-record" @@ -109,7 +110,7 @@ func (dht *IpfsDHT) GetValue(ctx context.Context, key string, opts ...routing.Op if err := cfg.Apply(opts...); err != nil { return nil, err } - opts = append(opts, Quorum(getQuorum(&cfg, defaultQuorum))) + opts = append(opts, Quorum(internalConfig.GetQuorum(&cfg))) responses, err := dht.SearchValue(ctx, key, opts...) if err != nil { @@ -145,7 +146,7 @@ func (dht *IpfsDHT) SearchValue(ctx context.Context, key string, opts ...routing responsesNeeded := 0 if !cfg.Offline { - responsesNeeded = getQuorum(&cfg, defaultQuorum) + responsesNeeded = internalConfig.GetQuorum(&cfg) } stopCh := make(chan struct{}) diff --git a/routing_options.go b/routing_options.go index a1e5935b9..7352c098b 100644 --- a/routing_options.go +++ b/routing_options.go @@ -1,10 +1,9 @@ package dht -import "github.com/libp2p/go-libp2p-core/routing" - -type quorumOptionKey struct{} - -const defaultQuorum = 0 +import ( + "github.com/libp2p/go-libp2p-core/routing" + internalConfig "github.com/libp2p/go-libp2p-kad-dht/internal/config" +) // Quorum is a DHT option that tells the DHT how many peers it needs to get // values from before returning the best one. Zero means the DHT query @@ -16,15 +15,7 @@ func Quorum(n int) routing.Option { if opts.Other == nil { opts.Other = make(map[interface{}]interface{}, 1) } - opts.Other[quorumOptionKey{}] = n + opts.Other[internalConfig.QuorumOptionKey{}] = n return nil } } - -func getQuorum(opts *routing.Options, ndefault int) int { - responsesNeeded, ok := opts.Other[quorumOptionKey{}].(int) - if !ok { - responsesNeeded = ndefault - } - return responsesNeeded -} diff --git a/subscriber_notifee.go b/subscriber_notifee.go index 7cc9018f7..13ef89ca1 100644 --- a/subscriber_notifee.go +++ b/subscriber_notifee.go @@ -1,6 +1,7 @@ package dht import ( + "context" "fmt" "github.com/libp2p/go-libp2p-core/event" @@ -154,6 +155,10 @@ func (dht *IpfsDHT) validRTPeer(p peer.ID) (bool, error) { return dht.routingTablePeerFilter == nil || dht.routingTablePeerFilter(dht, dht.Host().Network().ConnsToPeer(p)), nil } +type disconnector interface { + StreamDisconnect(ctx context.Context, p peer.ID) +} + func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { dht := nn.dht select { @@ -173,7 +178,9 @@ func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { return } - dht.msgSender.streamDisconnect(dht.Context(), p) + if ms, ok := dht.msgSender.(disconnector); ok { + ms.StreamDisconnect(dht.Context(), p) + } } func (nn *subscriberNotifee) Connected(network.Network, network.Conn) {} From 365c18c05fc6af0c14cfa305f08ee3ff61c1a35e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 8 Apr 2021 01:49:31 -0400 Subject: [PATCH 02/29] bug fixes and add some options --- fullrt/dht.go | 227 ++++++++++++++++++++++++---------------------- fullrt/options.go | 42 +++++++++ 2 files changed, 163 insertions(+), 106 deletions(-) create mode 100644 fullrt/options.go diff --git a/fullrt/dht.go b/fullrt/dht.go index c7b9a7403..76fc4d827 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -4,36 +4,39 @@ import ( "bytes" "context" "fmt" - "github.com/gogo/protobuf/proto" - "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - u "github.com/ipfs/go-ipfs-util" - logging "github.com/ipfs/go-log" + "sync" + "sync/atomic" + "time" + + "github.com/multiformats/go-base32" + "github.com/multiformats/go-multiaddr" + "github.com/multiformats/go-multihash" + "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-core/routing" - dht "github.com/libp2p/go-libp2p-kad-dht" + + "github.com/gogo/protobuf/proto" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + u "github.com/ipfs/go-ipfs-util" + logging "github.com/ipfs/go-log" + + kaddht "github.com/libp2p/go-libp2p-kad-dht" "github.com/libp2p/go-libp2p-kad-dht/crawler" "github.com/libp2p/go-libp2p-kad-dht/internal" + internalConfig "github.com/libp2p/go-libp2p-kad-dht/internal/config" "github.com/libp2p/go-libp2p-kad-dht/internal/net" dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb" "github.com/libp2p/go-libp2p-kad-dht/providers" "github.com/libp2p/go-libp2p-kad-dht/qpeerset" kb "github.com/libp2p/go-libp2p-kbucket" + record "github.com/libp2p/go-libp2p-record" recpb "github.com/libp2p/go-libp2p-record/pb" - "github.com/multiformats/go-base32" - "github.com/multiformats/go-multiaddr" - "github.com/multiformats/go-multihash" - "sync" - "sync/atomic" - "time" - - kaddht "github.com/libp2p/go-libp2p-kad-dht" - internalConfig "github.com/libp2p/go-libp2p-kad-dht/internal/config" "github.com/libp2p/go-libp2p-xor/kademlia" kadkey "github.com/libp2p/go-libp2p-xor/key" @@ -51,10 +54,11 @@ type FullRT struct { datastore ds.Datastore h host.Host + crawlerInterval time.Duration crawler *crawler.Crawler protoMessenger *dht_pb.ProtocolMessenger - filterFromTable dht.QueryFilterFunc + filterFromTable kaddht.QueryFilterFunc rtLk sync.RWMutex rt *trie.Trie @@ -69,16 +73,30 @@ type FullRT struct { bucketSize int triggerRefresh chan struct{} + + waitFrac float64 + timeoutPerOp time.Duration + + provideManyParallelism int } -func NewFullRT(ctx context.Context, h host.Host, validator record.Validator, ds ds.Batching) (*FullRT, error) { - ms := net.NewMessageSenderImpl(h, []protocol.ID{"/ipfs/kad/1.0.0"}) - protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(validator)) +// NewFullRT creates a DHT client that tracks the full network. It takes a protocol prefix for the given network, +// For example, the protocol /ipfs/kad/1.0.0 has the prefix /ipfs. +func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opts ... Option) (*FullRT, error) { + cfg := &config{} + for _, o := range opts { + if err := o(cfg); err != nil { + return nil, err + } + } + + ms := net.NewMessageSenderImpl(h, []protocol.ID{protocolPrefix + "/kad/1.0.0"}) + protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(cfg.validator)) if err != nil { return nil, err } - pm, err := providers.NewProviderManager(ctx, h.ID(), ds) + pm, err := providers.NewProviderManager(ctx, h.ID(), cfg.datastore) if err != nil { return nil, err } @@ -88,20 +106,20 @@ func NewFullRT(ctx context.Context, h host.Host, validator record.Validator, ds return nil, err } - bsPeersNormal := dht.GetDefaultBootstrapPeerAddrInfos() - bsPeers := make([]*peer.AddrInfo, len(bsPeersNormal)) - for i, ai := range bsPeersNormal { + var bsPeers []*peer.AddrInfo + + for _, ai := range cfg.bootstrapPeers { tmpai := ai - bsPeers[i] = &tmpai + bsPeers = append(bsPeers, &tmpai) } rt := &FullRT{ ctx: ctx, enableValues: true, enableProviders: true, - Validator: validator, + Validator: cfg.validator, ProviderManager: pm, - datastore: ds, + datastore: cfg.datastore, h: h, crawler: c, protoMessenger: protoMessenger, @@ -114,6 +132,13 @@ func NewFullRT(ctx context.Context, h host.Host, validator record.Validator, ds bootstrapPeers: bsPeers, triggerRefresh: make(chan struct{}), + + waitFrac: 0.3, + timeoutPerOp: 5 * time.Second, + + crawlerInterval: time.Minute * 60, + + provideManyParallelism: 10, } go rt.runCrawler(ctx) @@ -147,7 +172,7 @@ func (dht *FullRT) Stat() map[string]peer.ID { } func (dht *FullRT) runCrawler(ctx context.Context) { - t := time.NewTicker(time.Minute * 60) + t := time.NewTicker(dht.crawlerInterval) m := make(map[peer.ID]*crawlVal) mxLk := sync.Mutex{} @@ -344,14 +369,14 @@ func (dht *FullRT) PutValue(ctx context.Context, key string, value []byte, opts return err } - successes := putToMany(ctx, func(ctx context.Context, p peer.ID) error { + successes := dht.execOnMany(ctx, func(ctx context.Context, p peer.ID) error { routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.Value, ID: p, }) err := dht.protoMessenger.PutValue(ctx, p, rec) return err - }, peers, 0.3, time.Second*5) + }, peers) if successes == 0 { return fmt.Errorf("failed to complete put") @@ -594,7 +619,7 @@ func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan str go func() { defer close(valCh) defer close(lookupResCh) - queryFn := func(ctx context.Context, p peer.ID) ([]*peer.AddrInfo, error) { + queryFn := func(ctx context.Context, p peer.ID) error { // For DHT query command routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.SendingQuery, @@ -611,9 +636,9 @@ func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan str Type: routing.PeerResponse, ID: p, }) - return nil, err + return nil default: - return nil, err + return err case nil, internal.ErrInvalidRecord: // in either of these cases, we want to keep going } @@ -629,7 +654,7 @@ func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan str select { case valCh <- rv: case <-ctx.Done(): - return nil, ctx.Err() + return ctx.Err() } } @@ -640,15 +665,10 @@ func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan str Responses: peers, }) - return peers, err + return nil } - for i := range peers { - p := peers[i] - go func() { - queryFn(ctx, p) - }() - } + dht.execOnMany(ctx, queryFn, peers) }() return valCh, lookupResCh } @@ -710,31 +730,31 @@ func (dht *FullRT) Provide(ctx context.Context, key cid.Cid, brdcst bool) (err e return err } - successes := putToMany(ctx, func(ctx context.Context, p peer.ID) error { + successes := dht.execOnMany(ctx, func(ctx context.Context, p peer.ID) error { err := dht.protoMessenger.PutProvider(ctx, p, keyMH, dht.h) return err - }, peers, 0.3, time.Second*5) + }, peers) if exceededDeadline { return context.DeadlineExceeded } if successes == 0 { - return fmt.Errorf("could not complete provide") + return fmt.Errorf("failed to complete provide") } return ctx.Err() } -func putToMany(ctx context.Context, fn func(context.Context, peer.ID) error, peers []peer.ID, waitFrac float64, timeoutPerOp time.Duration) int { +func (dht *FullRT) execOnMany(ctx context.Context, fn func(context.Context, peer.ID) error, peers []peer.ID) int { putctx, cancel := context.WithCancel(ctx) waitAllCh := make(chan struct{}, len(peers)) - numSuccessfulToWaitFor := int(float64(len(peers)) * waitFrac) + numSuccessfulToWaitFor := int(float64(len(peers)) * dht.waitFrac) waitSuccessCh := make(chan struct{}, numSuccessfulToWaitFor) for _, p := range peers { go func(p peer.ID) { - fnCtx, fnCancel := context.WithTimeout(putctx, timeoutPerOp) + fnCtx, fnCancel := context.WithTimeout(putctx, dht.timeoutPerOp) defer fnCancel() err := fn(fnCtx, p) if err != nil { @@ -775,52 +795,51 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) } sortedKeys := kb.SortClosestPeers(keysAsPeerIDs, kb.ID(make([]byte, 32))) - for _, k := range sortedKeys { + + var anyProvidesSuccessful uint64 = 0 + + fn := func(k peer.ID) error { peers, err := dht.GetClosestPeers(ctx, string(k)) if err != nil { return err } - sendMsgToSome(ctx, peers, func(ctx context.Context, p peer.ID) error { + successes := dht.execOnMany(ctx, func(ctx context.Context, p peer.ID) error { return dht.protoMessenger.PutProvider(ctx, p, multihash.Multihash(k), dht.h) - }) + }, peers) + if successes == 0 { + return fmt.Errorf("no successful provides") + } + return nil } - return nil -} - -func sendMsgToSome(ctx context.Context, peers []peer.ID, sendFn func(context.Context, peer.ID) error) { - groupSendCtx, cancel := context.WithCancel(ctx) - waitAllCh := make(chan struct{}, len(peers)) - numSuccessfulToWaitFor := int(float64(len(peers)) * 0.3) - waitSuccessCh := make(chan struct{}, numSuccessfulToWaitFor) - for _, p := range peers { - go func(p peer.ID) { - err := sendFn(groupSendCtx, p) - if err != nil { - logger.Debug(err) - } else { - waitSuccessCh <- struct{}{} + wg := sync.WaitGroup{} + wg.Add(dht.provideManyParallelism) + chunkSize := len(sortedKeys)/dht.provideManyParallelism + for i := 0; i < dht.provideManyParallelism; i++ { + var chunk []peer.ID + if i == dht.provideManyParallelism - 1 { + chunk = sortedKeys[i*chunkSize:] + } else { + chunk = sortedKeys[i*chunkSize:i*(chunkSize+1)] + } + go func() { + defer wg.Done() + for _, key := range chunk { + if err := fn(key); err != nil { + logger.Infow("failed to complete provide of key :%v. %w", internal.LoggableProviderRecordBytes(key), err) + } else { + atomic.CompareAndSwapUint64(&anyProvidesSuccessful, 0, 1) + } } - waitAllCh <- struct{}{} - }(p) + }() } + wg.Wait() - numSuccess, numDone := 0, 0 - t := time.NewTimer(time.Hour) - for numDone != len(peers) { - select { - case <-waitAllCh: - numDone++ - case <-waitSuccessCh: - if numSuccess >= numSuccessfulToWaitFor { - t.Reset(time.Millisecond * 500) - } - numSuccess++ - numDone++ - case <-t.C: - cancel() - } + if anyProvidesSuccessful == 0 { + return fmt.Errorf("failed to complete provides") } + + return nil } // FindProviders searches until the context expires. @@ -901,18 +920,17 @@ func (dht *FullRT) findProvidersAsyncRoutine(ctx context.Context, key multihash. queryctx, cancelquery := context.WithCancel(ctx) defer cancelquery() - for i := range peers { - p := peers[i] - go func() { + + fn := func(ctx context.Context, p peer.ID) error { // For DHT query command - routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.SendingQuery, ID: p, }) - provs, closest, err := dht.protoMessenger.GetProviders(queryctx, p, key) + provs, closest, err := dht.protoMessenger.GetProviders(ctx, p, key) if err != nil { - return + return err } logger.Debugf("%d provider entries", len(provs)) @@ -925,15 +943,15 @@ func (dht *FullRT) findProvidersAsyncRoutine(ctx context.Context, key multihash. logger.Debugf("using provider: %s", prov) select { case peerOut <- *prov: - case <-queryctx.Done(): + case <-ctx.Done(): logger.Debug("context timed out sending more providers") - return + return ctx.Err() } } if !findAll && ps.Size() >= count { logger.Debugf("got enough providers (%d/%d)", ps.Size(), count) cancelquery() - return + return nil } } @@ -945,8 +963,10 @@ func (dht *FullRT) findProvidersAsyncRoutine(ctx context.Context, key multihash. ID: p, Responses: closest, }) - }() + return nil } + + dht.execOnMany(queryctx, fn, peers) } // FindPeer searches for a peer with given ID. @@ -1003,26 +1023,21 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e } }() - wg := sync.WaitGroup{} - wg.Add(len(peers)) - for i := range peers { - p := peers[i] - go func() { - defer wg.Done() + fn := func(ctx context.Context, p peer.ID) error { // For DHT query command - routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.SendingQuery, ID: p, }) - peers, err := dht.protoMessenger.GetClosestPeers(queryctx, p, id) + peers, err := dht.protoMessenger.GetClosestPeers(ctx, p, id) if err != nil { logger.Debugf("error getting closer peers: %s", err) - return + return err } // For DHT query command - routing.PublishQueryEvent(queryctx, &routing.QueryEvent{ + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ Type: routing.PeerResponse, ID: p, Responses: peers, @@ -1032,16 +1047,16 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e if a.ID == id { select { case addrsCh <- a: - case <-queryctx.Done(): - return + case <-ctx.Done(): + return ctx.Err() } - return + return nil } } - }() + return nil } - wg.Wait() + dht.execOnMany(queryctx, fn, peers) // Return peer information if we tried to dial the peer during the query or we are (or recently were) connected // to the peer. diff --git a/fullrt/options.go b/fullrt/options.go new file mode 100644 index 000000000..d4f5163af --- /dev/null +++ b/fullrt/options.go @@ -0,0 +1,42 @@ +package fullrt + +import ( + ds "github.com/ipfs/go-datastore" + "github.com/libp2p/go-libp2p-core/peer" + record "github.com/libp2p/go-libp2p-record" +) + +type config struct { + validator record.Validator + datastore ds.Batching + bootstrapPeers []peer.AddrInfo +} + +// Option DHT option type. +type Option func(*config) error + +func Validator(validator record.Validator) Option { + return func(c *config) error { + c.validator = validator + return nil + } +} + +// BootstrapPeers configures the bootstrapping nodes that we will connect to to seed +// and refresh our Routing Table if it becomes empty. +func BootstrapPeers(bootstrappers ...peer.AddrInfo) Option { + return func(c *config) error { + c.bootstrapPeers = bootstrappers + return nil + } +} + +// Datastore configures the DHT to use the specified datastore. +// +// Defaults to an in-memory (temporary) map. +func Datastore(ds ds.Batching) Option { + return func(c *config) error { + c.datastore = ds + return nil + } +} From 753a1ea56985a436ee02b006d677c24adde6e7e9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 9 Apr 2021 13:05:01 -0400 Subject: [PATCH 03/29] go fmt + prov fix --- fullrt/dht.go | 153 ++++++++++++++++++++++++---------------------- fullrt/options.go | 4 +- 2 files changed, 83 insertions(+), 74 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 76fc4d827..42413856c 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -55,8 +55,8 @@ type FullRT struct { h host.Host crawlerInterval time.Duration - crawler *crawler.Crawler - protoMessenger *dht_pb.ProtocolMessenger + crawler *crawler.Crawler + protoMessenger *dht_pb.ProtocolMessenger filterFromTable kaddht.QueryFilterFunc rtLk sync.RWMutex @@ -74,7 +74,7 @@ type FullRT struct { triggerRefresh chan struct{} - waitFrac float64 + waitFrac float64 timeoutPerOp time.Duration provideManyParallelism int @@ -82,7 +82,7 @@ type FullRT struct { // NewFullRT creates a DHT client that tracks the full network. It takes a protocol prefix for the given network, // For example, the protocol /ipfs/kad/1.0.0 has the prefix /ipfs. -func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opts ... Option) (*FullRT, error) { +func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opts ...Option) (*FullRT, error) { cfg := &config{} for _, o := range opts { if err := o(cfg); err != nil { @@ -133,7 +133,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt triggerRefresh: make(chan struct{}), - waitFrac: 0.3, + waitFrac: 0.3, timeoutPerOp: 5 * time.Second, crawlerInterval: time.Minute * 60, @@ -748,6 +748,7 @@ func (dht *FullRT) Provide(ctx context.Context, key cid.Cid, brdcst bool) (err e func (dht *FullRT) execOnMany(ctx context.Context, fn func(context.Context, peer.ID) error, peers []peer.ID) int { putctx, cancel := context.WithCancel(ctx) + defer cancel() waitAllCh := make(chan struct{}, len(peers)) numSuccessfulToWaitFor := int(float64(len(peers)) * dht.waitFrac) @@ -790,7 +791,7 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) return routing.ErrNotSupported } - keysAsPeerIDs := make([]peer.ID, len(keys)) + keysAsPeerIDs := make([]peer.ID, 0, len(keys)) for _, k := range keys { keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) } @@ -814,19 +815,28 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) wg := sync.WaitGroup{} wg.Add(dht.provideManyParallelism) - chunkSize := len(sortedKeys)/dht.provideManyParallelism + chunkSize := len(sortedKeys) / dht.provideManyParallelism for i := 0; i < dht.provideManyParallelism; i++ { var chunk []peer.ID - if i == dht.provideManyParallelism - 1 { + end := (i + 1) * chunkSize + if end > len(sortedKeys) { chunk = sortedKeys[i*chunkSize:] } else { - chunk = sortedKeys[i*chunkSize:i*(chunkSize+1)] + chunk = sortedKeys[i*chunkSize : end] } + + loopIndex := i + go func() { defer wg.Done() - for _, key := range chunk { + for ki, key := range chunk { + if loopIndex == 0 { + if ki%100 == 0 { + logger.Infof("reprovide goroutine: %v pct done - %d/%d done - %d total", (ki*100)/len(chunk), ki, len(chunk), len(sortedKeys)) + } + } if err := fn(key); err != nil { - logger.Infow("failed to complete provide of key :%v. %w", internal.LoggableProviderRecordBytes(key), err) + logger.Infof("failed to complete provide of key :%v. %v", internal.LoggableProviderRecordBytes(key), err) } else { atomic.CompareAndSwapUint64(&anyProvidesSuccessful, 0, 1) } @@ -920,49 +930,48 @@ func (dht *FullRT) findProvidersAsyncRoutine(ctx context.Context, key multihash. queryctx, cancelquery := context.WithCancel(ctx) defer cancelquery() - fn := func(ctx context.Context, p peer.ID) error { - // For DHT query command - routing.PublishQueryEvent(ctx, &routing.QueryEvent{ - Type: routing.SendingQuery, - ID: p, - }) + // For DHT query command + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.SendingQuery, + ID: p, + }) - provs, closest, err := dht.protoMessenger.GetProviders(ctx, p, key) - if err != nil { - return err - } + provs, closest, err := dht.protoMessenger.GetProviders(ctx, p, key) + if err != nil { + return err + } - logger.Debugf("%d provider entries", len(provs)) - - // Add unique providers from request, up to 'count' - for _, prov := range provs { - dht.maybeAddAddrs(prov.ID, prov.Addrs, peerstore.TempAddrTTL) - logger.Debugf("got provider: %s", prov) - if ps.TryAdd(prov.ID) { - logger.Debugf("using provider: %s", prov) - select { - case peerOut <- *prov: - case <-ctx.Done(): - logger.Debug("context timed out sending more providers") - return ctx.Err() - } - } - if !findAll && ps.Size() >= count { - logger.Debugf("got enough providers (%d/%d)", ps.Size(), count) - cancelquery() - return nil + logger.Debugf("%d provider entries", len(provs)) + + // Add unique providers from request, up to 'count' + for _, prov := range provs { + dht.maybeAddAddrs(prov.ID, prov.Addrs, peerstore.TempAddrTTL) + logger.Debugf("got provider: %s", prov) + if ps.TryAdd(prov.ID) { + logger.Debugf("using provider: %s", prov) + select { + case peerOut <- *prov: + case <-ctx.Done(): + logger.Debug("context timed out sending more providers") + return ctx.Err() } } + if !findAll && ps.Size() >= count { + logger.Debugf("got enough providers (%d/%d)", ps.Size(), count) + cancelquery() + return nil + } + } - // Give closer peers back to the query to be queried - logger.Debugf("got closer peers: %d %s", len(closest), closest) + // Give closer peers back to the query to be queried + logger.Debugf("got closer peers: %d %s", len(closest), closest) - routing.PublishQueryEvent(ctx, &routing.QueryEvent{ - Type: routing.PeerResponse, - ID: p, - Responses: closest, - }) + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + Responses: closest, + }) return nil } @@ -1024,36 +1033,36 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e }() fn := func(ctx context.Context, p peer.ID) error { - // For DHT query command - routing.PublishQueryEvent(ctx, &routing.QueryEvent{ - Type: routing.SendingQuery, - ID: p, - }) + // For DHT query command + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.SendingQuery, + ID: p, + }) - peers, err := dht.protoMessenger.GetClosestPeers(ctx, p, id) - if err != nil { - logger.Debugf("error getting closer peers: %s", err) - return err - } + peers, err := dht.protoMessenger.GetClosestPeers(ctx, p, id) + if err != nil { + logger.Debugf("error getting closer peers: %s", err) + return err + } - // For DHT query command - routing.PublishQueryEvent(ctx, &routing.QueryEvent{ - Type: routing.PeerResponse, - ID: p, - Responses: peers, - }) + // For DHT query command + routing.PublishQueryEvent(ctx, &routing.QueryEvent{ + Type: routing.PeerResponse, + ID: p, + Responses: peers, + }) - for _, a := range peers { - if a.ID == id { - select { - case addrsCh <- a: - case <-ctx.Done(): - return ctx.Err() - } - return nil + for _, a := range peers { + if a.ID == id { + select { + case addrsCh <- a: + case <-ctx.Done(): + return ctx.Err() } + return nil } - return nil + } + return nil } dht.execOnMany(queryctx, fn, peers) diff --git a/fullrt/options.go b/fullrt/options.go index d4f5163af..1327b6969 100644 --- a/fullrt/options.go +++ b/fullrt/options.go @@ -7,8 +7,8 @@ import ( ) type config struct { - validator record.Validator - datastore ds.Batching + validator record.Validator + datastore ds.Batching bootstrapPeers []peer.AddrInfo } From 9bc8a2fac6d7da789fd07018f7ec0f8e39d75222 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 13 Apr 2021 15:44:30 -0400 Subject: [PATCH 04/29] optimize provide many to only calculate peer info once --- fullrt/dht.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 42413856c..6d51266b5 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -57,6 +57,7 @@ type FullRT struct { crawlerInterval time.Duration crawler *crawler.Crawler protoMessenger *dht_pb.ProtocolMessenger + messageSender dht_pb.MessageSender filterFromTable kaddht.QueryFilterFunc rtLk sync.RWMutex @@ -122,6 +123,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt datastore: cfg.datastore, h: h, crawler: c, + messageSender: ms, protoMessenger: protoMessenger, filterFromTable: kaddht.PublicQueryFilter, rt: trie.New(), @@ -799,13 +801,29 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) var anyProvidesSuccessful uint64 = 0 + // Compute addresses once for all provides + pi := peer.AddrInfo{ + ID: dht.h.ID(), + Addrs: dht.h.Addrs(), + } + pbPeers := dht_pb.RawPeerInfosToPBPeers([]peer.AddrInfo{pi}) + + // TODO: We may want to limit the type of addresses in our provider records + // For example, in a WAN-only DHT prohibit sharing non-WAN addresses (e.g. 192.168.0.100) + if len(pi.Addrs) < 1 { + return fmt.Errorf("no known addresses for self, cannot put provider") + } + fn := func(k peer.ID) error { peers, err := dht.GetClosestPeers(ctx, string(k)) if err != nil { return err } successes := dht.execOnMany(ctx, func(ctx context.Context, p peer.ID) error { - return dht.protoMessenger.PutProvider(ctx, p, multihash.Multihash(k), dht.h) + pmes := dht_pb.NewMessage(dht_pb.Message_ADD_PROVIDER, multihash.Multihash(k), 0) + pmes.ProviderPeers = pbPeers + + return dht.messageSender.SendMessage(ctx, p, pmes) }, peers) if successes == 0 { return fmt.Errorf("no successful provides") From 41d6f4413ed50529b81991e9e0a8ade06d354c79 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 10:12:09 -0400 Subject: [PATCH 05/29] a bit of cleanup and added a Ready function --- fullrt/dht.go | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 6d51266b5..c37423cc2 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -45,6 +45,8 @@ import ( var logger = logging.Logger("fullrtdht") +// FullRT is an experimental DHT client that is under development. Expect breaking changes to occur in this client +// until it stabilizes. type FullRT struct { ctx context.Context @@ -55,9 +57,11 @@ type FullRT struct { h host.Host crawlerInterval time.Duration - crawler *crawler.Crawler - protoMessenger *dht_pb.ProtocolMessenger - messageSender dht_pb.MessageSender + lastCrawlTime time.Time + + crawler *crawler.Crawler + protoMessenger *dht_pb.ProtocolMessenger + messageSender dht_pb.MessageSender filterFromTable kaddht.QueryFilterFunc rtLk sync.RWMutex @@ -173,6 +177,28 @@ func (dht *FullRT) Stat() map[string]peer.ID { return newMap } +func (dht *FullRT) Ready() bool { + dht.rtLk.RLock() + lastCrawlTime := dht.lastCrawlTime + dht.rtLk.RUnlock() + + if time.Since(lastCrawlTime) > dht.crawlerInterval { + return false + } + + // TODO: This function needs to be better defined. Perhaps based on going through the peer map and seeing when the + // last time we were connected to any of them was. + dht.peerAddrsLk.RLock() + rtSize := len(dht.keyToPeerMap) + dht.peerAddrsLk.RUnlock() + + if rtSize > len(dht.bootstrapPeers)+1 { + return true + } + + return false +} + func (dht *FullRT) runCrawler(ctx context.Context) { t := time.NewTicker(dht.crawlerInterval) @@ -215,8 +241,7 @@ func (dht *FullRT) runCrawler(ctx context.Context) { return }) dur := time.Since(start) - fmt.Printf("crawl took %v\n", dur) - start = time.Now() + logger.Infof("crawl took %v", dur) peerAddrs := make(map[peer.ID][]multiaddr.Multiaddr) kPeerMap := make(map[string]peer.ID) @@ -238,10 +263,8 @@ func (dht *FullRT) runCrawler(ctx context.Context) { dht.rtLk.Lock() dht.rt = newRt + dht.lastCrawlTime = time.Now() dht.rtLk.Unlock() - - dur = time.Since(start) - fmt.Printf("processing crawl took %v\n", dur) } } From e7a2e579330fcfb7639df7d1c943fc8eecbbaddc Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 13:35:00 -0400 Subject: [PATCH 06/29] moved TestInvalidMessageSenderTracking to the internal package containing the component it is testing --- dht_test.go | 22 ------------------ internal/net/message_manager_test.go | 34 ++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 22 deletions(-) create mode 100644 internal/net/message_manager_test.go diff --git a/dht_test.go b/dht_test.go index ea26b1bd3..ac34d503f 100644 --- a/dht_test.go +++ b/dht_test.go @@ -562,28 +562,6 @@ func TestValueGetInvalid(t *testing.T) { testSetGet("valid", "newer", nil) } -func TestInvalidMessageSenderTracking(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dht := setupDHT(ctx, t, false) - defer dht.Close() - - foo := peer.ID("asdasd") - _, err := dht.msgSender.messageSenderForPeer(ctx, foo) - if err == nil { - t.Fatal("that shouldnt have succeeded") - } - - dht.msgSender.smlk.Lock() - mscnt := len(dht.msgSender.strmap) - dht.msgSender.smlk.Unlock() - - if mscnt > 0 { - t.Fatal("should have no message senders in map") - } -} - func TestProvides(t *testing.T) { // t.Skip("skipping test to debug another") ctx, cancel := context.WithCancel(context.Background()) diff --git a/internal/net/message_manager_test.go b/internal/net/message_manager_test.go new file mode 100644 index 000000000..3eeb06f8f --- /dev/null +++ b/internal/net/message_manager_test.go @@ -0,0 +1,34 @@ +package net + +import ( + "context" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" + swarmt "github.com/libp2p/go-libp2p-swarm/testing" + bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + "testing" +) + +func TestInvalidMessageSenderTracking(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + foo := peer.ID("asdasd") + + h := bhost.New(swarmt.GenSwarm(t, ctx, swarmt.OptDisableReuseport)) + + msgSender := NewMessageSenderImpl(h, []protocol.ID{"/test/kad/1.0.0"}).(*messageSenderImpl) + + _, err := msgSender.messageSenderForPeer(ctx, foo) + if err == nil { + t.Fatal("that shouldnt have succeeded") + } + + msgSender.smlk.Lock() + mscnt := len(msgSender.strmap) + msgSender.smlk.Unlock() + + if mscnt > 0 { + t.Fatal("should have no message senders in map") + } +} From 7702392979ce6d743a4a5346115e267332bcaa72 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 13:36:02 -0400 Subject: [PATCH 07/29] move IpfsDHT options to the internal package. Make a breaking change to the filter interfaces to support more DHT implementations --- dht.go | 77 +++++++------ dht_filters.go | 26 +++-- dht_options.go | 237 +++++++++----------------------------- dht_test.go | 2 +- dual/dual_test.go | 4 +- internal/config/config.go | 162 ++++++++++++++++++++++++++ 6 files changed, 274 insertions(+), 234 deletions(-) create mode 100644 internal/config/config.go diff --git a/dht.go b/dht.go index 0f24c5f34..ae82c1396 100644 --- a/dht.go +++ b/dht.go @@ -16,6 +16,7 @@ import ( "github.com/libp2p/go-libp2p-core/routing" "github.com/libp2p/go-libp2p-kad-dht/internal" + dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" "github.com/libp2p/go-libp2p-kad-dht/internal/net" "github.com/libp2p/go-libp2p-kad-dht/metrics" pb "github.com/libp2p/go-libp2p-kad-dht/pb" @@ -164,15 +165,15 @@ var ( // If the Routing Table has more than "minRTRefreshThreshold" peers, we consider a peer as a Routing Table candidate ONLY when // we successfully get a query response from it OR if it send us a query. func New(ctx context.Context, h host.Host, options ...Option) (*IpfsDHT, error) { - var cfg config - if err := cfg.apply(append([]Option{defaults}, options...)...); err != nil { + var cfg dhtcfg.Config + if err := cfg.Apply(append([]Option{dhtcfg.Defaults}, options...)...); err != nil { return nil, err } - if err := cfg.applyFallbacks(h); err != nil { + if err := cfg.ApplyFallbacks(h); err != nil { return nil, err } - if err := cfg.validate(); err != nil { + if err := cfg.Validate(); err != nil { return nil, err } @@ -181,30 +182,30 @@ func New(ctx context.Context, h host.Host, options ...Option) (*IpfsDHT, error) return nil, fmt.Errorf("failed to create DHT, err=%s", err) } - dht.autoRefresh = cfg.routingTable.autoRefresh + dht.autoRefresh = cfg.RoutingTable.AutoRefresh - dht.maxRecordAge = cfg.maxRecordAge - dht.enableProviders = cfg.enableProviders - dht.enableValues = cfg.enableValues - dht.disableFixLowPeers = cfg.disableFixLowPeers + dht.maxRecordAge = cfg.MaxRecordAge + dht.enableProviders = cfg.EnableProviders + dht.enableValues = cfg.EnableValues + dht.disableFixLowPeers = cfg.DisableFixLowPeers - dht.Validator = cfg.validator + dht.Validator = cfg.Validator dht.msgSender = net.NewMessageSenderImpl(h, dht.protocols) dht.protoMessenger, err = pb.NewProtocolMessenger(dht.msgSender, pb.WithValidator(dht.Validator)) if err != nil { return nil, err } - dht.testAddressUpdateProcessing = cfg.testAddressUpdateProcessing + dht.testAddressUpdateProcessing = cfg.TestAddressUpdateProcessing - dht.auto = cfg.mode - switch cfg.mode { + dht.auto = cfg.Mode + switch cfg.Mode { case ModeAuto, ModeClient: dht.mode = modeClient case ModeAutoServer, ModeServer: dht.mode = modeServer default: - return nil, fmt.Errorf("invalid dht mode %d", cfg.mode) + return nil, fmt.Errorf("invalid dht mode %d", cfg.Mode) } if dht.mode == modeServer { @@ -262,20 +263,20 @@ func NewDHTClient(ctx context.Context, h host.Host, dstore ds.Batching) *IpfsDHT return dht } -func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { +func makeDHT(ctx context.Context, h host.Host, cfg dhtcfg.Config) (*IpfsDHT, error) { var protocols, serverProtocols []protocol.ID - v1proto := cfg.protocolPrefix + kad1 + v1proto := cfg.ProtocolPrefix + kad1 - if cfg.v1ProtocolOverride != "" { - v1proto = cfg.v1ProtocolOverride + if cfg.V1ProtocolOverride != "" { + v1proto = cfg.V1ProtocolOverride } protocols = []protocol.ID{v1proto} serverProtocols = []protocol.ID{v1proto} dht := &IpfsDHT{ - datastore: cfg.datastore, + datastore: cfg.Datastore, self: h.ID(), selfKey: kb.ConvertPeerID(h.ID()), peerstore: h.Peerstore(), @@ -284,12 +285,12 @@ func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { protocols: protocols, protocolsStrs: protocol.ConvertToStrings(protocols), serverProtocols: serverProtocols, - bucketSize: cfg.bucketSize, - alpha: cfg.concurrency, - beta: cfg.resiliency, - queryPeerFilter: cfg.queryPeerFilter, - routingTablePeerFilter: cfg.routingTable.peerFilter, - rtPeerDiversityFilter: cfg.routingTable.diversityFilter, + bucketSize: cfg.BucketSize, + alpha: cfg.Concurrency, + beta: cfg.Resiliency, + queryPeerFilter: cfg.QueryPeerFilter, + routingTablePeerFilter: cfg.RoutingTable.PeerFilter, + rtPeerDiversityFilter: cfg.RoutingTable.DiversityFilter, fixLowPeersChan: make(chan struct{}, 1), @@ -303,12 +304,12 @@ func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { // query a peer as part of our refresh cycle. // To grok the Math Wizardy that produced these exact equations, please be patient as a document explaining it will // be published soon. - if cfg.concurrency < cfg.bucketSize { // (alpha < K) - l1 := math.Log(float64(1) / float64(cfg.bucketSize)) //(Log(1/K)) - l2 := math.Log(float64(1) - (float64(cfg.concurrency) / float64(cfg.bucketSize))) // Log(1 - (alpha / K)) - maxLastSuccessfulOutboundThreshold = time.Duration(l1 / l2 * float64(cfg.routingTable.refreshInterval)) + if cfg.Concurrency < cfg.BucketSize { // (alpha < K) + l1 := math.Log(float64(1) / float64(cfg.BucketSize)) //(Log(1/K)) + l2 := math.Log(float64(1) - (float64(cfg.Concurrency) / float64(cfg.BucketSize))) // Log(1 - (alpha / K)) + maxLastSuccessfulOutboundThreshold = time.Duration(l1 / l2 * float64(cfg.RoutingTable.RefreshInterval)) } else { - maxLastSuccessfulOutboundThreshold = cfg.routingTable.refreshInterval + maxLastSuccessfulOutboundThreshold = cfg.RoutingTable.RefreshInterval } // construct routing table @@ -318,7 +319,7 @@ func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { return nil, fmt.Errorf("failed to construct routing table,err=%s", err) } dht.routingTable = rt - dht.bootstrapPeers = cfg.bootstrapPeers + dht.bootstrapPeers = cfg.BootstrapPeers // rt refresh manager rtRefresh, err := makeRtRefreshManager(dht, cfg, maxLastSuccessfulOutboundThreshold) @@ -337,7 +338,7 @@ func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { // the DHT context should be done when the process is closed dht.ctx = goprocessctx.WithProcessClosing(ctxTags, dht.proc) - pm, err := providers.NewProviderManager(dht.ctx, h.ID(), cfg.datastore, cfg.providersOptions...) + pm, err := providers.NewProviderManager(dht.ctx, h.ID(), cfg.Datastore, cfg.ProvidersOptions...) if err != nil { return nil, err } @@ -348,7 +349,7 @@ func makeDHT(ctx context.Context, h host.Host, cfg config) (*IpfsDHT, error) { return dht, nil } -func makeRtRefreshManager(dht *IpfsDHT, cfg config, maxLastSuccessfulOutboundThreshold time.Duration) (*rtrefresh.RtRefreshManager, error) { +func makeRtRefreshManager(dht *IpfsDHT, cfg dhtcfg.Config, maxLastSuccessfulOutboundThreshold time.Duration) (*rtrefresh.RtRefreshManager, error) { keyGenFnc := func(cpl uint) (string, error) { p, err := dht.routingTable.GenRandPeerID(cpl) return string(p), err @@ -360,18 +361,18 @@ func makeRtRefreshManager(dht *IpfsDHT, cfg config, maxLastSuccessfulOutboundThr } r, err := rtrefresh.NewRtRefreshManager( - dht.host, dht.routingTable, cfg.routingTable.autoRefresh, + dht.host, dht.routingTable, cfg.RoutingTable.AutoRefresh, keyGenFnc, queryFnc, - cfg.routingTable.refreshQueryTimeout, - cfg.routingTable.refreshInterval, + cfg.RoutingTable.RefreshQueryTimeout, + cfg.RoutingTable.RefreshInterval, maxLastSuccessfulOutboundThreshold, dht.refreshFinishedCh) return r, err } -func makeRoutingTable(dht *IpfsDHT, cfg config, maxLastSuccessfulOutboundThreshold time.Duration) (*kb.RoutingTable, error) { +func makeRoutingTable(dht *IpfsDHT, cfg dhtcfg.Config, maxLastSuccessfulOutboundThreshold time.Duration) (*kb.RoutingTable, error) { // make a Routing Table Diversity Filter var filter *peerdiversity.Filter if dht.rtPeerDiversityFilter != nil { @@ -386,7 +387,7 @@ func makeRoutingTable(dht *IpfsDHT, cfg config, maxLastSuccessfulOutboundThresho filter = df } - rt, err := kb.NewRoutingTable(cfg.bucketSize, dht.selfKey, time.Minute, dht.host.Peerstore(), maxLastSuccessfulOutboundThreshold, filter) + rt, err := kb.NewRoutingTable(cfg.BucketSize, dht.selfKey, time.Minute, dht.host.Peerstore(), maxLastSuccessfulOutboundThreshold, filter) if err != nil { return nil, err } diff --git a/dht_filters.go b/dht_filters.go index df921963a..d56e50738 100644 --- a/dht_filters.go +++ b/dht_filters.go @@ -2,6 +2,8 @@ package dht import ( "bytes" + "github.com/libp2p/go-libp2p-core/host" + dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" "net" "sync" "time" @@ -17,11 +19,11 @@ import ( ) // QueryFilterFunc is a filter applied when considering peers to dial when querying -type QueryFilterFunc func(dht *IpfsDHT, ai peer.AddrInfo) bool +type QueryFilterFunc = dhtcfg.QueryFilterFunc // RouteTableFilterFunc is a filter applied when considering connections to keep in // the local route table. -type RouteTableFilterFunc func(dht *IpfsDHT, conns []network.Conn) bool +type RouteTableFilterFunc = dhtcfg.RouteTableFilterFunc var publicCIDR6 = "2000::/3" var public6 *net.IPNet @@ -59,7 +61,7 @@ func isPrivateAddr(a ma.Multiaddr) bool { } // PublicQueryFilter returns true if the peer is suspected of being publicly accessible -func PublicQueryFilter(_ *IpfsDHT, ai peer.AddrInfo) bool { +func PublicQueryFilter(_ interface{}, ai peer.AddrInfo) bool { if len(ai.Addrs) == 0 { return false } @@ -73,18 +75,24 @@ func PublicQueryFilter(_ *IpfsDHT, ai peer.AddrInfo) bool { return hasPublicAddr } +type hasHost interface { + Host() host.Host +} + var _ QueryFilterFunc = PublicQueryFilter // PublicRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate // that it is on a public network -func PublicRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool { +func PublicRoutingTableFilter(dht interface{}, conns []network.Conn) bool { if len(conns) == 0 { return false } + d := dht.(hasHost) + // Do we have a public address for this peer? id := conns[0].RemotePeer() - known := dht.peerstore.PeerInfo(id) + known := d.Host().Peerstore().PeerInfo(id) for _, a := range known.Addrs { if !isRelayAddr(a) && isPublicAddr(a) { return true @@ -97,7 +105,7 @@ func PublicRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool { var _ RouteTableFilterFunc = PublicRoutingTableFilter // PrivateQueryFilter doens't currently restrict which peers we are willing to query from the local DHT. -func PrivateQueryFilter(dht *IpfsDHT, ai peer.AddrInfo) bool { +func PrivateQueryFilter(_ interface{}, ai peer.AddrInfo) bool { return len(ai.Addrs) > 0 } @@ -137,10 +145,12 @@ func getCachedRouter() routing.Router { // PrivateRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate // that it is on a private network -func PrivateRoutingTableFilter(dht *IpfsDHT, conns []network.Conn) bool { +func PrivateRoutingTableFilter(dht interface{}, conns []network.Conn) bool { + d := dht.(hasHost) + router := getCachedRouter() myAdvertisedIPs := make([]net.IP, 0) - for _, a := range dht.Host().Addrs() { + for _, a := range d.Host().Addrs() { if isPublicAddr(a) && !isRelayAddr(a) { ip, err := manet.ToIP(a) if err != nil { diff --git a/dht_options.go b/dht_options.go index 820145451..1f4e47afe 100644 --- a/dht_options.go +++ b/dht_options.go @@ -5,22 +5,19 @@ import ( "testing" "time" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" + dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" "github.com/libp2p/go-libp2p-kad-dht/providers" "github.com/libp2p/go-libp2p-kbucket/peerdiversity" record "github.com/libp2p/go-libp2p-record" ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - "github.com/ipfs/go-ipns" ) // ModeOpt describes what mode the dht should operate in -type ModeOpt int +type ModeOpt = dhtcfg.ModeOpt const ( // ModeAuto utilizes EvtLocalReachabilityChanged events sent over the event bus to dynamically switch the DHT @@ -37,143 +34,13 @@ const ( // DefaultPrefix is the application specific prefix attached to all DHT protocols by default. const DefaultPrefix protocol.ID = "/ipfs" -// Options is a structure containing all the options that can be used when constructing a DHT. -type config struct { - datastore ds.Batching - validator record.Validator - validatorChanged bool // if true implies that the validator has been changed and that defaults should not be used - mode ModeOpt - protocolPrefix protocol.ID - v1ProtocolOverride protocol.ID - bucketSize int - concurrency int - resiliency int - maxRecordAge time.Duration - enableProviders bool - enableValues bool - providersOptions []providers.Option - queryPeerFilter QueryFilterFunc - - routingTable struct { - refreshQueryTimeout time.Duration - refreshInterval time.Duration - autoRefresh bool - latencyTolerance time.Duration - checkInterval time.Duration - peerFilter RouteTableFilterFunc - diversityFilter peerdiversity.PeerIPGroupFilter - } - - bootstrapPeers []peer.AddrInfo - - // test specific config options - disableFixLowPeers bool - testAddressUpdateProcessing bool -} - -func emptyQueryFilter(_ *IpfsDHT, ai peer.AddrInfo) bool { return true } -func emptyRTFilter(_ *IpfsDHT, conns []network.Conn) bool { return true } - -// apply applies the given options to this Option -func (c *config) apply(opts ...Option) error { - for i, opt := range opts { - if err := opt(c); err != nil { - return fmt.Errorf("dht option %d failed: %s", i, err) - } - } - return nil -} - -// applyFallbacks sets default values that could not be applied during config creation since they are dependent -// on other configuration parameters (e.g. optA is by default 2x optB) and/or on the Host -func (c *config) applyFallbacks(h host.Host) error { - if !c.validatorChanged { - nsval, ok := c.validator.(record.NamespacedValidator) - if ok { - if _, pkFound := nsval["pk"]; !pkFound { - nsval["pk"] = record.PublicKeyValidator{} - } - if _, ipnsFound := nsval["ipns"]; !ipnsFound { - nsval["ipns"] = ipns.Validator{KeyBook: h.Peerstore()} - } - } else { - return fmt.Errorf("the default validator was changed without being marked as changed") - } - } - return nil -} - -// Option DHT option type. -type Option func(*config) error - -const defaultBucketSize = 20 - -// defaults are the default DHT options. This option will be automatically -// prepended to any options you pass to the DHT constructor. -var defaults = func(o *config) error { - o.validator = record.NamespacedValidator{} - o.datastore = dssync.MutexWrap(ds.NewMapDatastore()) - o.protocolPrefix = DefaultPrefix - o.enableProviders = true - o.enableValues = true - o.queryPeerFilter = emptyQueryFilter - - o.routingTable.latencyTolerance = time.Minute - o.routingTable.refreshQueryTimeout = 1 * time.Minute - o.routingTable.refreshInterval = 10 * time.Minute - o.routingTable.autoRefresh = true - o.routingTable.peerFilter = emptyRTFilter - o.maxRecordAge = time.Hour * 36 - - o.bucketSize = defaultBucketSize - o.concurrency = 10 - o.resiliency = 3 - - return nil -} - -func (c *config) validate() error { - if c.protocolPrefix != DefaultPrefix { - return nil - } - if c.bucketSize != defaultBucketSize { - return fmt.Errorf("protocol prefix %s must use bucket size %d", DefaultPrefix, defaultBucketSize) - } - if !c.enableProviders { - return fmt.Errorf("protocol prefix %s must have providers enabled", DefaultPrefix) - } - if !c.enableValues { - return fmt.Errorf("protocol prefix %s must have values enabled", DefaultPrefix) - } - - nsval, isNSVal := c.validator.(record.NamespacedValidator) - if !isNSVal { - return fmt.Errorf("protocol prefix %s must use a namespaced validator", DefaultPrefix) - } - - if len(nsval) != 2 { - return fmt.Errorf("protocol prefix %s must have exactly two namespaced validators - /pk and /ipns", DefaultPrefix) - } - - if pkVal, pkValFound := nsval["pk"]; !pkValFound { - return fmt.Errorf("protocol prefix %s must support the /pk namespaced validator", DefaultPrefix) - } else if _, ok := pkVal.(record.PublicKeyValidator); !ok { - return fmt.Errorf("protocol prefix %s must use the record.PublicKeyValidator for the /pk namespace", DefaultPrefix) - } - - if ipnsVal, ipnsValFound := nsval["ipns"]; !ipnsValFound { - return fmt.Errorf("protocol prefix %s must support the /ipns namespaced validator", DefaultPrefix) - } else if _, ok := ipnsVal.(ipns.Validator); !ok { - return fmt.Errorf("protocol prefix %s must use ipns.Validator for the /ipns namespace", DefaultPrefix) - } - return nil -} +type Option = dhtcfg.Option // RoutingTableLatencyTolerance sets the maximum acceptable latency for peers // in the routing table's cluster. func RoutingTableLatencyTolerance(latency time.Duration) Option { - return func(c *config) error { - c.routingTable.latencyTolerance = latency + return func(c *dhtcfg.Config) error { + c.RoutingTable.LatencyTolerance = latency return nil } } @@ -181,8 +48,8 @@ func RoutingTableLatencyTolerance(latency time.Duration) Option { // RoutingTableRefreshQueryTimeout sets the timeout for routing table refresh // queries. func RoutingTableRefreshQueryTimeout(timeout time.Duration) Option { - return func(c *config) error { - c.routingTable.refreshQueryTimeout = timeout + return func(c *dhtcfg.Config) error { + c.RoutingTable.RefreshQueryTimeout = timeout return nil } } @@ -194,8 +61,8 @@ func RoutingTableRefreshQueryTimeout(timeout time.Duration) Option { // 1. Then searching for a random key in each bucket that hasn't been queried in // the last refresh period. func RoutingTableRefreshPeriod(period time.Duration) Option { - return func(c *config) error { - c.routingTable.refreshInterval = period + return func(c *dhtcfg.Config) error { + c.RoutingTable.RefreshInterval = period return nil } } @@ -204,8 +71,8 @@ func RoutingTableRefreshPeriod(period time.Duration) Option { // // Defaults to an in-memory (temporary) map. func Datastore(ds ds.Batching) Option { - return func(c *config) error { - c.datastore = ds + return func(c *dhtcfg.Config) error { + c.Datastore = ds return nil } } @@ -214,8 +81,8 @@ func Datastore(ds ds.Batching) Option { // // Defaults to ModeAuto. func Mode(m ModeOpt) Option { - return func(c *config) error { - c.mode = m + return func(c *dhtcfg.Config) error { + c.Mode = m return nil } } @@ -227,9 +94,9 @@ func Mode(m ModeOpt) Option { // implies that the user wants to control the validators and therefore the default // public key and IPNS validators will not be added. func Validator(v record.Validator) Option { - return func(c *config) error { - c.validator = v - c.validatorChanged = true + return func(c *dhtcfg.Config) error { + c.Validator = v + c.ValidatorChanged = true return nil } } @@ -246,8 +113,8 @@ func Validator(v record.Validator) Option { // myValidator)`, all records with keys starting with `/ipns/` will be validated // with `myValidator`. func NamespacedValidator(ns string, v record.Validator) Option { - return func(c *config) error { - nsval, ok := c.validator.(record.NamespacedValidator) + return func(c *dhtcfg.Config) error { + nsval, ok := c.Validator.(record.NamespacedValidator) if !ok { return fmt.Errorf("can only add namespaced validators to a NamespacedValidator") } @@ -261,8 +128,8 @@ func NamespacedValidator(ns string, v record.Validator) Option { // // Defaults to dht.DefaultPrefix func ProtocolPrefix(prefix protocol.ID) Option { - return func(c *config) error { - c.protocolPrefix = prefix + return func(c *dhtcfg.Config) error { + c.ProtocolPrefix = prefix return nil } } @@ -270,8 +137,8 @@ func ProtocolPrefix(prefix protocol.ID) Option { // ProtocolExtension adds an application specific protocol to the DHT protocol. For example, // /ipfs/lan/kad/1.0.0 instead of /ipfs/kad/1.0.0. extension should be of the form /lan. func ProtocolExtension(ext protocol.ID) Option { - return func(c *config) error { - c.protocolPrefix += ext + return func(c *dhtcfg.Config) error { + c.ProtocolPrefix += ext return nil } } @@ -282,8 +149,8 @@ func ProtocolExtension(ext protocol.ID) Option { // // This option will override and ignore the ProtocolPrefix and ProtocolExtension options func V1ProtocolOverride(proto protocol.ID) Option { - return func(c *config) error { - c.v1ProtocolOverride = proto + return func(c *dhtcfg.Config) error { + c.V1ProtocolOverride = proto return nil } } @@ -292,8 +159,8 @@ func V1ProtocolOverride(proto protocol.ID) Option { // // The default value is 20. func BucketSize(bucketSize int) Option { - return func(c *config) error { - c.bucketSize = bucketSize + return func(c *dhtcfg.Config) error { + c.BucketSize = bucketSize return nil } } @@ -302,8 +169,8 @@ func BucketSize(bucketSize int) Option { // // The default value is 10. func Concurrency(alpha int) Option { - return func(c *config) error { - c.concurrency = alpha + return func(c *dhtcfg.Config) error { + c.Concurrency = alpha return nil } } @@ -313,8 +180,8 @@ func Concurrency(alpha int) Option { // // The default value is 3. func Resiliency(beta int) Option { - return func(c *config) error { - c.resiliency = beta + return func(c *dhtcfg.Config) error { + c.Resiliency = beta return nil } } @@ -326,8 +193,8 @@ func Resiliency(beta int) Option { // until the year 2020 (a great time in the future). For that record to stick around // it must be rebroadcasted more frequently than once every 'MaxRecordAge' func MaxRecordAge(maxAge time.Duration) Option { - return func(c *config) error { - c.maxRecordAge = maxAge + return func(c *dhtcfg.Config) error { + c.MaxRecordAge = maxAge return nil } } @@ -336,8 +203,8 @@ func MaxRecordAge(maxAge time.Duration) Option { // table. This means that we will neither refresh the routing table periodically // nor when the routing table size goes below the minimum threshold. func DisableAutoRefresh() Option { - return func(c *config) error { - c.routingTable.autoRefresh = false + return func(c *dhtcfg.Config) error { + c.RoutingTable.AutoRefresh = false return nil } } @@ -349,8 +216,8 @@ func DisableAutoRefresh() Option { // WARNING: do not change this unless you're using a forked DHT (i.e., a private // network and/or distinct DHT protocols with the `Protocols` option). func DisableProviders() Option { - return func(c *config) error { - c.enableProviders = false + return func(c *dhtcfg.Config) error { + c.EnableProviders = false return nil } } @@ -363,8 +230,8 @@ func DisableProviders() Option { // WARNING: do not change this unless you're using a forked DHT (i.e., a private // network and/or distinct DHT protocols with the `Protocols` option). func DisableValues() Option { - return func(c *config) error { - c.enableValues = false + return func(c *dhtcfg.Config) error { + c.EnableValues = false return nil } } @@ -375,16 +242,16 @@ func DisableValues() Option { // them in between. These options are passed to the provider manager allowing // customisation of things like the GC interval and cache implementation. func ProvidersOptions(opts []providers.Option) Option { - return func(c *config) error { - c.providersOptions = opts + return func(c *dhtcfg.Config) error { + c.ProvidersOptions = opts return nil } } // QueryFilter sets a function that approves which peers may be dialed in a query func QueryFilter(filter QueryFilterFunc) Option { - return func(c *config) error { - c.queryPeerFilter = filter + return func(c *dhtcfg.Config) error { + c.QueryPeerFilter = filter return nil } } @@ -392,8 +259,8 @@ func QueryFilter(filter QueryFilterFunc) Option { // RoutingTableFilter sets a function that approves which peers may be added to the routing table. The host should // already have at least one connection to the peer under consideration. func RoutingTableFilter(filter RouteTableFilterFunc) Option { - return func(c *config) error { - c.routingTable.peerFilter = filter + return func(c *dhtcfg.Config) error { + c.RoutingTable.PeerFilter = filter return nil } } @@ -401,8 +268,8 @@ func RoutingTableFilter(filter RouteTableFilterFunc) Option { // BootstrapPeers configures the bootstrapping nodes that we will connect to to seed // and refresh our Routing Table if it becomes empty. func BootstrapPeers(bootstrappers ...peer.AddrInfo) Option { - return func(c *config) error { - c.bootstrapPeers = bootstrappers + return func(c *dhtcfg.Config) error { + c.BootstrapPeers = bootstrappers return nil } } @@ -411,8 +278,8 @@ func BootstrapPeers(bootstrappers ...peer.AddrInfo) Option { // to construct the diversity filter for the Routing Table. // Please see the docs for `peerdiversity.PeerIPGroupFilter` AND `peerdiversity.Filter` for more details. func RoutingTablePeerDiversityFilter(pg peerdiversity.PeerIPGroupFilter) Option { - return func(c *config) error { - c.routingTable.diversityFilter = pg + return func(c *dhtcfg.Config) error { + c.RoutingTable.DiversityFilter = pg return nil } } @@ -420,8 +287,8 @@ func RoutingTablePeerDiversityFilter(pg peerdiversity.PeerIPGroupFilter) Option // disableFixLowPeersRoutine disables the "fixLowPeers" routine in the DHT. // This is ONLY for tests. func disableFixLowPeersRoutine(t *testing.T) Option { - return func(c *config) error { - c.disableFixLowPeers = true + return func(c *dhtcfg.Config) error { + c.DisableFixLowPeers = true return nil } } @@ -430,8 +297,8 @@ func disableFixLowPeersRoutine(t *testing.T) Option { // This occurs even when AutoRefresh has been disabled. // This is ONLY for tests. func forceAddressUpdateProcessing(t *testing.T) Option { - return func(c *config) error { - c.testAddressUpdateProcessing = true + return func(c *dhtcfg.Config) error { + c.TestAddressUpdateProcessing = true return nil } } diff --git a/dht_test.go b/dht_test.go index ac34d503f..9791d9638 100644 --- a/dht_test.go +++ b/dht_test.go @@ -1165,7 +1165,7 @@ func TestFindPeerWithQueryFilter(t *testing.T) { defer cancel() filteredPeer := bhost.New(swarmt.GenSwarm(t, ctx, swarmt.OptDisableReuseport)) - dhts := setupDHTS(t, ctx, 4, QueryFilter(func(_ *IpfsDHT, ai peer.AddrInfo) bool { + dhts := setupDHTS(t, ctx, 4, QueryFilter(func(_ interface{}, ai peer.AddrInfo) bool { return ai.ID != filteredPeer.ID() })) defer func() { diff --git a/dual/dual_test.go b/dual/dual_test.go index 41c8a112b..24742a505 100644 --- a/dual/dual_test.go +++ b/dual/dual_test.go @@ -34,9 +34,9 @@ type customRtHelper struct { allow peer.ID } -func MkFilterForPeer() (func(d *dht.IpfsDHT, conns []network.Conn) bool, *customRtHelper) { +func MkFilterForPeer() (func(_ interface{}, conns []network.Conn) bool, *customRtHelper) { helper := customRtHelper{} - f := func(_ *dht.IpfsDHT, conns []network.Conn) bool { + f := func(_ interface{}, conns []network.Conn) bool { for _, c := range conns { if c.RemotePeer() == helper.allow { return true diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 000000000..274e24d64 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,162 @@ +package config + +import ( + "fmt" + "time" + + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + "github.com/ipfs/go-ipns" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/protocol" + "github.com/libp2p/go-libp2p-kad-dht/providers" + "github.com/libp2p/go-libp2p-kbucket/peerdiversity" + record "github.com/libp2p/go-libp2p-record" +) + +// DefaultPrefix is the application specific prefix attached to all DHT protocols by default. +const DefaultPrefix protocol.ID = "/ipfs" + +const defaultBucketSize = 20 + +// ModeOpt describes what mode the dht should operate in +type ModeOpt int + +// QueryFilterFunc is a filter applied when considering peers to dial when querying +type QueryFilterFunc func(dht interface{}, ai peer.AddrInfo) bool + +// RouteTableFilterFunc is a filter applied when considering connections to keep in +// the local route table. +type RouteTableFilterFunc func(dht interface{}, conns []network.Conn) bool + +// Config is a structure containing all the options that can be used when constructing a DHT. +type Config struct { + Datastore ds.Batching + Validator record.Validator + ValidatorChanged bool // if true implies that the validator has been changed and that Defaults should not be used + Mode ModeOpt + ProtocolPrefix protocol.ID + V1ProtocolOverride protocol.ID + BucketSize int + Concurrency int + Resiliency int + MaxRecordAge time.Duration + EnableProviders bool + EnableValues bool + ProvidersOptions []providers.Option + QueryPeerFilter QueryFilterFunc + + RoutingTable struct { + RefreshQueryTimeout time.Duration + RefreshInterval time.Duration + AutoRefresh bool + LatencyTolerance time.Duration + CheckInterval time.Duration + PeerFilter RouteTableFilterFunc + DiversityFilter peerdiversity.PeerIPGroupFilter + } + + BootstrapPeers []peer.AddrInfo + + // test specific Config options + DisableFixLowPeers bool + TestAddressUpdateProcessing bool +} + +func EmptyQueryFilter(_ interface{}, ai peer.AddrInfo) bool { return true } +func EmptyRTFilter(_ interface{}, conns []network.Conn) bool { return true } + +// Apply applies the given options to this Option +func (c *Config) Apply(opts ...Option) error { + for i, opt := range opts { + if err := opt(c); err != nil { + return fmt.Errorf("dht option %d failed: %s", i, err) + } + } + return nil +} + +// ApplyFallbacks sets default values that could not be applied during config creation since they are dependent +// on other configuration parameters (e.g. optA is by default 2x optB) and/or on the Host +func (c *Config) ApplyFallbacks(h host.Host) error { + if !c.ValidatorChanged { + nsval, ok := c.Validator.(record.NamespacedValidator) + if ok { + if _, pkFound := nsval["pk"]; !pkFound { + nsval["pk"] = record.PublicKeyValidator{} + } + if _, ipnsFound := nsval["ipns"]; !ipnsFound { + nsval["ipns"] = ipns.Validator{KeyBook: h.Peerstore()} + } + } else { + return fmt.Errorf("the default Validator was changed without being marked as changed") + } + } + return nil +} + +// Option DHT option type. +type Option func(*Config) error + +// Defaults are the default DHT options. This option will be automatically +// prepended to any options you pass to the DHT constructor. +var Defaults = func(o *Config) error { + o.Validator = record.NamespacedValidator{} + o.Datastore = dssync.MutexWrap(ds.NewMapDatastore()) + o.ProtocolPrefix = DefaultPrefix + o.EnableProviders = true + o.EnableValues = true + o.QueryPeerFilter = EmptyQueryFilter + + o.RoutingTable.LatencyTolerance = time.Minute + o.RoutingTable.RefreshQueryTimeout = 1 * time.Minute + o.RoutingTable.RefreshInterval = 10 * time.Minute + o.RoutingTable.AutoRefresh = true + o.RoutingTable.PeerFilter = EmptyRTFilter + o.MaxRecordAge = time.Hour * 36 + + o.BucketSize = defaultBucketSize + o.Concurrency = 10 + o.Resiliency = 3 + + return nil +} + +func (c *Config) Validate() error { + if c.ProtocolPrefix != DefaultPrefix { + return nil + } + if c.BucketSize != defaultBucketSize { + return fmt.Errorf("protocol prefix %s must use bucket size %d", DefaultPrefix, defaultBucketSize) + } + if !c.EnableProviders { + return fmt.Errorf("protocol prefix %s must have providers enabled", DefaultPrefix) + } + if !c.EnableValues { + return fmt.Errorf("protocol prefix %s must have values enabled", DefaultPrefix) + } + + nsval, isNSVal := c.Validator.(record.NamespacedValidator) + if !isNSVal { + return fmt.Errorf("protocol prefix %s must use a namespaced Validator", DefaultPrefix) + } + + if len(nsval) != 2 { + return fmt.Errorf("protocol prefix %s must have exactly two namespaced validators - /pk and /ipns", DefaultPrefix) + } + + if pkVal, pkValFound := nsval["pk"]; !pkValFound { + return fmt.Errorf("protocol prefix %s must support the /pk namespaced Validator", DefaultPrefix) + } else if _, ok := pkVal.(record.PublicKeyValidator); !ok { + return fmt.Errorf("protocol prefix %s must use the record.PublicKeyValidator for the /pk namespace", DefaultPrefix) + } + + if ipnsVal, ipnsValFound := nsval["ipns"]; !ipnsValFound { + return fmt.Errorf("protocol prefix %s must support the /ipns namespaced Validator", DefaultPrefix) + } else if _, ok := ipnsVal.(ipns.Validator); !ok { + return fmt.Errorf("protocol prefix %s must use ipns.Validator for the /ipns namespace", DefaultPrefix) + } + return nil +} From b26c889a7c14206b4029ce3b01884ab6a5352be4 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 14:00:15 -0400 Subject: [PATCH 08/29] switch experimental client to share options with the standard client --- fullrt/dht.go | 49 ++++++++++++++++++++++++++++++++--------------- fullrt/options.go | 42 ---------------------------------------- 2 files changed, 34 insertions(+), 57 deletions(-) delete mode 100644 fullrt/options.go diff --git a/fullrt/dht.go b/fullrt/dht.go index c37423cc2..d2a642caa 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -22,6 +22,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" u "github.com/ipfs/go-ipfs-util" logging "github.com/ipfs/go-log" @@ -87,21 +88,39 @@ type FullRT struct { // NewFullRT creates a DHT client that tracks the full network. It takes a protocol prefix for the given network, // For example, the protocol /ipfs/kad/1.0.0 has the prefix /ipfs. -func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opts ...Option) (*FullRT, error) { - cfg := &config{} - for _, o := range opts { - if err := o(cfg); err != nil { - return nil, err - } +// +// FullRT is an experimental DHT client that is under development. Expect breaking changes to occur in this client +// until it stabilizes. +// +// Not all of the standard DHT options are supported in this DHT. +func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, options ...kaddht.Option) (*FullRT, error) { + cfg := &internalConfig.Config{ + Datastore: dssync.MutexWrap(ds.NewMapDatastore()), + Validator: record.NamespacedValidator{}, + ValidatorChanged: false, + EnableProviders: true, + EnableValues: true, + ProtocolPrefix: protocolPrefix, + } + + if err := cfg.Apply(options...); err != nil { + return nil, err + } + if err := cfg.ApplyFallbacks(h); err != nil { + return nil, err + } + + if err := cfg.Validate(); err != nil { + return nil, err } - ms := net.NewMessageSenderImpl(h, []protocol.ID{protocolPrefix + "/kad/1.0.0"}) - protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(cfg.validator)) + ms := net.NewMessageSenderImpl(h, []protocol.ID{cfg.ProtocolPrefix + "/kad/1.0.0"}) + protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(cfg.Validator)) if err != nil { return nil, err } - pm, err := providers.NewProviderManager(ctx, h.ID(), cfg.datastore) + pm, err := providers.NewProviderManager(ctx, h.ID(), cfg.Datastore) if err != nil { return nil, err } @@ -113,18 +132,18 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt var bsPeers []*peer.AddrInfo - for _, ai := range cfg.bootstrapPeers { + for _, ai := range cfg.BootstrapPeers { tmpai := ai bsPeers = append(bsPeers, &tmpai) } rt := &FullRT{ ctx: ctx, - enableValues: true, - enableProviders: true, - Validator: cfg.validator, + enableValues: cfg.EnableValues, + enableProviders: cfg.EnableProviders, + Validator: cfg.Validator, ProviderManager: pm, - datastore: cfg.datastore, + datastore: cfg.Datastore, h: h, crawler: c, messageSender: ms, @@ -132,7 +151,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt filterFromTable: kaddht.PublicQueryFilter, rt: trie.New(), keyToPeerMap: make(map[string]peer.ID), - bucketSize: 20, + bucketSize: cfg.BucketSize, peerAddrs: make(map[peer.ID][]multiaddr.Multiaddr), bootstrapPeers: bsPeers, diff --git a/fullrt/options.go b/fullrt/options.go deleted file mode 100644 index 1327b6969..000000000 --- a/fullrt/options.go +++ /dev/null @@ -1,42 +0,0 @@ -package fullrt - -import ( - ds "github.com/ipfs/go-datastore" - "github.com/libp2p/go-libp2p-core/peer" - record "github.com/libp2p/go-libp2p-record" -) - -type config struct { - validator record.Validator - datastore ds.Batching - bootstrapPeers []peer.AddrInfo -} - -// Option DHT option type. -type Option func(*config) error - -func Validator(validator record.Validator) Option { - return func(c *config) error { - c.validator = validator - return nil - } -} - -// BootstrapPeers configures the bootstrapping nodes that we will connect to to seed -// and refresh our Routing Table if it becomes empty. -func BootstrapPeers(bootstrappers ...peer.AddrInfo) Option { - return func(c *config) error { - c.bootstrapPeers = bootstrappers - return nil - } -} - -// Datastore configures the DHT to use the specified datastore. -// -// Defaults to an in-memory (temporary) map. -func Datastore(ds ds.Batching) Option { - return func(c *config) error { - c.datastore = ds - return nil - } -} From c8f585716f5c7432abe94becd751195d5fd1ac1d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 14:36:02 -0400 Subject: [PATCH 09/29] add support for bulk puts --- fullrt/dht.go | 77 +++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index d2a642caa..89fec4969 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -83,7 +83,7 @@ type FullRT struct { waitFrac float64 timeoutPerOp time.Duration - provideManyParallelism int + bulkSendParallelism int } // NewFullRT creates a DHT client that tracks the full network. It takes a protocol prefix for the given network, @@ -163,7 +163,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt crawlerInterval: time.Minute * 60, - provideManyParallelism: 10, + bulkSendParallelism: 10, } go rt.runCrawler(ctx) @@ -835,14 +835,6 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) return routing.ErrNotSupported } - keysAsPeerIDs := make([]peer.ID, 0, len(keys)) - for _, k := range keys { - keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) - } - sortedKeys := kb.SortClosestPeers(keysAsPeerIDs, kb.ID(make([]byte, 32))) - - var anyProvidesSuccessful uint64 = 0 - // Compute addresses once for all provides pi := peer.AddrInfo{ ID: dht.h.ID(), @@ -856,7 +848,7 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) return fmt.Errorf("no known addresses for self, cannot put provider") } - fn := func(k peer.ID) error { + fn := func(ctx context.Context, k peer.ID) error { peers, err := dht.GetClosestPeers(ctx, string(k)) if err != nil { return err @@ -873,10 +865,61 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) return nil } + keysAsPeerIDs := make([]peer.ID, 0, len(keys)) + for _, k := range keys { + keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) + } + + return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn) +} + +func (dht *FullRT) PutMany(ctx context.Context, keys []string, values [][]byte) error { + if !dht.enableValues { + return routing.ErrNotSupported + } + + if len(keys) != len(values) { + return fmt.Errorf("number of keys does not match the number of values") + } + + keysAsPeerIDs := make([]peer.ID, 0, len(keys)) + keyRecMap := make(map[string][]byte) + for i, k := range keys { + keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) + keyRecMap[k] = values[i] + } + + if len(keys) != len(keyRecMap) { + return fmt.Errorf("does not support duplicate keys") + } + + fn := func(ctx context.Context, k peer.ID) error { + peers, err := dht.GetClosestPeers(ctx, string(k)) + if err != nil { + return err + } + successes := dht.execOnMany(ctx, func(ctx context.Context, p peer.ID) error { + keyStr := string(k) + return dht.protoMessenger.PutValue(ctx, p, record.MakePutRecord(keyStr, keyRecMap[keyStr])) + }, peers) + if successes == 0 { + return fmt.Errorf("no successful puts") + } + return nil + } + + return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn) +} + +func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func(ctx context.Context, k peer.ID) error) error { + sortedKeys := kb.SortClosestPeers(keys, kb.ID(make([]byte, 32))) + + var anySendsSuccessful uint64 = 0 + wg := sync.WaitGroup{} - wg.Add(dht.provideManyParallelism) - chunkSize := len(sortedKeys) / dht.provideManyParallelism - for i := 0; i < dht.provideManyParallelism; i++ { + wg.Add(dht.bulkSendParallelism) + chunkSize := len(sortedKeys) / dht.bulkSendParallelism + for i := 0; i < dht.bulkSendParallelism; i++ { var chunk []peer.ID end := (i + 1) * chunkSize if end > len(sortedKeys) { @@ -895,17 +938,17 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) logger.Infof("reprovide goroutine: %v pct done - %d/%d done - %d total", (ki*100)/len(chunk), ki, len(chunk), len(sortedKeys)) } } - if err := fn(key); err != nil { + if err := fn(ctx, key); err != nil { logger.Infof("failed to complete provide of key :%v. %v", internal.LoggableProviderRecordBytes(key), err) } else { - atomic.CompareAndSwapUint64(&anyProvidesSuccessful, 0, 1) + atomic.CompareAndSwapUint64(&anySendsSuccessful, 0, 1) } } }() } wg.Wait() - if anyProvidesSuccessful == 0 { + if anySendsSuccessful == 0 { return fmt.Errorf("failed to complete provides") } From 4880f626b7559f1e3d2a33272b313fe8f71e6c26 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 22 Apr 2021 16:51:02 -0400 Subject: [PATCH 10/29] cleanup some linting errors and make findpeer only dial the peer once we've already finished the query to help us deal with backoffs + invalid addresses --- fullrt/dht.go | 49 ++++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 89fec4969..7c0ac2ac4 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -33,7 +33,6 @@ import ( "github.com/libp2p/go-libp2p-kad-dht/internal/net" dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb" "github.com/libp2p/go-libp2p-kad-dht/providers" - "github.com/libp2p/go-libp2p-kad-dht/qpeerset" kb "github.com/libp2p/go-libp2p-kbucket" record "github.com/libp2p/go-libp2p-record" @@ -211,11 +210,7 @@ func (dht *FullRT) Ready() bool { rtSize := len(dht.keyToPeerMap) dht.peerAddrsLk.RUnlock() - if rtSize > len(dht.bootstrapPeers)+1 { - return true - } - - return false + return rtSize > len(dht.bootstrapPeers)+1 } func (dht *FullRT) runCrawler(ctx context.Context) { @@ -241,9 +236,8 @@ func (dht *FullRT) runCrawler(ctx context.Context) { for k, v := range m { addrs = append(addrs, &peer.AddrInfo{ID: k, Addrs: v.addrs}) } - for _, ai := range dht.bootstrapPeers { - addrs = append(addrs, ai) - } + + addrs = append(addrs, dht.bootstrapPeers...) dht.peerAddrsLk.Unlock() start := time.Now() @@ -256,9 +250,7 @@ func (dht *FullRT) runCrawler(ctx context.Context) { addrs: addrs, } }, - func(p peer.ID, err error) { - return - }) + func(p peer.ID, err error) {}) dur := time.Since(start) logger.Infof("crawl took %v", dur) @@ -618,7 +610,7 @@ func (dht *FullRT) updatePeerValues(ctx context.Context, key string, val []byte, } return } - ctx, cancel := context.WithTimeout(ctx, time.Second*30) + ctx, cancel := context.WithTimeout(ctx, time.Second*5) defer cancel() err := dht.protoMessenger.PutValue(ctx, p, fixupRec) if err != nil { @@ -629,12 +621,7 @@ func (dht *FullRT) updatePeerValues(ctx context.Context, key string, val []byte, } type lookupWithFollowupResult struct { - peers []peer.ID // the top K not unreachable peers at the end of the query - state []qpeerset.PeerState // the peer states at the end of the query - - // indicates that neither the lookup nor the followup has been prematurely terminated by an external condition such - // as context cancellation or the stop function being called. - completed bool + peers []peer.ID // the top K not unreachable peers at the end of the query } func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan struct{}) (<-chan RecvdVal, <-chan *lookupWithFollowupResult) { @@ -713,6 +700,7 @@ func (dht *FullRT) getValues(ctx context.Context, key string, stopQuery chan str } dht.execOnMany(ctx, queryFn, peers) + lookupResCh <- &lookupWithFollowupResult{peers: peers} }() return valCh, lookupResCh } @@ -1103,7 +1091,12 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e defer cancelquery() addrsCh := make(chan *peer.AddrInfo, 1) + newAddrs := make([]multiaddr.Multiaddr, 0) + + wg := sync.WaitGroup{} + wg.Add(1) go func() { + defer wg.Done() addrsSoFar := make(map[multiaddr.Multiaddr]struct{}) for { select { @@ -1112,7 +1105,6 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e return } - newAddrs := make([]multiaddr.Multiaddr, 0) for _, a := range ai.Addrs { _, found := addrsSoFar[a] if !found { @@ -1120,15 +1112,6 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e addrsSoFar[a] = struct{}{} } } - - err := dht.h.Connect(ctx, peer.AddrInfo{ - ID: id, - Addrs: ai.Addrs, - }) - if err == nil { - cancelquery() - return - } case <-ctx.Done(): return } @@ -1170,6 +1153,14 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e dht.execOnMany(queryctx, fn, peers) + close(addrsCh) + wg.Wait() + + _ = dht.h.Connect(ctx, peer.AddrInfo{ + ID: id, + Addrs: newAddrs, + }) + // Return peer information if we tried to dial the peer during the query or we are (or recently were) connected // to the peer. connectedness := dht.h.Network().Connectedness(id) From b62d4afbefc36c066ef9f57a6b38b79f94066415 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 23 Apr 2021 02:12:45 -0400 Subject: [PATCH 11/29] cleanup + add more logging to bulk sending --- fullrt/dht.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 7c0ac2ac4..fc5340545 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -858,7 +858,7 @@ func (dht *FullRT) ProvideMany(ctx context.Context, keys []multihash.Multihash) keysAsPeerIDs = append(keysAsPeerIDs, peer.ID(k)) } - return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn) + return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn, true) } func (dht *FullRT) PutMany(ctx context.Context, keys []string, values [][]byte) error { @@ -896,13 +896,14 @@ func (dht *FullRT) PutMany(ctx context.Context, keys []string, values [][]byte) return nil } - return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn) + return dht.bulkMessageSend(ctx, keysAsPeerIDs, fn, false) } -func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func(ctx context.Context, k peer.ID) error) error { +func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func(ctx context.Context, k peer.ID) error, isProvRec bool) error { sortedKeys := kb.SortClosestPeers(keys, kb.ID(make([]byte, 32))) var anySendsSuccessful uint64 = 0 + var numSendsSuccessful uint64 = 0 wg := sync.WaitGroup{} wg.Add(dht.bulkSendParallelism) @@ -923,13 +924,20 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( for ki, key := range chunk { if loopIndex == 0 { if ki%100 == 0 { - logger.Infof("reprovide goroutine: %v pct done - %d/%d done - %d total", (ki*100)/len(chunk), ki, len(chunk), len(sortedKeys)) + logger.Infof("bulk sending goroutine: %v pct done - %d/%d done - %d total", (ki*100)/len(chunk), ki, len(chunk), len(sortedKeys)) } } if err := fn(ctx, key); err != nil { - logger.Infof("failed to complete provide of key :%v. %v", internal.LoggableProviderRecordBytes(key), err) + var l interface{} + if isProvRec { + l = internal.LoggableProviderRecordBytes(key) + } else { + l = internal.LoggableRecordKeyString(key) + } + logger.Infof("failed to complete bulk sending of key :%v. %v", l, err) } else { atomic.CompareAndSwapUint64(&anySendsSuccessful, 0, 1) + atomic.AddUint64(&numSendsSuccessful, 1) } } }() @@ -937,9 +945,11 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( wg.Wait() if anySendsSuccessful == 0 { - return fmt.Errorf("failed to complete provides") + return fmt.Errorf("failed to complete bulk sending") } + logger.Infof("bulk send complete: %d of %d successful", numSendsSuccessful, len(keys)) + return nil } From 4532fb0577a6eb6d8826d04c596a576788ab75f1 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 23 Apr 2021 10:46:19 -0400 Subject: [PATCH 12/29] more findpeer changes --- fullrt/dht.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index fc5340545..b4f96cc84 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -1166,10 +1166,14 @@ func (dht *FullRT) FindPeer(ctx context.Context, id peer.ID) (_ peer.AddrInfo, e close(addrsCh) wg.Wait() - _ = dht.h.Connect(ctx, peer.AddrInfo{ - ID: id, - Addrs: newAddrs, - }) + if len(newAddrs) > 0 { + connctx, cancelconn := context.WithTimeout(ctx, time.Second*5) + defer cancelconn() + _ = dht.h.Connect(connctx, peer.AddrInfo{ + ID: id, + Addrs: newAddrs, + }) + } // Return peer information if we tried to dial the peer during the query or we are (or recently were) connected // to the peer. From 7e799836bea1c4ed3824b57af623484b48978fd2 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 30 Apr 2021 11:46:27 -0400 Subject: [PATCH 13/29] changed signature of DHT filter options to be more general --- dht_filters.go | 20 ++++++++++++++------ dht_filters_test.go | 2 +- dual/dual_test.go | 14 +++++++++++--- internal/config/config.go | 7 +++---- subscriber_notifee.go | 2 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/dht_filters.go b/dht_filters.go index d56e50738..aff9f5601 100644 --- a/dht_filters.go +++ b/dht_filters.go @@ -3,12 +3,12 @@ package dht import ( "bytes" "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" "net" "sync" "time" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/google/gopacket/routing" @@ -83,13 +83,14 @@ var _ QueryFilterFunc = PublicQueryFilter // PublicRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate // that it is on a public network -func PublicRoutingTableFilter(dht interface{}, conns []network.Conn) bool { +func PublicRoutingTableFilter(dht interface{}, p peer.ID) bool { + d := dht.(hasHost) + + conns := d.Host().Network().ConnsToPeer(p) if len(conns) == 0 { return false } - d := dht.(hasHost) - // Do we have a public address for this peer? id := conns[0].RemotePeer() known := d.Host().Peerstore().PeerInfo(id) @@ -145,12 +146,19 @@ func getCachedRouter() routing.Router { // PrivateRoutingTableFilter allows a peer to be added to the routing table if the connections to that peer indicate // that it is on a private network -func PrivateRoutingTableFilter(dht interface{}, conns []network.Conn) bool { +func PrivateRoutingTableFilter(dht interface{}, p peer.ID) bool { + d := dht.(hasHost) + conns := d.Host().Network().ConnsToPeer(p) + return privRTFilter(d, conns) +} + +func privRTFilter(dht interface{}, conns []network.Conn) bool { d := dht.(hasHost) + h := d.Host() router := getCachedRouter() myAdvertisedIPs := make([]net.IP, 0) - for _, a := range d.Host().Addrs() { + for _, a := range h.Addrs() { if isPublicAddr(a) && !isRelayAddr(a) { ip, err := manet.ToIP(a) if err != nil { diff --git a/dht_filters_test.go b/dht_filters_test.go index e4b098afd..000d4572f 100644 --- a/dht_filters_test.go +++ b/dht_filters_test.go @@ -53,7 +53,7 @@ func TestFilterCaching(t *testing.T) { d := setupDHT(ctx, t, true) remote, _ := manet.FromIP(net.IPv4(8, 8, 8, 8)) - if PrivateRoutingTableFilter(d, []network.Conn{&mockConn{ + if privRTFilter(d, []network.Conn{&mockConn{ local: d.Host().Peerstore().PeerInfo(d.Host().ID()), remote: peer.AddrInfo{ID: "", Addrs: []ma.Multiaddr{remote}}, }}) { diff --git a/dual/dual_test.go b/dual/dual_test.go index 24742a505..66cb57bc2 100644 --- a/dual/dual_test.go +++ b/dual/dual_test.go @@ -2,12 +2,12 @@ package dual import ( "context" + "github.com/libp2p/go-libp2p-core/host" "testing" "time" "github.com/ipfs/go-cid" u "github.com/ipfs/go-ipfs-util" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" peerstore "github.com/libp2p/go-libp2p-core/peerstore" dht "github.com/libp2p/go-libp2p-kad-dht" @@ -34,9 +34,17 @@ type customRtHelper struct { allow peer.ID } -func MkFilterForPeer() (func(_ interface{}, conns []network.Conn) bool, *customRtHelper) { +func MkFilterForPeer() (func(_ interface{}, p peer.ID) bool, *customRtHelper) { helper := customRtHelper{} - f := func(_ interface{}, conns []network.Conn) bool { + + type hasHost interface { + Host() host.Host + } + + f := func(dht interface{}, p peer.ID) bool { + d := dht.(hasHost) + conns := d.Host().Network().ConnsToPeer(p) + for _, c := range conns { if c.RemotePeer() == helper.allow { return true diff --git a/internal/config/config.go b/internal/config/config.go index 274e24d64..8e805688c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,7 +8,6 @@ import ( dssync "github.com/ipfs/go-datastore/sync" "github.com/ipfs/go-ipns" "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" "github.com/libp2p/go-libp2p-kad-dht/providers" @@ -29,7 +28,7 @@ type QueryFilterFunc func(dht interface{}, ai peer.AddrInfo) bool // RouteTableFilterFunc is a filter applied when considering connections to keep in // the local route table. -type RouteTableFilterFunc func(dht interface{}, conns []network.Conn) bool +type RouteTableFilterFunc func(dht interface{}, p peer.ID) bool // Config is a structure containing all the options that can be used when constructing a DHT. type Config struct { @@ -65,8 +64,8 @@ type Config struct { TestAddressUpdateProcessing bool } -func EmptyQueryFilter(_ interface{}, ai peer.AddrInfo) bool { return true } -func EmptyRTFilter(_ interface{}, conns []network.Conn) bool { return true } +func EmptyQueryFilter(_ interface{}, ai peer.AddrInfo) bool { return true } +func EmptyRTFilter(_ interface{}, p peer.ID) bool { return true } // Apply applies the given options to this Option func (c *Config) Apply(opts ...Option) error { diff --git a/subscriber_notifee.go b/subscriber_notifee.go index 13ef89ca1..60cff85a9 100644 --- a/subscriber_notifee.go +++ b/subscriber_notifee.go @@ -152,7 +152,7 @@ func (dht *IpfsDHT) validRTPeer(p peer.ID) (bool, error) { return false, err } - return dht.routingTablePeerFilter == nil || dht.routingTablePeerFilter(dht, dht.Host().Network().ConnsToPeer(p)), nil + return dht.routingTablePeerFilter == nil || dht.routingTablePeerFilter(dht, p), nil } type disconnector interface { From d543baaa80911d0dd1c180be3545c3d3d309d54e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 30 Apr 2021 11:52:55 -0400 Subject: [PATCH 14/29] experimental dht: add stored addresses to peerstore temporarily during GetClosestPeers calls. Only keep the backup addresses for peers found durin a crawl that we actually connected with. Properly clear out peermap between crawls --- crawler/crawler.go | 26 ++++++++++++++++++++------ fullrt/dht.go | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/crawler/crawler.go b/crawler/crawler.go index 5f4ec7732..b164ec103 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -119,6 +119,8 @@ type HandleQueryResult func(p peer.ID, rtPeers []*peer.AddrInfo) // HandleQueryFail is a callback on failed peer query type HandleQueryFail func(p peer.ID, err error) +const startAddressDur time.Duration = time.Minute * 30 + // Run crawls dht peers from an initial seed of `startingPeers` func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handleSuccess HandleQueryResult, handleFail HandleQueryFail) { jobs := make(chan peer.ID, 1) @@ -140,15 +142,27 @@ func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handl defer wg.Wait() defer close(jobs) - toDial := make([]*peer.AddrInfo, 0, len(startingPeers)) + var toDial []*peer.AddrInfo peersSeen := make(map[peer.ID]struct{}) + numSkipped := 0 for _, ai := range startingPeers { + extendAddrs := c.host.Peerstore().Addrs(ai.ID) + if len(ai.Addrs) > 0 { + extendAddrs = append(extendAddrs, ai.Addrs...) + c.host.Peerstore().AddAddrs(ai.ID, extendAddrs, startAddressDur) + } + if len(extendAddrs) == 0 { + numSkipped++ + continue + } + toDial = append(toDial, ai) peersSeen[ai.ID] = struct{}{} - extendAddrs := c.host.Peerstore().Addrs(ai.ID) - extendAddrs = append(extendAddrs, ai.Addrs...) - c.host.Peerstore().AddAddrs(ai.ID, extendAddrs, time.Hour) + } + + if numSkipped > 0 { + logger.Infof("%d starting peers were skipped due to lack of addresses. Starting crawl with %d peers", numSkipped, len(toDial)) } numQueried := 0 @@ -168,7 +182,7 @@ func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handl logger.Debugf("peer %v had %d peers", res.peer, len(res.data)) rtPeers := make([]*peer.AddrInfo, 0, len(res.data)) for p, ai := range res.data { - c.host.Peerstore().AddAddrs(p, ai.Addrs, time.Hour) + c.host.Peerstore().AddAddrs(p, ai.Addrs, time.Minute*30) if _, ok := peersSeen[p]; !ok { peersSeen[p] = struct{}{} toDial = append(toDial, ai) @@ -208,7 +222,7 @@ func (c *Crawler) queryPeer(ctx context.Context, nextPeer peer.ID) *queryResult defer cancel() err = c.host.Connect(connCtx, peer.AddrInfo{ID: nextPeer}) if err != nil { - logger.Infof("could not connect to peer %v: %v", nextPeer, err) + logger.Debugf("could not connect to peer %v: %v", nextPeer, err) return &queryResult{nextPeer, nil, err} } diff --git a/fullrt/dht.go b/fullrt/dht.go index b4f96cc84..19df7eed9 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -213,6 +213,10 @@ func (dht *FullRT) Ready() bool { return rtSize > len(dht.bootstrapPeers)+1 } +func (dht *FullRT) Host() host.Host { + return dht.h +} + func (dht *FullRT) runCrawler(ctx context.Context) { t := time.NewTicker(dht.crawlerInterval) @@ -233,17 +237,37 @@ func (dht *FullRT) runCrawler(ctx context.Context) { var addrs []*peer.AddrInfo dht.peerAddrsLk.Lock() - for k, v := range m { - addrs = append(addrs, &peer.AddrInfo{ID: k, Addrs: v.addrs}) + for k := range m { + addrs = append(addrs, &peer.AddrInfo{ID: k}) // Addrs: v.addrs } addrs = append(addrs, dht.bootstrapPeers...) dht.peerAddrsLk.Unlock() + for k := range m { + delete(m, k) + } + start := time.Now() dht.crawler.Run(ctx, addrs, func(p peer.ID, rtPeers []*peer.AddrInfo) { - addrs := dht.h.Peerstore().Addrs(p) + conns := dht.h.Network().ConnsToPeer(p) + var addrs []multiaddr.Multiaddr + for _, conn := range conns { + addr := conn.RemoteMultiaddr() + addrs = append(addrs, addr) + } + + if len(addrs) == 0 { + logger.Debugf("no connections to %v after successful query. keeping addresses from the peerstore", p) + addrs = dht.h.Peerstore().Addrs(p) + } + + keep := kaddht.PublicRoutingTableFilter(dht, p) + if !keep { + return + } + mxLk.Lock() defer mxLk.Unlock() m[p] = &crawlVal{ @@ -356,6 +380,11 @@ func (dht *FullRT) GetClosestPeers(ctx context.Context, key string) ([]peer.ID, logger.Errorf("key not found in map") } dht.kMapLk.RUnlock() + dht.peerAddrsLk.RLock() + peerAddrs := dht.peerAddrs[p] + dht.peerAddrsLk.RUnlock() + + dht.h.Peerstore().AddAddrs(p, peerAddrs, peerstore.TempAddrTTL) peers = append(peers, p) } return peers, nil From ee4a44ec5ff411c992c999628f2252690e20912d Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 30 Apr 2021 12:19:35 -0400 Subject: [PATCH 15/29] dht: switch GetClosestPeers to return a slice of peers instead of a channel --- dht_test.go | 18 ++++++++---------- lookup.go | 11 ++--------- routing.go | 6 +++--- 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/dht_test.go b/dht_test.go index 9791d9638..807867262 100644 --- a/dht_test.go +++ b/dht_test.go @@ -1483,7 +1483,7 @@ func testFindPeerQuery(t *testing.T, require.NoError(t, err) var outpeers []peer.ID - for p := range out { + for _, p := range out { outpeers = append(outpeers, p) } @@ -1521,7 +1521,7 @@ func TestFindClosestPeers(t *testing.T) { } var out []peer.ID - for p := range peers { + for _, p := range peers { out = append(out, p) } @@ -2112,18 +2112,16 @@ func TestPreconnectedNodes(t *testing.T) { } // See if it works - peerCh, err := d2.GetClosestPeers(ctx, "testkey") + peers, err := d2.GetClosestPeers(ctx, "testkey") if err != nil { t.Fatal(err) } - select { - case p := <-peerCh: - if p == h1.ID() { - break - } + if len(peers) != 1 { + t.Fatal("why is there more than one peer?") + } + + if peers[0] != h1.ID() { t.Fatal("could not find peer") - case <-ctx.Done(): - t.Fatal("test hung") } } diff --git a/lookup.go b/lookup.go index dff8bb244..88695dc4a 100644 --- a/lookup.go +++ b/lookup.go @@ -16,7 +16,7 @@ import ( // // If the context is canceled, this function will return the context error along // with the closest K peers it has found so far. -func (dht *IpfsDHT) GetClosestPeers(ctx context.Context, key string) (<-chan peer.ID, error) { +func (dht *IpfsDHT) GetClosestPeers(ctx context.Context, key string) ([]peer.ID, error) { if key == "" { return nil, fmt.Errorf("can't lookup empty key") } @@ -51,17 +51,10 @@ func (dht *IpfsDHT) GetClosestPeers(ctx context.Context, key string) (<-chan pee return nil, err } - out := make(chan peer.ID, dht.bucketSize) - defer close(out) - - for _, p := range lookupRes.peers { - out <- p - } - if ctx.Err() == nil && lookupRes.completed { // refresh the cpl for this key as the query was successful dht.routingTable.ResetCplRefreshedAtForID(kb.ConvertKey(key), time.Now()) } - return out, ctx.Err() + return lookupRes.peers, ctx.Err() } diff --git a/routing.go b/routing.go index 9204b367d..7793bebb4 100644 --- a/routing.go +++ b/routing.go @@ -65,13 +65,13 @@ func (dht *IpfsDHT) PutValue(ctx context.Context, key string, value []byte, opts return err } - pchan, err := dht.GetClosestPeers(ctx, key) + peers, err := dht.GetClosestPeers(ctx, key) if err != nil { return err } wg := sync.WaitGroup{} - for p := range pchan { + for _, p := range peers { wg.Add(1) go func(p peer.ID) { ctx, cancel := context.WithCancel(ctx) @@ -446,7 +446,7 @@ func (dht *IpfsDHT) Provide(ctx context.Context, key cid.Cid, brdcst bool) (err } wg := sync.WaitGroup{} - for p := range peers { + for _, p := range peers { wg.Add(1) go func(p peer.ID) { defer wg.Done() From 43f5e30d69f7ff1330afffe09a5248df857990e8 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 30 Apr 2021 14:00:16 -0400 Subject: [PATCH 16/29] linter cleanup --- dht_test.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/dht_test.go b/dht_test.go index 807867262..3bc45426b 100644 --- a/dht_test.go +++ b/dht_test.go @@ -1479,14 +1479,9 @@ func testFindPeerQuery(t *testing.T, val := "foobar" rtval := kb.ConvertKey(val) - out, err := guy.GetClosestPeers(ctx, val) + outpeers, err := guy.GetClosestPeers(ctx, val) require.NoError(t, err) - var outpeers []peer.ID - for _, p := range out { - outpeers = append(outpeers, p) - } - sort.Sort(peer.IDSlice(outpeers)) exp := kb.SortClosestPeers(peers, rtval)[:minInt(guy.bucketSize, len(peers))] @@ -1520,13 +1515,8 @@ func TestFindClosestPeers(t *testing.T) { t.Fatal(err) } - var out []peer.ID - for _, p := range peers { - out = append(out, p) - } - - if len(out) < querier.beta { - t.Fatalf("got wrong number of peers (got %d, expected at least %d)", len(out), querier.beta) + if len(peers) < querier.beta { + t.Fatalf("got wrong number of peers (got %d, expected at least %d)", len(peers), querier.beta) } } From de6e38093185fea30a32fcf55d38b54bed40aaea Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 11 May 2021 14:23:57 -0400 Subject: [PATCH 17/29] fullrtdht: better progress logging on bulk operations --- fullrt/dht.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 19df7eed9..f98ca57c9 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -931,7 +931,7 @@ func (dht *FullRT) PutMany(ctx context.Context, keys []string, values [][]byte) func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func(ctx context.Context, k peer.ID) error, isProvRec bool) error { sortedKeys := kb.SortClosestPeers(keys, kb.ID(make([]byte, 32))) - var anySendsSuccessful uint64 = 0 + var numSends uint64 = 0 var numSendsSuccessful uint64 = 0 wg := sync.WaitGroup{} @@ -946,15 +946,12 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( chunk = sortedKeys[i*chunkSize : end] } - loopIndex := i - go func() { defer wg.Done() - for ki, key := range chunk { - if loopIndex == 0 { - if ki%100 == 0 { - logger.Infof("bulk sending goroutine: %v pct done - %d/%d done - %d total", (ki*100)/len(chunk), ki, len(chunk), len(sortedKeys)) - } + for _, key := range chunk { + sendsSoFar := atomic.AddUint64(&numSends, 1) + if uint64(len(sortedKeys))%sendsSoFar == 0 { + logger.Infof("bulk sending goroutine: %v pct done - %d/%d done", sendsSoFar/uint64(len(sortedKeys)), sendsSoFar, len(sortedKeys)) } if err := fn(ctx, key); err != nil { var l interface{} @@ -965,7 +962,6 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( } logger.Infof("failed to complete bulk sending of key :%v. %v", l, err) } else { - atomic.CompareAndSwapUint64(&anySendsSuccessful, 0, 1) atomic.AddUint64(&numSendsSuccessful, 1) } } @@ -973,7 +969,7 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( } wg.Wait() - if anySendsSuccessful == 0 { + if numSendsSuccessful == 0 { return fmt.Errorf("failed to complete bulk sending") } From 359d3def873429a80dfde39afe8382fcace3af2b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 11 May 2021 14:24:51 -0400 Subject: [PATCH 18/29] fullrtdht: do not error on bulk operations if we are not asked to do any operations --- fullrt/dht.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fullrt/dht.go b/fullrt/dht.go index f98ca57c9..58a070da6 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -929,6 +929,10 @@ func (dht *FullRT) PutMany(ctx context.Context, keys []string, values [][]byte) } func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func(ctx context.Context, k peer.ID) error, isProvRec bool) error { + if len(keys) == 0 { + return nil + } + sortedKeys := kb.SortClosestPeers(keys, kb.ID(make([]byte, 32))) var numSends uint64 = 0 From aefc008dab2255128f672fb596e9460f665dede2 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Tue, 11 May 2021 16:00:03 -0400 Subject: [PATCH 19/29] fix logging output --- fullrt/dht.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 58a070da6..5585860a0 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -941,6 +941,7 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( wg := sync.WaitGroup{} wg.Add(dht.bulkSendParallelism) chunkSize := len(sortedKeys) / dht.bulkSendParallelism + onePctKeys := uint64(len(sortedKeys)) / 100 for i := 0; i < dht.bulkSendParallelism; i++ { var chunk []peer.ID end := (i + 1) * chunkSize @@ -954,7 +955,7 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( defer wg.Done() for _, key := range chunk { sendsSoFar := atomic.AddUint64(&numSends, 1) - if uint64(len(sortedKeys))%sendsSoFar == 0 { + if sendsSoFar%onePctKeys == 0 { logger.Infof("bulk sending goroutine: %v pct done - %d/%d done", sendsSoFar/uint64(len(sortedKeys)), sendsSoFar, len(sortedKeys)) } if err := fn(ctx, key); err != nil { From 77e740817115c0762b353c2b5cd907c1e6fdf50a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 May 2021 13:00:31 -0400 Subject: [PATCH 20/29] fullrt: have support for both standard and fullrt options --- fullrt/dht.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 5585860a0..5dd9c708a 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -92,8 +92,13 @@ type FullRT struct { // until it stabilizes. // // Not all of the standard DHT options are supported in this DHT. -func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, options ...kaddht.Option) (*FullRT, error) { - cfg := &internalConfig.Config{ +func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, options ...Option) (*FullRT, error) { + var fullrtcfg config + if err := fullrtcfg.apply(options...); err != nil { + return nil, err + } + + dhtcfg := &internalConfig.Config{ Datastore: dssync.MutexWrap(ds.NewMapDatastore()), Validator: record.NamespacedValidator{}, ValidatorChanged: false, @@ -102,24 +107,24 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt ProtocolPrefix: protocolPrefix, } - if err := cfg.Apply(options...); err != nil { + if err := dhtcfg.Apply(fullrtcfg.dhtOpts...); err != nil { return nil, err } - if err := cfg.ApplyFallbacks(h); err != nil { + if err := dhtcfg.ApplyFallbacks(h); err != nil { return nil, err } - if err := cfg.Validate(); err != nil { + if err := dhtcfg.Validate(); err != nil { return nil, err } - ms := net.NewMessageSenderImpl(h, []protocol.ID{cfg.ProtocolPrefix + "/kad/1.0.0"}) - protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(cfg.Validator)) + ms := net.NewMessageSenderImpl(h, []protocol.ID{dhtcfg.ProtocolPrefix + "/kad/1.0.0"}) + protoMessenger, err := dht_pb.NewProtocolMessenger(ms, dht_pb.WithValidator(dhtcfg.Validator)) if err != nil { return nil, err } - pm, err := providers.NewProviderManager(ctx, h.ID(), cfg.Datastore) + pm, err := providers.NewProviderManager(ctx, h.ID(), dhtcfg.Datastore) if err != nil { return nil, err } @@ -131,18 +136,18 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt var bsPeers []*peer.AddrInfo - for _, ai := range cfg.BootstrapPeers { + for _, ai := range dhtcfg.BootstrapPeers { tmpai := ai bsPeers = append(bsPeers, &tmpai) } rt := &FullRT{ ctx: ctx, - enableValues: cfg.EnableValues, - enableProviders: cfg.EnableProviders, - Validator: cfg.Validator, + enableValues: dhtcfg.EnableValues, + enableProviders: dhtcfg.EnableProviders, + Validator: dhtcfg.Validator, ProviderManager: pm, - datastore: cfg.Datastore, + datastore: dhtcfg.Datastore, h: h, crawler: c, messageSender: ms, @@ -150,7 +155,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt filterFromTable: kaddht.PublicQueryFilter, rt: trie.New(), keyToPeerMap: make(map[string]peer.ID), - bucketSize: cfg.BucketSize, + bucketSize: dhtcfg.BucketSize, peerAddrs: make(map[peer.ID][]multiaddr.Multiaddr), bootstrapPeers: bsPeers, From d0236f058367d979e602790936389b52f26004a0 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 May 2021 14:57:57 -0400 Subject: [PATCH 21/29] oops --- fullrt/options.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 fullrt/options.go diff --git a/fullrt/options.go b/fullrt/options.go new file mode 100644 index 000000000..cd0f9ba59 --- /dev/null +++ b/fullrt/options.go @@ -0,0 +1,28 @@ +package fullrt + +import ( + "fmt" + kaddht "github.com/libp2p/go-libp2p-kad-dht" +) + +type config struct { + dhtOpts []kaddht.Option +} + +func (cfg *config) apply(opts ...Option) error { + for i, o := range opts { + if err := o(cfg); err != nil { + return fmt.Errorf("fullrt dht option %d failed: %w", i, err) + } + } + return nil +} + +type Option func(opt *config) error + +func DHTOption(opts ...kaddht.Option) Option { + return func(c *config) error { + c.dhtOpts = append(c.dhtOpts, opts...) + return nil + } +} From 1b0922d948557395a68228da366532742557f8ee Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 May 2021 13:03:29 -0400 Subject: [PATCH 22/29] reorder imports --- internal/net/message_manager.go | 13 +++++++------ internal/net/message_manager_test.go | 4 +++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/net/message_manager.go b/internal/net/message_manager.go index d808803e5..b2709d9ef 100644 --- a/internal/net/message_manager.go +++ b/internal/net/message_manager.go @@ -4,10 +4,6 @@ import ( "bufio" "context" "fmt" - logging "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-kad-dht/internal" - pb "github.com/libp2p/go-libp2p-kad-dht/pb" - "github.com/libp2p/go-msgio/protoio" "io" "sync" "time" @@ -17,11 +13,16 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" - "github.com/libp2p/go-libp2p-kad-dht/metrics" - + logging "github.com/ipfs/go-log" "github.com/libp2p/go-msgio" + "github.com/libp2p/go-msgio/protoio" + "go.opencensus.io/stats" "go.opencensus.io/tag" + + "github.com/libp2p/go-libp2p-kad-dht/internal" + "github.com/libp2p/go-libp2p-kad-dht/metrics" + pb "github.com/libp2p/go-libp2p-kad-dht/pb" ) var dhtReadMessageTimeout = 10 * time.Second diff --git a/internal/net/message_manager_test.go b/internal/net/message_manager_test.go index 3eeb06f8f..3bd6d2aea 100644 --- a/internal/net/message_manager_test.go +++ b/internal/net/message_manager_test.go @@ -2,11 +2,13 @@ package net import ( "context" + "testing" + "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" + swarmt "github.com/libp2p/go-libp2p-swarm/testing" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" - "testing" ) func TestInvalidMessageSenderTracking(t *testing.T) { From c22b4bfac9c0ead6d238dd864750a11ab2c98f4a Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 May 2021 13:06:10 -0400 Subject: [PATCH 23/29] renamed StreamDisconnect to OnDisconnect --- internal/net/message_manager.go | 2 +- subscriber_notifee.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/net/message_manager.go b/internal/net/message_manager.go index b2709d9ef..627c47a5b 100644 --- a/internal/net/message_manager.go +++ b/internal/net/message_manager.go @@ -49,7 +49,7 @@ func NewMessageSenderImpl(h host.Host, protos []protocol.ID) pb.MessageSender { } } -func (m *messageSenderImpl) StreamDisconnect(ctx context.Context, p peer.ID) { +func (m *messageSenderImpl) OnDisconnect(ctx context.Context, p peer.ID) { m.smlk.Lock() defer m.smlk.Unlock() ms, ok := m.strmap[p] diff --git a/subscriber_notifee.go b/subscriber_notifee.go index 60cff85a9..d80f4d297 100644 --- a/subscriber_notifee.go +++ b/subscriber_notifee.go @@ -156,7 +156,7 @@ func (dht *IpfsDHT) validRTPeer(p peer.ID) (bool, error) { } type disconnector interface { - StreamDisconnect(ctx context.Context, p peer.ID) + OnDisconnect(ctx context.Context, p peer.ID) } func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { @@ -179,7 +179,7 @@ func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { } if ms, ok := dht.msgSender.(disconnector); ok { - ms.StreamDisconnect(dht.Context(), p) + ms.OnDisconnect(dht.Context(), p) } } From 57eeffeacd663875a31954c1f0f4bedf1fb4d16b Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 12 May 2021 13:08:48 -0400 Subject: [PATCH 24/29] skip disconnect events if the message sender does not need them --- subscriber_notifee.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/subscriber_notifee.go b/subscriber_notifee.go index d80f4d297..00ff4ba03 100644 --- a/subscriber_notifee.go +++ b/subscriber_notifee.go @@ -161,6 +161,12 @@ type disconnector interface { func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { dht := nn.dht + + ms, ok := dht.msgSender.(disconnector) + if !ok { + return + } + select { case <-dht.Process().Closing(): return @@ -178,9 +184,7 @@ func (nn *subscriberNotifee) Disconnected(n network.Network, v network.Conn) { return } - if ms, ok := dht.msgSender.(disconnector); ok { - ms.OnDisconnect(dht.Context(), p) - } + ms.OnDisconnect(dht.Context(), p) } func (nn *subscriberNotifee) Connected(network.Network, network.Conn) {} From ff66a31ed04bbe8cf6be9e00516042b113d58406 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 13 May 2021 02:51:07 -0400 Subject: [PATCH 25/29] fullrtdht: add Close function and remove passed in context --- fullrt/dht.go | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 5dd9c708a..64a3087a0 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -48,7 +48,9 @@ var logger = logging.Logger("fullrtdht") // FullRT is an experimental DHT client that is under development. Expect breaking changes to occur in this client // until it stabilizes. type FullRT struct { - ctx context.Context + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup enableValues, enableProviders bool Validator record.Validator @@ -92,7 +94,7 @@ type FullRT struct { // until it stabilizes. // // Not all of the standard DHT options are supported in this DHT. -func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, options ...Option) (*FullRT, error) { +func NewFullRT(h host.Host, protocolPrefix protocol.ID, options ...Option) (*FullRT, error) { var fullrtcfg config if err := fullrtcfg.apply(options...); err != nil { return nil, err @@ -124,13 +126,16 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt return nil, err } - pm, err := providers.NewProviderManager(ctx, h.ID(), dhtcfg.Datastore) + c, err := crawler.New(h, crawler.WithParallelism(200)) if err != nil { return nil, err } - c, err := crawler.New(h, crawler.WithParallelism(200)) + ctx, cancel := context.WithCancel(context.Background()) + + pm, err := providers.NewProviderManager(ctx, h.ID(), dhtcfg.Datastore) if err != nil { + cancel() return nil, err } @@ -142,7 +147,9 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt } rt := &FullRT{ - ctx: ctx, + ctx: ctx, + cancel: cancel, + enableValues: dhtcfg.EnableValues, enableProviders: dhtcfg.EnableProviders, Validator: dhtcfg.Validator, @@ -170,6 +177,7 @@ func NewFullRT(ctx context.Context, h host.Host, protocolPrefix protocol.ID, opt bulkSendParallelism: 10, } + rt.wg.Add(1) go rt.runCrawler(ctx) return rt, nil @@ -186,6 +194,8 @@ func (dht *FullRT) TriggerRefresh(ctx context.Context) error { return ctx.Err() case dht.triggerRefresh <- struct{}{}: return nil + case <-dht.ctx.Done(): + return fmt.Errorf("dht is closed") } } @@ -223,6 +233,7 @@ func (dht *FullRT) Host() host.Host { } func (dht *FullRT) runCrawler(ctx context.Context) { + defer dht.wg.Done() t := time.NewTicker(dht.crawlerInterval) m := make(map[peer.ID]*crawlVal) @@ -308,6 +319,13 @@ func (dht *FullRT) runCrawler(ctx context.Context) { } } +func (dht *FullRT) Close() error { + dht.cancel() + err := dht.ProviderManager.Process().Close() + dht.wg.Wait() + return err +} + func (dht *FullRT) Bootstrap(ctx context.Context) error { return nil } From 03d275f0b2adae8c36221e89dfe2c3f534745c5e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Thu, 13 May 2021 03:07:36 -0400 Subject: [PATCH 26/29] fix percentage logging during bulk sends --- fullrt/dht.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fullrt/dht.go b/fullrt/dht.go index 64a3087a0..f01662737 100644 --- a/fullrt/dht.go +++ b/fullrt/dht.go @@ -979,7 +979,7 @@ func (dht *FullRT) bulkMessageSend(ctx context.Context, keys []peer.ID, fn func( for _, key := range chunk { sendsSoFar := atomic.AddUint64(&numSends, 1) if sendsSoFar%onePctKeys == 0 { - logger.Infof("bulk sending goroutine: %v pct done - %d/%d done", sendsSoFar/uint64(len(sortedKeys)), sendsSoFar, len(sortedKeys)) + logger.Infof("bulk sending goroutine: %.1f%% done - %d/%d done", 100*float64(sendsSoFar)/float64(len(sortedKeys)), sendsSoFar, len(sortedKeys)) } if err := fn(ctx, key); err != nil { var l interface{} From 8b9a22eddf14ff23c6bb76f34b58bbe424705610 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 14 May 2021 01:29:25 -0400 Subject: [PATCH 27/29] crawler: starting peers and peers found during crawl have their addresses extended --- crawler/crawler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crawler/crawler.go b/crawler/crawler.go index b164ec103..dae145382 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -119,7 +119,7 @@ type HandleQueryResult func(p peer.ID, rtPeers []*peer.AddrInfo) // HandleQueryFail is a callback on failed peer query type HandleQueryFail func(p peer.ID, err error) -const startAddressDur time.Duration = time.Minute * 30 +const dialAddressExtendDur time.Duration = time.Minute * 30 // Run crawls dht peers from an initial seed of `startingPeers` func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handleSuccess HandleQueryResult, handleFail HandleQueryFail) { @@ -150,7 +150,7 @@ func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handl extendAddrs := c.host.Peerstore().Addrs(ai.ID) if len(ai.Addrs) > 0 { extendAddrs = append(extendAddrs, ai.Addrs...) - c.host.Peerstore().AddAddrs(ai.ID, extendAddrs, startAddressDur) + c.host.Peerstore().AddAddrs(ai.ID, extendAddrs, dialAddressExtendDur) } if len(extendAddrs) == 0 { numSkipped++ @@ -182,7 +182,7 @@ func (c *Crawler) Run(ctx context.Context, startingPeers []*peer.AddrInfo, handl logger.Debugf("peer %v had %d peers", res.peer, len(res.data)) rtPeers := make([]*peer.AddrInfo, 0, len(res.data)) for p, ai := range res.data { - c.host.Peerstore().AddAddrs(p, ai.Addrs, time.Minute*30) + c.host.Peerstore().AddAddrs(p, ai.Addrs, dialAddressExtendDur) if _, ok := peersSeen[p]; !ok { peersSeen[p] = struct{}{} toDial = append(toDial, ai) From 74d4e7e7bf2bafe23125ba1c2c052453299aa9a9 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 14 May 2021 01:33:38 -0400 Subject: [PATCH 28/29] import ordering --- dht_filters.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dht_filters.go b/dht_filters.go index aff9f5601..b6c041ea1 100644 --- a/dht_filters.go +++ b/dht_filters.go @@ -2,13 +2,12 @@ package dht import ( "bytes" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" "net" "sync" "time" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" "github.com/google/gopacket/routing" @@ -16,6 +15,8 @@ import ( ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr/net" + + dhtcfg "github.com/libp2p/go-libp2p-kad-dht/internal/config" ) // QueryFilterFunc is a filter applied when considering peers to dial when querying From 6e5e5d6209746843d707aa9ad8f00f53f542a4ce Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Fri, 14 May 2021 01:36:17 -0400 Subject: [PATCH 29/29] import ordering --- dual/dual_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dual/dual_test.go b/dual/dual_test.go index 66cb57bc2..bab6d726d 100644 --- a/dual/dual_test.go +++ b/dual/dual_test.go @@ -2,12 +2,12 @@ package dual import ( "context" - "github.com/libp2p/go-libp2p-core/host" "testing" "time" "github.com/ipfs/go-cid" u "github.com/ipfs/go-ipfs-util" + "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/peer" peerstore "github.com/libp2p/go-libp2p-core/peerstore" dht "github.com/libp2p/go-libp2p-kad-dht"