fix_: ensure individual store queries do not exceed 24h while still allowing to retrieve more than one day of messages

go.mod

@@ -97,7 +97,7 @@ require (
 	github.com/schollz/peerdiscovery v1.7.0
 	github.com/siphiuel/lc-proxy-wrapper v0.0.0-20230516150924-246507cee8c7
 	github.com/urfave/cli/v2 v2.27.2
-	github.com/waku-org/go-waku v0.8.1-0.20241021202955-3c4e40c729a0
+	github.com/waku-org/go-waku v0.8.1-0.20241128183857-1608cf2b0b90
 	github.com/wk8/go-ordered-map/v2 v2.1.7
 	github.com/yeqown/go-qrcode/v2 v2.2.1
 	github.com/yeqown/go-qrcode/writer/standard v1.2.1

go.sum

@@ -2152,8 +2152,8 @@ github.com/waku-org/go-libp2p-pubsub v0.12.0-gowaku.0.20240823143342-b0f2429ca27
 github.com/waku-org/go-libp2p-pubsub v0.12.0-gowaku.0.20240823143342-b0f2429ca27f/go.mod h1:Oi0zw9aw8/Y5GC99zt+Ef2gYAl+0nZlwdJonDyOz/sE=
 github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0 h1:R4YYx2QamhBRl/moIxkDCNW+OP7AHbyWLBygDc/xIMo=
 github.com/waku-org/go-libp2p-rendezvous v0.0.0-20240110193335-a67d1cc760a0/go.mod h1:EhZP9fee0DYjKH/IOQvoNSy1tSHp2iZadsHGphcAJgY=
-github.com/waku-org/go-waku v0.8.1-0.20241021202955-3c4e40c729a0 h1:PNKcOPMn0yoC2NQaJPPB8FvHT/YtaU8hZAoovSl42KM=
-github.com/waku-org/go-waku v0.8.1-0.20241021202955-3c4e40c729a0/go.mod h1:1BRnyg2mQ2aBNLTBaPq6vEvobzywGykPOhGQFbHGf74=
+github.com/waku-org/go-waku v0.8.1-0.20241128183857-1608cf2b0b90 h1:p7tehUW7f+D6pvMJYop2yJV03SJU2fFUusmSnKL3uow=
+github.com/waku-org/go-waku v0.8.1-0.20241128183857-1608cf2b0b90/go.mod h1:1BRnyg2mQ2aBNLTBaPq6vEvobzywGykPOhGQFbHGf74=
 github.com/waku-org/go-zerokit-rln v0.1.14-0.20240102145250-fa738c0bdf59 h1:jisj+OCI6QydLtFq3Pyhu49wl9ytPN7oAHjMfepHDrA=
 github.com/waku-org/go-zerokit-rln v0.1.14-0.20240102145250-fa738c0bdf59/go.mod h1:1PdBdPzyTaKt3VnpAHk3zj+r9dXPFOr3IHZP9nFle6E=
 github.com/waku-org/go-zerokit-rln-apple v0.0.0-20230916172309-ee0ee61dde2b h1:KgZVhsLkxsj5gb/FfndSCQu6VYwALrCOgYI3poR95yE=

protocol/messenger_mailserver.go

@@ -8,6 +8,7 @@ import (
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
+	"golang.org/x/exp/maps"
 
 	"github.com/waku-org/go-waku/waku/v2/api/history"
 
@@ -542,55 +543,15 @@ func (m *Messenger) syncFiltersFrom(peerID peer.ID, filters []*transport.Filter,
 		m.config.messengerSignalsHandler.HistoryRequestStarted(len(batches))
 	}
 
-	var batches24h []types.MailserverBatch
 	for pubsubTopic := range batches {
-		batchKeys := make([]int, 0, len(batches[pubsubTopic]))
-		for k := range batches[pubsubTopic] {
-			batchKeys = append(batchKeys, k)
-		}
+		batchKeys := maps.Keys(batches[pubsubTopic])
 		sort.Ints(batchKeys)
-
-		keysToIterate := append([]int{}, batchKeys...)
-		for {
-			// For all batches
-			var tmpKeysToIterate []int
-			for _, k := range keysToIterate {
-				batch := batches[pubsubTopic][k]
-
-				dayBatch := types.MailserverBatch{
-					To:          batch.To,
-					Cursor:      batch.Cursor,
-					PubsubTopic: batch.PubsubTopic,
-					Topics:      batch.Topics,
-					ChatIDs:     batch.ChatIDs,
-				}
-
-				from := batch.To.Add(-oneDayDuration)
-				if from.After(batch.From) {
-					dayBatch.From = from
-					batches24h = append(batches24h, dayBatch)
-
-					// Replace og batch with new dates
-					batch.To = from
-					batches[pubsubTopic][k] = batch
-					tmpKeysToIterate = append(tmpKeysToIterate, k)
-				} else {
-					batches24h = append(batches24h, batch)
-				}
-			}
-
-			if len(tmpKeysToIterate) == 0 {
-				break
+		for _, k := range batchKeys {
+			err := m.processMailserverBatch(peerID, batches[pubsubTopic][k])
+			if err != nil {
+				m.logger.Error("error syncing topics", zap.Error(err))
+				return nil, err
 			}
-			keysToIterate = tmpKeysToIterate
-		}
-	}
-
-	for _, batch := range batches24h {
-		err := m.processMailserverBatch(peerID, batch)
-		if err != nil {
-			m.logger.Error("error syncing topics", zap.Error(err))
-			return nil, err
 		}
 	}
 

protocol/messenger_mailserver_cycle.go

@@ -93,63 +93,64 @@ func (m *Messenger) asyncRequestAllHistoricMessages() {
 	}()
 }
 
-func (m *Messenger) GetPinnedStorenode() (peer.ID, error) {
+func (m *Messenger) GetPinnedStorenode() (peer.AddrInfo, error) {
 	fleet, err := m.getFleet()
 	if err != nil {
-		return "", err
+		return peer.AddrInfo{}, err
 	}
 
 	pinnedMailservers, err := m.settings.GetPinnedMailservers()
 	if err != nil {
-		return "", err
+		return peer.AddrInfo{}, err
 	}
 
 	pinnedMailserver, ok := pinnedMailservers[fleet]
 	if !ok {
-		return "", nil
+		return peer.AddrInfo{}, nil
 	}
 
 	fleetMailservers := mailservers.DefaultMailservers()
 
 	for _, c := range fleetMailservers {
 		if c.Fleet == fleet && c.ID == pinnedMailserver {
-			return c.PeerID()
+			return c.PeerInfo()
 		}
 	}
 
 	if m.mailserversDatabase != nil {
 		customMailservers, err := m.mailserversDatabase.Mailservers()
 		if err != nil {
-			return "", err
+			return peer.AddrInfo{}, err
 		}
 
 		for _, c := range customMailservers {
 			if c.Fleet == fleet && c.ID == pinnedMailserver {
-				return c.PeerID()
+				return c.PeerInfo()
 			}
 		}
 	}
 
-	return "", nil
+	return peer.AddrInfo{}, nil
 }
 
 func (m *Messenger) UseStorenodes() (bool, error) {
 	return m.settings.CanUseMailservers()
 }
 
-func (m *Messenger) Storenodes() ([]peer.ID, error) {
+func (m *Messenger) Storenodes() ([]peer.AddrInfo, error) {
 	mailservers, err := m.AllMailservers()
 	if err != nil {
 		return nil, err
 	}
 
-	var result []peer.ID
+	var result []peer.AddrInfo
 	for _, m := range mailservers {
-		peerID, err := m.PeerID()
+
+		peerInfo, err := m.PeerInfo()
 		if err != nil {
 			return nil, err
 		}
-		result = append(result, peerID)
+		result = append(result, peerInfo)
 	}
 
 	return result, nil

services/mailservers/database.go

@@ -49,13 +49,13 @@ type Mailserver struct {
 	FailedRequests uint   `json:"-"`
 }
 
-func (m Mailserver) PeerInfo() (*peer.AddrInfo, error) {
+func (m Mailserver) PeerInfo() (peer.AddrInfo, error) {
 	var maddrs []multiaddr.Multiaddr
 
 	if m.ENR != nil {
 		addrInfo, err := enr.EnodeToPeerInfo(m.ENR)
 		if err != nil {
-			return nil, err
+			return peer.AddrInfo{}, err
 		}
 		addrInfo.Addrs = utils.EncapsulatePeerID(addrInfo.ID, addrInfo.Addrs...)
 		maddrs = append(maddrs, addrInfo.Addrs...)
@@ -67,14 +67,14 @@ func (m Mailserver) PeerInfo() (*peer.AddrInfo, error) {
 
 	p, err := peer.AddrInfosFromP2pAddrs(maddrs...)
 	if err != nil {
-		return nil, err
+		return peer.AddrInfo{}, err
 	}
 
 	if len(p) != 1 {
-		return nil, errors.New("invalid mailserver setup")
+		return peer.AddrInfo{}, errors.New("invalid mailserver setup")
 	}
 
-	return &p[0], nil
+	return p[0], nil
 }
 
 func (m Mailserver) PeerID() (peer.ID, error) {