Discovery: split node records to a sepatate DB table (#3581)

Problem:
QuerySeeds will poke 150 random entries in the whole node DB and ignore hitting "field" entries.
In a bootstrap scenario it might hit hundreds of :lastping :lastpong entries,
and very few true "node record" entries.
After running for 15 minutes I've got totalEntryCount=1508 nodeRecordCount=114 entries.
There's a 1/16 chance of hitting a "node record" entry.
It means finding just about 10 nodes of 114 total on average from 150 attempts.

Solution:
Split "node record" entries to a separate table such that QuerySeeds doesn't do idle cycle hits.

go.mod

@@ -40,7 +40,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kevinburke/go-bindata v3.21.0+incompatible
-	github.com/ledgerwatch/erigon-lib v0.0.0-20220310120408-f54e4c7eaeb0
+	github.com/ledgerwatch/erigon-lib v0.0.0-20220310121515-3123b6d895c5
 	github.com/ledgerwatch/log/v3 v3.4.1
 	github.com/ledgerwatch/secp256k1 v1.0.0
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect

go.sum

@@ -641,8 +641,8 @@ github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3P
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220310120408-f54e4c7eaeb0 h1:7fvcGOHcEAGX/QAtcPF6wqdxzt5vq01fpnA8LS8JoZc=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220310120408-f54e4c7eaeb0/go.mod h1:mag5WaGOUTVOLvFkT4wpjR5YHMmm4hynWJ3YfQ44Elg=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220310121515-3123b6d895c5 h1:f81JYvbRxP0T//AG+wmfZNxiwz2mDPK1cVJuhwcAiYc=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220310121515-3123b6d895c5/go.mod h1:mag5WaGOUTVOLvFkT4wpjR5YHMmm4hynWJ3YfQ44Elg=
 github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
 github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=

p2p/discover/table.go

@@ -305,6 +305,7 @@ func (tab *Table) doRefresh(done chan struct{}) {
 
 func (tab *Table) loadSeedNodes() {
 	seeds := wrapNodes(tab.db.QuerySeeds(seedCount, seedMaxAge))
+	tab.log.Debug("QuerySeeds read nodes from the node DB", "count", len(seeds))
 	seeds = append(seeds, tab.nursery...)
 	for i := range seeds {
 		seed := seeds[i]

p2p/enode/nodedb.go

@@ -63,7 +63,7 @@ const (
 const (
 	dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped.
 	dbCleanupCycle   = time.Hour      // Time period for running the expiration task.
-	dbVersion        = 9
+	dbVersion        = 10
 )
 
 var (
@@ -90,9 +90,10 @@ func OpenDB(path string) (*DB, error) {
 	return newPersistentDB(logger, path)
 }
 
-var bucketsConfig = func(defaultBuckets kv.TableCfg) kv.TableCfg {
+func bucketsConfig(_ kv.TableCfg) kv.TableCfg {
 	return kv.TableCfg{
-		kv.Inodes: {},
+		kv.Inodes:      {},
+		kv.NodeRecords: {},
 	}
 }
 
@@ -150,7 +151,7 @@ func newPersistentDB(logger log.Logger, path string) (*DB, error) {
 	}
 	if blob != nil && !bytes.Equal(blob, currentVer) {
 		db.Close()
-		if err := os.Remove(path); err != nil {
+		if err := os.RemoveAll(path); err != nil {
 			return nil, err
 		}
 		return newPersistentDB(logger, path)
@@ -283,7 +284,7 @@ func (db *DB) storeUint64(key []byte, n uint64) error {
 func (db *DB) Node(id ID) *Node {
 	var blob []byte
 	if err := db.kv.View(context.Background(), func(tx kv.Tx) error {
-		v, errGet := tx.GetOne(kv.Inodes, nodeKey(id))
+		v, errGet := tx.GetOne(kv.NodeRecords, nodeKey(id))
 		if errGet != nil {
 			return errGet
 		}
@@ -321,7 +322,7 @@ func (db *DB) UpdateNode(node *Node) error {
 		return err
 	}
 	if err := db.kv.Update(context.Background(), func(tx kv.RwTx) error {
-		return tx.Put(kv.Inodes, nodeKey(node.ID()), blob)
+		return tx.Put(kv.NodeRecords, nodeKey(node.ID()), blob)
 	}); err != nil {
 		return err
 	}
@@ -349,24 +350,31 @@ func (db *DB) DeleteNode(id ID) {
 
 func deleteRange(db kv.RwDB, prefix []byte) {
 	if err := db.Update(context.Background(), func(tx kv.RwTx) error {
-		c, err := tx.RwCursor(kv.Inodes)
-		if err != nil {
-			return err
-		}
-		for k, _, err := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, _, err = c.Next() {
-			if err != nil {
+		for bucket := range bucketsConfig(nil) {
+			if err := deleteRangeInBucket(tx, prefix, bucket); err != nil {
 				return err
 			}
-			if err := c.Delete(k, nil); err != nil {
-				return nil
-			}
 		}
 		return nil
 	}); err != nil {
 		log.Warn("nodeDB.deleteRange failed", "err", err)
 	}
 }
 
+func deleteRangeInBucket(tx kv.RwTx, prefix []byte, bucket string) error {
+	c, err := tx.RwCursor(bucket)
+	if err != nil {
+		return err
+	}
+	var k []byte
+	for k, _, err = c.Seek(prefix); (err == nil) && (k != nil) && bytes.HasPrefix(k, prefix); k, _, err = c.Next() {
+		if err = c.DeleteCurrent(); err != nil {
+			break
+		}
+	}
+	return err
+}
+
 // ensureExpirer is a small helper method ensuring that the data expiration
 // mechanism is running. If the expiration goroutine is already running, this
 // method simply returns.
@@ -537,7 +545,7 @@ func (db *DB) QuerySeeds(n int, maxAge time.Duration) []*Node {
 	)
 
 	if err := db.kv.View(context.Background(), func(tx kv.Tx) error {
-		c, err := tx.Cursor(kv.Inodes)
+		c, err := tx.Cursor(kv.NodeRecords)
 		if err != nil {
 			return err
 		}