fix bug in pinsets and add a stress test for the scenario

License: MIT
Signed-off-by: Jeromy <why@ipfs.io>

pin/set.go

@@ -143,7 +143,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
 		if !ok {
 			break
 		}
-		h := hash(seed, k)
+		h := hash(seed, k) % defaultFanout
 		hashed[h] = append(hashed[h], item{k, data})
 	}
 	for h, items := range hashed {

pin/set_test.go

@@ -1,13 +1,66 @@
 package pin
 
-import "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
 
-func ignoreKeys(key.Key) {}
+	dag "github.com/ipfs/go-ipfs/merkledag"
+	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
 
-func copyMap(m map[key.Key]uint16) map[key.Key]uint64 {
-	c := make(map[key.Key]uint64, len(m))
-	for k, v := range m {
-		c[k] = uint64(v)
+	cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
+)
+
+func ignoreCids(_ *cid.Cid) {}
+
+func TestSet(t *testing.T) {
+	ds := mdtest.Mock()
+	limit := 10000 // 10000 reproduces the pinloss issue fairly reliably
+
+	if os.Getenv("STRESS_IT_OUT_YO") != "" {
+		limit = 10000000
+	}
+	var inputs []*cid.Cid
+	for i := 0; i < limit; i++ {
+		c, err := ds.Add(dag.NodeWithData([]byte(fmt.Sprint(i))))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		inputs = append(inputs, c)
+	}
+
+	out, err := storeSet(context.Background(), ds, inputs, ignoreCids)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// weird wrapper node because loadSet expects us to pass an
+	// object pointing to multiple named sets
+	setroot := &dag.Node{}
+	err = setroot.AddNodeLinkClean("foo", out)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(outset) != limit {
+		t.Fatal("got wrong number", len(outset), limit)
+	}
+
+	seen := cid.NewSet()
+	for _, c := range outset {
+		seen.Add(c)
+	}
+
+	for _, c := range inputs {
+		if !seen.Has(c) {
+			t.Fatalf("expected to have %s, didnt find it")
+		}
 	}
-	return c
 }