Merge pull request ethereum#363 from ethersphere/remove-oldest

new impl for MemStore ; refactor of collectGarbage ; tests for MemStore, LDBStore and collectGarbage

cmd/swarm/db.go

@@ -112,6 +112,8 @@ func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)
 	}
-	storeParams := storage.NewLDBStoreParams(path, 10000000, nil, nil)
-	return storage.NewLDBStore(storeParams)
+
+	storeparams := storage.NewDefaultStoreParams()
+	ldbparams := storage.NewLDBStoreParams(storeparams, path)
+	return storage.NewLDBStore(ldbparams)
 }

metrics/timer_test.go

@@ -47,8 +47,8 @@ func TestTimerStop(t *testing.T) {
 func TestTimerFunc(t *testing.T) {
 	tm := NewTimer()
 	tm.Time(func() { time.Sleep(50e6) })
-	if max := tm.Max(); 35e6 > max || max > 95e6 {
-		t.Errorf("tm.Max(): 35e6 > %v || %v > 95e6\n", max, max)
+	if max := tm.Max(); 35e6 > max || max > 145e6 {
+		t.Errorf("tm.Max(): 35e6 > %v || %v > 145e6\n", max, max)
 	}
 }
 

swarm/network/stream/common_test.go

@@ -93,7 +93,7 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
 	db := storage.NewDBAPI(store)
 	delivery := NewDelivery(kad, db)
 	deliveries[id] = delivery
-	r := NewRegistry(addr, delivery, db, state.NewMemStore(), &RegistryOptions{
+	r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
 		SkipCheck:  defaultSkipCheck,
 		DoRetrieve: false,
 	})
@@ -153,7 +153,7 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora
 
 	db := storage.NewDBAPI(localStore)
 	delivery := NewDelivery(to, db)
-	streamer := NewRegistry(addr, delivery, db, state.NewMemStore(), &RegistryOptions{
+	streamer := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
 		SkipCheck: defaultSkipCheck,
 	})
 	teardown := func() {

swarm/network/stream/intervals/store_test.go

@@ -25,9 +25,9 @@ import (
 
 var ErrNotFound = errors.New("not found")
 
-// TestMemStore tests basic functionality of MemStore.
-func TestMemStore(t *testing.T) {
-	testStore(t, state.NewMemStore())
+// TestInmemoryStore tests basic functionality of InmemoryStore.
+func TestInmemoryStore(t *testing.T) {
+	testStore(t, state.NewInmemoryStore())
 }
 
 // testStore is a helper function to test various Store implementations.

swarm/network/stream/intervals_test.go

@@ -51,7 +51,7 @@ func newIntervalsStreamerService(ctx *adapters.ServiceContext) (node.Service, er
 	db := storage.NewDBAPI(store)
 	delivery := NewDelivery(kad, db)
 	deliveries[id] = delivery
-	r := NewRegistry(addr, delivery, db, state.NewMemStore(), &RegistryOptions{
+	r := NewRegistry(addr, delivery, db, state.NewInmemoryStore(), &RegistryOptions{
 		SkipCheck: defaultSkipCheck,
 	})
 

swarm/pss/client/client_test.go

@@ -210,7 +210,7 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
 }
 
 func newServices() adapters.Services {
-	stateStore := state.NewMemStore()
+	stateStore := state.NewInmemoryStore()
 	kademlias := make(map[discover.NodeID]*network.Kademlia)
 	kademlia := func(id discover.NodeID) *network.Kademlia {
 		if k, ok := kademlias[id]; ok {

swarm/pss/pss_test.go

@@ -1326,7 +1326,7 @@ func setupNetwork(numnodes int, allowRaw bool) (clients []*rpc.Client, err error
 }
 
 func newServices(allowRaw bool) adapters.Services {
-	stateStore := state.NewMemStore()
+	stateStore := state.NewInmemoryStore()
 	kademlias := make(map[discover.NodeID]*network.Kademlia)
 	kademlia := func(id discover.NodeID) *network.Kademlia {
 		if k, ok := kademlias[id]; ok {

swarm/state/memstore.go → swarm/state/inmemorystore.go

@@ -22,23 +22,23 @@ import (
 	"sync"
 )
 
-// MemStore is the reference implementation of Store interface that is supposed
+// InmemoryStore is the reference implementation of Store interface that is supposed
 // to be used in tests.
-type MemStore struct {
+type InmemoryStore struct {
 	db map[string][]byte
 	mu sync.RWMutex
 }
 
-// NewMemStore returns a new instance of MemStore.
-func NewMemStore() *MemStore {
-	return &MemStore{
+// NewInmemoryStore returns a new instance of InmemoryStore.
+func NewInmemoryStore() *InmemoryStore {
+	return &InmemoryStore{
 		db: make(map[string][]byte),
 	}
 }
 
 // Get retrieves a value stored for a specific key. If there is no value found,
 // ErrNotFound is returned.
-func (s *MemStore) Get(key string, i interface{}) (err error) {
+func (s *InmemoryStore) Get(key string, i interface{}) (err error) {
 	s.mu.RLock()
 	defer s.mu.RUnlock()
 
@@ -56,7 +56,7 @@ func (s *MemStore) Get(key string, i interface{}) (err error) {
 }
 
 // Put stores a value for a specific key.
-func (s *MemStore) Put(key string, i interface{}) (err error) {
+func (s *InmemoryStore) Put(key string, i interface{}) (err error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	bytes := []byte{}
@@ -77,7 +77,7 @@ func (s *MemStore) Put(key string, i interface{}) (err error) {
 }
 
 // Delete removes value stored under a specific key.
-func (s *MemStore) Delete(key string) (err error) {
+func (s *InmemoryStore) Delete(key string) (err error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -89,6 +89,6 @@ func (s *MemStore) Delete(key string) (err error) {
 }
 
 // Close does not do anything.
-func (s *MemStore) Close() error {
+func (s *InmemoryStore) Close() error {
 	return nil
 }

swarm/storage/common_test.go

@@ -34,7 +34,7 @@ import (
 )
 
 var (
-	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
+	loglevel = flag.Int("loglevel", 3, "verbosity of logs")
 )
 
 func init() {

swarm/storage/dpa.go

@@ -34,8 +34,9 @@ implementation for storage or retrieval.
 */
 
 const (
-	singletonSwarmDbCapacity    = 50000
-	singletonSwarmCacheCapacity = 500
+	defaultLDBCapacity                = 5000000 // capacity for LevelDB, by default 5*10^6*4096 bytes == 20GB
+	defaultCacheCapacity              = 500     // capacity for in-memory chunks' cache
+	defaultChunkRequestsCacheCapacity = 5000000 // capacity for container holding outgoing requests for chunks. should be set to LevelDB capacity
 )
 
 var (

swarm/storage/dpa_test.go

@@ -39,8 +39,7 @@ func testDpaRandom(toEncrypt bool, t *testing.T) {
 	defer tdb.close()
 	db := tdb.LDBStore
 	db.setCapacity(50000)
-	storeParams := NewStoreParams(defaultCacheCapacity, nil, nil)
-	memStore := NewMemStore(storeParams, db)
+	memStore := NewMemStore(NewDefaultStoreParams(), db)
 	localStore := &LocalStore{
 		memStore: memStore,
 		DbStore:  db,
@@ -72,7 +71,7 @@ func testDpaRandom(toEncrypt bool, t *testing.T) {
 	}
 	ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
 	ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
-	localStore.memStore = NewMemStore(storeParams, db)
+	localStore.memStore = NewMemStore(NewDefaultStoreParams(), db)
 	resultReader, isEncrypted = dpa.Retrieve(key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
@@ -104,9 +103,7 @@ func testDPA_capacity(toEncrypt bool, t *testing.T) {
 	}
 	defer tdb.close()
 	db := tdb.LDBStore
-	storeParams := NewStoreParams(0, nil, nil)
-	storeParams.CacheCapacity = 10000000
-	memStore := NewMemStore(storeParams, db)
+	memStore := NewMemStore(NewDefaultStoreParams(), db)
 	localStore := &LocalStore{
 		memStore: memStore,
 		DbStore:  db,