implement DB options for customizing maxopenfiles

backend_test.go

@@ -12,7 +12,7 @@ import (
 
 // Register a test backend for PrefixDB as well, with some unrelated junk data
 func init() {
-	registerDBCreator("prefixdb", func(name, dir string) (DB, error) {
+	registerDBCreator("prefixdb", func(name, dir string, opts DBOptions) (DB, error) {
 		mdb := NewMemDB()
 		mdb.Set([]byte("a"), []byte{1})    //nolint:errcheck
 		mdb.Set([]byte("b"), []byte{2})    //nolint:errcheck

cleveldb.go

@@ -8,11 +8,12 @@ import (
 	"path/filepath"
 
 	"github.com/jmhodges/levigo"
+	"github.com/spf13/cast"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewCLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts DBOptions) (DB, error) {
+		return NewCLevelDB(name, dir, opts)
 	}
 	registerDBCreator(CLevelDBBackend, dbCreator, false)
 }
@@ -27,14 +28,26 @@ type CLevelDB struct {
 
 var _ DB = (*CLevelDB)(nil)
 
-// NewCLevelDB creates a new CLevelDB.
-func NewCLevelDB(name string, dir string) (*CLevelDB, error) {
-	dbPath := filepath.Join(dir, name+".db")
-
+func defaultCleveldbOptions() *levigo.Options {
 	opts := levigo.NewOptions()
 	opts.SetCache(levigo.NewLRUCache(1 << 30))
 	opts.SetCreateIfMissing(true)
-	db, err := levigo.Open(dbPath, opts)
+	return opts
+}
+
+// NewCLevelDB creates a new CLevelDB.
+func NewCLevelDB(name string, dir string, opts DBOptions) (*CLevelDB, error) {
+	do := defaultCleveldbOptions()
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			do.SetMaxOpenFiles(files)
+		}
+	}
+
+	dbPath := filepath.Join(dir, name+".db")
+	db, err := levigo.Open(dbPath, do)
 	if err != nil {
 		return nil, err
 	}

cleveldb_test.go

@@ -19,7 +19,7 @@ func TestWithClevelDB(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "cleveldb")
 
-	db, err := NewCLevelDB(path, "")
+	db, err := NewCLevelDB(path, "", nil)
 	require.NoError(t, err)
 
 	t.Run("ClevelDB", func(t *testing.T) { Run(t, db) })
@@ -33,7 +33,7 @@ func BenchmarkRandomReadsWrites2(b *testing.B) {
 	for i := 0; i < int(numItems); i++ {
 		internal[int64(i)] = int64(0)
 	}
-	db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "")
+	db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "", nil)
 	if err != nil {
 		b.Fatal(err.Error())
 		return

db.go

@@ -31,7 +31,13 @@ const (
 	PebbleDBBackend BackendType = "pebbledb"
 )
 
-type dbCreator func(name string, dir string) (DB, error)
+type (
+	dbCreator func(name string, dir string, opts DBOptions) (DB, error)
+
+	DBOptions interface {
+		Get(string) interface{}
+	}
+)
 
 var backends = map[BackendType]dbCreator{}
 
@@ -45,6 +51,10 @@ func registerDBCreator(backend BackendType, creator dbCreator, force bool) {
 
 // NewDB creates a new database of type backend with the given name.
 func NewDB(name string, backend BackendType, dir string) (DB, error) {
+	return NewDBwithOptions(name, backend, dir, nil)
+}
+
+func NewDBwithOptions(name string, backend BackendType, dir string, opts DBOptions) (DB, error) {
 	dbCreator, ok := backends[backend]
 	if !ok {
 		keys := make([]string, 0, len(backends))
@@ -55,7 +65,7 @@ func NewDB(name string, backend BackendType, dir string) (DB, error) {
 			backend, strings.Join(keys, ","))
 	}
 
-	db, err := dbCreator(name, dir)
+	db, err := dbCreator(name, dir, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize database: %w", err)
 	}

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/google/btree v1.1.2
 	github.com/jmhodges/levigo v1.0.0
 	github.com/linxGnu/grocksdb v1.7.5
+	github.com/spf13/cast v1.3.0
 	github.com/stretchr/testify v1.8.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 )

go.sum

@@ -200,6 +200,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=

goleveldb.go

@@ -4,15 +4,16 @@ import (
 	"fmt"
 	"path/filepath"
 
+	"github.com/spf13/cast"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewGoLevelDB(name, dir)
+	dbCreator := func(name string, dir string, opts DBOptions) (DB, error) {
+		return NewGoLevelDB(name, dir, opts)
 	}
 	registerDBCreator(GoLevelDBBackend, dbCreator, false)
 }
@@ -23,8 +24,17 @@ type GoLevelDB struct {
 
 var _ DB = (*GoLevelDB)(nil)
 
-func NewGoLevelDB(name string, dir string) (*GoLevelDB, error) {
-	return NewGoLevelDBWithOpts(name, dir, nil)
+func NewGoLevelDB(name string, dir string, opts DBOptions) (*GoLevelDB, error) {
+	defaultOpts := &opt.Options{}
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			defaultOpts.OpenFilesCacheCapacity = files
+		}
+	}
+
+	return NewGoLevelDBWithOpts(name, dir, defaultOpts)
 }
 
 func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, error) {

goleveldb_test.go

@@ -13,9 +13,9 @@ func TestGoLevelDBNewGoLevelDB(t *testing.T) {
 	defer cleanupDBDir("", name)
 
 	// Test we can't open the db twice for writing
-	wr1, err := NewGoLevelDB(name, "")
+	wr1, err := NewGoLevelDB(name, "", nil)
 	require.Nil(t, err)
-	_, err = NewGoLevelDB(name, "")
+	_, err = NewGoLevelDB(name, "", nil)
 	require.NotNil(t, err)
 	wr1.Close() // Close the db to release the lock
 
@@ -30,7 +30,7 @@ func TestGoLevelDBNewGoLevelDB(t *testing.T) {
 
 func BenchmarkGoLevelDBRandomReadsWrites(b *testing.B) {
 	name := fmt.Sprintf("test_%x", randStr(12))
-	db, err := NewGoLevelDB(name, "")
+	db, err := NewGoLevelDB(name, "", nil)
 	if err != nil {
 		b.Fatal(err)
 	}

memdb.go

@@ -14,7 +14,7 @@ const (
 )
 
 func init() {
-	registerDBCreator(MemDBBackend, func(name, dir string) (DB, error) {
+	registerDBCreator(MemDBBackend, func(name, dir string, opts DBOptions) (DB, error) {
 		return NewMemDB(), nil
 	}, false)
 }

pebble.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	"github.com/cockroachdb/pebble"
+	"github.com/spf13/cast"
 )
 
 // ForceSync
@@ -54,8 +55,8 @@ var (
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewPebbleDB(name, dir)
+	dbCreator := func(name string, dir string, opts DBOptions) (DB, error) {
+		return NewPebbleDB(name, dir, opts)
 	}
 	registerDBCreator(PebbleDBBackend, dbCreator, false)
 
@@ -71,15 +72,22 @@ type PebbleDB struct {
 
 var _ DB = (*PebbleDB)(nil)
 
-func NewPebbleDB(name string, dir string) (DB, error) {
-	dbPath := filepath.Join(dir, name+".db")
-	opts := &pebble.Options{
+func NewPebbleDB(name string, dir string, opts DBOptions) (DB, error) {
+	do := &pebble.Options{
 		MaxConcurrentCompactions: func() int { return 3 }, // default 1
 	}
 
-	opts.EnsureDefaults()
+	do.EnsureDefaults()
 
-	p, err := pebble.Open(dbPath, opts)
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			do.MaxOpenFiles = files
+		}
+	}
+
+	dbPath := filepath.Join(dir, name+".db")
+	p, err := pebble.Open(dbPath, do)
 	if err != nil {
 		return nil, err
 	}

prefixdb_test.go

@@ -39,7 +39,7 @@ func randomValue() []byte {
 func TestGolevelDB(t *testing.T) {
 	path := filepath.Join(t.TempDir(), "goleveldb")
 
-	db, err := NewGoLevelDB(path, "")
+	db, err := NewGoLevelDB(path, "", nil)
 	require.NoError(t, err)
 
 	Run(t, db)

rocksdb.go

@@ -9,11 +9,12 @@ import (
 	"runtime"
 
 	"github.com/linxGnu/grocksdb"
+	"github.com/spf13/cast"
 )
 
 func init() {
-	dbCreator := func(name string, dir string) (DB, error) {
-		return NewRocksDB(name, dir)
+	dbCreator := func(name string, dir string, opts DBOptions) (DB, error) {
+		return NewRocksDB(name, dir, opts)
 	}
 	registerDBCreator(RocksDBBackend, dbCreator, false)
 }
@@ -28,23 +29,36 @@ type RocksDB struct {
 
 var _ DB = (*RocksDB)(nil)
 
-func NewRocksDB(name string, dir string) (*RocksDB, error) {
-	// default rocksdb option, good enough for most cases, including heavy workloads.
-	// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
-	// compression: snappy as default, need to -lsnappy to enable.
+// defaultRocksdbOptions, good enough for most cases, including heavy workloads.
+// 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads).
+// compression: snappy as default, need to -lsnappy to enable.
+func defaultRocksdbOptions() *grocksdb.Options {
 	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
 	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
 	bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10))
 
-	opts := grocksdb.NewDefaultOptions()
-	opts.SetBlockBasedTableFactory(bbto)
+	rocksdbOpts := grocksdb.NewDefaultOptions()
+	rocksdbOpts.SetBlockBasedTableFactory(bbto)
 	// SetMaxOpenFiles to 4096 seems to provide a reliable performance boost
-	opts.SetMaxOpenFiles(4096)
-	opts.SetCreateIfMissing(true)
-	opts.IncreaseParallelism(runtime.NumCPU())
+	rocksdbOpts.SetMaxOpenFiles(4096)
+	rocksdbOpts.SetCreateIfMissing(true)
+	rocksdbOpts.IncreaseParallelism(runtime.NumCPU())
 	// 1.5GB maximum memory use for writebuffer.
-	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
-	return NewRocksDBWithOptions(name, dir, opts)
+	rocksdbOpts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
+	return rocksdbOpts
+}
+
+func NewRocksDB(name string, dir string, opts DBOptions) (*RocksDB, error) {
+	defaultOpts := defaultRocksdbOptions()
+
+	if opts != nil {
+		files := cast.ToInt(opts.Get("maxopenfiles"))
+		if files > 0 {
+			defaultOpts.SetMaxOpenFiles(files)
+		}
+	}
+
+	return NewRocksDBWithOptions(name, dir, defaultOpts)
 }
 
 func NewRocksDBWithOptions(name string, dir string, opts *grocksdb.Options) (*RocksDB, error) {

rocksdb_test.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/spf13/cast"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -28,7 +29,7 @@ func TestWithRocksDB(t *testing.T) {
 	dir := t.TempDir()
 	path := filepath.Join(dir, "rocksdb")
 
-	db, err := NewRocksDB(path, "")
+	db, err := NewRocksDB(path, "", nil)
 	require.NoError(t, err)
 
 	t.Run("RocksDB", func(t *testing.T) { Run(t, db) })
@@ -44,4 +45,20 @@ func TestRocksDBStats(t *testing.T) {
 	assert.NotEmpty(t, db.Stats())
 }
 
-// TODO: Add tests for rocksdb
+func TestRocksDBWithOptions(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "rocksdb")
+
+	opts := make(DBOptionsMap, 0)
+	opts["maxopenfiles"] = 1000
+
+	defaultOpts := defaultRocksdbOptions()
+	files := cast.ToInt(opts.Get("maxopenfiles"))
+	defaultOpts.SetMaxOpenFiles(files)
+	require.Equal(t, opts["maxopenfiles"], defaultOpts.GetMaxOpenFiles())
+
+	db, err := NewRocksDB(path, "", opts)
+	require.NoError(t, err)
+
+	t.Run("RocksDB", func(t *testing.T) { Run(t, db) })
+}

util.go

@@ -49,3 +49,15 @@ func FileExists(filePath string) bool {
 	_, err := os.Stat(filePath)
 	return !os.IsNotExist(err)
 }
+
+// DBOptionsMap is a stub implementing DBOptions which can get data from a map
+type DBOptionsMap map[string]interface{}
+
+func (m DBOptionsMap) Get(key string) interface{} {
+	v, ok := m[key]
+	if !ok {
+		return interface{}(nil)
+	}
+
+	return v
+}