Skip to content

Commit

Permalink
Add new option for revision based auto compaction
Browse files Browse the repository at this point in the history
  • Loading branch information
Iwasaki Yudai committed Jun 20, 2017
1 parent d6caf54 commit 8b5f70b
Show file tree
Hide file tree
Showing 7 changed files with 42 additions and 5 deletions.
28 changes: 28 additions & 0 deletions compactor/compactor.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package compactor

import (
"fmt"
"time"

"github.com/coreos/pkg/capnslog"
Expand All @@ -30,12 +31,39 @@ var (
const (
checkCompactionInterval = 5 * time.Minute
executeCompactionInterval = time.Hour

ModePeriodic = "period"
ModeRevisional = "revision"
)

// Compactor purges old log from the storage periodically.
type Compactor interface {
// Run starts the main loop of the compactor in background.
// Use Stop() to halt the loop and release the resource.
Run()
// Stop halts the main loop of the compactor.
Stop()
// Pause temporally suspend the compactor not to run compaction. Resume() to unpose.
Pause()
// Resume restarts the compactor suspended by Pause().
Resume()
}

type Compactable interface {
Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error)
}

type RevGetter interface {
Rev() int64
}

func New(mode string, retention int, rg RevGetter, c Compactable) (Compactor, error) {
switch mode {
case ModePeriodic:
return NewPeriodic(retention, rg, c), nil
case ModeRevisional:
return NewRevisional(int64(retention), rg, c), nil
default:
return nil, fmt.Errorf("unsupported compaction mode %s", mode)
}
}
1 change: 1 addition & 0 deletions embed/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ type Config struct {
Name string `json:"name"`
SnapCount uint64 `json:"snapshot-count"`
AutoCompactionRetention int `json:"auto-compaction-retention"`
AutoCompactionMode string `json:"auto-compaction-mode`

// TickMs is the number of milliseconds between heartbeat ticks.
// TODO: decouple tickMs and heartbeat tick (current heartbeat tick = 1).
Expand Down
1 change: 1 addition & 0 deletions embed/etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
TickMs: cfg.TickMs,
ElectionTicks: cfg.ElectionTicks(),
AutoCompactionRetention: cfg.AutoCompactionRetention,
AutoCompactionMode: cfg.AutoCompactionMode,
QuotaBackendBytes: cfg.QuotaBackendBytes,
MaxTxnOps: cfg.MaxTxnOps,
MaxRequestBytes: cfg.MaxRequestBytes,
Expand Down
3 changes: 2 additions & 1 deletion etcdmain/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,8 @@ func newConfig() *config {
// version
fs.BoolVar(&cfg.printVersion, "version", false, "Print the version and exit.")

fs.IntVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", 0, "Auto compaction retention for mvcc key value store in hour. 0 means disable auto compaction.")
fs.IntVar(&cfg.AutoCompactionRetention, "auto-compaction-retention", 0, "Auto compaction retention for mvcc key value store. 0 means disable auto compaction.")
fs.StringVar(&cfg.AutoCompactionMode, "auto-compaction-mode", "period", "Interpreted 'auto-comapction-retention' as hours when 'period', as revision numbers when 'revision'.")

// pprof profiler via HTTP
fs.BoolVar(&cfg.EnablePprof, "enable-pprof", false, "Enable runtime profiling data via HTTP server. Address is at client URL + \"/debug/pprof/\"")
Expand Down
4 changes: 3 additions & 1 deletion etcdmain/help.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,9 @@ clustering flags:
--strict-reconfig-check
reject reconfiguration requests that would cause quorum loss.
--auto-compaction-retention '0'
auto compaction retention in hour. 0 means disable auto compaction.
auto compaction retention length. 0 means disable auto compaction.
--auto-compaction-mode 'period'
'period' means hours, 'revision' means revision numbers to retain by auto compaction
--enable-v2
Accept etcd V2 client requests.
Expand Down
1 change: 1 addition & 0 deletions etcdserver/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ type ServerConfig struct {
BootstrapTimeout time.Duration

AutoCompactionRetention int
AutoCompactionMode string
QuotaBackendBytes int64
MaxTxnOps uint

Expand Down
9 changes: 6 additions & 3 deletions etcdserver/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,7 @@ type EtcdServer struct {

SyncTicker *time.Ticker
// compactor is used to auto-compact the KV.
compactor *compactor.Periodic
compactor compactor.Compactor

// peerRt used to send requests (version, lease) to peers.
peerRt http.RoundTripper
Expand Down Expand Up @@ -469,8 +469,11 @@ func NewServer(cfg ServerConfig) (srv *EtcdServer, err error) {
return nil, err
}
srv.authStore = auth.NewAuthStore(srv.be, tp)
if h := cfg.AutoCompactionRetention; h != 0 {
srv.compactor = compactor.NewPeriodic(h, srv.kv, srv)
if num := cfg.AutoCompactionRetention; num != 0 {
srv.compactor, err = compactor.New(cfg.AutoCompactionMode, num, srv.kv, srv)
if err != nil {
return nil, err
}
srv.compactor.Run()
}

Expand Down

0 comments on commit 8b5f70b

Please sign in to comment.