pkg/receive: remove flushed WAL

This commit ensures that we delete the WAL after it has been flushed to
a block. Flushing the WAL simply creates a block but does not remove the
WAL directory or its contents. This means that once the DB is re-opened,
new samples are added to the same WAL. Flushing the WAL again does not
result in blocks with overlapping time ranges because the flushing logic
guards against this
(https://github.com/prometheus/prometheus/blob/master/tsdb/db.go#L300).
Nevertheless, we should delete the WAL after flushing it to ensure that
flushed samples are not needlessly re-processed. Also, once multi-TSDB
support is added, holding old samples in the WAL could cause problems.

Signed-off-by: Lucas Servén Marín <lserven@gmail.com>

pkg/receive/tsdb.go

@@ -1,6 +1,8 @@
 package receive
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/go-kit/kit/log"
@@ -86,6 +88,9 @@ func (f *FlushableStorage) Flush() error {
 	if err := ro.FlushWAL(f.Dir()); err != nil {
 		return errors.Wrap(err, "flushing WAL")
 	}
+	if err := os.RemoveAll(filepath.Join(f.Dir(), "wal")); err != nil {
+		return errors.Wrap(err, "removing stale WAL")
+	}
 	if reopen {
 		return errors.Wrap(f.open(), "re-starting storage")
 	}