tweaks config values

storage/storage.go

@@ -1337,6 +1337,7 @@ func NewStorage(storagePath string, ingestFlushTick, searchFlushTick time.Durati
 				if err != nil {
 					log.Printf("terms merge fail: %s", err)
 					time.Sleep(time.Second * 10)
+					continue
 				}
 				err = termsDir.Cleanup()
 				if err != nil {
@@ -1412,7 +1413,7 @@ func NewStorage(storagePath string, ingestFlushTick, searchFlushTick time.Durati
 	s.incomingQueryMessages = make(chan common.MatchedMessage)
 	go s.ingestQueryMessages(searchFlushTick)
 
-	memoryReportCh = make(chan time.Duration)
+	memoryReportCh = make(chan time.Duration, 10)
 	go reportMemory(s.db)
 
 	return s, nil

storage/terms_dir.go

@@ -240,36 +240,50 @@ func (d *TermsDir) GetMatchedTermIds(match func(term string) bool) (terms []stri
 // writeNewFile prepares a callback with the Writer,
 // after the callback return it closes the writer.
 func (d *TermsDir) writeNewFile(writeF func(w io.Writer) error) error {
+
+	// Prepare
 	targetFilePath := path.Join(d.dir, fmt.Sprintf("%d.fst", time.Now().UnixNano()))
 	f, err := os.Create(targetFilePath)
 	if err != nil {
 		return xerrors.Errorf("terms: unable to put to a file: %w", err)
 	}
-	w := bufio.NewWriterSize(f, 4096*100)
 	defer func() {
-		err2 := w.Flush()
-		if err2 != nil {
-			log.Printf("unable to wrire a terms file: %s", err2)
-		}
-
-		var fStat os.FileInfo
-		fStat, err = f.Stat()
-		err2 = f.Close()
-		if err2 != nil {
-			log.Printf("unable to close a terms file: %s", err2)
-			_ = os.Remove(targetFilePath)
-		} else {
-			// Insert to the main list:
-			d.mainList.safeWrite(func() {
-				d.mainList.putFile(&termsFile{
-					path: targetFilePath,
-					len:  fStat.Size(),
-				})
-			})
+		if err != nil {
+			_ = os.Remove(targetFilePath) // cleanup
 		}
 	}()
 
-	return writeF(w)
+	w := bufio.NewWriterSize(f, 4096*100)
+
+	// Write
+	err = writeF(w)
+	if err != nil {
+		return err
+	}
+
+	// Complete writing
+	err2 := w.Flush()
+	if err2 != nil {
+		log.Printf("unable to wrire a terms file: %s", err2)
+	}
+
+	var fStat os.FileInfo
+	fStat, err = f.Stat()
+	err = f.Close()
+	if err != nil {
+		log.Printf("unable to close a terms file: %s", err2)
+		return err
+	}
+
+	// Insert to the main list:
+	d.mainList.safeWrite(func() {
+		d.mainList.putFile(&termsFile{
+			path: targetFilePath,
+			len:  fStat.Size(),
+		})
+	})
+
+	return nil
 }
 
 func NewTermsDir(dir string) (*TermsDir, error) {

ui/config.go

@@ -26,6 +26,10 @@ type Config struct {
 	// sets parallel degree of ingesting.
 	// defaults to the number of cores is omitted or 0.
 	IngestWorkers uint
+	// Terms are extracted from messages and indexed.
+	// These control how fast ingestion goes (and space taken for the inverted index),
+	// as well as how fast search goes (as shorter terms may duplicate in the index).
+	MinTermLen, MaxTermLen uint
 }
 
 var DefaultCfg = Config{

ui/console.go

@@ -19,8 +19,18 @@ import (
 )
 
 func buildHeaplog(cfg Config) *heaplog.Heaplog {
-	maxTokenLen := 20
-	tokenizerFunc := func(input string) []string { return tokenizer.TokenizeS2(input, 4, maxTokenLen) }
+	// controls the max length of tokens,
+	// easier to index (consume less space), could be slower to search due to collisions.
+	minTokenLen := 4
+	if cfg.MinTermLen > 0 {
+		minTokenLen = int(cfg.MinTermLen)
+	}
+	maxTokenLen := max(10, minTokenLen+1)
+	if cfg.MaxTermLen > 0 {
+		maxTokenLen = max(minTokenLen+1, int(cfg.MaxTermLen))
+	}
+
+	tokenizerFunc := func(input string) []string { return tokenizer.TokenizeS2(input, minTokenLen, maxTokenLen) }
 	unboundTokenizerFunc := func(input string) []string { return tokenizer.TokenizeS2(input, 1, maxTokenLen) }
 
 	ingestWorkers := int(cfg.IngestWorkers)

ui/web_templates/theme.gohtml

@@ -114,9 +114,12 @@
 
     // Manage a WIP label:
     isWIP = 0
+    loading = ' Loading...'
     htmx.on("htmx:beforeRequest", function(){
         isWIP++
-        $("title").html($("title").html() + ' Loading...');
+        if(!$("title").html().includes(loading)) {
+            $("title").html($("title").html() + loading);
+        }
     });
     htmx.on("htmx:afterSettle", function(){
         isWIP--
@@ -125,7 +128,7 @@
         if(isWIP > 0) {
             return;
         }
-        $("title").html($("title").html().replace(/ Loading...$/, ''));
+        $("title").html($("title").html().replace(loading, ''));
     }, 500)
 </script>
 </body>