memory revision

scanner/scanner.go

@@ -171,7 +171,7 @@ func (sc *Scanner) Scan(
 			msgStartPos, msgStartMatchEnd = 0, msgStartMatchEnd-msgStartPos
 		}
 		// if buf has no spare space after the workable area - extend the buffer
-		if bufLen == cap(buf) {
+		if bufLen == len(buf) {
 			// respect max buf size
 			if len(buf) >= sc.maxBufSize {
 				return xerrors.Errorf("%w: %d bytes (consider increasing the max buffer size)", MaxBufSizeReached, sc.maxBufSize)

search/query_test.go

@@ -5,55 +5,50 @@ import (
 	"github.com/stretchr/testify/require"
 	"golang.org/x/exp/maps"
 	"heaplog/common"
-	"heaplog/indexer"
-	"heaplog/ingest"
-	"heaplog/scanner"
 	"heaplog/test"
-	"heaplog/tokenizer"
-	"os"
-	"regexp"
 	"slices"
 	"testing"
 	"time"
 )
 
-func TestQueryPerformance(t *testing.T) {
-
-	segmentSize := int64(50_000_000)
-
-	storage, _, storageRoot := test.PrepareServices(t, segmentSize)
-	defer os.Remove(storageRoot)
-
-	files := map[string]int64{
-		"/home/dmitry/Code/go/src/heaplog2/local/logs/500Mb.log": 2905976243,
-		// "/home/dmitry/Code/go/src/heaplog2/local/logs/laravel-2024-01-17.log": 346914816,
-	}
-	_, _, err := storage.CheckInFiles(files)
-	require.NoError(t, err)
-
-	messageStartPattern := regexp.MustCompile(`(?m)^\[(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}\.?(\d{6}([+-]\d\d:\d\d)?)?)]`)
-	dateLayout := "2006-01-02T15:04:05.000000-07:00"
-	_scanner := scanner.NewScanner(dateLayout, messageStartPattern, 1_000_000, 50_000_000)
-
-	tokenizerFunc := func(input string) []string {
-		return tokenizer.TokenizeS2(input, 4, 40)
-	}
-	_indexer := indexer.NewIndexer(_scanner, tokenizerFunc)
-
-	ingestor := ingest.NewIngestor(storage, _indexer, segmentSize, 1)
-
-	require.NoError(t, ingestor.Ingest())
-
-	// _, unboundTokenizerFunc := test.PrepareTokenizers()
-	// _selector := NewSegmentSelector(storage, unboundTokenizerFunc, tokenizerFunc)
-	// querySearch := NewQuerySearch(_selector, _selector.storage, _scanner)
-	//
-	// _, c, err := querySearch.NewQuery("error !debug", nil, nil, 100)
-	// require.NoError(t, err)
-	// <-c
-
-	time.Sleep(5 * time.Second)
-}
+//
+// func TestQueryPerformance(t *testing.T) {
+//
+// 	segmentSize := int64(50_000_000)
+//
+// 	storage, _, storageRoot := test.PrepareServices(t, segmentSize)
+// 	defer os.Remove(storageRoot)
+//
+// 	files := map[string]int64{
+// 		"/home/dmitry/Code/go/src/heaplog2/local/logs/500Mb.log":              2905976243,
+// 		"/home/dmitry/Code/go/src/heaplog2/local/logs/laravel-2024-01-17.log": 346914816,
+// 	}
+// 	_, _, err := storage.CheckInFiles(files)
+// 	require.NoError(t, err)
+//
+// 	messageStartPattern := regexp.MustCompile(`(?m)^\[(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}\.?(\d{6}([+-]\d\d:\d\d)?)?)]`)
+// 	dateLayout := "2006-01-02T15:04:05.000000-07:00"
+// 	_scanner := scanner.NewScanner(dateLayout, messageStartPattern, 1_000_000, 50_000_000)
+//
+// 	tokenizerFunc := func(input string) []string {
+// 		return tokenizer.TokenizeS2(input, 4, 40)
+// 	}
+// 	_indexer := indexer.NewIndexer(_scanner, tokenizerFunc)
+//
+// 	ingestor := ingest.NewIngestor(storage, _indexer, segmentSize, runtime.NumCPU())
+//
+// 	require.NoError(t, ingestor.Ingest())
+//
+// 	// _, unboundTokenizerFunc := test.PrepareTokenizers()
+// 	// _selector := NewSegmentSelector(storage, unboundTokenizerFunc, tokenizerFunc)
+// 	// querySearch := NewQuerySearch(_selector, _selector.storage, _scanner)
+// 	//
+// 	// _, c, err := querySearch.NewQuery("error !debug", nil, nil, 100)
+// 	// require.NoError(t, err)
+// 	// <-c
+//
+// 	time.Sleep(5 * time.Second)
+// }
 
 func TestBuildQuerySuccess(t *testing.T) {
 	selector, files := ingestFiles(t, 10)

ui/console.go

@@ -26,6 +26,11 @@ func buildHeaplog(cfg Config) *heaplog.Heaplog {
 	ingestWorkers := int(cfg.IngestWorkers)
 	if ingestWorkers == 0 {
 		ingestWorkers = runtime.NumCPU()
+		if ingestWorkers > 4 {
+			// This is to reduce chances of concurrency for resources
+			// a magick number :)
+			ingestWorkers = ingestWorkers - 2
+		}
 	}
 
 	hl, err := heaplog.NewHeaplog(