move log file manager inside logs db client

engine/server/engine/centralized_logs/client_implementations/kurtosis_backend/kurtosis_backend_logs_database_client.go

@@ -156,6 +156,20 @@ func (client *kurtosisBackendLogsDatabaseClient) FilterExistingServiceUuids(
 	return filteredServiceUuidsSet, nil
 }
 
+func (client *kurtosisBackendLogsDatabaseClient) StartLogFileManagement(ctx context.Context) {
+	// no log file management needs to be done for this logs db client
+}
+
+func (client *kurtosisBackendLogsDatabaseClient) RemoveEnclaveLogs(enclaveUuid string) error {
+	// no log file management needs to be done for this logs db client
+	return nil
+}
+
+func (client *kurtosisBackendLogsDatabaseClient) RemoveAllLogs() error {
+	// no log file management needs to be done for this logs db client
+	return nil
+}
+
 // ====================================================================================================
 //
 //	Private helper functions

engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client.go

@@ -5,6 +5,7 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_filesystem"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline"
@@ -22,17 +23,21 @@ type persistentVolumeLogsDatabaseClient struct {
 
 	filesystem volume_filesystem.VolumeFilesystem
 
+	logFileManager *log_file_manager.LogFileManager
+
 	streamStrategy stream_logs_strategy.StreamLogsStrategy
 }
 
 func NewPersistentVolumeLogsDatabaseClient(
 	kurtosisBackend backend_interface.KurtosisBackend,
 	filesystem volume_filesystem.VolumeFilesystem,
+	logFileManager *log_file_manager.LogFileManager,
 	streamStrategy stream_logs_strategy.StreamLogsStrategy,
 ) *persistentVolumeLogsDatabaseClient {
 	return &persistentVolumeLogsDatabaseClient{
 		kurtosisBackend: kurtosisBackend,
 		filesystem:      filesystem,
+		logFileManager:  logFileManager,
 		streamStrategy:  streamStrategy,
 	}
 }
@@ -127,6 +132,18 @@ func (client *persistentVolumeLogsDatabaseClient) FilterExistingServiceUuids(
 	return filteredServiceUuidsSet, nil
 }
 
+func (client *persistentVolumeLogsDatabaseClient) StartLogFileManagement(ctx context.Context) {
+	client.logFileManager.StartLogFileManagement(ctx)
+}
+
+func (client *persistentVolumeLogsDatabaseClient) RemoveEnclaveLogs(enclaveUuid string) error {
+	return client.logFileManager.RemoveEnclaveLogs(enclaveUuid)
+}
+
+func (client *persistentVolumeLogsDatabaseClient) RemoveAllLogs() error {
+	return client.logFileManager.RemoveAllLogs()
+}
+
 // ====================================================================================================
 //
 //	Private helper functions

engine/server/engine/centralized_logs/client_implementations/persistent_volume/persistent_volume_logs_database_client_test.go

@@ -6,6 +6,8 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave"
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/file_layout"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/logs_clock"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/stream_logs_strategy"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/volume_consts"
@@ -694,7 +696,11 @@ func executeStreamCallAndGetReceivedServiceLogLines(
 
 	kurtosisBackend := backend_interface.NewMockKurtosisBackend(t)
 
-	logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, underlyingFs, streamStrategy)
+	// no log file management is done in these tests so values for logFileManager aren't important
+	mockTime := logs_clock.NewMockLogsClock(0, 0, 0)
+	fileLayout := file_layout.NewPerWeekFileLayout(mockTime)
+	logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, underlyingFs, fileLayout, mockTime, 0)
+	logsDatabaseClient := NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, underlyingFs, logFileManager, streamStrategy)
 
 	userServiceLogsByUuidChan, errChan, receivedCancelCtxFunc, err := logsDatabaseClient.StreamUserServiceLogs(ctx, enclaveUuid, userServiceUuids, logLinesFilters, shouldFollowLogs, defaultShouldReturnAllLogs, defaultNumLogLines)
 	if err != nil {

engine/server/engine/centralized_logs/logs_database_client.go

@@ -31,4 +31,10 @@ type LogsDatabaseClient interface {
 		map[service.ServiceUUID]bool,
 		error,
 	)
+
+	StartLogFileManagement(ctx context.Context)
+
+	RemoveEnclaveLogs(enclaveUuid string) error
+
+	RemoveAllLogs() error
 }

engine/server/engine/enclave_manager/enclave_manager.go

@@ -3,12 +3,12 @@ package enclave_manager
 import (
 	"context"
 	"fmt"
+	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager"
 	"github.com/kurtosis-tech/kurtosis/metrics-library/golang/lib/metrics_client"
 
 	dockerTypes "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_impls/docker/docker_manager/types"
@@ -64,10 +64,11 @@ type EnclaveManager struct {
 	// this is an append only list
 	allExistingAndHistoricalIdentifiers []*types.EnclaveIdentifiers
 
-	enclaveCreator        *EnclaveCreator
-	enclavePool           *EnclavePool
-	enclaveEnvVars        string
-	enclaveLogFileManager *log_file_manager.LogFileManager
+	enclaveCreator *EnclaveCreator
+	enclavePool    *EnclavePool
+	enclaveEnvVars string
+
+	logsDbClient centralized_logs.LogsDatabaseClient
 
 	metricsUserID               string
 	didUserAcceptSendingMetrics bool
@@ -83,7 +84,7 @@ func CreateEnclaveManager(
 	engineVersion string,
 	poolSize uint8,
 	enclaveEnvVars string,
-	enclaveLogFileManager *log_file_manager.LogFileManager,
+	logsDbClient centralized_logs.LogsDatabaseClient,
 	metricsUserID string,
 	didUserAcceptSendingMetrics bool,
 	isCI bool,
@@ -114,7 +115,7 @@ func CreateEnclaveManager(
 		enclaveCreator:                            enclaveCreator,
 		enclavePool:                               enclavePool,
 		enclaveEnvVars:                            enclaveEnvVars,
-		enclaveLogFileManager:                     enclaveLogFileManager,
+		logsDbClient:                              logsDbClient,
 		metricsUserID:                             metricsUserID,
 		didUserAcceptSendingMetrics:               didUserAcceptSendingMetrics,
 		isCI:                                      isCI,
@@ -277,7 +278,7 @@ func (manager *EnclaveManager) DestroyEnclave(ctx context.Context, enclaveIdenti
 		return stacktrace.Propagate(err, "An error occurred destroying the enclave")
 	}
 	if _, found := successfullyDestroyedEnclaves[enclaveUuid]; found {
-		if err = manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveUuid)); err != nil {
+		if err = manager.logsDbClient.RemoveEnclaveLogs(string(enclaveUuid)); err != nil {
 			return stacktrace.Propagate(err, "An error occurred attempting to remove enclave '%v' logs after it was destroyed.", enclaveIdentifier)
 		}
 		return nil
@@ -601,7 +602,7 @@ func (manager *EnclaveManager) cleanEnclaves(
 	for enclaveId := range successfullyDestroyedEnclaves {
 		successfullyDestroyedEnclaveIdStrs = append(successfullyDestroyedEnclaveIdStrs, string(enclaveId))
 
-		if err := manager.enclaveLogFileManager.RemoveEnclaveLogs(string(enclaveId)); err != nil {
+		if err := manager.logsDbClient.RemoveEnclaveLogs(string(enclaveId)); err != nil {
 			logRemovalErr := stacktrace.Propagate(err, "An error occurred removing enclave '%v' logs.", enclaveId)
 			enclaveDestructionErrors = append(enclaveDestructionErrors, logRemovalErr)
 		}

engine/server/engine/main.go

@@ -175,23 +175,16 @@ func runMain() error {
 	if err != nil {
 		return stacktrace.Propagate(err, "An error occurred getting the Kurtosis backend for backend type '%v' and config '%+v'", serverArgs.KurtosisBackendType, backendConfig)
 	}
-
 	logsDatabaseClient := getLogsDatabaseClient(serverArgs.KurtosisBackendType, kurtosisBackend)
-
-	// TODO: Move log file management into LogsDatabaseClient
-	osFs := volume_filesystem.NewOsVolumeFilesystem()
-	realTime := logs_clock.NewRealClock()
-	perWeekFileLayout := file_layout.NewPerWeekFileLayout(realTime)
-	logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, perWeekFileLayout, realTime, volume_consts.LogRetentionPeriodInWeeks)
-	logFileManager.StartLogFileManagement(ctx)
+	logsDatabaseClient.StartLogFileManagement(ctx)
 
 	enclaveManager, err := getEnclaveManager(
 		kurtosisBackend,
 		serverArgs.KurtosisBackendType,
 		serverArgs.ImageVersionTag,
 		serverArgs.PoolSize,
 		serverArgs.EnclaveEnvVars,
-		logFileManager,
+		logsDatabaseClient,
 		serverArgs.MetricsUserID,
 		serverArgs.DidUserAcceptSendingMetrics,
 		serverArgs.IsCI,
@@ -292,7 +285,6 @@ func runMain() error {
 			serverArgs,
 			enclaveManager,
 			logsDatabaseClient,
-			logFileManager,
 			metricsClient,
 		)
 		if err != nil {
@@ -308,7 +300,6 @@ func runMain() error {
 		serverArgs.MetricsUserID,
 		serverArgs.DidUserAcceptSendingMetrics,
 		logsDatabaseClient,
-		logFileManager,
 		metricsClient)
 	apiPath, handler := kurtosis_engine_rpc_api_bindingsconnect.NewEngineServiceHandler(engineConnectServer)
 	defer func() {
@@ -331,7 +322,7 @@ func getEnclaveManager(
 	engineVersion string,
 	poolSize uint8,
 	enclaveEnvVars string,
-	enclaveLogFileManager *log_file_manager.LogFileManager,
+	logsDbClient centralized_logs.LogsDatabaseClient,
 	metricsUserID string,
 	didUserAcceptSendingMetrics bool,
 	isCI bool,
@@ -360,7 +351,7 @@ func getEnclaveManager(
 		engineVersion,
 		poolSize,
 		enclaveEnvVars,
-		enclaveLogFileManager,
+		logsDbClient,
 		metricsUserID,
 		didUserAcceptSendingMetrics,
 		isCI,
@@ -413,10 +404,14 @@ func getLogsDatabaseClient(kurtosisBackendType args.KurtosisBackendType, kurtosi
 	var logsDatabaseClient centralized_logs.LogsDatabaseClient
 	switch kurtosisBackendType {
 	case args.KurtosisBackendType_Docker:
-		osFs := volume_filesystem.NewOsVolumeFilesystem()
 		realTime := logs_clock.NewRealClock()
+
+		osFs := volume_filesystem.NewOsVolumeFilesystem()
+		perWeekFileLayout := file_layout.NewPerWeekFileLayout(realTime)
+		logFileManager := log_file_manager.NewLogFileManager(kurtosisBackend, osFs, perWeekFileLayout, realTime, volume_consts.LogRetentionPeriodInWeeks)
 		perWeekStreamLogsStrategy := stream_logs_strategy.NewPerWeekStreamLogsStrategy(realTime, volume_consts.LogRetentionPeriodInWeeks)
-		logsDatabaseClient = persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, perWeekStreamLogsStrategy)
+
+		logsDatabaseClient = persistent_volume.NewPersistentVolumeLogsDatabaseClient(kurtosisBackend, osFs, logFileManager, perWeekStreamLogsStrategy)
 	case args.KurtosisBackendType_Kubernetes:
 		logsDatabaseClient = kurtosis_backend.NewKurtosisBackendLogsDatabaseClient(kurtosisBackend)
 	}
@@ -438,7 +433,6 @@ func restApiServer(
 	serverArgs *args.EngineServerArgs,
 	enclave_manager *enclave_manager.EnclaveManager,
 	logsDatabaseClient centralized_logs.LogsDatabaseClient,
-	logFileManager *log_file_manager.LogFileManager,
 	metricsClient metrics_client.MetricsClient,
 ) error {
 
@@ -466,7 +460,6 @@ func restApiServer(
 	engineRuntime := restApi.EngineRuntime{
 		ImageVersionTag: serverArgs.ImageVersionTag,
 		EnclaveManager:  enclave_manager,
-		LogFileManager:  logFileManager,
 		MetricsClient:   metricsClient,
 	}
 	engineApi.RegisterHandlers(echoApiRouter, engineApi.NewStrictHandler(engineRuntime, nil))
@@ -483,7 +476,6 @@ func restApiServer(
 		MetricsUserID:               serverArgs.MetricsUserID,
 		DidUserAcceptSendingMetrics: serverArgs.DidUserAcceptSendingMetrics,
 		LogsDatabaseClient:          logsDatabaseClient,
-		LogFileManager:              logFileManager,
 		MetricsClient:               metricsClient,
 		AsyncStarlarkLogs:           asyncStarlarkLogs,
 		CorsConfig:                  *corsConfig,

engine/server/engine/server/engine_connect_server_service.go

@@ -10,7 +10,6 @@ import (
 	"github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/enclave"
 	user_service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs"
-	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/logline"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/types"
@@ -41,8 +40,6 @@ type EngineConnectServerService struct {
 	// The client for consuming container logs from the logs database
 	logsDatabaseClient centralized_logs.LogsDatabaseClient
 
-	logFileManager *log_file_manager.LogFileManager
-
 	metricsClient metrics_client.MetricsClient
 }
 
@@ -52,7 +49,6 @@ func NewEngineConnectServerService(
 	metricsUserId string,
 	didUserAcceptSendingMetrics bool,
 	logsDatabaseClient centralized_logs.LogsDatabaseClient,
-	logFileManager *log_file_manager.LogFileManager,
 	metricsClient metrics_client.MetricsClient,
 ) *EngineConnectServerService {
 	service := &EngineConnectServerService{
@@ -61,7 +57,6 @@ func NewEngineConnectServerService(
 		metricsUserID:               metricsUserId,
 		didUserAcceptSendingMetrics: didUserAcceptSendingMetrics,
 		logsDatabaseClient:          logsDatabaseClient,
-		logFileManager:              logFileManager,
 		metricsClient:               metricsClient,
 	}
 	return service
@@ -265,7 +260,7 @@ func (service *EngineConnectServerService) Clean(ctx context.Context, connectArg
 		return nil, stacktrace.Propagate(err, "An error occurred while cleaning enclaves")
 	}
 	if args.GetShouldCleanAll() {
-		if err = service.logFileManager.RemoveAllLogs(); err != nil {
+		if err = service.logsDatabaseClient.RemoveAllLogs(); err != nil {
 			return nil, stacktrace.Propagate(err, "An error occurred removing all logs.")
 		}
 	}

engine/server/engine/server/websocket_api_handler.go

@@ -7,7 +7,6 @@ import (
 
 	user_service "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs"
-	"github.com/kurtosis-tech/kurtosis/engine/server/engine/centralized_logs/client_implementations/persistent_volume/log_file_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/enclave_manager"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/mapping/to_http"
 	"github.com/kurtosis-tech/kurtosis/engine/server/engine/streaming"
@@ -37,8 +36,6 @@ type WebSocketRuntime struct {
 	// The clients for consuming container logs from the logs' database server
 	LogsDatabaseClient centralized_logs.LogsDatabaseClient
 
-	LogFileManager *log_file_manager.LogFileManager
-
 	MetricsClient metrics_client.MetricsClient
 
 	// Pool of Starlark log streamers create by package/script runs