add option to use AWS S3 inplace of existing file server

deepfence_kafka/kafka-broker-Dockerfile

@@ -23,7 +23,8 @@ ENV KAFKA_BROKER_ID=1 \
     # KAFKA_LOG_DIRS='/data/kafka' \
     KAFKA_LOG_RETENTION_MS=86400000 \
     KAFKA_LOG_RETENTION_BYTES=-1 \
-    KAFKA_MESSAGE_MAX_BYTES=52428800
+    KAFKA_MESSAGE_MAX_BYTES=52428800 \
+    KAFKA_AUTO_CREATE_TOPICS_ENABLE='false'
 
 COPY kafka_update_run.sh /home/appuser/kafka_update_run.sh
 CMD ["bash","-c", "/home/appuser/kafka_update_run.sh" ]

deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
+	"path/filepath"
 	"time"
 
 	"github.com/deepfence/ThreatMapper/deepfence_server/diagnosis"
@@ -155,7 +156,8 @@ func GenerateAgentDiagnosticLogs(ctx context.Context, nodeIdentifiers []diagnosi
 			continue
 		}
 		fileName := "deepfence-agent-logs-" + nodeIdentifier.NodeId + fileNameSuffix
-		uploadUrl, err := mc.CreatePublicUploadURL(ctx, diagnosis.AgentDiagnosisFileServerPrefix+fileName, true, time.Minute*10, url.Values{})
+		uploadUrl, err := mc.CreatePublicUploadURL(ctx,
+			filepath.Join(diagnosis.AgentDiagnosisFileServerPrefix, fileName), true, time.Minute*10, url.Values{})
 		if err != nil {
 			return err
 		}

deepfence_server/diagnosis/common.go

@@ -10,14 +10,15 @@ import (
 	"time"
 
 	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
+	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
 	"github.com/neo4j/neo4j-go-driver/v4/neo4j"
 )
 
 const (
 	DiagnosisLinkExpiry              = 5 * time.Minute
-	ConsoleDiagnosisFileServerPrefix = "/diagnosis/console-diagnosis/"
-	AgentDiagnosisFileServerPrefix   = "/diagnosis/agent-diagnosis/"
+	ConsoleDiagnosisFileServerPrefix = "diagnosis/console-diagnosis/"
+	AgentDiagnosisFileServerPrefix   = "diagnosis/agent-diagnosis/"
 )
 
 type DiagnosticNotification struct {
@@ -76,11 +77,13 @@ func GetDiagnosticLogs(ctx context.Context) (*GetDiagnosticLogsResponse, error)
 func getDiagnosticLogsHelper(ctx context.Context, mc directory.FileManager, pathPrefix string) []DiagnosticLogsLink {
 	// Get completed files from minio
 	objects := mc.ListFiles(ctx, pathPrefix, false, 0, true)
+	log.Debug().Msgf("diagnosis logs at %s: %v", pathPrefix, objects)
 	diagnosticLogsResponse := make([]DiagnosticLogsLink, len(objects))
 	for i, obj := range objects {
 		message := ""
 		urlLink, err := mc.ExposeFile(ctx, obj.Key, false, DiagnosisLinkExpiry, url.Values{})
 		if err != nil {
+			log.Error().Err(err).Msg("failed to list console diagnosis logs")
 			var minioError utils.MinioError
 			xmlErr := xml.Unmarshal([]byte(err.Error()), &minioError)
 			if xmlErr != nil {
@@ -122,6 +125,10 @@ func getAgentDiagnosticLogs(ctx context.Context, mc directory.FileManager, pathP
 	}
 	defer session.Close()
 	tx, err := session.BeginTransaction(neo4j.WithTxTimeout(30 * time.Second))
+	if err != nil {
+		log.Error().Msg(err.Error())
+		return diagnosticLogs
+	}
 	defer tx.Close()
 
 	r, err := tx.Run(`
@@ -133,6 +140,10 @@ func getAgentDiagnosticLogs(ctx context.Context, mc directory.FileManager, pathP
 
 	nodeIdToName := make(map[string]string)
 	records, err := r.Collect()
+	if err != nil {
+		log.Error().Msg(err.Error())
+		return diagnosticLogs
+	}
 	for _, rec := range records {
 		var nodeId, fileName, message, status, updatedAt, nodeName interface{}
 		var ok bool

deepfence_server/diagnosis/console-diagnosis/docker.go

@@ -73,7 +73,9 @@ func (d *DockerConsoleDiagnosisHandler) GenerateDiagnosticLogs(ctx context.Conte
 	if err != nil {
 		return err
 	}
-	_, err = mc.UploadLocalFile(ctx, diagnosis.ConsoleDiagnosisFileServerPrefix+filepath.Base(zipFile.Name()), zipFile.Name(),
+	_, err = mc.UploadLocalFile(ctx,
+		filepath.Join(diagnosis.ConsoleDiagnosisFileServerPrefix, filepath.Base(zipFile.Name())),
+		zipFile.Name(),
 		minio.PutObjectOptions{ContentType: "application/zip"})
 	if err != nil {
 		return err

deepfence_server/diagnosis/console-diagnosis/kubernetes.go

@@ -122,7 +122,9 @@ func (k *KubernetesConsoleDiagnosisHandler) GenerateDiagnosticLogs(ctx context.C
 	if err != nil {
 		return err
 	}
-	_, err = mc.UploadLocalFile(ctx, diagnosis.ConsoleDiagnosisFileServerPrefix+filepath.Base(zipFile.Name()), zipFile.Name(),
+	_, err = mc.UploadLocalFile(ctx,
+		filepath.Join(diagnosis.ConsoleDiagnosisFileServerPrefix, filepath.Base(zipFile.Name())),
+		zipFile.Name(),
 		minio.PutObjectOptions{ContentType: "application/zip"})
 	if err != nil {
 		return err

deepfence_server/entrypoint.sh

@@ -32,11 +32,15 @@ do
 done
 
 # wait for file server to start
-until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT};
-do
-  echo "file server is unavailable - sleeping"
-  sleep 5;
-done
+if [ "$DEEPFENCE_MINIO_HOST" != "s3.amazonaws.com" ]; then
+  until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT};
+  do
+    echo "file server is unavailable - sleeping"
+    sleep 5;
+  done
+else
+  echo "S3 mode skip file server health check"
+fi
 
 sed -i "s/https:\/\/petstore.swagger.io\/v2\/swagger.json/\/deepfence\/openapi.json/g" /usr/local/share/swagger-ui/swagger-initializer.js
 

deepfence_utils/directory/directory.go

@@ -45,6 +45,7 @@ type MinioConfig struct {
 	Password   string
 	BucketName string
 	Secure     bool
+	Region     string
 }
 
 type DBConfigs struct {
@@ -195,11 +196,18 @@ func initMinio() MinioConfig {
 		minioPort = "9000"
 		log.Warn().Msgf("DEEPFENCE_MINIO_PORT defaults to: %v", minioPort)
 	}
-	minioEndpoint := minioHost + ":" + minioPort
+
 	minioUser := os.Getenv("DEEPFENCE_MINIO_USER")
 	minioPassword := os.Getenv("DEEPFENCE_MINIO_PASSWORD")
 	minioBucket := os.Getenv("DEEPFENCE_MINIO_BUCKET")
+	minioRegion := os.Getenv("DEEPFENCE_MINIO_REGION")
 	minioSecure := os.Getenv("DEEPFENCE_MINIO_SECURE")
+
+	minioEndpoint := minioHost
+	if minioHost != "s3.amazonaws.com" {
+		minioEndpoint = minioHost + ":" + minioPort
+	}
+
 	if minioSecure == "" {
 		minioSecure = "false"
 	}
@@ -214,6 +222,7 @@ func initMinio() MinioConfig {
 		Password:   minioPassword,
 		BucketName: minioBucket,
 		Secure:     isSecure,
+		Region:     minioRegion,
 	}
 }
 

deepfence_utils/directory/minio.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -39,6 +40,14 @@ func (e AlreadyPresentError) Error() string {
 	return fmt.Sprintf("Already exists here: %s", e.Path)
 }
 
+type PathDoesNotExistsError struct {
+	Path string
+}
+
+func (e PathDoesNotExistsError) Error() string {
+	return fmt.Sprintf("Path doesnot exists here: %s", e.Path)
+}
+
 type FileManager interface {
 	ListFiles(ctx context.Context, pathPrefix string, recursive bool, maxKeys int, skipDir bool) []ObjectInfo
 	UploadLocalFile(ctx context.Context, filename string, localFilename string, extra interface{}) (UploadResult, error)
@@ -92,31 +101,28 @@ func (mfm *MinioFileManager) optionallyAddNamespacePrefix(filePath string, addFi
 	if addFilePathPrefix {
 		return mfm.addNamespacePrefix(filePath)
 	} else {
-		if strings.HasPrefix(filePath, "/") {
-			return filePath
-		} else {
-			return "/" + filePath
-		}
+		return strings.TrimPrefix(filePath, "/")
 	}
 }
 
 func (mfm *MinioFileManager) addNamespacePrefix(filePath string) string {
-	if !strings.HasPrefix(filePath, "/") {
-		filePath = "/" + filePath
-	}
-	return mfm.namespace + filePath
+	return filepath.Join(mfm.namespace, filePath)
 }
 
 func (mfm *MinioFileManager) ListFiles(ctx context.Context, pathPrefix string, recursive bool, maxKeys int, skipDir bool) []ObjectInfo {
-	objects := mfm.client.ListObjects(ctx, mfm.bucket, minio.ListObjectsOptions{
-		WithVersions: false,
-		WithMetadata: false,
-		Prefix:       mfm.addNamespacePrefix(pathPrefix),
-		Recursive:    recursive,
-		MaxKeys:      maxKeys,
-		StartAfter:   "",
-		UseV1:        false,
-	})
+	prefix := mfm.addNamespacePrefix(pathPrefix) + "/"
+
+	objects := mfm.client.ListObjects(ctx, mfm.bucket,
+		minio.ListObjectsOptions{
+			WithVersions: false,
+			WithMetadata: false,
+			Prefix:       prefix,
+			Recursive:    recursive,
+			MaxKeys:      maxKeys,
+			StartAfter:   "",
+			UseV1:        false,
+		})
+
 	var objectsInfo []ObjectInfo
 	for obj := range objects {
 		isDir := strings.HasSuffix(obj.Key, "/")
@@ -233,17 +239,27 @@ func (mfm *MinioFileManager) ExposeFile(ctx context.Context, filePath string, ad
 		return "", err
 	}
 
+	actualPath := mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix)
+
+	key, has := checkIfFileExists(ctx, mfm.client, mfm.bucket, actualPath)
+	if !has {
+		return "", PathDoesNotExistsError{Path: actualPath}
+	}
+
 	headers := http.Header{}
-	headers.Add("Host", consoleIp)
+	if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") {
+		headers.Add("Host", consoleIp)
+	}
 
 	urlLink, err := mfm.client.PresignHeader(
 		ctx,
 		"GET",
 		mfm.bucket,
-		mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix),
+		key,
 		expires,
 		reqParams,
-		headers)
+		headers,
+	)
 	if err != nil {
 		return "", err
 	}
@@ -258,7 +274,9 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath
 	}
 
 	headers := http.Header{}
-	headers.Add("Host", consoleIp)
+	if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") {
+		headers.Add("Host", consoleIp)
+	}
 
 	urlLink, err := mfm.client.PresignHeader(
 		ctx,
@@ -267,7 +285,8 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath
 		mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix),
 		expires,
 		reqParams,
-		headers)
+		headers,
+	)
 	if err != nil {
 		return "", err
 	}
@@ -346,6 +365,7 @@ func newMinioClient(endpoints DBConfigs) (*minio.Client, error) {
 	minioClient, err := minio.New(endpoints.Minio.Endpoint, &minio.Options{
 		Creds:  credentials.NewStaticV4(endpoints.Minio.Username, endpoints.Minio.Password, ""),
 		Secure: endpoints.Minio.Secure,
+		Region: endpoints.Minio.Region,
 	})
 	return minioClient, err
 }

deepfence_utils/utils/utils.go

@@ -455,27 +455,25 @@ func RecursiveZip(pathsToZip []string, excludePathPrefixes []string, destination
 }
 
 func UploadFile(url string, fileName string) ([]byte, int, error) {
-	r, err := os.Open(fileName)
-	if err != nil {
-		return nil, 0, err
-	}
-	buf := make([]byte, 512)
-	_, err = r.Read(buf)
+
+	buff, err := os.ReadFile(fileName)
 	if err != nil {
 		return nil, 0, err
 	}
-	r.Close()
 
 	client, err := NewHTTPClient()
 	if err != nil {
 		return nil, 0, err
 	}
-	r, err = os.Open(fileName)
-	req, err := http.NewRequest("PUT", url, r)
+
+	req, err := http.NewRequest("PUT", url, bytes.NewReader(buff))
 	if err != nil {
 		return nil, 0, err
 	}
-	req.Header.Add("Content-Type", http.DetectContentType(buf))
+
+	req.Header.Add("Content-Type", http.DetectContentType(buff))
+	req.Header.Add("Content-Length", strconv.Itoa(len(buff)))
+
 	res, err := client.Do(req)
 	if err != nil {
 		return nil, 0, err

deepfence_utils/vulnerability_db/database.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"path"
 	"sort"
 	"time"
@@ -136,16 +137,21 @@ func UpdateListing(newFile, newFileCheckSum string, buildTime time.Time) {
 
 	minioHost := utils.GetEnvOrDefault("DEEPFENCE_MINIO_HOST", "deepfence-file-server")
 	minioPort := utils.GetEnvOrDefault("DEEPFENCE_MINIO_PORT", "9000")
+	minioRegion := os.Getenv("DEEPFENCE_MINIO_REGION")
+	minioBucket := os.Getenv("DEEPFENCE_MINIO_DB_BUCKET")
+
+	// for aws s3
+	fileURL := fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", minioBucket, minioRegion, newFile)
+	if minioHost != "s3.amazonaws.com" {
+		fileURL = fmt.Sprintf("http://%s:%s/%s",
+			minioHost, minioPort, path.Join(string(directory.DatabaseDirKey), newFile))
+	}
 
 	listing.Append(
 		Database{
-			Built:   buildTime,
-			Version: 5,
-			URL: fmt.Sprintf(
-				"http://%s/%s",
-				minioHost+":"+minioPort,
-				path.Join(string(directory.DatabaseDirKey), newFile),
-			),
+			Built:    buildTime,
+			Version:  5,
+			URL:      fileURL,
 			Checksum: newFileCheckSum,
 		},
 		Version5,

deepfence_worker/cronscheduler/init_db.go

@@ -73,12 +73,12 @@ func initSqlDatabase(ctx context.Context) error {
 	return nil
 }
 
-func InitMinioDatabase() error {
+func InitMinioDatabase() {
 	ctx := directory.NewContextWithNameSpace("database")
 	mc, err := directory.MinioClient(ctx)
 	if err != nil {
 		log.Error().Msg(err.Error())
-		return err
+		return
 	}
 	retries := 3
 	for {
@@ -88,13 +88,13 @@ func InitMinioDatabase() error {
 			if retries != 0 {
 				continue
 			}
-			return err
+			// donot continue we need this step succesfull
+			panic(err)
 		}
 		break
 	}
 
 	// download vulnerability database once on init
 	vulnerability_db.DownloadDatabase()
 
-	return nil
 }