From e08bedcff20e328a128ae8d6a5addfe635814270 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Fri, 20 Oct 2023 16:38:55 +0000 Subject: [PATCH 1/6] add option to use AWS S3 inplace of existing file server --- deepfence_utils/directory/directory.go | 11 +++++- deepfence_utils/directory/minio.go | 40 +++++++++++--------- deepfence_utils/utils/utils.go | 28 +++++++++----- deepfence_utils/vulnerability_db/database.go | 20 ++++++---- deepfence_worker/cronscheduler/init_db.go | 7 ++-- deepfence_worker/entrypoint.sh | 9 ++++- deepfence_worker/tasks/sbom/scan_sbom.go | 12 +++++- deployment-scripts/docker-compose.yml | 13 ++++++- 8 files changed, 96 insertions(+), 44 deletions(-) diff --git a/deepfence_utils/directory/directory.go b/deepfence_utils/directory/directory.go index 6cd41df86a..a87a21f25f 100644 --- a/deepfence_utils/directory/directory.go +++ b/deepfence_utils/directory/directory.go @@ -45,6 +45,7 @@ type MinioConfig struct { Password string BucketName string Secure bool + Region string } type DBConfigs struct { @@ -195,11 +196,18 @@ func initMinio() MinioConfig { minioPort = "9000" log.Warn().Msgf("DEEPFENCE_MINIO_PORT defaults to: %v", minioPort) } - minioEndpoint := minioHost + ":" + minioPort + minioUser := os.Getenv("DEEPFENCE_MINIO_USER") minioPassword := os.Getenv("DEEPFENCE_MINIO_PASSWORD") minioBucket := os.Getenv("DEEPFENCE_MINIO_BUCKET") + minioRegion := os.Getenv("DEEPFENCE_MINIO_REGION") minioSecure := os.Getenv("DEEPFENCE_MINIO_SECURE") + + minioEndpoint := minioHost + if minioHost != "s3.amazonaws.com" { + minioEndpoint = minioHost + ":" + minioPort + } + if minioSecure == "" { minioSecure = "false" } @@ -214,6 +222,7 @@ func initMinio() MinioConfig { Password: minioPassword, BucketName: minioBucket, Secure: isSecure, + Region: minioRegion, } } diff --git a/deepfence_utils/directory/minio.go b/deepfence_utils/directory/minio.go index d55cd7b4dd..bbdd7df32c 100644 --- a/deepfence_utils/directory/minio.go +++ b/deepfence_utils/directory/minio.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "path/filepath" "strings" "sync" "time" @@ -39,6 +40,14 @@ func (e AlreadyPresentError) Error() string { return fmt.Sprintf("Already exists here: %s", e.Path) } +type PathDoesNotExistsError struct { + Path string +} + +func (e PathDoesNotExistsError) Error() string { + return fmt.Sprintf("Path doesnot exists here: %s", e.Path) +} + type FileManager interface { ListFiles(ctx context.Context, pathPrefix string, recursive bool, maxKeys int, skipDir bool) []ObjectInfo UploadLocalFile(ctx context.Context, filename string, localFilename string, extra interface{}) (UploadResult, error) @@ -92,19 +101,12 @@ func (mfm *MinioFileManager) optionallyAddNamespacePrefix(filePath string, addFi if addFilePathPrefix { return mfm.addNamespacePrefix(filePath) } else { - if strings.HasPrefix(filePath, "/") { - return filePath - } else { - return "/" + filePath - } + return strings.TrimPrefix(filePath, "/") } } func (mfm *MinioFileManager) addNamespacePrefix(filePath string) string { - if !strings.HasPrefix(filePath, "/") { - filePath = "/" + filePath - } - return mfm.namespace + filePath + return filepath.Join(mfm.namespace, filePath) } func (mfm *MinioFileManager) ListFiles(ctx context.Context, pathPrefix string, recursive bool, maxKeys int, skipDir bool) []ObjectInfo { @@ -233,17 +235,22 @@ func (mfm *MinioFileManager) ExposeFile(ctx context.Context, filePath string, ad return "", err } - headers := http.Header{} - headers.Add("Host", consoleIp) + actualPath := mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix) + + key, has := checkIfFileExists(ctx, mfm.client, mfm.bucket, actualPath) + if !has { + return "", PathDoesNotExistsError{Path: actualPath} + } urlLink, err := mfm.client.PresignHeader( ctx, "GET", mfm.bucket, - mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix), + key, expires, reqParams, - headers) + http.Header{}, + ) if err != nil { return "", err } @@ -257,9 +264,6 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath return "", err } - headers := http.Header{} - headers.Add("Host", consoleIp) - urlLink, err := mfm.client.PresignHeader( ctx, "PUT", @@ -267,7 +271,8 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix), expires, reqParams, - headers) + http.Header{}, + ) if err != nil { return "", err } @@ -346,6 +351,7 @@ func newMinioClient(endpoints DBConfigs) (*minio.Client, error) { minioClient, err := minio.New(endpoints.Minio.Endpoint, &minio.Options{ Creds: credentials.NewStaticV4(endpoints.Minio.Username, endpoints.Minio.Password, ""), Secure: endpoints.Minio.Secure, + Region: endpoints.Minio.Region, }) return minioClient, err } diff --git a/deepfence_utils/utils/utils.go b/deepfence_utils/utils/utils.go index 4a6714df10..5c7acafe16 100644 --- a/deepfence_utils/utils/utils.go +++ b/deepfence_utils/utils/utils.go @@ -455,27 +455,35 @@ func RecursiveZip(pathsToZip []string, excludePathPrefixes []string, destination } func UploadFile(url string, fileName string) ([]byte, int, error) { - r, err := os.Open(fileName) + // r, err := os.Open(fileName) + // if err != nil { + // return nil, 0, err + // } + // buf := make([]byte, 512) + // _, err = r.Read(buf) + // if err != nil { + // return nil, 0, err + // } + // r.Close() + + buff, err := os.ReadFile(fileName) if err != nil { return nil, 0, err } - buf := make([]byte, 512) - _, err = r.Read(buf) - if err != nil { - return nil, 0, err - } - r.Close() client, err := NewHTTPClient() if err != nil { return nil, 0, err } - r, err = os.Open(fileName) - req, err := http.NewRequest("PUT", url, r) + + req, err := http.NewRequest("PUT", url, bytes.NewReader(buff)) if err != nil { return nil, 0, err } - req.Header.Add("Content-Type", http.DetectContentType(buf)) + + req.Header.Add("Content-Type", http.DetectContentType(buff)) + req.Header.Add("Content-Length", strconv.Itoa(len(buff))) + res, err := client.Do(req) if err != nil { return nil, 0, err diff --git a/deepfence_utils/vulnerability_db/database.go b/deepfence_utils/vulnerability_db/database.go index af96436e78..d106d8abb3 100644 --- a/deepfence_utils/vulnerability_db/database.go +++ b/deepfence_utils/vulnerability_db/database.go @@ -9,6 +9,7 @@ import ( "io" "mime/multipart" "net/http" + "os" "path" "sort" "time" @@ -136,16 +137,21 @@ func UpdateListing(newFile, newFileCheckSum string, buildTime time.Time) { minioHost := utils.GetEnvOrDefault("DEEPFENCE_MINIO_HOST", "deepfence-file-server") minioPort := utils.GetEnvOrDefault("DEEPFENCE_MINIO_PORT", "9000") + minioRegion := os.Getenv("DEEPFENCE_MINIO_REGION") + minioBucket := os.Getenv("DEEPFENCE_MINIO_DB_BUCKET") + + // for aws s3 + fileURL := fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s", minioBucket, minioRegion, newFile) + if minioHost != "s3.amazonaws.com" { + fileURL = fmt.Sprintf("http://%s:%s/%s", + minioHost, minioPort, path.Join(string(directory.DatabaseDirKey), newFile)) + } listing.Append( Database{ - Built: buildTime, - Version: 5, - URL: fmt.Sprintf( - "http://%s/%s", - minioHost+":"+minioPort, - path.Join(string(directory.DatabaseDirKey), newFile), - ), + Built: buildTime, + Version: 5, + URL: fileURL, Checksum: newFileCheckSum, }, Version5, diff --git a/deepfence_worker/cronscheduler/init_db.go b/deepfence_worker/cronscheduler/init_db.go index e422b3af26..5ed0194ea7 100644 --- a/deepfence_worker/cronscheduler/init_db.go +++ b/deepfence_worker/cronscheduler/init_db.go @@ -73,12 +73,12 @@ func initSqlDatabase(ctx context.Context) error { return nil } -func InitMinioDatabase() error { +func InitMinioDatabase() { ctx := directory.NewContextWithNameSpace("database") mc, err := directory.MinioClient(ctx) if err != nil { log.Error().Msg(err.Error()) - return err + return } retries := 3 for { @@ -88,7 +88,7 @@ func InitMinioDatabase() error { if retries != 0 { continue } - return err + return } break } @@ -96,5 +96,4 @@ func InitMinioDatabase() error { // download vulnerability database once on init vulnerability_db.DownloadDatabase() - return nil } diff --git a/deepfence_worker/entrypoint.sh b/deepfence_worker/entrypoint.sh index a1c4d2fe99..fe092db29b 100644 --- a/deepfence_worker/entrypoint.sh +++ b/deepfence_worker/entrypoint.sh @@ -29,13 +29,18 @@ do sleep 5; done +# for aws s3 +export GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_MINIO_HOST}:${DEEPFENCE_MINIO_PORT}/database/database/vulnerability/listing.json" +if [ "$DEEPFENCE_MINIO_HOST" == "s3.amazonaws.com" ]; then + export GRYPE_DB_UPDATE_URL="https://${DEEPFENCE_MINIO_DB_BUCKET}.s3.${DEEPFENCE_MINIO_REGION}.amazonaws.com/database/vulnerability/listing.json" +fi + # update vulnerability databae if [ "$DEEPFENCE_MODE" == "worker" ]; then echo "update vulnerability database" - export GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_MINIO_HOST}:${DEEPFENCE_MINIO_PORT}/database/database/vulnerability/listing.json" echo "db update url $GRYPE_DB_UPDATE_URL" /usr/local/bin/grype db update - echo "0 */2 * * * export GRYPE_DB_UPDATE_URL=http://${DEEPFENCE_MINIO_HOST}:${DEEPFENCE_MINIO_PORT}/database/database/vulnerability/listing.json && /usr/local/bin/grype db update" >> /etc/crontabs/root + echo "0 */2 * * * export GRYPE_DB_UPDATE_URL=${GRYPE_DB_UPDATE_URL} && /usr/local/bin/grype db update" >> /etc/crontabs/root /usr/sbin/crond fi diff --git a/deepfence_worker/tasks/sbom/scan_sbom.go b/deepfence_worker/tasks/sbom/scan_sbom.go index d89ddfbad9..84de2f1308 100644 --- a/deepfence_worker/tasks/sbom/scan_sbom.go +++ b/deepfence_worker/tasks/sbom/scan_sbom.go @@ -34,9 +34,19 @@ var ( grypeBin = "grype" minioHost = utils.GetEnvOrDefault("DEEPFENCE_MINIO_HOST", "deepfence-file-server") minioPort = utils.GetEnvOrDefault("DEEPFENCE_MINIO_PORT", "9000") - GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=http://%s:%s/database/database/vulnerability/listing.json", minioHost, minioPort) + minioRegion = os.Getenv("DEEPFENCE_MINIO_REGION") + minioBucket = os.Getenv("DEEPFENCE_MINIO_DB_BUCKET") + GRYPE_DB_UPDATE_URL string ) +func init() { + // for aws s3 + GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=https://%s.s3.%s.amazonaws.com/database/vulnerability/listing.json", minioBucket, minioRegion) + if minioHost != "s3.amazonaws.com" { + GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=http://%s:%s/database/database/vulnerability/listing.json", minioHost, minioPort) + } +} + type SbomParser struct { ingestC chan *kgo.Record } diff --git a/deployment-scripts/docker-compose.yml b/deployment-scripts/docker-compose.yml index 5564eaffe0..d6f969fb64 100644 --- a/deployment-scripts/docker-compose.yml +++ b/deployment-scripts/docker-compose.yml @@ -13,12 +13,21 @@ x-service-variables: &common-creds DEEPFENCE_POSTGRES_USER_DB_PORT: 5432 DEEPFENCE_POSTGRES_USER_DB_SSLMODE: disable DEEPFENCE_KAFKA_BROKERS: deepfence-kafka-broker:9092 + # public bucket with read permisons on objects for hosting vulnerability database + DEEPFENCE_MINIO_DB_BUCKET: database + # prvate bucket to host reports, sbom, etc. DEEPFENCE_MINIO_BUCKET: default + # set s3.amazonaws.com if using s3 buckets DEEPFENCE_MINIO_HOST: deepfence-file-server - DEEPFENCE_MINIO_PASSWORD: deepfence DEEPFENCE_MINIO_PORT: 9000 - DEEPFENCE_MINIO_SECURE: "false" + # set access key if using s3 buckets DEEPFENCE_MINIO_USER: deepfence + # set secret key if using s3 buckets + DEEPFENCE_MINIO_PASSWORD: deepfence + # set true if using s3 buckets + DEEPFENCE_MINIO_SECURE: "false" + # set aws s3 buckets region if using s3 buckets + DEEPFENCE_MINIO_REGION: "" DEEPFENCE_REDIS_DB_NUMBER: 0 DEEPFENCE_REDIS_HOST: deepfence-redis DEEPFENCE_REDIS_PORT: 6379 From f7b46f32461b9e0e343210f913875ac7fdea5945 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Sat, 21 Oct 2023 16:39:18 +0000 Subject: [PATCH 2/6] fix listing diagnostic logs --- .../agent-diagnosis/agent_diagnosis.go | 4 +++- deepfence_server/diagnosis/common.go | 15 +++++++++++-- .../diagnosis/console-diagnosis/docker.go | 4 +++- .../diagnosis/console-diagnosis/kubernetes.go | 4 +++- deepfence_utils/directory/minio.go | 22 +++++++++++-------- deepfence_utils/utils/utils.go | 10 --------- deepfence_worker/cronscheduler/init_db.go | 3 ++- deepfence_worker/tasks/sbom/scan_sbom.go | 1 + 8 files changed, 38 insertions(+), 25 deletions(-) diff --git a/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go b/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go index dddf167500..2df8e41ed5 100644 --- a/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go +++ b/deepfence_server/diagnosis/agent-diagnosis/agent_diagnosis.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/url" + "path/filepath" "time" "github.com/deepfence/ThreatMapper/deepfence_server/diagnosis" @@ -155,7 +156,8 @@ func GenerateAgentDiagnosticLogs(ctx context.Context, nodeIdentifiers []diagnosi continue } fileName := "deepfence-agent-logs-" + nodeIdentifier.NodeId + fileNameSuffix - uploadUrl, err := mc.CreatePublicUploadURL(ctx, diagnosis.AgentDiagnosisFileServerPrefix+fileName, true, time.Minute*10, url.Values{}) + uploadUrl, err := mc.CreatePublicUploadURL(ctx, + filepath.Join(diagnosis.AgentDiagnosisFileServerPrefix, fileName), true, time.Minute*10, url.Values{}) if err != nil { return err } diff --git a/deepfence_server/diagnosis/common.go b/deepfence_server/diagnosis/common.go index 473f5435c3..1927dde3b3 100644 --- a/deepfence_server/diagnosis/common.go +++ b/deepfence_server/diagnosis/common.go @@ -10,14 +10,15 @@ import ( "time" "github.com/deepfence/ThreatMapper/deepfence_utils/directory" + "github.com/deepfence/ThreatMapper/deepfence_utils/log" "github.com/deepfence/ThreatMapper/deepfence_utils/utils" "github.com/neo4j/neo4j-go-driver/v4/neo4j" ) const ( DiagnosisLinkExpiry = 5 * time.Minute - ConsoleDiagnosisFileServerPrefix = "/diagnosis/console-diagnosis/" - AgentDiagnosisFileServerPrefix = "/diagnosis/agent-diagnosis/" + ConsoleDiagnosisFileServerPrefix = "diagnosis/console-diagnosis/" + AgentDiagnosisFileServerPrefix = "diagnosis/agent-diagnosis/" ) type DiagnosticNotification struct { @@ -76,11 +77,13 @@ func GetDiagnosticLogs(ctx context.Context) (*GetDiagnosticLogsResponse, error) func getDiagnosticLogsHelper(ctx context.Context, mc directory.FileManager, pathPrefix string) []DiagnosticLogsLink { // Get completed files from minio objects := mc.ListFiles(ctx, pathPrefix, false, 0, true) + log.Debug().Msgf("diagnosis logs at %s: %v", pathPrefix, objects) diagnosticLogsResponse := make([]DiagnosticLogsLink, len(objects)) for i, obj := range objects { message := "" urlLink, err := mc.ExposeFile(ctx, obj.Key, false, DiagnosisLinkExpiry, url.Values{}) if err != nil { + log.Error().Err(err).Msg("failed to list console diagnosis logs") var minioError utils.MinioError xmlErr := xml.Unmarshal([]byte(err.Error()), &minioError) if xmlErr != nil { @@ -122,6 +125,10 @@ func getAgentDiagnosticLogs(ctx context.Context, mc directory.FileManager, pathP } defer session.Close() tx, err := session.BeginTransaction(neo4j.WithTxTimeout(30 * time.Second)) + if err != nil { + log.Error().Msg(err.Error()) + return diagnosticLogs + } defer tx.Close() r, err := tx.Run(` @@ -133,6 +140,10 @@ func getAgentDiagnosticLogs(ctx context.Context, mc directory.FileManager, pathP nodeIdToName := make(map[string]string) records, err := r.Collect() + if err != nil { + log.Error().Msg(err.Error()) + return diagnosticLogs + } for _, rec := range records { var nodeId, fileName, message, status, updatedAt, nodeName interface{} var ok bool diff --git a/deepfence_server/diagnosis/console-diagnosis/docker.go b/deepfence_server/diagnosis/console-diagnosis/docker.go index 3edab67068..71ae008669 100644 --- a/deepfence_server/diagnosis/console-diagnosis/docker.go +++ b/deepfence_server/diagnosis/console-diagnosis/docker.go @@ -73,7 +73,9 @@ func (d *DockerConsoleDiagnosisHandler) GenerateDiagnosticLogs(ctx context.Conte if err != nil { return err } - _, err = mc.UploadLocalFile(ctx, diagnosis.ConsoleDiagnosisFileServerPrefix+filepath.Base(zipFile.Name()), zipFile.Name(), + _, err = mc.UploadLocalFile(ctx, + filepath.Join(diagnosis.ConsoleDiagnosisFileServerPrefix, filepath.Base(zipFile.Name())), + zipFile.Name(), minio.PutObjectOptions{ContentType: "application/zip"}) if err != nil { return err diff --git a/deepfence_server/diagnosis/console-diagnosis/kubernetes.go b/deepfence_server/diagnosis/console-diagnosis/kubernetes.go index 1837bcaa24..67f007bf86 100644 --- a/deepfence_server/diagnosis/console-diagnosis/kubernetes.go +++ b/deepfence_server/diagnosis/console-diagnosis/kubernetes.go @@ -122,7 +122,9 @@ func (k *KubernetesConsoleDiagnosisHandler) GenerateDiagnosticLogs(ctx context.C if err != nil { return err } - _, err = mc.UploadLocalFile(ctx, diagnosis.ConsoleDiagnosisFileServerPrefix+filepath.Base(zipFile.Name()), zipFile.Name(), + _, err = mc.UploadLocalFile(ctx, + filepath.Join(diagnosis.ConsoleDiagnosisFileServerPrefix, filepath.Base(zipFile.Name())), + zipFile.Name(), minio.PutObjectOptions{ContentType: "application/zip"}) if err != nil { return err diff --git a/deepfence_utils/directory/minio.go b/deepfence_utils/directory/minio.go index bbdd7df32c..5fa7ba78c8 100644 --- a/deepfence_utils/directory/minio.go +++ b/deepfence_utils/directory/minio.go @@ -110,15 +110,19 @@ func (mfm *MinioFileManager) addNamespacePrefix(filePath string) string { } func (mfm *MinioFileManager) ListFiles(ctx context.Context, pathPrefix string, recursive bool, maxKeys int, skipDir bool) []ObjectInfo { - objects := mfm.client.ListObjects(ctx, mfm.bucket, minio.ListObjectsOptions{ - WithVersions: false, - WithMetadata: false, - Prefix: mfm.addNamespacePrefix(pathPrefix), - Recursive: recursive, - MaxKeys: maxKeys, - StartAfter: "", - UseV1: false, - }) + prefix := mfm.addNamespacePrefix(pathPrefix) + "/" + + objects := mfm.client.ListObjects(ctx, mfm.bucket, + minio.ListObjectsOptions{ + WithVersions: false, + WithMetadata: false, + Prefix: prefix, + Recursive: recursive, + MaxKeys: maxKeys, + StartAfter: "", + UseV1: false, + }) + var objectsInfo []ObjectInfo for obj := range objects { isDir := strings.HasSuffix(obj.Key, "/") diff --git a/deepfence_utils/utils/utils.go b/deepfence_utils/utils/utils.go index 5c7acafe16..18540ef340 100644 --- a/deepfence_utils/utils/utils.go +++ b/deepfence_utils/utils/utils.go @@ -455,16 +455,6 @@ func RecursiveZip(pathsToZip []string, excludePathPrefixes []string, destination } func UploadFile(url string, fileName string) ([]byte, int, error) { - // r, err := os.Open(fileName) - // if err != nil { - // return nil, 0, err - // } - // buf := make([]byte, 512) - // _, err = r.Read(buf) - // if err != nil { - // return nil, 0, err - // } - // r.Close() buff, err := os.ReadFile(fileName) if err != nil { diff --git a/deepfence_worker/cronscheduler/init_db.go b/deepfence_worker/cronscheduler/init_db.go index 5ed0194ea7..88f26bab47 100644 --- a/deepfence_worker/cronscheduler/init_db.go +++ b/deepfence_worker/cronscheduler/init_db.go @@ -88,7 +88,8 @@ func InitMinioDatabase() { if retries != 0 { continue } - return + // donot continue we need this step succesfull + panic(err) } break } diff --git a/deepfence_worker/tasks/sbom/scan_sbom.go b/deepfence_worker/tasks/sbom/scan_sbom.go index 84de2f1308..756f74db71 100644 --- a/deepfence_worker/tasks/sbom/scan_sbom.go +++ b/deepfence_worker/tasks/sbom/scan_sbom.go @@ -45,6 +45,7 @@ func init() { if minioHost != "s3.amazonaws.com" { GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=http://%s:%s/database/database/vulnerability/listing.json", minioHost, minioPort) } + log.Info().Msg(GRYPE_DB_UPDATE_URL) } type SbomParser struct { From b8611cc9a2e86f12396d3e5f92a7c547f68a4de2 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Tue, 24 Oct 2023 05:16:59 +0000 Subject: [PATCH 3/6] add header while singing request for local file server --- deepfence_utils/directory/minio.go | 14 ++++++++++++-- deployment-scripts/docker-compose.yml | 1 + 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/deepfence_utils/directory/minio.go b/deepfence_utils/directory/minio.go index 5fa7ba78c8..64fd6dbdf6 100644 --- a/deepfence_utils/directory/minio.go +++ b/deepfence_utils/directory/minio.go @@ -246,6 +246,11 @@ func (mfm *MinioFileManager) ExposeFile(ctx context.Context, filePath string, ad return "", PathDoesNotExistsError{Path: actualPath} } + headers := http.Header{} + if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") { + headers.Add("Host", consoleIp) + } + urlLink, err := mfm.client.PresignHeader( ctx, "GET", @@ -253,7 +258,7 @@ func (mfm *MinioFileManager) ExposeFile(ctx context.Context, filePath string, ad key, expires, reqParams, - http.Header{}, + headers, ) if err != nil { return "", err @@ -268,6 +273,11 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath return "", err } + headers := http.Header{} + if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") { + headers.Add("Host", consoleIp) + } + urlLink, err := mfm.client.PresignHeader( ctx, "PUT", @@ -275,7 +285,7 @@ func (mfm *MinioFileManager) CreatePublicUploadURL(ctx context.Context, filePath mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix), expires, reqParams, - http.Header{}, + headers, ) if err != nil { return "", err diff --git a/deployment-scripts/docker-compose.yml b/deployment-scripts/docker-compose.yml index d6f969fb64..e56919c079 100644 --- a/deployment-scripts/docker-compose.yml +++ b/deployment-scripts/docker-compose.yml @@ -14,6 +14,7 @@ x-service-variables: &common-creds DEEPFENCE_POSTGRES_USER_DB_SSLMODE: disable DEEPFENCE_KAFKA_BROKERS: deepfence-kafka-broker:9092 # public bucket with read permisons on objects for hosting vulnerability database + # required permissions {"Version":"2012-10-17","Statement":[{"Sid":"public_objects","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::/*","arn:aws:s3:::"]}]} DEEPFENCE_MINIO_DB_BUCKET: database # prvate bucket to host reports, sbom, etc. DEEPFENCE_MINIO_BUCKET: default From 10f0eef61ddf5da586c1cd043b05885c00ae1bf3 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Tue, 24 Oct 2023 09:07:01 +0000 Subject: [PATCH 4/6] disable filserver check when S3 is used --- deepfence_server/entrypoint.sh | 14 +++++++++----- deepfence_worker/entrypoint.sh | 14 +++++++++----- deployment-scripts/docker-compose.yml | 3 ++- haproxy/haproxy.cfg | 1 + 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/deepfence_server/entrypoint.sh b/deepfence_server/entrypoint.sh index 51c5df694b..5d53818dae 100644 --- a/deepfence_server/entrypoint.sh +++ b/deepfence_server/entrypoint.sh @@ -32,11 +32,15 @@ do done # wait for file server to start -until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT}; -do - echo "file server is unavailable - sleeping" - sleep 5; -done +if [ "$DEEPFENCE_MINIO_HOST" != "s3.amazonaws.com" ]; then + until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT}; + do + echo "file server is unavailable - sleeping" + sleep 5; + done +else + echo "S3 mode skip file server health check" +fi sed -i "s/https:\/\/petstore.swagger.io\/v2\/swagger.json/\/deepfence\/openapi.json/g" /usr/local/share/swagger-ui/swagger-initializer.js diff --git a/deepfence_worker/entrypoint.sh b/deepfence_worker/entrypoint.sh index fe092db29b..40ca81cdc4 100644 --- a/deepfence_worker/entrypoint.sh +++ b/deepfence_worker/entrypoint.sh @@ -23,11 +23,15 @@ do done # wait for file server to start -until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT}; -do - echo "file server is unavailable - sleeping" - sleep 5; -done +if [ "$DEEPFENCE_MINIO_HOST" != "s3.amazonaws.com" ]; then + until nc -z ${DEEPFENCE_MINIO_HOST} ${DEEPFENCE_MINIO_PORT}; + do + echo "file server is unavailable - sleeping" + sleep 5; + done +else + echo "S3 mode skip file server health check" +fi # for aws s3 export GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_MINIO_HOST}:${DEEPFENCE_MINIO_PORT}/database/database/vulnerability/listing.json" diff --git a/deployment-scripts/docker-compose.yml b/deployment-scripts/docker-compose.yml index e56919c079..1e49c4abea 100644 --- a/deployment-scripts/docker-compose.yml +++ b/deployment-scripts/docker-compose.yml @@ -14,7 +14,7 @@ x-service-variables: &common-creds DEEPFENCE_POSTGRES_USER_DB_SSLMODE: disable DEEPFENCE_KAFKA_BROKERS: deepfence-kafka-broker:9092 # public bucket with read permisons on objects for hosting vulnerability database - # required permissions {"Version":"2012-10-17","Statement":[{"Sid":"public_objects","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::/*","arn:aws:s3:::"]}]} + # S3 bucket permissions {"Version":"2012-10-17","Statement":[{"Sid":"database","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::/database/*","arn:aws:s3:::/database"]}]} DEEPFENCE_MINIO_DB_BUCKET: database # prvate bucket to host reports, sbom, etc. DEEPFENCE_MINIO_BUCKET: default @@ -154,6 +154,7 @@ services: options: max-size: "100m" + # this service can be commented in case S3 is used deepfence-file-server: container_name: deepfence-file-server image: ${IMAGE_REPOSITORY:-deepfenceio}/deepfence_file_server_ce:${DF_IMG_TAG:-2.0.0} diff --git a/haproxy/haproxy.cfg b/haproxy/haproxy.cfg index af75511bdf..437744f4b6 100644 --- a/haproxy/haproxy.cfg +++ b/haproxy/haproxy.cfg @@ -33,6 +33,7 @@ defaults # CLF log format # https://www.haproxy.com/blog/haproxy-log-customization/ log-format "%{+Q}o %{-Q}ci - - [%trg] %r %ST %B %cp %ft %b %s %TR %Tw %Tc %Tr %Ta %fc %bc %sc %rc %sq %bq" + default-server init-addr last,libc,none frontend port_80 From d7f237ada51354e1ddd652b13620646d5dc1f3ec Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Tue, 24 Oct 2023 12:21:23 +0000 Subject: [PATCH 5/6] update helm charts to support S3 instead of fileserver --- .../deepfence-console-secrets/fileserver.yaml | 1 + .../deepfence-console-secrets/s3.yaml | 19 +++++++++++++++++++ .../templates/deepfence-ingester.yaml | 8 +++++--- .../templates/deepfence-scheduler.yaml | 8 +++++--- .../templates/deepfence-server.yaml | 8 +++++--- .../templates/deepfence-ui.yaml | 8 +++++--- .../templates/deepfence-worker.yaml | 8 +++++--- .../helm-charts/deepfence-console/values.yaml | 19 +++++++++++++++++++ 8 files changed, 64 insertions(+), 15 deletions(-) create mode 100644 deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/s3.yaml diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/fileserver.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/fileserver.yaml index 9a770b318c..dcee11f42f 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/fileserver.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/fileserver.yaml @@ -15,5 +15,6 @@ stringData: DEEPFENCE_MINIO_BUCKET: default DEEPFENCE_MINIO_PASSWORD: {{ .Values.fileserver.secrets.MINIO_ROOT_PASSWORD | quote }} DEEPFENCE_MINIO_USER: {{ .Values.fileserver.secrets.MINIO_ROOT_USER | quote }} + DEEPFENCE_MINIO_DB_BUCKET: database {{- end }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/s3.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/s3.yaml new file mode 100644 index 0000000000..41d9720c7a --- /dev/null +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-secrets/s3.yaml @@ -0,0 +1,19 @@ +{{- if and (eq .Values.fileserver.create false) .Values.aws_s3_buckets.create }} +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: {{ include "deepfence-console.fullname" . }}-secrets-fileserver + labels: + {{- include "deepfence-console.labels" . | nindent 4 }} + component: file-server +stringData: + DEEPFENCE_MINIO_HOST: "s3.amazonaws.com" + DEEPFENCE_MINIO_SECURE: "true" + DEEPFENCE_MINIO_DB_BUCKET: {{ .Values.aws_s3_buckets.vulnerability_db_bucket | quote }} + DEEPFENCE_MINIO_BUCKET: {{ .Values.aws_s3_buckets.data_bucket | quote }} + DEEPFENCE_MINIO_PASSWORD: {{ .Values.aws_s3_buckets.secret_access_key | quote }} + DEEPFENCE_MINIO_USER: {{ .Values.aws_s3_buckets.access_key_id | quote }} + DEEPFENCE_MINIO_REGION: {{.Values.aws_s3_buckets.region | quote }} +{{- end }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ingester.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ingester.yaml index 516c5d61cd..ee4f8d33a0 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ingester.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ingester.yaml @@ -45,10 +45,12 @@ spec: name: {{ .Values.redis.secretName }} {{- end }} - secretRef: - {{- if .Values.fileserver.create }} + {{- if or .Values.fileserver.create .Values.aws_s3_buckets.create }} name: "{{ include "deepfence-console.fullname" . }}-secrets-fileserver" - {{- else }} - name: {{ .Values.minio.secretName }} + {{- else if .Values.fileserver.secretName }} + name: {{ .Values.fileserver.secretName }} + {{- else if .Values.aws_s3_buckets.secretName }} + name: {{ .Values.aws_s3_buckets.secretName }} {{- end }} - secretRef: {{- if .Values.kafka.create }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-scheduler.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-scheduler.yaml index 4b657d5097..2748963a49 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-scheduler.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-scheduler.yaml @@ -43,10 +43,12 @@ spec: name: {{ .Values.redis.secretName }} {{- end }} - secretRef: - {{- if .Values.fileserver.create }} + {{- if or .Values.fileserver.create .Values.aws_s3_buckets.create }} name: "{{ include "deepfence-console.fullname" . }}-secrets-fileserver" - {{- else }} - name: {{ .Values.minio.secretName }} + {{- else if .Values.fileserver.secretName }} + name: {{ .Values.fileserver.secretName }} + {{- else if .Values.aws_s3_buckets.secretName }} + name: {{ .Values.aws_s3_buckets.secretName }} {{- end }} - secretRef: {{- if .Values.kafka.create }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-server.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-server.yaml index 645de83c32..300b2f4451 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-server.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-server.yaml @@ -52,10 +52,12 @@ spec: name: {{ .Values.redis.secretName }} {{- end }} - secretRef: - {{- if .Values.fileserver.create }} + {{- if or .Values.fileserver.create .Values.aws_s3_buckets.create }} name: "{{ include "deepfence-console.fullname" . }}-secrets-fileserver" - {{- else }} - name: {{ .Values.minio.secretName }} + {{- else if .Values.fileserver.secretName }} + name: {{ .Values.fileserver.secretName }} + {{- else if .Values.aws_s3_buckets.secretName }} + name: {{ .Values.aws_s3_buckets.secretName }} {{- end }} - secretRef: {{- if .Values.kafka.create }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ui.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ui.yaml index b8a112d9ca..914ac3966b 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ui.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-ui.yaml @@ -42,10 +42,12 @@ spec: name: {{ .Values.redis.secretName }} {{- end }} - secretRef: - {{- if .Values.fileserver.create }} + {{- if or .Values.fileserver.create .Values.aws_s3_buckets.create }} name: "{{ include "deepfence-console.fullname" . }}-secrets-fileserver" - {{- else }} - name: {{ .Values.minio.secretName }} + {{- else if .Values.fileserver.secretName }} + name: {{ .Values.fileserver.secretName }} + {{- else if .Values.aws_s3_buckets.secretName }} + name: {{ .Values.aws_s3_buckets.secretName }} {{- end }} - secretRef: {{- if .Values.kafka.create }} diff --git a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-worker.yaml b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-worker.yaml index e86c1cc69d..4c9e74798b 100644 --- a/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-worker.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/templates/deepfence-worker.yaml @@ -45,10 +45,12 @@ spec: name: {{ .Values.redis.secretName }} {{- end }} - secretRef: - {{- if .Values.fileserver.create }} + {{- if or .Values.fileserver.create .Values.aws_s3_buckets.create }} name: "{{ include "deepfence-console.fullname" . }}-secrets-fileserver" - {{- else }} - name: {{ .Values.minio.secretName }} + {{- else if .Values.fileserver.secretName }} + name: {{ .Values.fileserver.secretName }} + {{- else if .Values.aws_s3_buckets.secretName }} + name: {{ .Values.aws_s3_buckets.secretName }} {{- end }} - secretRef: {{- if .Values.kafka.create }} diff --git a/deployment-scripts/helm-charts/deepfence-console/values.yaml b/deployment-scripts/helm-charts/deepfence-console/values.yaml index 2c92857781..071c2aa2fb 100644 --- a/deployment-scripts/helm-charts/deepfence-console/values.yaml +++ b/deployment-scripts/helm-charts/deepfence-console/values.yaml @@ -125,6 +125,7 @@ redis: fileserver: # Specifies whether a file server instance should be created + # set this to false if using S3 create: true # if create false provide name of the existing secret # secret format refer templates/console-secrets/minio.yaml @@ -154,6 +155,24 @@ fileserver: tolerations: [] affinity: {} +# these values are used if fileserver.create=false +aws_s3_buckets: + # Specifies whether secret should be created + create: false + # if create false provide name of the existing secret + # secret format refer templates/deepfence-console-secrets/s3.yaml + secretName: "" + # public bucket with read permisons on objects for hosting vulnerability database + # S3 bucket permissions {"Version":"2012-10-17","Statement":[{"Sid":"database","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::/database/*","arn:aws:s3:::/database"]}]} + vulnerability_db_bucket: "" + # prvate bucket to host reports, sbom, etc. + data_bucket: "" + # aws credentials to access buckets + access_key_id : "" + secret_access_key: "" + # region where the buckets are hosted ex: ap-south-1 + region: "" + neo4j: # Specifies whether a neo4j database instance should be created create: true From 7faf3ea3681c98e5fd2e80b87fcfca94e8530d36 Mon Sep 17 00:00:00 2001 From: gnmahanth Date: Tue, 24 Oct 2023 12:22:02 +0000 Subject: [PATCH 6/6] disable automatic kafka topics creation --- deepfence_kafka/kafka-broker-Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deepfence_kafka/kafka-broker-Dockerfile b/deepfence_kafka/kafka-broker-Dockerfile index c8815dc201..72d96051d1 100644 --- a/deepfence_kafka/kafka-broker-Dockerfile +++ b/deepfence_kafka/kafka-broker-Dockerfile @@ -23,7 +23,8 @@ ENV KAFKA_BROKER_ID=1 \ # KAFKA_LOG_DIRS='/data/kafka' \ KAFKA_LOG_RETENTION_MS=86400000 \ KAFKA_LOG_RETENTION_BYTES=-1 \ - KAFKA_MESSAGE_MAX_BYTES=52428800 + KAFKA_MESSAGE_MAX_BYTES=52428800 \ + KAFKA_AUTO_CREATE_TOPICS_ENABLE='false' COPY kafka_update_run.sh /home/appuser/kafka_update_run.sh CMD ["bash","-c", "/home/appuser/kafka_update_run.sh" ]