Option to use external file server #2025 #2011 (#2125)

deepfence_server/entrypoint.sh

@@ -41,7 +41,7 @@ done
 
 # wait for file server to start
 if [ "$DEEPFENCE_FILE_SERVER_HOST" != "s3.amazonaws.com" ]; then
-  until nc -z ${DEEPFENCE_FILE_SERVER_HOST} ${DEEPFENCE_FILE_SERVER_PORT};
+  until nc -z "${DEEPFENCE_FILE_SERVER_HOST}" "${DEEPFENCE_FILE_SERVER_PORT}";
   do
     echo "file server is unavailable - sleeping"
     sleep 5;

deepfence_utils/directory/fileserver.go

@@ -24,13 +24,25 @@ import (
 )
 
 var (
+	FileServerExternal       = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_EXTERNAL", "false") == "true"
+	FileServerProtocol       string
+	FileServerSecure         = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_SECURE", "false") == "true"
+	FileServerHost           = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_HOST", "deepfence-file-server")
+	FileServerPort           = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_PORT", "9000")
+	FileServerRegion         = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_REGION", "")
 	FileServerBucket         = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_BUCKET", string(NonSaaSDirKey))
 	FileServerDatabaseBucket = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_DB_BUCKET", string(DatabaseDirKey))
-	fileServerClientMap      sync.Map
+
+	fileServerClientMap sync.Map
 )
 
 func init() {
 	fileServerClientMap = sync.Map{}
+
+	FileServerProtocol = "http"
+	if FileServerSecure {
+		FileServerProtocol = "https"
+	}
 }
 
 type AlreadyPresentError struct {
@@ -303,17 +315,20 @@ func (mfm *FileServerFileManager) ExposeFile(ctx context.Context, filePath strin
 
 	var consoleIP string
 	var err error
-	// consoleURL can optionally be set based on the host header of the request, in case it's different
-	// from the Console URL saved in global settings.
-	// Format: deepfence.customer.com:8080 or 56.56.56.56
-	if consoleURL == "" {
-		consoleIP, err = GetFileServerHost(ctx)
-	} else {
-		consoleIP = consoleURL
-	}
-	if err != nil {
-		span.EndWithErr(err)
-		return "", err
+
+	if !FileServerExternal {
+		// consoleURL can optionally be set based on the host header of the request, in case it's different
+		// from the Console URL saved in global settings.
+		// Format: deepfence.customer.com:8080 or 56.56.56.56
+		if consoleURL == "" {
+			consoleIP, err = GetFileServerHost(ctx)
+		} else {
+			consoleIP = consoleURL
+		}
+		if err != nil {
+			span.EndWithErr(err)
+			return "", err
+		}
 	}
 
 	actualPath := mfm.optionallyAddNamespacePrefix(filePath, addFilePathPrefix)
@@ -326,7 +341,7 @@ func (mfm *FileServerFileManager) ExposeFile(ctx context.Context, filePath strin
 	}
 
 	headers := http.Header{}
-	if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") {
+	if !FileServerExternal {
 		headers.Add("Host", consoleIP)
 	}
 
@@ -344,6 +359,10 @@ func (mfm *FileServerFileManager) ExposeFile(ctx context.Context, filePath strin
 		return "", err
 	}
 
+	if FileServerExternal {
+		return urlLink.String(), nil
+	}
+
 	return updateURL(urlLink.String(), consoleIP), nil
 }
 
@@ -354,21 +373,24 @@ func (mfm *FileServerFileManager) CreatePublicUploadURL(ctx context.Context, fil
 
 	var consoleIP string
 	var err error
-	// consoleURL can optionally be set based on the host header of the request, in case it's different
-	// from the Console URL saved in global settings.
-	// Format: deepfence.customer.com:8080 or 56.56.56.56
-	if consoleURL == "" {
-		consoleIP, err = GetFileServerHost(ctx)
-	} else {
-		consoleIP = consoleURL
-	}
-	if err != nil {
-		span.EndWithErr(err)
-		return "", err
+
+	if !FileServerExternal {
+		// consoleURL can optionally be set based on the host header of the request, in case it's different
+		// from the Console URL saved in global settings.
+		// Format: deepfence.customer.com:8080 or 56.56.56.56
+		if consoleURL == "" {
+			consoleIP, err = GetFileServerHost(ctx)
+		} else {
+			consoleIP = consoleURL
+		}
+		if err != nil {
+			span.EndWithErr(err)
+			return "", err
+		}
 	}
 
 	headers := http.Header{}
-	if !strings.Contains(mfm.client.EndpointURL().Hostname(), "s3.amazonaws.com") {
+	if !FileServerExternal {
 		headers.Add("Host", consoleIP)
 	}
 
@@ -386,6 +408,10 @@ func (mfm *FileServerFileManager) CreatePublicUploadURL(ctx context.Context, fil
 		return "", err
 	}
 
+	if FileServerExternal {
+		return urlLink.String(), nil
+	}
+
 	return updateURL(urlLink.String(), consoleIP), nil
 }
 
@@ -455,11 +481,8 @@ func (mfm *FileServerFileManager) CleanNamespace(ctx context.Context) error {
 }
 
 func updateURL(url string, consoleIP string) string {
-	fileServerHost := utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_HOST", "deepfence-file-server")
-	fileServerPort := utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_PORT", "9000")
-
 	updated := strings.ReplaceAll(url,
-		fmt.Sprintf("%s:%s", fileServerHost, fileServerPort),
+		fmt.Sprintf("%s:%s", FileServerHost, FileServerPort),
 		fmt.Sprintf("%s/file-server", consoleIP),
 	)
 

deepfence_utils/threatintel/vulndb.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 	"path"
 	"path/filepath"
 	"sort"
@@ -13,7 +12,6 @@ import (
 	"github.com/deepfence/ThreatMapper/deepfence_utils/directory"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/log"
 	"github.com/deepfence/ThreatMapper/deepfence_utils/telemetry"
-	"github.com/deepfence/ThreatMapper/deepfence_utils/utils"
 	"github.com/minio/minio-go/v7"
 )
 
@@ -26,7 +24,6 @@ var (
 	ListingJSON          = "listing.json"
 	VulnerabilityDBStore = "vulnerability"
 	ListingPath          = path.Join(VulnerabilityDBStore, ListingJSON)
-	// DeepfenceVulnDBURL   = "https://threat-intel.deepfence.io/vulnerability-db/listing.json"
 )
 
 type VulnerabilityDBListing struct {
@@ -151,17 +148,13 @@ func VulnDBUpdateListing(ctx context.Context, newFile, newFileCheckSum string, b
 		listing = NewVulnerabilityDBListing()
 	}
 
-	minioHost := utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_HOST", "deepfence-file-server")
-	minioPort := utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_PORT", "9000")
-	minioRegion := os.Getenv("DEEPFENCE_FILE_SERVER_REGION")
-	minioBucket := os.Getenv("DEEPFENCE_FILE_SERVER_DB_BUCKET")
-
 	// for aws s3
-	fileURL := fmt.Sprintf("https://%s.s3.%s.amazonaws.com/%s",
-		minioBucket, minioRegion, newFile)
-	if minioHost != "s3.amazonaws.com" {
-		fileURL = fmt.Sprintf("http://%s:%s/%s",
-			minioHost, minioPort, path.Join(string(directory.DatabaseDirKey), newFile))
+	fileURL := fmt.Sprintf("%s://%s.s3.%s.amazonaws.com/%s",
+		directory.FileServerProtocol, directory.FileServerDatabaseBucket, directory.FileServerRegion, newFile)
+	if directory.FileServerHost != "s3.amazonaws.com" {
+		fileURL = fmt.Sprintf("%s://%s:%s/%s",
+			directory.FileServerProtocol, directory.FileServerHost, directory.FileServerPort,
+			path.Join(string(directory.DatabaseDirKey), newFile))
 	}
 
 	listing.Append(

deepfence_worker/Dockerfile

@@ -14,7 +14,6 @@ ARG AGENT_BINARY_DIST_RELATIVE
 LABEL MAINTAINER="Deepfence Inc"
 LABEL deepfence.role=system
 
-# ADD deepfence_server/cloud_controls /cloud_controls
 ADD deepfence_utils/postgresql/migrate /usr/local/postgresql-migrate
 
 RUN apk add --no-cache curl kafkacat docker-cli openrc bash skopeo jansson-dev \
@@ -31,12 +30,6 @@ ENV DEEPFENCE_KAFKA_TOPIC_PARTITIONS=3 \
     DEEPFENCE_MODE=worker \
     LD_LIBRARY_PATH=/usr/local/yara/lib
 
-# ENV GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_FILE_SERVER_HOST}:${DEEPFENCE_FILE_SERVER_PORT}/database/database/vulnerability/listing.json"
-
-# RUN apk add --no-cache --update bash curl \
-#     && apk upgrade \
-#     && curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v0.55.0
-
 COPY --from=wkhtmltopdf /bin/wkhtmltopdf /bin/libwkhtmltox.so /bin/
 
 COPY --from=packagescanner /usr/local/bin/syft /usr/local/bin/syft
@@ -58,9 +51,7 @@ RUN tar -xzf /usr/local/yara.tar.gz -C /usr/local/ \
     && rm /usr/local/yara.tar.gz \
     && chmod +x /entrypoint.sh \
     && mkdir -p /opt/deepfence
-    # && sed -i 's/auto-update: true/auto-update: false/g' /usr/local/bin/grype.yaml
 
-# RUN export GRYPE_DB_UPDATE_URL=https://threat-intel.deepfence.io/vulnerability-db/listing.json && grype db update
 COPY ./deepfence_agent/binary-install-scripts/* /opt/deepfence
 COPY ./${AGENT_BINARY_DIST_RELATIVE}/* /opt/deepfence
 

deepfence_worker/entrypoint.sh

@@ -24,7 +24,7 @@ done
 
 # wait for file server to start
 if [ "$DEEPFENCE_FILE_SERVER_HOST" != "s3.amazonaws.com" ]; then
-  until nc -z ${DEEPFENCE_FILE_SERVER_HOST} ${DEEPFENCE_FILE_SERVER_PORT};
+  until nc -z "${DEEPFENCE_FILE_SERVER_HOST}" "${DEEPFENCE_FILE_SERVER_PORT}";
   do
     echo "file server is unavailable - sleeping"
     sleep 5;
@@ -33,16 +33,15 @@ else
   echo "S3 mode skip file server health check"
 fi
 
-# threat intel urls
-# DEEPFENCE_VULN_DB_URL=""
-# DEEPFENCE_CLOUD_CONTROLS_URL=""
-# DEEPFENCE_SECRETS_RULES_URL=""
-# DEEPFENCE_MALWARE_RULES_URL=""
-
 # for aws s3
-export GRYPE_DB_UPDATE_URL="http://${DEEPFENCE_FILE_SERVER_HOST}:${DEEPFENCE_FILE_SERVER_PORT}/database/database/vulnerability/listing.json"
+fileServerProtocol="http"
+if [ "$DEEPFENCE_FILE_SERVER_SECURE" == "true" ]; then
+  fileServerProtocol="https"
+fi
+
+export GRYPE_DB_UPDATE_URL="${fileServerProtocol}://${DEEPFENCE_FILE_SERVER_HOST}:${DEEPFENCE_FILE_SERVER_PORT}/database/database/vulnerability/listing.json"
 if [ "$DEEPFENCE_FILE_SERVER_HOST" == "s3.amazonaws.com" ]; then
-  export GRYPE_DB_UPDATE_URL="https://${DEEPFENCE_FILE_SERVER_DB_BUCKET}.s3.${DEEPFENCE_FILE_SERVER_REGION}.amazonaws.com/database/vulnerability/listing.json"
+  export GRYPE_DB_UPDATE_URL="${fileServerProtocol}://${DEEPFENCE_FILE_SERVER_DB_BUCKET}.s3.${DEEPFENCE_FILE_SERVER_REGION}.amazonaws.com/database/vulnerability/listing.json"
 fi
 
 # update vulnerability databae

deepfence_worker/tasks/sbom/scan_sbom.go

@@ -33,18 +33,16 @@ import (
 var (
 	grypeConfig         = "/usr/local/bin/grype.yaml"
 	grypeBin            = "grype"
-	minioHost           = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_HOST", "deepfence-file-server")
-	minioPort           = utils.GetEnvOrDefault("DEEPFENCE_FILE_SERVER_PORT", "9000")
-	minioRegion         = os.Getenv("DEEPFENCE_FILE_SERVER_REGION")
-	minioBucket         = os.Getenv("DEEPFENCE_FILE_SERVER_DB_BUCKET")
 	GRYPE_DB_UPDATE_URL string
 )
 
 func init() {
 	// for aws s3
-	GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=https://%s.s3.%s.amazonaws.com/database/vulnerability/listing.json", minioBucket, minioRegion)
-	if minioHost != "s3.amazonaws.com" {
-		GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=http://%s:%s/database/database/vulnerability/listing.json", minioHost, minioPort)
+	GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=%s://%s.s3.%s.amazonaws.com/database/vulnerability/listing.json",
+		directory.FileServerProtocol, directory.FileServerDatabaseBucket, directory.FileServerRegion)
+	if directory.FileServerHost != "s3.amazonaws.com" {
+		GRYPE_DB_UPDATE_URL = fmt.Sprintf("GRYPE_DB_UPDATE_URL=%s://%s:%s/database/database/vulnerability/listing.json",
+			directory.FileServerProtocol, directory.FileServerHost, directory.FileServerPort)
 	}
 	log.Info().Msg(GRYPE_DB_UPDATE_URL)
 }

deployment-scripts/docker-compose.yml

@@ -14,21 +14,23 @@ x-service-variables: &common-creds
     DEEPFENCE_POSTGRES_USER_DB_PORT: 5432
     DEEPFENCE_POSTGRES_USER_DB_SSLMODE: disable
     DEEPFENCE_KAFKA_BROKERS: deepfence-kafka-broker:9092
-    # public bucket with read permisons on objects for hosting vulnerability database
+    # public bucket with read permissions on objects for hosting vulnerability database
     # S3 bucket permissions {"Version":"2012-10-17","Statement":[{"Sid":"database","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::<bucket-name>/database/*","arn:aws:s3:::<bucket-name>/database"]}]}
     DEEPFENCE_FILE_SERVER_DB_BUCKET: database
-    # prvate bucket to host reports, sbom, etc.
+    # private bucket to host reports, sbom, etc.
     DEEPFENCE_FILE_SERVER_BUCKET: default
+    # If using S3 or other external file server (MinIO/SeaweedFS), set this to true
+    DEEPFENCE_FILE_SERVER_EXTERNAL: "false"
     # set s3.amazonaws.com if using s3 buckets
     DEEPFENCE_FILE_SERVER_HOST: deepfence-file-server
     DEEPFENCE_FILE_SERVER_PORT: 9000
     # uncomment to set access key if using s3 buckets
     # DEEPFENCE_FILE_SERVER_USER: fileserveruser
     # uncomment to set secret key if using s3 buckets
     # DEEPFENCE_FILE_SERVER_PASSWORD: changeme
-    # set true if using s3 buckets
+    # set true if https
     DEEPFENCE_FILE_SERVER_SECURE: "false"
-    # set aws s3 buckets region if using s3 buckets
+    # set aws s3 bucket region if using s3 buckets
     DEEPFENCE_FILE_SERVER_REGION: ""
     DEEPFENCE_REDIS_DB_NUMBER: 0
     DEEPFENCE_REDIS_HOST: deepfence-redis
@@ -168,7 +170,7 @@ services:
       options:
         max-size: "100m"
 
-  # this service can be commented in case S3 is used
+  # this service can be commented in case S3 or external file server is used
   deepfence-file-server:
     container_name: deepfence-file-server
     image: ${IMAGE_REPOSITORY:-quay.io/deepfenceio}/deepfence_file_server_ce:${DF_IMG_TAG:-2.2.0}

deployment-scripts/helm-charts/deepfence-console/templates/deepfence-console-config.yaml

@@ -10,6 +10,16 @@ data:
   DEEPFENCE_DEBUG: "false"
   DEEPFENCE_SAAS_DEPLOYMENT: "false"
   DEEPFENCE_TELEMETRY_ENABLED: "false"
+  {{- if .Values.fileserver.create }}
+  DEEPFENCE_FILE_SERVER_HOST: {{ include "deepfence-console.fullname" . }}-file-server.{{ .Release.Namespace }}.svc.{{ .Values.router.cluster_domain }}
+  DEEPFENCE_FILE_SERVER_PORT: "9000"
+  DEEPFENCE_FILE_SERVER_EXTERNAL: "false"
+  {{- else }}
+  DEEPFENCE_FILE_SERVER_HOST: {{ .Values.fileserver.fileServerHost }}
+  DEEPFENCE_FILE_SERVER_PORT: {{ .Values.fileserver.fileServerPort }}
+  DEEPFENCE_FILE_SERVER_EXTERNAL: "true"
+  {{- end }}
+
   {{- if .Values.proxy.enabled }}
   http_proxy: {{ .Values.proxy.http_proxy }}
   https_proxy: {{ .Values.proxy.https_proxy }}

deployment-scripts/helm-charts/deepfence-console/templates/deepfence-router.yaml

@@ -46,10 +46,6 @@ spec:
               value: {{ include "deepfence-console.fullname" . }}-server.{{ .Release.Namespace }}.svc.{{ .Values.router.cluster_domain }}
             - name: API_SERVICE_PORT
               value: {{ .Values.server.service.port | quote }}
-            - name: DEEPFENCE_FILE_SERVER_HOST
-              value: {{ include "deepfence-console.fullname" . }}-file-server.{{ .Release.Namespace }}.svc.{{ .Values.router.cluster_domain }}
-            - name: DEEPFENCE_FILE_SERVER_PORT
-              value: "9000"
           envFrom:
             - configMapRef:
                 name: "{{ include "deepfence-console.fullname" . }}-config"

deployment-scripts/helm-charts/deepfence-console/values.yaml

@@ -144,9 +144,14 @@ fileserver:
   # Specifies whether a file server instance should be created
   # set this to false if using S3
   create: true
-  # if create false provide name of the existing secret
-  # secret format refer templates/console-secrets/minio.yaml
+  # if create false provide name of the existing secret.
+  # Secret format refer templates/deepfence-console-secrets/s3.yaml or templates/deepfence-console-secrets/fileserver.yaml
   secretName: ""
+
+  # Set this if external file server is used and create=false
+  fileServerHost: ""
+  fileServerPort: ""
+
   # if create true then below values are used to create postgres database instance
   # set the below to change the default credentials
   # secrets:
@@ -178,8 +183,8 @@ fileserver:
 aws_s3_buckets:
   # Specifies whether secret should be created
   create: false
-  # if create false provide name of the existing secret
-  # secret format refer templates/deepfence-console-secrets/s3.yaml
+  # if create false provide name of the existing secret.
+  # Secret format refer templates/deepfence-console-secrets/s3.yaml
   secretName: ""
   # public bucket with read permisons on objects for hosting vulnerability database
   # S3 bucket permissions {"Version":"2012-10-17","Statement":[{"Sid":"database","Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":["arn:aws:s3:::<bucket-name>/database/*","arn:aws:s3:::<bucket-name>/database"]}]}