From e5ef9940f4fcfbd12b3d619f7109a00eb8e2d89a Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Sat, 21 Dec 2024 13:00:55 +0200 Subject: [PATCH 1/6] Internal API - Visibility System support (#1060) --- .../usage/{reportusage.go => call_home.go} | 22 +++++---- ...{reportusage_test.go => call_home_test.go} | 0 go.mod | 2 +- go.sum | 13 ++---- jfconnect/auth/jfconnectdetails.go | 17 +++++++ jfconnect/manager.go | 45 +++++++++++++++++++ jfconnect/services/metrics.go | 39 ++++++++++++++++ utils/utils.go | 2 +- 8 files changed, 120 insertions(+), 20 deletions(-) rename artifactory/usage/{reportusage.go => call_home.go} (72%) rename artifactory/usage/{reportusage_test.go => call_home_test.go} (100%) create mode 100644 jfconnect/auth/jfconnectdetails.go create mode 100644 jfconnect/manager.go create mode 100644 jfconnect/services/metrics.go diff --git a/artifactory/usage/reportusage.go b/artifactory/usage/call_home.go similarity index 72% rename from artifactory/usage/reportusage.go rename to artifactory/usage/call_home.go index bda41b225..cdcb76eae 100644 --- a/artifactory/usage/reportusage.go +++ b/artifactory/usage/call_home.go @@ -18,11 +18,17 @@ type ReportUsageAttribute struct { AttributeValue string } +type ArtifactoryCallHome struct{} + +func NewArtifactoryCallHome() *ArtifactoryCallHome { + return &ArtifactoryCallHome{} +} + func (rua *ReportUsageAttribute) isEmpty() bool { return rua.AttributeName == "" } -func validateAndGetUsageServerInfo(serviceManager artifactory.ArtifactoryServicesManager) (url string, clientDetails httputils.HttpClientDetails, err error) { +func (ach *ArtifactoryCallHome) validateAndGetUsageServerInfo(serviceManager artifactory.ArtifactoryServicesManager) (url string, clientDetails httputils.HttpClientDetails, err error) { config := serviceManager.GetConfig() if config == nil { err = errorutils.CheckErrorf("expected full config, but no configuration exists.") @@ -50,7 +56,7 @@ func validateAndGetUsageServerInfo(serviceManager artifactory.ArtifactoryService return } -func sendReport(url string, serviceManager artifactory.ArtifactoryServicesManager, clientDetails httputils.HttpClientDetails, bodyContent []byte) error { +func (ach *ArtifactoryCallHome) sendReport(url string, serviceManager artifactory.ArtifactoryServicesManager, clientDetails httputils.HttpClientDetails, bodyContent []byte) error { clientDetails.SetContentTypeApplicationJson() resp, body, err := serviceManager.Client().SendPost(url, bodyContent, &clientDetails) if err != nil { @@ -63,8 +69,8 @@ func sendReport(url string, serviceManager artifactory.ArtifactoryServicesManage return nil } -func ReportUsageToArtifactory(productId string, serviceManager artifactory.ArtifactoryServicesManager, features ...Feature) error { - url, clientDetails, err := validateAndGetUsageServerInfo(serviceManager) +func (ach *ArtifactoryCallHome) SendUsageToArtifactory(productId string, serviceManager artifactory.ArtifactoryServicesManager, features ...Feature) error { + url, clientDetails, err := ach.validateAndGetUsageServerInfo(serviceManager) if err != nil || url == "" { return err } @@ -72,11 +78,11 @@ func ReportUsageToArtifactory(productId string, serviceManager artifactory.Artif if err != nil { return err } - return sendReport(url, serviceManager, clientDetails, bodyContent) + return ach.sendReport(url, serviceManager, clientDetails, bodyContent) } -func SendReportUsage(productId, commandName string, serviceManager artifactory.ArtifactoryServicesManager, attributes ...ReportUsageAttribute) error { - url, clientDetails, err := validateAndGetUsageServerInfo(serviceManager) +func (ach *ArtifactoryCallHome) SendUsage(productId, commandName string, serviceManager artifactory.ArtifactoryServicesManager, attributes ...ReportUsageAttribute) error { + url, clientDetails, err := ach.validateAndGetUsageServerInfo(serviceManager) if err != nil || url == "" { return err } @@ -84,7 +90,7 @@ func SendReportUsage(productId, commandName string, serviceManager artifactory.A if err != nil { return err } - return sendReport(url, serviceManager, clientDetails, bodyContent) + return ach.sendReport(url, serviceManager, clientDetails, bodyContent) } func usageFeaturesToJson(productId string, features ...Feature) ([]byte, error) { diff --git a/artifactory/usage/reportusage_test.go b/artifactory/usage/call_home_test.go similarity index 100% rename from artifactory/usage/reportusage_test.go rename to artifactory/usage/call_home_test.go diff --git a/go.mod b/go.mod index 8385a0305..25d108b10 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.1 github.com/gookit/color v1.5.4 github.com/jfrog/archiver/v3 v3.6.1 - github.com/jfrog/build-info-go v1.10.5 + github.com/jfrog/build-info-go v1.10.7 github.com/jfrog/gofrog v1.7.6 github.com/minio/sha256-simd v1.0.1 github.com/stretchr/testify v1.9.0 diff --git a/go.sum b/go.sum index 87554483e..d0fc3425f 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI= github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw= -github.com/jfrog/build-info-go v1.10.5 h1:cW03JlPlKv7RMUU896uLUxyLWXAmCgR5Y5QX0fwgz0Q= -github.com/jfrog/build-info-go v1.10.5/go.mod h1:JcISnovFXKx3wWf3p1fcMmlPdt6adxScXvoJN4WXqIE= +github.com/jfrog/build-info-go v1.10.7 h1:10NVHYg0193gJpQft+S4WQfvYMtj5jlwwhJRvkFJtBE= +github.com/jfrog/build-info-go v1.10.7/go.mod h1:JcISnovFXKx3wWf3p1fcMmlPdt6adxScXvoJN4WXqIE= github.com/jfrog/gofrog v1.7.6 h1:QmfAiRzVyaI7JYGsB7cxfAJePAZTzFz0gRWZSE27c6s= github.com/jfrog/gofrog v1.7.6/go.mod h1:ntr1txqNOZtHplmaNd7rS4f8jpA5Apx8em70oYEe7+4= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -124,8 +124,6 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMx github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= -golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= @@ -144,19 +142,14 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= -golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= -golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= diff --git a/jfconnect/auth/jfconnectdetails.go b/jfconnect/auth/jfconnectdetails.go new file mode 100644 index 000000000..734eb37e0 --- /dev/null +++ b/jfconnect/auth/jfconnectdetails.go @@ -0,0 +1,17 @@ +package auth + +import ( + "github.com/jfrog/jfrog-client-go/auth" +) + +func NewJfConnectDetails() auth.ServiceDetails { + return &jfConnectDetails{} +} + +type jfConnectDetails struct { + auth.CommonConfigFields +} + +func (jc *jfConnectDetails) GetVersion() (string, error) { + panic("Failed: Method is not implemented") +} diff --git a/jfconnect/manager.go b/jfconnect/manager.go new file mode 100644 index 000000000..08fffb052 --- /dev/null +++ b/jfconnect/manager.go @@ -0,0 +1,45 @@ +package jfconnect + +import ( + "github.com/jfrog/jfrog-client-go/config" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/jfconnect/services" +) + +type Manager interface { + PostMetric([]byte) error +} + +type jfConnectManager struct { + client *jfroghttpclient.JfrogHttpClient + config config.Config +} + +func NewManager(config config.Config) (Manager, error) { + details := config.GetServiceDetails() + var err error + manager := &jfConnectManager{config: config} + manager.client, err = jfroghttpclient.JfrogClientBuilder(). + SetCertificatesPath(config.GetCertificatesPath()). + SetInsecureTls(config.IsInsecureTls()). + SetClientCertPath(details.GetClientCertPath()). + SetClientCertKeyPath(details.GetClientCertKeyPath()). + AppendPreRequestInterceptor(details.RunPreRequestFunctions). + SetContext(config.GetContext()). + SetDialTimeout(config.GetDialTimeout()). + SetOverallRequestTimeout(config.GetOverallRequestTimeout()). + SetRetries(config.GetHttpRetries()). + SetRetryWaitMilliSecs(config.GetHttpRetryWaitMilliSecs()). + Build() + + return manager, err +} + +func (jm *jfConnectManager) Client() *jfroghttpclient.JfrogHttpClient { + return jm.client +} + +func (jm *jfConnectManager) PostMetric(metric []byte) error { + jfConnectService := services.NewJfConnectService(jm.config.GetServiceDetails(), jm.client) + return jfConnectService.PostMetric(metric) +} diff --git a/jfconnect/services/metrics.go b/jfconnect/services/metrics.go new file mode 100644 index 000000000..cb3fa9ecb --- /dev/null +++ b/jfconnect/services/metrics.go @@ -0,0 +1,39 @@ +package services + +import ( + "net/http" + + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" +) + +const LogMetricApiEndpoint = "jfconnect/api/v1/backoffice/metrics/log" + +type JfConnectService struct { + client *jfroghttpclient.JfrogHttpClient + serviceDetails *auth.ServiceDetails +} + +func NewJfConnectService(serviceDetails auth.ServiceDetails, client *jfroghttpclient.JfrogHttpClient) *JfConnectService { + return &JfConnectService{serviceDetails: &serviceDetails, client: client} +} + +func (jcs *JfConnectService) GetJfConnectDetails() auth.ServiceDetails { + return *jcs.serviceDetails +} + +func (jcs *JfConnectService) PostMetric(metric []byte) error { + details := jcs.GetJfConnectDetails() + httpClientDetails := details.CreateHttpClientDetails() + httpClientDetails.SetContentTypeApplicationJson() + + url := clientutils.AddTrailingSlashIfNeeded(details.GetUrl()) + url += LogMetricApiEndpoint + resp, body, err := jcs.client.SendPost(url, metric, &httpClientDetails) + if err != nil { + return err + } + return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusCreated, http.StatusOK) +} diff --git a/utils/utils.go b/utils/utils.go index 17bec088e..b9c13f3c6 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -28,7 +28,7 @@ import ( const ( Development = "development" Agent = "jfrog-client-go" - Version = "1.48.1" + Version = "1.48.4" ) const xrayDevVersion = "3.x-dev" From d77d8c8ba7c473acf16d08024ad088737d6f8851 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Sun, 22 Dec 2024 20:15:56 +0200 Subject: [PATCH 2/6] Internal API - Visibility System fix & improvements (#1061) --- jfconnect/manager.go | 6 +++--- jfconnect/services/metrics.go | 27 ++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/jfconnect/manager.go b/jfconnect/manager.go index 08fffb052..20aee5e87 100644 --- a/jfconnect/manager.go +++ b/jfconnect/manager.go @@ -7,7 +7,7 @@ import ( ) type Manager interface { - PostMetric([]byte) error + PostVisibilityMetric(services.VisibilityMetric) error } type jfConnectManager struct { @@ -39,7 +39,7 @@ func (jm *jfConnectManager) Client() *jfroghttpclient.JfrogHttpClient { return jm.client } -func (jm *jfConnectManager) PostMetric(metric []byte) error { +func (jm *jfConnectManager) PostVisibilityMetric(metric services.VisibilityMetric) error { jfConnectService := services.NewJfConnectService(jm.config.GetServiceDetails(), jm.client) - return jfConnectService.PostMetric(metric) + return jfConnectService.PostVisibilityMetric(metric) } diff --git a/jfconnect/services/metrics.go b/jfconnect/services/metrics.go index cb3fa9ecb..e7ff49543 100644 --- a/jfconnect/services/metrics.go +++ b/jfconnect/services/metrics.go @@ -1,6 +1,7 @@ package services import ( + "encoding/json" "net/http" "github.com/jfrog/jfrog-client-go/auth" @@ -9,7 +10,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/errorutils" ) -const LogMetricApiEndpoint = "jfconnect/api/v1/backoffice/metrics/log" +const LogMetricApiEndpoint = "api/v1/backoffice/metrics/log" type JfConnectService struct { client *jfroghttpclient.JfrogHttpClient @@ -24,16 +25,36 @@ func (jcs *JfConnectService) GetJfConnectDetails() auth.ServiceDetails { return *jcs.serviceDetails } -func (jcs *JfConnectService) PostMetric(metric []byte) error { +func (jcs *JfConnectService) PostVisibilityMetric(metric VisibilityMetric) error { + metricJson, err := json.Marshal(metric) + if err != nil { + return errorutils.CheckError(err) + } details := jcs.GetJfConnectDetails() httpClientDetails := details.CreateHttpClientDetails() httpClientDetails.SetContentTypeApplicationJson() url := clientutils.AddTrailingSlashIfNeeded(details.GetUrl()) url += LogMetricApiEndpoint - resp, body, err := jcs.client.SendPost(url, metric, &httpClientDetails) + resp, body, err := jcs.client.SendPost(url, metricJson, &httpClientDetails) if err != nil { return err } return errorutils.CheckResponseStatusWithBody(resp, body, http.StatusCreated, http.StatusOK) } + +type Labels struct { + ProductID string `json:"product_id"` + FeatureID string `json:"feature_id"` + OIDCUsed string `json:"oidc_used"` + JobID string `json:"job_id"` + RunID string `json:"run_id"` + GitRepo string `json:"git_repo"` + GhTokenForCodeScanningAlertsProvided string `json:"gh_token_for_code_scanning_alerts_provided"` +} + +type VisibilityMetric struct { + Value int `json:"value"` + MetricsName string `json:"metrics_name"` + Labels Labels `json:"labels"` +} From 88f1089d0694b16f0ca8db4eb3335e436a5415b5 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Mon, 23 Dec 2024 19:54:48 +0200 Subject: [PATCH 3/6] Add product_version to VisibilityMetric (#1062) --- jfconnect/services/metrics.go | 1 + 1 file changed, 1 insertion(+) diff --git a/jfconnect/services/metrics.go b/jfconnect/services/metrics.go index e7ff49543..847656755 100644 --- a/jfconnect/services/metrics.go +++ b/jfconnect/services/metrics.go @@ -45,6 +45,7 @@ func (jcs *JfConnectService) PostVisibilityMetric(metric VisibilityMetric) error type Labels struct { ProductID string `json:"product_id"` + ProductVersion string `json:"product_version"` FeatureID string `json:"feature_id"` OIDCUsed string `json:"oidc_used"` JobID string `json:"job_id"` From 80a5e1ba7a2c4ed605e6e0f8c78832f7df046443 Mon Sep 17 00:00:00 2001 From: Eyal Ben Moshe Date: Wed, 25 Dec 2024 20:37:33 +0200 Subject: [PATCH 4/6] Internal Remove Call Home version validation (#1063) --- artifactory/usage/call_home.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/artifactory/usage/call_home.go b/artifactory/usage/call_home.go index cdcb76eae..77e5dd86d 100644 --- a/artifactory/usage/call_home.go +++ b/artifactory/usage/call_home.go @@ -11,8 +11,6 @@ import ( "github.com/jfrog/jfrog-client-go/utils/io/httputils" ) -const minArtifactoryVersion = "6.9.0" - type ReportUsageAttribute struct { AttributeName string AttributeValue string @@ -28,7 +26,7 @@ func (rua *ReportUsageAttribute) isEmpty() bool { return rua.AttributeName == "" } -func (ach *ArtifactoryCallHome) validateAndGetUsageServerInfo(serviceManager artifactory.ArtifactoryServicesManager) (url string, clientDetails httputils.HttpClientDetails, err error) { +func (ach *ArtifactoryCallHome) getUsageServerInfo(serviceManager artifactory.ArtifactoryServicesManager) (url string, clientDetails httputils.HttpClientDetails, err error) { config := serviceManager.GetConfig() if config == nil { err = errorutils.CheckErrorf("expected full config, but no configuration exists.") @@ -39,15 +37,6 @@ func (ach *ArtifactoryCallHome) validateAndGetUsageServerInfo(serviceManager art err = errorutils.CheckErrorf("Artifactory details not configured.") return } - // Check Artifactory version - artifactoryVersion, err := rtDetails.GetVersion() - if err != nil { - err = errors.New("Couldn't get Artifactory version. Error: " + err.Error()) - return - } - if err = clientutils.ValidateMinimumVersion(clientutils.Artifactory, artifactoryVersion, minArtifactoryVersion); err != nil { - return - } url, err = clientutils.BuildUrl(rtDetails.GetUrl(), "api/system/usage", make(map[string]string)) if err != nil { return @@ -69,8 +58,8 @@ func (ach *ArtifactoryCallHome) sendReport(url string, serviceManager artifactor return nil } -func (ach *ArtifactoryCallHome) SendUsageToArtifactory(productId string, serviceManager artifactory.ArtifactoryServicesManager, features ...Feature) error { - url, clientDetails, err := ach.validateAndGetUsageServerInfo(serviceManager) +func (ach *ArtifactoryCallHome) SendToArtifactory(productId string, serviceManager artifactory.ArtifactoryServicesManager, features ...Feature) error { + url, clientDetails, err := ach.getUsageServerInfo(serviceManager) if err != nil || url == "" { return err } @@ -81,8 +70,8 @@ func (ach *ArtifactoryCallHome) SendUsageToArtifactory(productId string, service return ach.sendReport(url, serviceManager, clientDetails, bodyContent) } -func (ach *ArtifactoryCallHome) SendUsage(productId, commandName string, serviceManager artifactory.ArtifactoryServicesManager, attributes ...ReportUsageAttribute) error { - url, clientDetails, err := ach.validateAndGetUsageServerInfo(serviceManager) +func (ach *ArtifactoryCallHome) Send(productId, commandName string, serviceManager artifactory.ArtifactoryServicesManager, attributes ...ReportUsageAttribute) error { + url, clientDetails, err := ach.getUsageServerInfo(serviceManager) if err != nil || url == "" { return err } From e342ed5065f156d265c31bf7bfe2eaddc0465eea Mon Sep 17 00:00:00 2001 From: Assaf Attias <49212512+attiasas@users.noreply.github.com> Date: Mon, 30 Dec 2024 17:46:16 +0200 Subject: [PATCH 5/6] Add Git Repository Resource in Xray API's (#1039) --- .../services/utils/tests/xray/consts.go | 18 ++-- xray/services/enrich.go | 7 +- xray/services/ignorerule.go | 23 +++++- xray/services/scan.go | 59 ++++++------- xray/services/scan_test.go | 65 ++++++++------- xray/services/utils/ignorerulebody.go | 25 ++++-- xray/services/utils/policybody.go | 44 +++++++++- xray/services/utils/watchbody.go | 31 ++++++- xray/services/watch.go | 10 ++- xray/services/xsc/xsc.go | 7 ++ xsc/services/analytics.go | 18 +++- xsc/services/utils/utils.go | 17 ++++ xsc/services/utils/utils_test.go | 22 ++++- xsc/services/watch.go | 82 +++++++++++++++++++ 14 files changed, 341 insertions(+), 87 deletions(-) create mode 100644 xsc/services/watch.go diff --git a/artifactory/services/utils/tests/xray/consts.go b/artifactory/services/utils/tests/xray/consts.go index 05692a839..2d8319278 100644 --- a/artifactory/services/utils/tests/xray/consts.go +++ b/artifactory/services/utils/tests/xray/consts.go @@ -1,6 +1,8 @@ package xray -import xrayServices "github.com/jfrog/jfrog-client-go/xray/services" +import ( + xscServices "github.com/jfrog/jfrog-client-go/xsc/services" +) const ScanResponse = ` { @@ -1437,15 +1439,15 @@ const XscGitInfoResponse = `{"multi_scan_id": "3472b4e2-bddc-11ee-a9c9-acde48001 const XscGitInfoBadResponse = `"failed create git info request: git_repo_url field must contain value"` -var GitInfoContextWithMinimalRequiredFields = xrayServices.XscGitInfoContext{ - GitRepoUrl: "https://git.jfrog.info/projects/XSC/repos/xsc-service", - BranchName: "feature/XRAY-123-cool-feature", - CommitHash: "acc5e24e69a-d3c1-4022-62eb-69e4a1e5", +var GitInfoContextWithMinimalRequiredFields = xscServices.XscGitInfoContext{ + GitRepoHttpsCloneUrl: "https://git.jfrog.info/projects/XSC/repos/xsc-service", + BranchName: "feature/XRAY-123-cool-feature", + LastCommitHash: "acc5e24e69a-d3c1-4022-62eb-69e4a1e5", } -var GitInfoContextWithMissingFields = xrayServices.XscGitInfoContext{ - GitRepoUrl: "https://git.jfrog.info/projects/XSC/repos/xsc-service", - BranchName: "feature/XRAY-123-cool-feature", +var GitInfoContextWithMissingFields = xscServices.XscGitInfoContext{ + GitRepoHttpsCloneUrl: "https://git.jfrog.info/projects/XSC/repos/xsc-service", + BranchName: "feature/XRAY-123-cool-feature", } const TestMultiScanId = "3472b4e2-bddc-11ee-a9c9-acde48001122" diff --git a/xray/services/enrich.go b/xray/services/enrich.go index 7c0924c84..346bb7cd4 100644 --- a/xray/services/enrich.go +++ b/xray/services/enrich.go @@ -91,8 +91,7 @@ func (es *EnrichService) GetImportGraphResults(scanId string) (*ScanResponse, er type XrayGraphImportParams struct { // A path in Artifactory that this Artifact is intended to be deployed to. // This will provide a way to extract the watches that should be applied on this graph - ScanType ScanType - SBOMInput []byte - XscGitInfoContext *XscGitInfoContext - XscVersion string + ScanType ScanType + SBOMInput []byte + XscVersion string } diff --git a/xray/services/ignorerule.go b/xray/services/ignorerule.go index 952332555..30ba30f15 100644 --- a/xray/services/ignorerule.go +++ b/xray/services/ignorerule.go @@ -64,9 +64,12 @@ func (xirs *IgnoreRuleService) Delete(ignoreRuleId string) error { } // Create will create a new Xray ignore rule -// The function creates the ignore rule and returns its id which is recieved after post +// The function creates the ignore rule and returns its id which is received after post func (xirs *IgnoreRuleService) Create(params utils.IgnoreRuleParams) (ignoreRuleId string, err error) { ignoreRuleBody := utils.CreateIgnoreRuleBody(params) + if err = validateIgnoreFilters(ignoreRuleBody.IgnoreFilters); err != nil { + return "", err + } content, err := json.Marshal(ignoreRuleBody) if err != nil { return "", errorutils.CheckError(err) @@ -98,6 +101,24 @@ func (xirs *IgnoreRuleService) Create(params utils.IgnoreRuleParams) (ignoreRule return ignoreRuleId, nil } +func validateIgnoreFilters(ignoreFilters utils.IgnoreFilters) error { + filters := []string{} + if len(ignoreFilters.CVEs) > 0 { + filters = append(filters, "CVEs") + } + if ignoreFilters.Exposures != nil { + filters = append(filters, "Exposures") + } + if ignoreFilters.Sast != nil { + filters = append(filters, "Sast") + } + // if more than one filter is set, notify the user + if len(filters) > 1 { + return errorutils.CheckErrorf("more than one ignore filter is set, split them to multiple ignore rules: %v", filters) + } + return nil +} + func getIgnoreRuleIdFromBody(body []byte) (string, error) { str := string(body) diff --git a/xray/services/scan.go b/xray/services/scan.go index 8c2a192a5..08d5bfe76 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -6,9 +6,10 @@ import ( "strings" "time" + clientUtils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" - "github.com/jfrog/jfrog-client-go/xsc/services/utils" + xscUtils "github.com/jfrog/jfrog-client-go/xsc/services/utils" "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" @@ -46,7 +47,8 @@ const ( scanTechQueryParam = "tech=" - XscVersionAPI = "system/version" + gitRepoKeyQueryParam = "git_repo=" + MinXrayVersionGitRepoKey = "3.111.0" ) type ScanType string @@ -75,15 +77,11 @@ func createScanGraphQueryParams(scanParams XrayGraphScanParams) string { } } } - + // Xsc params are used only when XSC is enabled and MultiScanId is provided if scanParams.XscVersion != "" && scanParams.MultiScanId != "" { params = append(params, multiScanIdParam+scanParams.MultiScanId) - gitInfoContext := scanParams.XscGitInfoContext - if gitInfoContext != nil { - if len(gitInfoContext.Technologies) > 0 { - // Append the tech type, each graph can contain only one tech type - params = append(params, scanTechQueryParam+gitInfoContext.Technologies[0]) - } + if scanParams.Technology != "" { + params = append(params, scanTechQueryParam+scanParams.Technology) } } @@ -91,12 +89,21 @@ func createScanGraphQueryParams(scanParams XrayGraphScanParams) string { params = append(params, scanTypeQueryParam+string(scanParams.ScanType)) } + if isGitRepoUrlSupported(scanParams.XrayVersion) && scanParams.GitRepoHttpsCloneUrl != "" { + // Add git repo key to the query params to produce violations defined in the git repo policy + params = append(params, gitRepoKeyQueryParam+xscUtils.GetGitRepoUrlKey(scanParams.GitRepoHttpsCloneUrl)) + } + if len(params) == 0 { return "" } return "?" + strings.Join(params, "&") } +func isGitRepoUrlSupported(xrayVersion string) bool { + return clientUtils.ValidateMinimumVersion(clientUtils.Xray, xrayVersion, MinXrayVersionGitRepoKey) == nil +} + func (ss *ScanService) ScanGraph(scanParams XrayGraphScanParams) (string, error) { httpClientsDetails := ss.XrayDetails.CreateHttpClientDetails() httpClientsDetails.SetContentTypeApplicationJson() @@ -114,7 +121,7 @@ func (ss *ScanService) ScanGraph(scanParams XrayGraphScanParams) (string, error) // When XSC is enabled and MultiScanId is provided, modify the URL to use XSC scan graph (analytics enabled) if scanParams.XrayVersion != "" && scanParams.XscVersion != "" && scanParams.MultiScanId != "" { - url = utils.XrayUrlToXscUrl(ss.XrayDetails.GetUrl(), scanParams.XrayVersion) + XscGraphAPI + url = xscUtils.XrayUrlToXscUrl(ss.XrayDetails.GetUrl(), scanParams.XrayVersion) + XscGraphAPI } url += createScanGraphQueryParams(scanParams) resp, body, err := ss.client.SendPost(url, requestBody, &httpClientsDetails) @@ -144,7 +151,7 @@ func (ss *ScanService) GetScanGraphResults(scanId, xrayVersion string, includeVu endPoint := ss.XrayDetails.GetUrl() + scanGraphAPI // Modify endpoint if XSC is enabled if xscEnabled { - endPoint = utils.XrayUrlToXscUrl(ss.XrayDetails.GetUrl(), xrayVersion) + XscGraphAPI + endPoint = xscUtils.XrayUrlToXscUrl(ss.XrayDetails.GetUrl(), xrayVersion) + XscGraphAPI } endPoint += "/" + scanId @@ -183,7 +190,10 @@ func (ss *ScanService) GetScanGraphResults(scanId, xrayVersion string, includeVu type XrayGraphScanParams struct { // A path in Artifactory that this Artifact is intended to be deployed to. // This will provide a way to extract the watches that should be applied on this graph - RepoPath string + RepoPath string + // This will provide a way to extract the watches that should be applied on this graph + GitRepoHttpsCloneUrl string + // This will provide a way to extract the watches that should be applied on this graph ProjectKey string Watches []string ScanType ScanType @@ -193,9 +203,9 @@ type XrayGraphScanParams struct { BinaryGraph *xrayUtils.BinaryGraphNode IncludeVulnerabilities bool IncludeLicenses bool - XscGitInfoContext *XscGitInfoContext XscVersion string XrayVersion string + Technology string MultiScanId string } @@ -231,6 +241,7 @@ type Violation struct { LicenseKey string `json:"license_key,omitempty"` LicenseName string `json:"license_name,omitempty"` IgnoreUrl string `json:"ignore_url,omitempty"` + Policies []Policy `json:"policies,omitempty"` RiskReason string `json:"risk_reason,omitempty"` IsEol *bool `json:"is_eol,omitempty"` EolMessage string `json:"eol_message,omitempty"` @@ -308,25 +319,9 @@ type JfrogResearchSeverityReason struct { IsPositive bool `json:"is_positive,omitempty"` } -type XscPostContextResponse struct { - MultiScanId string `json:"multi_scan_id,omitempty"` -} - -type XscVersionResponse struct { - Version string `json:"xsc_version"` -} - -type XscGitInfoContext struct { - GitRepoUrl string `json:"git_repo_url"` - GitRepoName string `json:"git_repo_name,omitempty"` - GitProject string `json:"git_project,omitempty"` - GitProvider string `json:"git_provider,omitempty"` - Technologies []string `json:"technologies,omitempty"` - BranchName string `json:"branch_name"` - LastCommit string `json:"last_commit,omitempty"` - CommitHash string `json:"commit_hash"` - CommitMessage string `json:"commit_message,omitempty"` - CommitAuthor string `json:"commit_author,omitempty"` +type Policy struct { + Policy string `json:"policy,omitempty"` + Rule string `json:"rule,omitempty"` } func (gp *XrayGraphScanParams) GetProjectKey() string { diff --git a/xray/services/scan_test.go b/xray/services/scan_test.go index 1eafe2a21..a7bdac2d1 100644 --- a/xray/services/scan_test.go +++ b/xray/services/scan_test.go @@ -8,39 +8,48 @@ import ( func TestCreateScanGraphQueryParams(t *testing.T) { tests := []struct { testName string - projectKey string - repoPath string - watches []string - scanType ScanType + params XrayGraphScanParams expectedQuery string }{ - {"with_project_key", "p1", "", nil, Binary, - fmt.Sprintf("?%s%s&%s%s", projectQueryParam, "p1", scanTypeQueryParam, Binary)}, - - {"with_repo_path", "", "r1", nil, Binary, - fmt.Sprintf("?%s%s&%s%s", repoPathQueryParam, "r1", scanTypeQueryParam, Binary)}, - - {"with_watches", "", "", []string{"w1", "w2"}, Binary, - fmt.Sprintf("?%s%s&%s%s&%s%s", watchesQueryParam, "w1", watchesQueryParam, "w2", scanTypeQueryParam, Binary)}, - - {"with_empty_watch_string", "", "", []string{""}, "", - ""}, - - {"without_context", "", "", nil, Dependency, - fmt.Sprintf("?%s%s", scanTypeQueryParam, Dependency)}, - - {"without_scan_type", "", "", []string{"w1", "w2"}, "", - fmt.Sprintf("?%s%s&%s%s", watchesQueryParam, "w1", watchesQueryParam, "w2")}, + { + testName: "with_project_key", + params: XrayGraphScanParams{ProjectKey: "p1", ScanType: Binary}, + expectedQuery: fmt.Sprintf("?%s%s&%s%s", projectQueryParam, "p1", scanTypeQueryParam, Binary), + }, + { + testName: "with_repo_path", + params: XrayGraphScanParams{RepoPath: "r1", ScanType: Binary}, + expectedQuery: fmt.Sprintf("?%s%s&%s%s", repoPathQueryParam, "r1", scanTypeQueryParam, Binary), + }, + { + testName: "with_watches", + params: XrayGraphScanParams{Watches: []string{"w1", "w2"}, ScanType: Binary}, + expectedQuery: fmt.Sprintf("?%s%s&%s%s&%s%s", watchesQueryParam, "w1", watchesQueryParam, "w2", scanTypeQueryParam, Binary), + }, + { + testName: "with_empty_watch_string", + params: XrayGraphScanParams{Watches: []string{""}}, + expectedQuery: "", + }, + { + testName: "without_context", + params: XrayGraphScanParams{ScanType: Dependency, XrayVersion: MinXrayVersionGitRepoKey}, + expectedQuery: fmt.Sprintf("?%s%s", scanTypeQueryParam, Dependency), + }, + { + testName: "without_scan_type", + params: XrayGraphScanParams{Watches: []string{"w1", "w2"}}, + expectedQuery: fmt.Sprintf("?%s%s&%s%s", watchesQueryParam, "w1", watchesQueryParam, "w2"), + }, + { + testName: "with_git_repo_url", + params: XrayGraphScanParams{GitRepoHttpsCloneUrl: "http://some-url", ScanType: Dependency, XrayVersion: MinXrayVersionGitRepoKey}, + expectedQuery: fmt.Sprintf("?%s%s&%s%s", scanTypeQueryParam, Dependency, gitRepoKeyQueryParam, "some-url.git"), + }, } for _, test := range tests { t.Run(test.testName, func(t *testing.T) { - params := XrayGraphScanParams{ - RepoPath: test.repoPath, - Watches: test.watches, - ProjectKey: test.projectKey, - ScanType: test.scanType, - } - actualQuery := createScanGraphQueryParams(params) + actualQuery := createScanGraphQueryParams(test.params) if actualQuery != test.expectedQuery { t.Error(test.testName, "Expecting:", test.expectedQuery, "Got:", actualQuery) } diff --git a/xray/services/utils/ignorerulebody.go b/xray/services/utils/ignorerulebody.go index 4b465a162..37e2d206b 100644 --- a/xray/services/utils/ignorerulebody.go +++ b/xray/services/utils/ignorerulebody.go @@ -2,6 +2,13 @@ package utils import "time" +const ( + SecretExposureType ExposureType = "secrets" + IacExposureType ExposureType = "iac" +) + +type ExposureType string + type IgnoreRuleParams struct { Notes string `json:"notes"` ExpiresAt time.Time `json:"expires_at,omitempty"` @@ -20,17 +27,25 @@ type IgnoreFilters struct { Vulnerabilities []string `json:"vulnerabilities,omitempty"` Licenses []string `json:"licenses,omitempty"` CVEs []string `json:"cves,omitempty"` + GitRepositories []string `json:"git_repositories,omitempty"` Policies []string `json:"policies,omitempty"` Watches []string `json:"watches,omitempty"` DockerLayers []string `json:"docker-layers,omitempty"` OperationalRisks []string `json:"operational_risk,omitempty"` - Exposures []ExposuresFilterName `json:"exposures,omitempty"` + Exposures *ExposuresFilterName `json:"exposures,omitempty"` + Sast *SastFilterName `json:"sast,omitempty"` ReleaseBundles []IgnoreFilterNameVersion `json:"release-bundles,omitempty"` Builds []IgnoreFilterNameVersion `json:"builds,omitempty"` Components []IgnoreFilterNameVersion `json:"components,omitempty"` Artifacts []IgnoreFilterNameVersionPath `json:"artifacts,omitempty"` } +type SastFilterName struct { + Fingerprint []string `json:"fingerprint,omitempty"` + Rule []string `json:"rule,omitempty"` + FilePath []string `json:"file_path,omitempty"` +} + type IgnoreFilterNameVersion struct { Name string `json:"name"` Version string `json:"version,omitempty"` @@ -42,12 +57,12 @@ type IgnoreFilterNameVersionPath struct { } type ExposuresFilterName struct { - Catagories []ExposuresCatagories `json:"catagories,omitempty"` - Scanners []string `json:"scanners,omitempty"` - FilePath []string `json:"file_path,omitempty"` + Categories []ExposureType `json:"categories,omitempty"` + Scanners []string `json:"scanners,omitempty"` + FilePath []string `json:"file_path,omitempty"` } -type ExposuresCatagories struct { +type ExposuresCategories struct { Secrets bool `json:"secrets,omitempty"` Services bool `json:"services,omitempty"` Applications bool `json:"applications,omitempty"` diff --git a/xray/services/utils/policybody.go b/xray/services/utils/policybody.go index b1b33eec1..cdb034d27 100644 --- a/xray/services/utils/policybody.go +++ b/xray/services/utils/policybody.go @@ -56,8 +56,10 @@ type PolicyRule struct { type PolicyCriteria struct { // Security - MinSeverity Severity `json:"min_severity,omitempty"` - CvssRange *PolicyCvssRange `json:"cvss_range,omitempty"` + MinSeverity Severity `json:"min_severity,omitempty"` + CvssRange *PolicyCvssRange `json:"cvss_range,omitempty"` + Exposures *PolicyExposureCriteria `json:"exposures,omitempty"` + Sast *PolicySastCriteria `json:"sast,omitempty"` // License AllowedLicenses []string `json:"allowed_licenses,omitempty"` @@ -66,6 +68,19 @@ type PolicyCriteria struct { MultiLicensePermissive *bool `json:"multi_license_permissive,omitempty"` } +type PolicyExposureCriteria struct { + MinSeverity Severity `json:"min_severity,omitempty"` + Secrets *bool `json:"secrets,omitempty"` + Applications *bool `json:"applications,omitempty"` + Services *bool `json:"services,omitempty"` + IaC *bool `json:"iac,omitempty"` + MaliciousCode *bool `json:"malicious_code,omitempty"` +} + +type PolicySastCriteria struct { + MinSeverity Severity `json:"min_severity,omitempty"` +} + type PolicyCvssRange struct { From float64 `json:"from,omitempty"` To float64 `json:"to,omitempty"` @@ -93,6 +108,31 @@ func CreateSeverityPolicyCriteria(minSeverity Severity) *PolicyCriteria { } } +func CreateExposuresPolicyCriteria(minSeverity Severity, secrets, applications, services, iac bool) *PolicyCriteria { + criteria := &PolicyCriteria{Exposures: &PolicyExposureCriteria{MinSeverity: minSeverity}} + if secrets { + criteria.Exposures.Secrets = &secrets + } + if applications { + criteria.Exposures.Applications = &applications + } + if services { + criteria.Exposures.Services = &services + } + if iac { + criteria.Exposures.IaC = &iac + } + return criteria +} + +func CreateSastPolicyCriteria(minSeverity Severity) *PolicyCriteria { + return &PolicyCriteria{ + Sast: &PolicySastCriteria{ + MinSeverity: minSeverity, + }, + } +} + // Create security policy criteria with range. // from - CVSS range from 0.0 to 10.0 // to - CVSS range from 0.0 to 10.0 diff --git a/xray/services/utils/watchbody.go b/xray/services/utils/watchbody.go index aee708ff3..fbee34400 100644 --- a/xray/services/utils/watchbody.go +++ b/xray/services/utils/watchbody.go @@ -22,6 +22,8 @@ const ( WatchRepositoriesAll WatchRepositoriesType = "all" // WatchRepositoriesByName is the option where repositories are selected by name to be watched WatchRepositoriesByName WatchRepositoriesType = "byname" + + WatchGitRepository = "gitRepository" ) // WatchBuildType defines the type of filter for a builds on a watch @@ -57,10 +59,13 @@ type WatchParams struct { Description string Active bool - Repositories WatchRepositoriesParams + Repositories WatchRepositoriesParams + GitRepositories WatchGitRepositoryParams + Builds WatchBuildsParams - Builds WatchBuildsParams Policies []AssignedPolicy + + ProjectKey string } // WatchRepositoriesParams is a struct that stores the repository configuration for watch @@ -76,6 +81,10 @@ type WatchRepositoryAll struct { Filters watchFilters } +type WatchGitRepositoryParams struct { + Resources []string +} + // WatchRepository is used to define a specific repository in a watch type WatchRepository struct { Name string @@ -161,6 +170,11 @@ type watchFilterPropertyValue struct { Value string `json:"value"` } +type ResourcesWatchesBody struct { + GitRepositoryWatches []string `json:"git_repository_watches,omitempty"` + ProjectWatches []string `json:"project_watches,omitempty"` +} + // CreateBody creates a payload to configure a Watch in Xray // This can configure repositories and builds // However, bundles are not supported. @@ -187,9 +201,22 @@ func CreateBody(params WatchParams) (*WatchBody, error) { return nil, err } + configureGitRepositories(&payloadBody, params) + return &payloadBody, nil } +func configureGitRepositories(payloadBody *WatchBody, params WatchParams) { + for _, gitRepoResource := range params.GitRepositories.Resources { + gitRepo := watchProjectResourcesElement{ + Type: WatchGitRepository, + BinMgrID: "default", + Name: gitRepoResource, + } + payloadBody.ProjectResources.Resources = append(payloadBody.ProjectResources.Resources, gitRepo) + } +} + func configureRepositories(payloadBody *WatchBody, params WatchParams) error { // Filters needs to be an empty array for Xray to accept the payload. diff --git a/xray/services/watch.go b/xray/services/watch.go index ac71623ea..433885049 100644 --- a/xray/services/watch.go +++ b/xray/services/watch.go @@ -54,6 +54,14 @@ func (xws *WatchService) getWatchURL() string { return clientutils.AddTrailingSlashIfNeeded(xws.XrayDetails.GetUrl()) + watchAPIURL } +func (xws *WatchService) getWatchUrlWithProjectKey(projectKey string) string { + baseUrl := xws.getWatchURL() + if projectKey == "" { + return baseUrl + } + return baseUrl + "?projectKey=" + projectKey +} + // Delete will delete an existing watch by name // It will error if no watch can be found by that name. func (xws *WatchService) Delete(watchName string) error { @@ -86,7 +94,7 @@ func (xws *WatchService) Create(params utils.WatchParams) error { httpClientsDetails := xws.XrayDetails.CreateHttpClientDetails() httpClientsDetails.SetContentTypeApplicationJson() - var url = xws.getWatchURL() + var url = xws.getWatchUrlWithProjectKey(params.ProjectKey) log.Info(fmt.Sprintf("Creating a new Watch named %s on JFrog Xray....", params.Name)) resp, body, err := xws.client.SendPost(url, content, &httpClientsDetails) diff --git a/xray/services/xsc/xsc.go b/xray/services/xsc/xsc.go index 1e3003ed4..7b5f232f1 100644 --- a/xray/services/xsc/xsc.go +++ b/xray/services/xsc/xsc.go @@ -3,6 +3,7 @@ package xsc import ( "github.com/jfrog/jfrog-client-go/auth" "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/xray/services/utils" "github.com/jfrog/jfrog-client-go/xsc/services" ) @@ -58,3 +59,9 @@ func (xs *XscInnerService) GetConfigProfileByUrl(repoUrl string) (*services.Conf configProfileService.XrayDetails = xs.XrayDetails return configProfileService.GetConfigurationProfileByUrl(repoUrl) } + +func (xs *XscInnerService) GetResourceWatches(gitRepo, project string) (watches *utils.ResourcesWatchesBody, err error) { + watchService := services.NewWatchService(xs.client) + watchService.XrayDetails = xs.XrayDetails + return watchService.GetResourceWatches(gitRepo, project) +} diff --git a/xsc/services/analytics.go b/xsc/services/analytics.go index 8603f7e9f..36ea510ca 100644 --- a/xsc/services/analytics.go +++ b/xsc/services/analytics.go @@ -9,7 +9,6 @@ import ( "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/xray/services" xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils" ) @@ -117,8 +116,21 @@ func (vs *AnalyticsEventService) GetGeneralEvent(msi string) (*XscAnalyticsGener // XscAnalyticsGeneralEvent extend the basic struct with Frogbot related info. type XscAnalyticsGeneralEvent struct { XscAnalyticsBasicGeneralEvent - GitInfo *services.XscGitInfoContext `json:"gitinfo,omitempty"` - IsGitInfoFlow bool `json:"is_gitinfo_flow,omitempty"` + GitInfo *XscGitInfoContext `json:"gitinfo,omitempty"` + IsGitInfoFlow bool `json:"is_gitinfo_flow,omitempty"` +} + +type XscGitInfoContext struct { + GitRepoHttpsCloneUrl string `json:"git_repo_url"` + GitRepoName string `json:"git_repo_name,omitempty"` + GitProject string `json:"git_project,omitempty"` + GitProvider string `json:"git_provider,omitempty"` + Technologies []string `json:"technologies,omitempty"` + BranchName string `json:"branch_name"` + LastCommitUrl string `json:"last_commit,omitempty"` + LastCommitHash string `json:"commit_hash"` + LastCommitMessage string `json:"commit_message,omitempty"` + LastCommitAuthor string `json:"commit_author,omitempty"` } type XscAnalyticsGeneralEventFinalize struct { diff --git a/xsc/services/utils/utils.go b/xsc/services/utils/utils.go index af62188b8..539a18379 100644 --- a/xsc/services/utils/utils.go +++ b/xsc/services/utils/utils.go @@ -32,3 +32,20 @@ func IsXscXrayInnerService(xrayVersion string) bool { } return true } + +// The platform expects the git repo key to be in the format of the https/http clone Git URL without the protocol. +func GetGitRepoUrlKey(gitRepoHttpUrl string) string { + if len(gitRepoHttpUrl) == 0 { + // No git context was provided + return "" + } + if !strings.HasSuffix(gitRepoHttpUrl, ".git") { + // Append .git to the URL if not included + gitRepoHttpUrl += ".git" + } + // Remove the Http/s protocol from the URL + if strings.HasPrefix(gitRepoHttpUrl, "http") { + return strings.TrimPrefix(strings.TrimPrefix(gitRepoHttpUrl, "https://"), "http://") + } + return gitRepoHttpUrl +} diff --git a/xsc/services/utils/utils_test.go b/xsc/services/utils/utils_test.go index 57cdaf8bb..a5d1db0af 100644 --- a/xsc/services/utils/utils_test.go +++ b/xsc/services/utils/utils_test.go @@ -1,6 +1,9 @@ package utils -import "testing" +import ( + "github.com/stretchr/testify/assert" + "testing" +) func TestXrayUrlToXscUrl(t *testing.T) { tests := []struct { @@ -21,3 +24,20 @@ func TestXrayUrlToXscUrl(t *testing.T) { }) } } + +func TestGetGitRepoUrlKey(t *testing.T) { + expected := "git.com/jfrog/jfrog-client-go.git" + tests := []struct { + testName string + gitRepoUrl string + }{ + {"with_http", "http://git.com/jfrog/jfrog-client-go.git"}, + {"with_https", "https://git.com/jfrog/jfrog-client-go.git"}, + {"without_protocol", "git.com/jfrog/jfrog-client-go"}, + } + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + assert.Equal(t, expected, GetGitRepoUrlKey(test.gitRepoUrl)) + }) + } +} diff --git a/xsc/services/watch.go b/xsc/services/watch.go new file mode 100644 index 000000000..d187f3ca6 --- /dev/null +++ b/xsc/services/watch.go @@ -0,0 +1,82 @@ +package services + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/jfrog/jfrog-client-go/auth" + "github.com/jfrog/jfrog-client-go/http/jfroghttpclient" + "github.com/jfrog/jfrog-client-go/utils" + "github.com/jfrog/jfrog-client-go/utils/errorutils" + "github.com/jfrog/jfrog-client-go/utils/log" + + api "github.com/jfrog/jfrog-client-go/xray/services/utils" + xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils" +) + +const ( + watchResourceAPIUrl = "watches/resource" + gitRepoResourceUrlKey = "git_repository" + projectResourceUrlKey = "project" +) + +// WatchService defines the http client and Xray details +type WatchService struct { + client *jfroghttpclient.JfrogHttpClient + XrayDetails auth.ServiceDetails +} + +// NewWatchService creates a new Xray Watch Service +func NewWatchService(client *jfroghttpclient.JfrogHttpClient) *WatchService { + return &WatchService{client: client} +} + +// GetResourceWatches retrieves the active watches that are associated with a specific git repository and project +func (xws *WatchService) GetResourceWatches(gitRepo, project string) (watches *api.ResourcesWatchesBody, err error) { + if gitRepo == "" && project == "" { + return nil, errors.New("no resources provided") + } + httpClientsDetails := xws.XrayDetails.CreateHttpClientDetails() + log.Info(fmt.Sprintf("Getting resources (%s) active watches...", getResourcesString(gitRepo, project))) + resp, body, _, err := xws.client.SendGet(xws.getWatchURL(gitRepo, project), true, &httpClientsDetails) + if err != nil { + return + } + if err = errorutils.CheckResponseStatusWithBody(resp, body, http.StatusOK); err != nil { + return + } + log.Debug(fmt.Sprintf("Xray response (status %s): %s", resp.Status, body)) + watches = &api.ResourcesWatchesBody{} + err = json.Unmarshal(body, &watches) + if err != nil { + return nil, errors.New("failed un-marshalling resources watches body") + } + log.Info(fmt.Sprintf("Found %d active watches", len(watches.GitRepositoryWatches)+len(watches.ProjectWatches))) + return +} + +func getResourcesString(gitRepo, project string) string { + providedResources := []string{} + if gitRepo != "" { + providedResources = append(providedResources, fmt.Sprintf("git repository: %s", gitRepo)) + } + if project != "" { + providedResources = append(providedResources, fmt.Sprintf("project: %s", project)) + } + return strings.Join(providedResources, ", ") +} + +func (xws *WatchService) getWatchURL(gitRepo, project string) string { + url := utils.AddTrailingSlashIfNeeded(xws.XrayDetails.GetUrl()) + xscutils.XscInXraySuffix + watchResourceAPIUrl + params := []string{} + if gitRepo != "" { + params = append(params, fmt.Sprintf("%s=%s", gitRepoResourceUrlKey, gitRepo)) + } + if project != "" { + params = append(params, fmt.Sprintf("%s=%s", projectResourceUrlKey, project)) + } + return url + "?" + strings.Join(params, "&") +} From de902d8b8495983013a1c8611920b306394f6f21 Mon Sep 17 00:00:00 2001 From: Eran Turgeman <81029514+eranturgeman@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:33:59 +0200 Subject: [PATCH 6/6] Adding fields to Policy struct to support skip not applicable (#1067) --- tests/xraypolicy_test.go | 25 +++++++++++++++++++------ tests/xraywatch_test.go | 2 +- xray/services/scan.go | 7 +++++-- xray/services/utils/policybody.go | 14 ++++++++------ 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/tests/xraypolicy_test.go b/tests/xraypolicy_test.go index 8097a48e4..e5310cfff 100644 --- a/tests/xraypolicy_test.go +++ b/tests/xraypolicy_test.go @@ -17,6 +17,7 @@ func TestXrayPolicy(t *testing.T) { t.Run("create2Priorities", create2Priorities) t.Run("createPolicyActions", createPolicyActions) t.Run("createUpdatePolicy", createUpdatePolicy) + t.Run("createSkipNonApplicablePolicy", createSkipNonApplicable) } func deletePolicy(t *testing.T, policyName string) { @@ -30,7 +31,7 @@ func createMinSeverity(t *testing.T) { policyRule := utils.PolicyRule{ Name: "min-severity" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low, false), Priority: 1, } createAndCheckPolicy(t, policyName, true, utils.Security, policyRule) @@ -78,12 +79,12 @@ func create2Priorities(t *testing.T) { policyRule1 := utils.PolicyRule{ Name: "priority-1" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low, false), Priority: 1, } policyRule2 := utils.PolicyRule{ Name: "priority-2" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium, false), Priority: 2, } createAndCheckPolicy(t, policyName, true, utils.Security, policyRule1, policyRule2) @@ -95,7 +96,7 @@ func createPolicyActions(t *testing.T) { policyRule := utils.PolicyRule{ Name: "policy-actions" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.High), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.High, false), Priority: 1, Actions: &utils.PolicyAction{ BlockDownload: utils.PolicyBlockDownload{ @@ -118,20 +119,32 @@ func createUpdatePolicy(t *testing.T) { policyRule := utils.PolicyRule{ Name: "low-severity" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low, false), Priority: 1, } createAndCheckPolicy(t, policyName, true, utils.Security, policyRule) policyRule = utils.PolicyRule{ Name: "medium-severity" + getRunId(), - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium, false), Priority: 1, } createAndCheckPolicy(t, policyName, false, utils.Security, policyRule) } +func createSkipNonApplicable(t *testing.T) { + policyName := "skip-non-applicable" + getRunId() + defer deletePolicy(t, policyName) + + policyRule := utils.PolicyRule{ + Name: "skip-non-applicable-rule" + getRunId(), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Low, true), + Priority: 1, + } + createAndCheckPolicy(t, policyName, true, utils.Security, policyRule) +} + func createPolicy(t *testing.T, policyName string, policyType utils.PolicyType, policyRules ...utils.PolicyRule) *utils.PolicyParams { policyParams := utils.PolicyParams{ Name: policyName, diff --git a/tests/xraywatch_test.go b/tests/xraywatch_test.go index e317ec63c..07a43be92 100644 --- a/tests/xraywatch_test.go +++ b/tests/xraywatch_test.go @@ -364,7 +364,7 @@ func createDummyPolicy(policyName string) error { Type: utils.Security, Rules: []utils.PolicyRule{{ Name: "sec_rule", - Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium), + Criteria: *utils.CreateSeverityPolicyCriteria(utils.Medium, false), Actions: &utils.PolicyAction{ Webhooks: []string{}, BlockDownload: utils.PolicyBlockDownload{ diff --git a/xray/services/scan.go b/xray/services/scan.go index 08d5bfe76..5e19b3f16 100644 --- a/xray/services/scan.go +++ b/xray/services/scan.go @@ -320,8 +320,11 @@ type JfrogResearchSeverityReason struct { } type Policy struct { - Policy string `json:"policy,omitempty"` - Rule string `json:"rule,omitempty"` + Policy string `json:"policy,omitempty"` + Rule string `json:"rule,omitempty"` + IsBlocking bool `json:"is_blocking,omitempty"` + IgnoreRuleId string `json:"ignore_rule_id,omitempty"` + SkipNotApplicable bool `json:"is_skip_not_applicable,omitempty"` } func (gp *XrayGraphScanParams) GetProjectKey() string { diff --git a/xray/services/utils/policybody.go b/xray/services/utils/policybody.go index cdb034d27..04dcc85d4 100644 --- a/xray/services/utils/policybody.go +++ b/xray/services/utils/policybody.go @@ -56,10 +56,11 @@ type PolicyRule struct { type PolicyCriteria struct { // Security - MinSeverity Severity `json:"min_severity,omitempty"` - CvssRange *PolicyCvssRange `json:"cvss_range,omitempty"` - Exposures *PolicyExposureCriteria `json:"exposures,omitempty"` - Sast *PolicySastCriteria `json:"sast,omitempty"` + MinSeverity Severity `json:"min_severity,omitempty"` + CvssRange *PolicyCvssRange `json:"cvss_range,omitempty"` + Exposures *PolicyExposureCriteria `json:"exposures,omitempty"` + Sast *PolicySastCriteria `json:"sast,omitempty"` + SkipNotApplicableCVEs bool `json:"applicable_cves_only,omitempty"` // License AllowedLicenses []string `json:"allowed_licenses,omitempty"` @@ -102,9 +103,10 @@ type PolicyBlockDownload struct { } // Create security policy criteria with min severity -func CreateSeverityPolicyCriteria(minSeverity Severity) *PolicyCriteria { +func CreateSeverityPolicyCriteria(minSeverity Severity, skipNotApplicableCves bool) *PolicyCriteria { return &PolicyCriteria{ - MinSeverity: minSeverity, + MinSeverity: minSeverity, + SkipNotApplicableCVEs: skipNotApplicableCves, } }