Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Audit - remove redundant releases remote repo code #851

Merged
merged 10 commits into from
Jul 12, 2023
67 changes: 30 additions & 37 deletions xray/audit/java/gradle.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,21 +102,20 @@ func (dtp *depTreeManager) appendDependenciesPaths(jsonDepTree []byte, fileName
return nil
}

func buildGradleDependencyTree(useWrapper bool, server *config.ServerDetails, depsRepo, releasesRepo string) (dependencyTree []*xrayUtils.GraphNode, err error) {
if (server != nil && server.IsEmpty()) || depsRepo == "" {
depsRepo, server, err = getGradleConfig()
func buildGradleDependencyTree(params *DependencyTreeParams) (dependencyTree []*xrayUtils.GraphNode, err error) {
manager := &depTreeManager{useWrapper: params.UseWrapper}
if params.IgnoreConfigFile {
// In case we don't need to use the gradle config file,
// use the server and depsRepo values that were usually given from Frogbot
manager.depsRepo = params.DepsRepo
manager.server = params.Server
} else {
manager.depsRepo, manager.server, err = getGradleConfig()
if err != nil {
return
}
}

manager := &depTreeManager{
server: server,
releasesRepo: releasesRepo,
depsRepo: depsRepo,
useWrapper: useWrapper,
}

outputFileContent, err := manager.runGradleDepTree()
if err != nil {
return nil, err
Expand All @@ -131,10 +130,7 @@ func (dtp *depTreeManager) runGradleDepTree() (outputFileContent []byte, err err
return
}
defer func() {
e := fileutils.RemoveTempDir(depTreeDir)
if err == nil {
err = e
}
err = errors.Join(err, fileutils.RemoveTempDir(depTreeDir))
}()

if dtp.useWrapper {
Expand All @@ -152,23 +148,20 @@ func (dtp *depTreeManager) createDepTreeScriptAndGetDir() (tmpDir string, err er
if err != nil {
return
}
if dtp.server != nil {
dtp.releasesRepo, dtp.depsRepo, err = getRemoteRepos(dtp.releasesRepo, dtp.depsRepo, dtp.server)
if err != nil {
return
}
dtp.releasesRepo, dtp.depsRepo, err = getRemoteRepos(dtp.depsRepo, dtp.server)
if err != nil {
return
}
depTreeInitScript := fmt.Sprintf(depTreeInitScript, dtp.releasesRepo, dtp.depsRepo)
return tmpDir, errorutils.CheckError(os.WriteFile(filepath.Join(tmpDir, depTreeInitFile), []byte(depTreeInitScript), 0666))
}

// getRemoteRepos constructs the sections of Artifactory's remote repositories in the gradle-dep-tree init script.
// releasesRepoName - name of the remote repository that proxies https://releases.jfrog.io
// depsRemoteRepo - name of the remote repository that proxies the dependencies server, e.g. maven central.
// server - the Artifactory server details on which the repositories reside in.
// Returns the constructed sections.
func getRemoteRepos(releasesRepo, depsRepo string, server *config.ServerDetails) (string, string, error) {
constructedReleasesRepo, err := constructReleasesRemoteRepo(releasesRepo, server)
func getRemoteRepos(depsRepo string, server *config.ServerDetails) (string, string, error) {
constructedReleasesRepo, err := constructReleasesRemoteRepo()
if err != nil {
return "", "", err
}
Expand All @@ -180,21 +173,20 @@ func getRemoteRepos(releasesRepo, depsRepo string, server *config.ServerDetails)
return constructedReleasesRepo, constructedDepsRepo, nil
}

func constructReleasesRemoteRepo(releasesRepo string, server *config.ServerDetails) (string, error) {
releasesServer := server
if releasesRepo == "" {
// Try to get releases repository from the environment variable
serverId, repoName, err := coreutils.GetServerIdAndRepo(coreutils.ReleasesRemoteEnv)
if err != nil || serverId == "" || repoName == "" {
return "", err
}
releasesServer, err = config.GetSpecificConfig(serverId, false, true)
if err != nil {
return "", err
}
releasesRepo = repoName
func constructReleasesRemoteRepo() (string, error) {
// Try to retrieve the serverID and remote repository that proxies https://releases.jfrog.io, from the environment variable
serverId, repoName, err := coreutils.GetServerIdAndRepo(coreutils.ReleasesRemoteEnv)
if err != nil || serverId == "" || repoName == "" {
return "", err
}
releasesPath := fmt.Sprintf("%s/%s", releasesRepo, remoteDepTreePath)

releasesServer, err := config.GetSpecificConfig(serverId, false, true)
if err != nil {
return "", err
}

releasesPath := fmt.Sprintf("%s/%s", repoName, remoteDepTreePath)
log.Debug("The `gradledeptree` will be resolved from", repoName)
return getDepTreeArtifactoryRepository(releasesPath, releasesServer)
}

Expand Down Expand Up @@ -263,7 +255,7 @@ func populateGradleDependencyTree(currNode *xrayUtils.GraphNode, currNodeChildre
}

func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDetails) (string, error) {
if remoteRepo == "" {
if remoteRepo == "" || server.IsEmpty() {
return "", nil
}
pass := server.Password
Expand All @@ -283,6 +275,7 @@ func getDepTreeArtifactoryRepository(remoteRepo string, server *config.ServerDet
}
return "", errors.New(errString)
}
log.Debug("The project dependencies will be resolved from", server.ArtifactoryUrl, "from the", remoteRepo, "repository")
return fmt.Sprintf(artifactoryRepository,
strings.TrimSuffix(server.ArtifactoryUrl, "/"),
remoteRepo,
Expand Down
30 changes: 8 additions & 22 deletions xray/audit/java/gradle_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ func TestGradleTreesWithoutConfig(t *testing.T) {
assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700))

// Run getModulesDependencyTrees
modulesDependencyTrees, err := buildGradleDependencyTree(false, nil, "", "")
modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{})
if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
assert.Len(t, modulesDependencyTrees, 5)
// Check module
Expand All @@ -46,7 +46,7 @@ func TestGradleTreesWithConfig(t *testing.T) {
assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700))

// Run getModulesDependencyTrees
modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "")
modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true})
if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
assert.Len(t, modulesDependencyTrees, 5)

Expand All @@ -70,7 +70,7 @@ func TestGradleTreesExcludeTestDeps(t *testing.T) {
assert.NoError(t, os.Chmod(filepath.Join(tempDirPath, "gradlew"), 0700))

// Run getModulesDependencyTrees
modulesDependencyTrees, err := buildGradleDependencyTree(true, nil, "", "")
modulesDependencyTrees, err := buildGradleDependencyTree(&DependencyTreeParams{UseWrapper: true})
if assert.NoError(t, err) && assert.NotNil(t, modulesDependencyTrees) {
assert.Len(t, modulesDependencyTrees, 5)

Expand Down Expand Up @@ -213,22 +213,15 @@ func TestCreateDepTreeScript(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, fmt.Sprintf(depTreeInitScript, "", ""), string(content))
manager.depsRepo = "deps-repo"
manager.releasesRepo = "release-repo"
manager.server = &config.ServerDetails{
Url: "https://myartifactory.com/",
ArtifactoryUrl: "https://myartifactory.com/artifactory",
AccessToken: "my-access-token",
}
tmpDir, err = manager.createDepTreeScriptAndGetDir()
assert.NoError(t, err)
expectedInitScript := `initscript {
repositories {
maven {
url "https://myartifactory.com/artifactory/release-repo/artifactory/oss-release-local"
credentials {
username = ''
password = 'my-access-token'
}
}
mavenCentral()
}
dependencies {
Expand Down Expand Up @@ -264,21 +257,14 @@ func TestConstructReleasesRemoteRepo(t *testing.T) {
err := config.SaveServersConf([]*config.ServerDetails{serverDetails})
assert.NoError(t, err)
defer cleanUp()
server := &config.ServerDetails{
ArtifactoryUrl: "https://myartifactory.com/artifactory",
User: "myuser",
Password: "mypass",
}
testCases := []struct {
releasesRepo string
envVar string
expectedRepo string
expectedErr error
}{
{releasesRepo: "", envVar: "", expectedRepo: "", expectedErr: nil},
{releasesRepo: "", envVar: "test/repo1", expectedRepo: "\n\t\tmaven {\n\t\t\turl \"https://domain.com/artifactory/repo1/artifactory/oss-release-local\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'user'\n\t\t\t\tpassword = 'pass'\n\t\t\t}\n\t\t}", expectedErr: nil},
{releasesRepo: "", envVar: "notexist/repo1", expectedRepo: "", expectedErr: errors.New("Server ID 'notexist' does not exist.")},
{releasesRepo: "repo2", envVar: "", expectedRepo: "\n\t\tmaven {\n\t\t\turl \"https://myartifactory.com/artifactory/repo2/artifactory/oss-release-local\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'myuser'\n\t\t\t\tpassword = 'mypass'\n\t\t\t}\n\t\t}", expectedErr: nil},
{envVar: "", expectedRepo: "", expectedErr: nil},
{envVar: "test/repo1", expectedRepo: "\n\t\tmaven {\n\t\t\turl \"https://domain.com/artifactory/repo1/artifactory/oss-release-local\"\n\t\t\tcredentials {\n\t\t\t\tusername = 'user'\n\t\t\t\tpassword = 'pass'\n\t\t\t}\n\t\t}", expectedErr: nil},
{envVar: "notexist/repo1", expectedRepo: "", expectedErr: errors.New("Server ID 'notexist' does not exist.")},
}

for _, tc := range testCases {
Expand All @@ -289,7 +275,7 @@ func TestConstructReleasesRemoteRepo(t *testing.T) {
// Reset the environment variable after each test case
assert.NoError(t, os.Unsetenv(coreutils.ReleasesRemoteEnv))
}()
actualRepo, actualErr := constructReleasesRemoteRepo(tc.releasesRepo, server)
actualRepo, actualErr := constructReleasesRemoteRepo()
assert.Equal(t, tc.expectedRepo, actualRepo)
assert.Equal(t, tc.expectedErr, actualErr)
}()
Expand Down
14 changes: 2 additions & 12 deletions xray/audit/java/javautils.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,8 @@ type DependencyTreeParams struct {
IgnoreConfigFile bool
ExcludeTestDeps bool
UseWrapper bool
JavaProps map[string]any
Server *config.ServerDetails
DepsRepo string
ReleasesRepo string
}

func createBuildConfiguration(buildName string) (*artifactoryUtils.BuildConfiguration, func() error) {
Expand Down Expand Up @@ -133,17 +131,9 @@ func hasLoop(idsAdded []string, idToAdd string) bool {

func BuildDependencyTree(params *DependencyTreeParams) (modules []*xrayUtils.GraphNode, err error) {
if params.Tool == coreutils.Maven {
return buildMvnDependencyTree(params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, params.JavaProps)
return buildMvnDependencyTree(params)
}
server := &config.ServerDetails{}
depsRepo := ""
releaseRepo := ""
if params.IgnoreConfigFile {
server = params.Server
depsRepo = params.DepsRepo
releaseRepo = params.ReleasesRepo
}
return buildGradleDependencyTree(params.UseWrapper, server, depsRepo, releaseRepo)
return buildGradleDependencyTree(params)
}

type dependencyMultimap struct {
Expand Down
36 changes: 30 additions & 6 deletions xray/audit/java/mvn.go
Original file line number Diff line number Diff line change
@@ -1,32 +1,56 @@
package java

import (
"errors"
"fmt"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
mvnutils "github.com/jfrog/jfrog-cli-core/v2/utils/mvn"
"github.com/jfrog/jfrog-client-go/auth"
"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
"github.com/jfrog/jfrog-client-go/utils/log"
xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
)

func buildMvnDependencyTree(insecureTls, ignoreConfigFile, useWrapper bool, mvnProps map[string]any) (modules []*xrayUtils.GraphNode, err error) {
func buildMvnDependencyTree(params *DependencyTreeParams) (modules []*xrayUtils.GraphNode, err error) {
buildConfiguration, cleanBuild := createBuildConfiguration("audit-mvn")
defer func() {
e := cleanBuild()
if err == nil {
err = e
}
err = errors.Join(err, cleanBuild())
}()

err = runMvn(buildConfiguration, insecureTls, ignoreConfigFile, useWrapper, mvnProps)
mvnProps := CreateMvnProps(params.DepsRepo, params.Server)
err = runMvn(buildConfiguration, params.InsecureTls, params.IgnoreConfigFile, params.UseWrapper, mvnProps)
if err != nil {
return
}

return createGavDependencyTree(buildConfiguration)
}

func CreateMvnProps(resolverRepo string, serverDetails *config.ServerDetails) map[string]any {
if serverDetails == nil || serverDetails.IsEmpty() {
return nil
}
authPass := serverDetails.Password
if serverDetails.AccessToken != "" {
authPass = serverDetails.AccessToken
}
authUser := serverDetails.User
if authUser == "" {
authUser = auth.ExtractUsernameFromAccessToken(serverDetails.AccessToken)
}
return map[string]any{
"resolver.username": authUser,
"resolver.password": authPass,
"resolver.url": serverDetails.ArtifactoryUrl,
"resolver.releaseRepo": resolverRepo,
"resolver.repo": resolverRepo,
"resolver.snapshotRepo": resolverRepo,
"buildInfoConfig.artifactoryResolutionEnabled": true,
}
}

func runMvn(buildConfiguration *utils.BuildConfiguration, insecureTls, ignoreConfigFile, useWrapper bool, mvnProps map[string]any) (err error) {
goals := []string{"-B", "compile", "test-compile", "-Dcheckstyle.skip", "-Denforcer.skip"}
log.Debug(fmt.Sprintf("mvn command goals: %v", goals))
Expand Down
4 changes: 2 additions & 2 deletions xray/audit/java/mvn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func TestMavenTreesMultiModule(t *testing.T) {
defer cleanUp()

// Run getModulesDependencyTrees
modulesDependencyTrees, err := buildMvnDependencyTree(false, true, false, nil)
modulesDependencyTrees, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true})
if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) {
// Check root module
multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT")
Expand All @@ -42,7 +42,7 @@ func TestMavenWrapperTrees(t *testing.T) {
err := os.Chmod("mvnw", 0700)
defer cleanUp()
assert.NoError(t, err)
modulesDependencyTrees, err := buildMvnDependencyTree(false, true, true, nil)
modulesDependencyTrees, err := buildMvnDependencyTree(&DependencyTreeParams{IgnoreConfigFile: true, UseWrapper: true})
if assert.NoError(t, err) && assert.NotEmpty(t, modulesDependencyTrees) {
// Check root module
multi := audit.GetAndAssertNode(t, modulesDependencyTrees, "org.jfrog.test:multi:3.7-SNAPSHOT")
Expand Down
6 changes: 0 additions & 6 deletions xray/commands/audit/generic/auditmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -305,24 +305,18 @@ func GetTechDependencyTree(params *clientUtils.GraphBasicParams, tech coreutils.
}

func getJavaDependencyTree(params *clientUtils.GraphBasicParams, tech coreutils.Technology) ([]*xrayCmdUtils.GraphNode, error) {
var javaProps map[string]any
serverDetails, err := params.ServerDetails()
if err != nil {
return nil, err
}
if params.DepsRepo() != "" && tech == coreutils.Maven {
javaProps = CreateJavaProps(params.DepsRepo(), serverDetails)
}
return java.BuildDependencyTree(&java.DependencyTreeParams{
Tool: tech,
InsecureTls: params.InsecureTls(),
IgnoreConfigFile: params.IgnoreConfigFile(),
ExcludeTestDeps: params.ExcludeTestDependencies(),
UseWrapper: params.UseWrapper(),
JavaProps: javaProps,
Server: serverDetails,
DepsRepo: params.DepsRepo(),
ReleasesRepo: params.ReleasesRepo(),
})
}

Expand Down
10 changes: 0 additions & 10 deletions xray/utils/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ type GraphBasicParams struct {
outputFormat OutputFormat
progress ioUtils.ProgressMgr
fullDependenciesTree []*xrayUtils.GraphNode
releasesRepo string
excludeTestDependencies bool
useWrapper bool
insecureTls bool
Expand All @@ -22,15 +21,6 @@ type GraphBasicParams struct {
ignoreConfigFile bool
}

func (gbp *GraphBasicParams) ReleasesRepo() string {
return gbp.releasesRepo
}

func (gbp *GraphBasicParams) SetReleasesRepo(releasesRepo string) *GraphBasicParams {
gbp.releasesRepo = releasesRepo
return gbp
}

func (gbp *GraphBasicParams) FullDependenciesTree() []*xrayUtils.GraphNode {
return gbp.fullDependenciesTree
}
Expand Down
Loading