diff --git a/artifactory/commands/generic/upload.go b/artifactory/commands/generic/upload.go index 86f0bf24a..5640c8188 100644 --- a/artifactory/commands/generic/upload.go +++ b/artifactory/commands/generic/upload.go @@ -102,19 +102,7 @@ func (uc *UploadCommand) upload() (err error) { } if toCollect && !uc.DryRun() { addVcsProps = true - buildName, err := uc.buildConfiguration.GetBuildName() - if err != nil { - return err - } - buildNumber, err := uc.buildConfiguration.GetBuildNumber() - if err != nil { - return err - } - err = utils.SaveBuildGeneralDetails(buildName, buildNumber, uc.buildConfiguration.GetProject()) - if err != nil { - return err - } - buildProps, err = utils.CreateBuildProperties(buildName, buildNumber, uc.buildConfiguration.GetProject()) + buildProps, err = utils.CreateBuildPropsFromConfiguration(uc.buildConfiguration) if err != nil { return err } @@ -203,20 +191,7 @@ func (uc *UploadCommand) upload() (err error) { if err != nil { return } - populateFunc := func(partial *buildInfo.Partial) { - partial.Artifacts = buildArtifacts - partial.ModuleId = uc.buildConfiguration.GetModule() - partial.ModuleType = buildInfo.Generic - } - buildName, err := uc.buildConfiguration.GetBuildName() - if err != nil { - return err - } - buildNumber, err := uc.buildConfiguration.GetBuildNumber() - if err != nil { - return err - } - return utils.SavePartialBuildInfo(buildName, buildNumber, uc.buildConfiguration.GetProject(), populateFunc) + return utils.PopulateBuildArtifactsAsPartials(buildArtifacts, uc.buildConfiguration, buildInfo.Generic) } return } diff --git a/artifactory/commands/terraform/terraformpublish.go b/artifactory/commands/terraform/terraformpublish.go index 2a26d1b5f..df5344b6b 100644 --- a/artifactory/commands/terraform/terraformpublish.go +++ b/artifactory/commands/terraform/terraformpublish.go @@ -1,6 +1,7 @@ package terraform import ( + buildInfo "github.com/jfrog/build-info-go/entities" "github.com/jfrog/gofrog/parallel" commandsUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils" "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -8,27 +9,27 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" "github.com/jfrog/jfrog-client-go/artifactory/services" servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" - clientutils "github.com/jfrog/jfrog-client-go/utils" + clientUtils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/errorutils" - "github.com/jfrog/jfrog-client-go/utils/io/content" "github.com/jfrog/jfrog-client-go/utils/log" "io" "io/fs" "os" "path" "path/filepath" - "strconv" "strings" ) const threads = 3 type TerraformPublishCommandArgs struct { - namespace string - moduleName string - provider string - tag string - exclusions []string + namespace string + provider string + tag string + exclusions []string + buildConfiguration *utils.BuildConfiguration + collectBuildInfo bool + buildProps string } type TerraformPublishCommand struct { @@ -48,10 +49,6 @@ func NewTerraformPublishCommandArgs() *TerraformPublishCommandArgs { return &TerraformPublishCommandArgs{} } -func (tpc *TerraformPublishCommand) GetArgs() []string { - return tpc.args -} - func (tpc *TerraformPublishCommand) SetArgs(terraformArg []string) *TerraformPublishCommand { tpc.args = terraformArg return tpc @@ -62,7 +59,7 @@ func (tpc *TerraformPublishCommand) SetServerDetails(serverDetails *config.Serve return tpc } -func (tpc *TerraformPublishCommand) SetRepo(repo string) *TerraformPublishCommand { +func (tpc *TerraformPublishCommand) setRepo(repo string) *TerraformPublishCommand { tpc.repo = repo return tpc } @@ -84,11 +81,6 @@ func (tpc *TerraformPublishCommand) Result() *commandsUtils.Result { return tpc.result } -func (tpc *TerraformPublishCommand) SetModuleName(moduleName string) *TerraformPublishCommand { - tpc.moduleName = moduleName - return tpc -} - func (tpc *TerraformPublishCommand) Run() error { log.Info("Running Terraform publish") err := tpc.preparePrerequisites() @@ -118,7 +110,17 @@ func (tpc *TerraformPublishCommand) preparePrerequisites() error { if err != nil { return err } - return utils.ValidateRepoExists(tpc.repo, artDetails) + if err := utils.ValidateRepoExists(tpc.repo, artDetails); err != nil { + return err + } + tpc.collectBuildInfo, err = tpc.buildConfiguration.IsCollectBuildInfo() + if err != nil { + return err + } + if tpc.collectBuildInfo { + tpc.buildProps, err = utils.CreateBuildPropsFromConfiguration(tpc.buildConfiguration) + } + return err } func (tpc *TerraformPublishCommand) publish() error { @@ -159,6 +161,10 @@ func (tpa *TerraformPublishCommandArgs) extractTerraformPublishOptionsFromArgs(a } tpa.exclusions = append(tpa.exclusions, strings.Split(exclusionsString, ";")...) coreutils.RemoveFlagFromCommand(&args, flagIndex, valueIndex) + args, tpa.buildConfiguration, err = utils.ExtractBuildDetailsFromArgs(args) + if err != nil { + return err + } if len(args) != 0 { err = errorutils.CheckErrorf("Unknown flag:" + strings.Split(args[0], "=")[0] + ". for a terraform publish command please provide --namespace, --provider, --tag and optionally --exclusions.") } @@ -166,20 +172,20 @@ func (tpa *TerraformPublishCommandArgs) extractTerraformPublishOptionsFromArgs(a } func (tpc *TerraformPublishCommand) terraformPublish() (int, int, error) { - uploadSummary := servicesUtils.NewResult(threads) + uploadSummary := getNewUploadSummaryMultiArray() producerConsumer := parallel.NewRunner(3, 20000, false) - errorsQueue := clientutils.NewErrorsQueue(threads) + errorsQueue := clientUtils.NewErrorsQueue(threads) tpc.prepareTerraformPublishTasks(producerConsumer, errorsQueue, uploadSummary) - totalUploaded, totalFailed := tpc.performTerraformPublishTasks(producerConsumer, uploadSummary) + tpc.performTerraformPublishTasks(producerConsumer) e := errorsQueue.GetError() if e != nil { return 0, 0, e } - return totalUploaded, totalFailed, nil + return tpc.aggregateSummaryResults(uploadSummary) } -func (tpc *TerraformPublishCommand) prepareTerraformPublishTasks(producer parallel.Runner, errorsQueue *clientutils.ErrorsQueue, uploadSummary *servicesUtils.Result) { +func (tpc *TerraformPublishCommand) prepareTerraformPublishTasks(producer parallel.Runner, errorsQueue *clientUtils.ErrorsQueue, uploadSummary *[][]*servicesUtils.OperationSummary) { go func() { defer producer.Done() pwd, err := os.Getwd() @@ -196,14 +202,34 @@ func (tpc *TerraformPublishCommand) prepareTerraformPublishTasks(producer parall }() } -// ProduceTaskFunk is provided as an argument to 'walkDirAndUploadTerraformModules' function for testing purposes. -type ProduceTaskFunk func(producer parallel.Runner, uploadService *services.UploadService, uploadSummary *servicesUtils.Result, target string, archiveData *services.ArchiveUploadData, errorsQueue *clientutils.ErrorsQueue) (int, error) +// ProduceTaskFunc is provided as an argument to 'walkDirAndUploadTerraformModules' function for testing purposes. +type ProduceTaskFunc func(producer parallel.Runner, serverDetails *config.ServerDetails, uploadSummary *[][]*servicesUtils.OperationSummary, uploadParams *services.UploadParams, errorsQueue *clientUtils.ErrorsQueue) (int, error) -func addTaskWithError(producer parallel.Runner, uploadService *services.UploadService, uploadSummary *servicesUtils.Result, target string, archiveData *services.ArchiveUploadData, errorsQueue *clientutils.ErrorsQueue) (int, error) { - return producer.AddTaskWithError(uploadService.CreateUploadAsZipFunc(uploadSummary, target, archiveData, errorsQueue), errorsQueue.AddError) +func addTaskWithError(producer parallel.Runner, serverDetails *config.ServerDetails, uploadSummary *[][]*servicesUtils.OperationSummary, uploadParams *services.UploadParams, errorsQueue *clientUtils.ErrorsQueue) (int, error) { + return producer.AddTaskWithError(uploadModuleTask(serverDetails, uploadSummary, uploadParams), errorsQueue.AddError) } -func (tpc *TerraformPublishCommand) walkDirAndUploadTerraformModules(pwd string, producer parallel.Runner, errorsQueue *clientutils.ErrorsQueue, uploadSummary *servicesUtils.Result, produceTaskFunk ProduceTaskFunk) error { +func uploadModuleTask(serverDetails *config.ServerDetails, uploadSummary *[][]*servicesUtils.OperationSummary, uploadParams *services.UploadParams) parallel.TaskFunc { + return func(threadId int) (err error) { + summary, err := createServiceManagerAndUpload(serverDetails, uploadParams, false) + if err != nil { + return err + } + // Add summary to the thread's summary array. + (*uploadSummary)[threadId] = append((*uploadSummary)[threadId], summary) + return nil + } +} + +func createServiceManagerAndUpload(serverDetails *config.ServerDetails, uploadParams *services.UploadParams, dryRun bool) (operationSummary *servicesUtils.OperationSummary, err error) { + serviceManager, err := utils.CreateServiceManagerWithThreads(serverDetails, dryRun, 1, -1, 0) + if err != nil { + return nil, err + } + return serviceManager.UploadFilesWithSummary(*uploadParams) +} + +func (tpc *TerraformPublishCommand) walkDirAndUploadTerraformModules(pwd string, producer parallel.Runner, errorsQueue *clientUtils.ErrorsQueue, uploadSummary *[][]*servicesUtils.OperationSummary, produceTaskFunc ProduceTaskFunc) error { return filepath.WalkDir(pwd, func(path string, info fs.DirEntry, err error) error { if err != nil { return err @@ -222,29 +248,12 @@ func (tpc *TerraformPublishCommand) walkDirAndUploadTerraformModules(pwd string, } if isTerraformModule { uploadParams := tpc.uploadParamsForTerraformPublish(pathInfo.Name(), strings.TrimPrefix(path, pwd+string(filepath.Separator))) - archiveData := createTerraformArchiveUploadData(uploadParams, errorsQueue) - dataHandlerFunc := GetSaveTaskInContentWriterFunc(archiveData, *uploadParams, errorsQueue) - // Collect files that match uploadParams and write them in archiveData's writer. - e = services.CollectFilesForUpload(*uploadParams, nil, nil, dataHandlerFunc) - if e != nil { - log.Error(e) - errorsQueue.AddError(e) - } - e = archiveData.GetWriter().Close() - if e != nil { - log.Error(e) - errorsQueue.AddError(e) - } - // In case all files were excluded and no files were written to writer - skip this module. - if archiveData.GetWriter().IsEmpty() { - return filepath.SkipDir - } - uploadService := createUploadServiceManager(tpc.serverDetails, errorsQueue) - _, e = produceTaskFunk(producer, uploadService, uploadSummary, uploadParams.Target, archiveData, errorsQueue) + _, e = produceTaskFunc(producer, tpc.serverDetails, uploadSummary, uploadParams, errorsQueue) if e != nil { log.Error(e) errorsQueue.AddError(e) } + // SkipDir will not stop the walk, but it will make us jump to the next directory. return filepath.SkipDir } @@ -252,50 +261,52 @@ func (tpc *TerraformPublishCommand) walkDirAndUploadTerraformModules(pwd string, }) } -func createTerraformArchiveUploadData(uploadParams *services.UploadParams, errorsQueue *clientutils.ErrorsQueue) *services.ArchiveUploadData { - archiveData := services.ArchiveUploadData{} - var err error - archiveData.SetUploadParams(services.DeepCopyUploadParams(uploadParams)) - writer, err := content.NewContentWriter("archive", true, false) - if err != nil { - log.Error(err) - errorsQueue.AddError(err) - } - archiveData.SetWriter(writer) - return &archiveData +func (tpc *TerraformPublishCommand) performTerraformPublishTasks(consumer parallel.Runner) { + // Blocking until consuming is finished. + consumer.Run() } -func createUploadServiceManager(serverDetails *config.ServerDetails, errorsQueue *clientutils.ErrorsQueue) *services.UploadService { - // Upload module using upload service - serviceManager, err := utils.CreateServiceManager(serverDetails, 0, 0, false) - if err != nil { - log.Error(err) - errorsQueue.AddError(err) +// Aggregate the operation summaries from all threads, to get the number of successful and failed uploads. +// If collecting build info, also aggregate the artifacts uploaded. +func (tpc *TerraformPublishCommand) aggregateSummaryResults(uploadSummary *[][]*servicesUtils.OperationSummary) (totalUploaded, totalFailed int, err error) { + var artifacts []buildInfo.Artifact + for i := 0; i < threads; i++ { + threadSummary := (*uploadSummary)[i] + for j := range threadSummary { + // Operation summary should always be returned. + if threadSummary[j] == nil { + return 0, 0, errorutils.CheckErrorf("unexpected nil operation summary") + } + totalUploaded += threadSummary[j].TotalSucceeded + totalFailed += threadSummary[j].TotalFailed + + if tpc.collectBuildInfo { + buildArtifacts, err := readArtifactsFromSummary(threadSummary[j]) + if err != nil { + return 0, 0, err + } + artifacts = append(artifacts, buildArtifacts...) + } + } } - uploadService := services.NewUploadService(serviceManager.Client()) - uploadService.ArtDetails = serviceManager.GetConfig().GetServiceDetails() - uploadService.Threads = serviceManager.GetConfig().GetThreads() - return uploadService -} - -func GetSaveTaskInContentWriterFunc(archiveData *services.ArchiveUploadData, uploadParams services.UploadParams, errorsQueue *clientutils.ErrorsQueue) services.UploadDataHandlerFunc { - return func(data services.UploadData) { - archiveData.GetWriter().Write(data) + if tpc.collectBuildInfo { + err = utils.PopulateBuildArtifactsAsPartials(artifacts, tpc.buildConfiguration, buildInfo.Terraform) } + return } -func (tpc *TerraformPublishCommand) performTerraformPublishTasks(consumer parallel.Runner, uploadSummary *servicesUtils.Result) (totalUploaded, totalFailed int) { - // Blocking until consuming is finished. - consumer.Run() - totalUploaded = servicesUtils.SumIntArray(uploadSummary.SuccessCount) - totalUploadAttempted := servicesUtils.SumIntArray(uploadSummary.TotalCount) - - log.Debug("Uploaded", strconv.Itoa(totalUploaded), "artifacts.") - totalFailed = totalUploadAttempted - totalUploaded - if totalFailed > 0 { - log.Error("Failed uploading", strconv.Itoa(totalFailed), "artifacts.") +func readArtifactsFromSummary(summary *servicesUtils.OperationSummary) (artifacts []buildInfo.Artifact, err error) { + artifactsDetailsReader := summary.ArtifactsDetailsReader + if artifactsDetailsReader == nil { + return []buildInfo.Artifact{}, nil } - return + defer func() { + e := artifactsDetailsReader.Close() + if err == nil { + err = e + } + }() + return servicesUtils.ConvertArtifactsDetailsToBuildInfoArtifacts(artifactsDetailsReader) } func (tpc *TerraformPublishCommand) uploadParamsForTerraformPublish(moduleName, dirPath string) *services.UploadParams { @@ -307,7 +318,7 @@ func (tpc *TerraformPublishCommand) uploadParamsForTerraformPublish(moduleName, uploadParams.Recursive = true uploadParams.CommonParams.TargetProps = servicesUtils.NewProperties() uploadParams.CommonParams.Exclusions = append(tpc.exclusions, "*.git", "*.DS_Store") - + uploadParams.BuildProps = tpc.buildProps return &uploadParams } @@ -316,8 +327,7 @@ func (tpc *TerraformPublishCommand) getPublishTarget(moduleName string) string { return path.Join(tpc.repo, tpc.namespace, moduleName, tpc.provider, tpc.tag+".zip") } -// We identify a Terraform module by the existence of a file with a ".tf" extension inside the module directory. -// isTerraformModule search for a file with a ".tf" extension inside and returns true it founds at least one. +// We identify a Terraform module by having at least one file with a ".tf" extension inside the module directory. func checkIfTerraformModule(path string) (isModule bool, err error) { dirname := path + string(filepath.Separator) d, err := os.Open(dirname) @@ -356,6 +366,12 @@ func (tpc *TerraformPublishCommand) setRepoFromConfiguration() error { if err != nil { return err } - tpc.SetRepo(deployerParams.TargetRepo()) + tpc.setRepo(deployerParams.TargetRepo()) return nil } + +// Each thread will save the summary of all its operations, in an array at its corresponding index. +func getNewUploadSummaryMultiArray() *[][]*servicesUtils.OperationSummary { + uploadSummary := make([][]*servicesUtils.OperationSummary, threads) + return &uploadSummary +} diff --git a/artifactory/commands/terraform/terraformpublish_test.go b/artifactory/commands/terraform/terraformpublish_test.go index 3d7726fc2..6fc707b95 100644 --- a/artifactory/commands/terraform/terraformpublish_test.go +++ b/artifactory/commands/terraform/terraformpublish_test.go @@ -4,11 +4,9 @@ import ( "errors" "github.com/jfrog/gofrog/parallel" "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-core/v2/utils/tests" "github.com/jfrog/jfrog-client-go/artifactory/services" clientServicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils" - clientutils "github.com/jfrog/jfrog-client-go/utils" - "github.com/jfrog/jfrog-client-go/utils/io/content" + clientUtils "github.com/jfrog/jfrog-client-go/utils" "github.com/stretchr/testify/assert" "path/filepath" "testing" @@ -28,7 +26,7 @@ func TestPreparePrerequisites(t *testing.T) { } func TestCheckIfTerraformModule(t *testing.T) { - dirPath := filepath.Join("..", "testdata", "terraform", "terraformproject") + dirPath := filepath.Join("..", "testdata", "terraform", "terraform_project") // Check terraform module directory which contain files with a ".tf" extension. isModule, err := checkIfTerraformModule(dirPath) assert.NoError(t, err) @@ -40,96 +38,68 @@ func TestCheckIfTerraformModule(t *testing.T) { assert.False(t, isModule) } -type terraformTests struct { - name string - path string - testFunc ProduceTaskFunk +func TestWalkDirAndUploadTerraformModules(t *testing.T) { + t.Run("testEmptyModule", func(t *testing.T) { runTerraformTest(t, "empty", mockEmptyModule) }) + t.Run("mockProduceTaskFunc", func(t *testing.T) { + runTerraformTest(t, "terraform_project", getTaskMock(t, []string{"a.tf", "test_dir/b.tf", "test_dir/submodules/test_submodule/c.tf"})) + }) + t.Run("testSpecialChar", func(t *testing.T) { + runTerraformTest(t, "terra$+~&^a#", getTaskMock(t, []string{"a.tf", "$+~&^a#@.tf"})) + }) + t.Run("testExcludeTestDirectory", func(t *testing.T) { + runTerraformTestWithExclusions(t, "terraform_project", getTaskMock(t, []string{"a.tf"}), []string{"*test_dir*", "*@*"}) + }) + t.Run("testExcludeTestSubmoduleModule", func(t *testing.T) { + runTerraformTestWithExclusions(t, filepath.Join("terraform_project", "test_dir"), getTaskMock(t, []string{"b.tf"}), []string{"*test_sub*", "*@*"}) + }) + t.Run("testExcludeSpecialChar", func(t *testing.T) { + runTerraformTestWithExclusions(t, "terra$+~&^a#", getTaskMock(t, []string{"a.tf"}), []string{"*test_dir*", "*@*"}) + }) } -func TestWalkDirAndUploadTerraformModules(t *testing.T) { - terraformTestDir := filepath.Join("..", "testdata", "terraform") - tests := []terraformTests{ - {name: "testEmptyModule", path: filepath.Join(terraformTestDir, "empty"), testFunc: mockEmptyModule}, - {name: "mockProduceTaskFunk", path: filepath.Join(terraformTestDir, "terraformproject"), testFunc: testTerraformModule}, - {name: "testSpecialChar", path: filepath.Join(terraformTestDir, "terra$+~&^a#"), testFunc: testSpecialChar}, - } - runTerraformTests(t, tests, []string{}) - // Test exclusions - exclusionsTests := []terraformTests{ - {name: "testExcludeTestDirectory", path: filepath.Join(terraformTestDir, "terraformproject"), testFunc: testExcludeTestDirectory}, - {name: "testExcludeTestSubmoduleModule", path: filepath.Join(terraformTestDir, "terraformproject", "test"), testFunc: testExcludeTestSubmoduleModule}, - {name: "testSpecialChar", path: filepath.Join(terraformTestDir, "terra$+~&^a#"), testFunc: testSpecialCharWithExclusions}, - } - runTerraformTests(t, exclusionsTests, []string{"*test*", "$*"}) +func getTerraformTestDir(path string) string { + return filepath.Join("..", "testdata", "terraform", path) +} + +func runTerraformTest(t *testing.T, subDir string, testFunc ProduceTaskFunc) { + runTerraformTestWithExclusions(t, subDir, testFunc, []string{}) } -func runTerraformTests(t *testing.T, tests []terraformTests, exclusions []string) { +func runTerraformTestWithExclusions(t *testing.T, subDir string, testFunc ProduceTaskFunc, exclusions []string) { terraformPublish := NewTerraformPublishCommand() terraformPublish.SetServerDetails(&config.ServerDetails{}) terraformPublish.exclusions = exclusions - uploadSummary := clientServicesUtils.NewResult(threads) + uploadSummary := getNewUploadSummaryMultiArray() producerConsumer := parallel.NewRunner(threads, 20000, false) - errorsQueue := clientutils.NewErrorsQueue(1) - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - assert.NoError(t, terraformPublish.walkDirAndUploadTerraformModules(test.path, producerConsumer, errorsQueue, uploadSummary, test.testFunc)) - assert.NoError(t, errorsQueue.GetError()) - }) - } -} - -func mockEmptyModule(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, _ *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - return 0, errors.New("Failed: testing empty directory. this function shouldn't be called. ") + errorsQueue := clientUtils.NewErrorsQueue(1) + assert.NoError(t, terraformPublish.walkDirAndUploadTerraformModules(getTerraformTestDir(subDir), producerConsumer, errorsQueue, uploadSummary, testFunc)) + assert.NoError(t, errorsQueue.GetError()) } -func testTerraformModule(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, archiveData *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - paths, err := mockProduceTaskFunk(archiveData) - if err != nil { - return 0, err - } - return 0, tests.ValidateListsIdentical([]string{"a.tf", "test/b.tf", "test/submodules/testSubmodule/c.tf"}, paths) -} +// In order to verify the walk function has visited the correct dirs, and that the correct files were collected/excluded, we run a dry run upload on the files without an archive. +func getTaskMock(t *testing.T, expectedPaths []string) func(parallel.Runner, *config.ServerDetails, *[][]*clientServicesUtils.OperationSummary, *services.UploadParams, *clientUtils.ErrorsQueue) (int, error) { + return func(_ parallel.Runner, serverDetails *config.ServerDetails, _ *[][]*clientServicesUtils.OperationSummary, uploadParams *services.UploadParams, _ *clientUtils.ErrorsQueue) (int, error) { + uploadParams.Target = uploadParams.TargetPathInArchive + uploadParams.TargetPathInArchive = "" + uploadParams.Archive = "" + uploadParams.Target = "dummy-repo/{1}" -func testExcludeTestSubmoduleModule(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, archiveData *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - paths, err := mockProduceTaskFunk(archiveData) - if err != nil { - return 0, err + summary, err := createServiceManagerAndUpload(serverDetails, uploadParams, true) + assert.NoError(t, err) + if !assert.NotNil(t, summary) { + return 0, nil + } + artifacts, err := readArtifactsFromSummary(summary) + assert.NoError(t, err) + var actualPaths []string + for _, artifact := range artifacts { + actualPaths = append(actualPaths, artifact.Path) + } + assert.ElementsMatch(t, actualPaths, expectedPaths) + return 0, nil } - return 0, tests.ValidateListsIdentical([]string{"b/file.tf"}, paths) } -func testExcludeTestDirectory(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, archiveData *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - paths, err := mockProduceTaskFunk(archiveData) - if err != nil { - return 0, err - } - return 0, tests.ValidateListsIdentical([]string{"a.tf"}, paths) -} - -func testSpecialChar(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, archiveData *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - paths, err := mockProduceTaskFunk(archiveData) - if err != nil { - return 0, err - } - return 0, tests.ValidateListsIdentical([]string{"a.tf", "$+~&^a#.tf"}, paths) -} - -func testSpecialCharWithExclusions(_ parallel.Runner, _ *services.UploadService, _ *clientServicesUtils.Result, _ string, archiveData *services.ArchiveUploadData, _ *clientutils.ErrorsQueue) (int, error) { - paths, err := mockProduceTaskFunk(archiveData) - if err != nil { - return 0, err - } - return 0, tests.ValidateListsIdentical([]string{"a.tf"}, paths) -} - -func mockProduceTaskFunk(archiveData *services.ArchiveUploadData) (paths []string, err error) { - archiveData.GetWriter().GetFilePath() - archiveDataReader := content.NewContentReader(archiveData.GetWriter().GetFilePath(), archiveData.GetWriter().GetArrayKey()) - defer func() { - err = archiveDataReader.Close() - }() - for uploadData := new(services.UploadData); archiveDataReader.NextRecord(uploadData) == nil; uploadData = new(services.UploadData) { - paths = append(paths, uploadData.Artifact.TargetPathInArchive) - } - return +func mockEmptyModule(_ parallel.Runner, _ *config.ServerDetails, _ *[][]*clientServicesUtils.OperationSummary, _ *services.UploadParams, _ *clientUtils.ErrorsQueue) (int, error) { + return 0, errors.New("failed: testing empty directory. this function shouldn't be called. ") } diff --git a/artifactory/commands/testdata/terraform/terra$+~&^a#/$+~&^a#.tf b/artifactory/commands/testdata/terraform/terra$+~&^a#/$+~&^a#@.tf similarity index 100% rename from artifactory/commands/testdata/terraform/terra$+~&^a#/$+~&^a#.tf rename to artifactory/commands/testdata/terraform/terra$+~&^a#/$+~&^a#@.tf diff --git a/artifactory/commands/testdata/terraform/terraformproject/a.tf b/artifactory/commands/testdata/terraform/terraform_project/a.tf similarity index 100% rename from artifactory/commands/testdata/terraform/terraformproject/a.tf rename to artifactory/commands/testdata/terraform/terraform_project/a.tf diff --git a/artifactory/commands/testdata/terraform/terraformproject/test/b.tf b/artifactory/commands/testdata/terraform/terraform_project/test_dir/b.tf similarity index 100% rename from artifactory/commands/testdata/terraform/terraformproject/test/b.tf rename to artifactory/commands/testdata/terraform/terraform_project/test_dir/b.tf diff --git a/artifactory/commands/testdata/terraform/terraformproject/test/submodules/testSubmodule/c.tf b/artifactory/commands/testdata/terraform/terraform_project/test_dir/submodules/test_submodule/c.tf similarity index 100% rename from artifactory/commands/testdata/terraform/terraformproject/test/submodules/testSubmodule/c.tf rename to artifactory/commands/testdata/terraform/terraform_project/test_dir/submodules/test_submodule/c.tf diff --git a/artifactory/utils/buildutils.go b/artifactory/utils/buildutils.go index d1870bbb9..cc9d5f443 100644 --- a/artifactory/utils/buildutils.go +++ b/artifactory/utils/buildutils.go @@ -450,3 +450,36 @@ func (bc *BuildConfiguration) IsCollectBuildInfo() (bool, error) { func (bc *BuildConfiguration) IsLoadedFromConfigFile() bool { return bc.loadedFromConfigFile } + +func PopulateBuildArtifactsAsPartials(buildArtifacts []buildInfo.Artifact, buildConfiguration *BuildConfiguration, moduleType buildInfo.ModuleType) error { + populateFunc := func(partial *buildInfo.Partial) { + partial.Artifacts = buildArtifacts + partial.ModuleId = buildConfiguration.GetModule() + partial.ModuleType = moduleType + } + buildName, err := buildConfiguration.GetBuildName() + if err != nil { + return err + } + buildNumber, err := buildConfiguration.GetBuildNumber() + if err != nil { + return err + } + return SavePartialBuildInfo(buildName, buildNumber, buildConfiguration.GetProject(), populateFunc) +} + +func CreateBuildPropsFromConfiguration(buildConfiguration *BuildConfiguration) (string, error) { + buildName, err := buildConfiguration.GetBuildName() + if err != nil { + return "", err + } + buildNumber, err := buildConfiguration.GetBuildNumber() + if err != nil { + return "", err + } + err = SaveBuildGeneralDetails(buildName, buildNumber, buildConfiguration.GetProject()) + if err != nil { + return "", err + } + return CreateBuildProperties(buildName, buildNumber, buildConfiguration.GetProject()) +} diff --git a/go.mod b/go.mod index 69cfc7bc2..8232ed6d7 100644 --- a/go.mod +++ b/go.mod @@ -95,6 +95,6 @@ require ( replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.24.6-0.20230102092554-c56fc329c8fc -replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.5-0.20230102085833-456b7aea6098 +replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.5-0.20230103131235-4993ad739dc6 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.5-0.20221107113836-a4c9225c690e diff --git a/go.sum b/go.sum index 397a30870..984273a8b 100644 --- a/go.sum +++ b/go.sum @@ -196,8 +196,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jedib0t/go-pretty/v6 v6.4.3 h1:2n9BZ0YQiXGESUSR+6FLg0WWWE80u+mIz35f0uHWcIE= github.com/jedib0t/go-pretty/v6 v6.4.3/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jfrog/build-info-go v1.8.5-0.20230102085833-456b7aea6098 h1:ESa8yV/3MrmHHKcO/wmZ0nk2SzLy9+DsB8hlsiVG2Fk= -github.com/jfrog/build-info-go v1.8.5-0.20230102085833-456b7aea6098/go.mod h1:iSTj26qEX3eUtyAGMH0qKsW4WJT+MceYxLWP9FfiAq4= +github.com/jfrog/build-info-go v1.8.5-0.20230103131235-4993ad739dc6 h1:4WeDAwB+tuHQY9Qow9kKe7J15zlrUHECpVs+gFHHJBs= +github.com/jfrog/build-info-go v1.8.5-0.20230103131235-4993ad739dc6/go.mod h1:iSTj26qEX3eUtyAGMH0qKsW4WJT+MceYxLWP9FfiAq4= github.com/jfrog/gofrog v1.2.5 h1:jCgJC0iGQ8bU7jCC+YEFJTNINyngApIrhd8BjZAVRIE= github.com/jfrog/gofrog v1.2.5/go.mod h1:o00tSRff6IapTgaCMuX1Cs9MH08Y1JqnsKgRtx91Gc4= github.com/jfrog/jfrog-client-go v1.24.6-0.20230102092554-c56fc329c8fc h1:EVp7jh405HHaor48wNVslSAi62uCdJPgPU1Cm391s3A=