diff --git a/README.md b/README.md index 5fc2374..e523481 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![shield_gh-workflow-test]][link_gh-workflow-test] [![shield_license]][license_file] +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=sonatype-nexus-community_sonatype-lifecycle-bulk-scm-onboarder&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=sonatype-nexus-community_sonatype-lifecycle-bulk-scm-onboarder) --- @@ -10,6 +11,8 @@ Introduce your project here. A short summary about what its purpose and scope is. - [What does this tool do?](#what-does-this-tool-do) + - [Organization Creation](#organization-creation) + - [Application Creation](#application-creation) - [Installation](#installation) - [Usage](#usage) - [Development](#development) @@ -19,9 +22,23 @@ Introduce your project here. A short summary about what its purpose and scope is This tool queries your Source Control Management (SCM) system and creates Organizations and Applications in Sonatype Lifecycle in bulk. +This tool is intended to be run against either an empty Sonatype Lifecycle installation or a specific Organization that represents a given SCM connection ONCE. + +This tool is NOT intended to be run multiple times to continuously import/keep Sonatype Lifecycle up to date with newly created Projects or Repositories in your SCM system - to do this (once you've run this tool once successfully), use [Easy SCM Onboarding](https://help.sonatype.com/en/easy-scm-onboarding.html). + Currently supports: - ✅ Azure DevOps +### Organization Creation + +Sub-organizations under either your Root Organization, or an existing Organization of your choosing (see `-org-name` flag) will be created where they do no exist matching your SCM organizations and/or projects. Organizations will not be re-created or duplicated by running this import process. This can lead in some cases to applications from more than one SCM organizations or project being creating in a single Sonatype Organization due to naming restrictions. + +### Application Creation + +Applications will be create where they cannot be determined to exist for the Repository in your SCM. There are sitations where, due to naming collisions, this cannot be determined and so if you run the import more than once, it is possible that you will have some applications duplicated. + +If an Application is determined to already exist, it's SCM configuration will be updated. SCM configuration is always set for newly created Applications. + ## Installation Obtain the binary for your Operating System and Architecture from the [GitHub Releases page](https://github.com/sonatype-nexus-community/nexus-repo-asset-lister/releases). diff --git a/iq/server.go b/iq/server.go index c01c3ce..f60ec2b 100644 --- a/iq/server.go +++ b/iq/server.go @@ -19,6 +19,8 @@ package iq import ( "context" "fmt" + "io" + "net/http" "os" "strings" @@ -29,12 +31,15 @@ import ( ) type NxiqServer struct { - baseUrl string - username string - password string - apiClient *sonatypeiq.APIClient - apiContext *context.Context - configuration *sonatypeiq.Configuration + baseUrl string + username string + password string + apiClient *sonatypeiq.APIClient + apiContext *context.Context + configuration *sonatypeiq.Configuration + cacheLoaded bool + existingApplications []*sonatypeiq.ApiApplicationDTO + existingOrganizations []*sonatypeiq.ApiOrganizationDTO } func NewNxiqServer(url string, username string, password string) *NxiqServer { @@ -66,37 +71,72 @@ func NewNxiqServer(url string, username string, password string) *NxiqServer { return server } -func (s *NxiqServer) ApplyOrgContents(orgContent scm.OrgContents, rootOrganization *sonatypeiq.ApiOrganizationDTO, scmConfig *scm.ScmConfiguration) error { - for _, o := range orgContent.Organizations { - org, err := s.CreateOrganization(o, *rootOrganization.Id) +func (s *NxiqServer) InitCache() error { + if !s.cacheLoaded { + err := s.cacheExistingOrganizations() if err != nil { return err } - log.Debug(fmt.Sprintf("Created Organization %s - %s", o.SafeName(), *org.Id)) - err = s.SetOrganizationScmConfiguration(org, scmConfig) + + err = s.cacheExistingApplications() if err != nil { return err } - log.Debug(fmt.Sprintf("Applied %s SCM Configuration to Organization %s - %s", scmConfig.Type, o.SafeName(), *org.Id)) - err = s.createAppsInOrg(org, o.Applications) + s.cacheLoaded = true + } + + return nil +} + +func (s *NxiqServer) cacheExistingApplications() error { + s.existingApplications = make([]*sonatypeiq.ApiApplicationDTO, 0) + + apiResponse, r, err := s.apiClient.ApplicationsAPI.GetApplications(*s.apiContext).Execute() + if err != nil { + log.Error(fmt.Sprintf("Failed to load existing Applications from Sonatype IQ: %s: %v: %v", r.Status, err, r.Body)) + return err + } + + for _, a := range apiResponse.Applications { + s.existingApplications = append(s.existingApplications, &a) + } + + log.Info(fmt.Sprintf("Loaded %d existing Applications from Sonatype Lifecycle", len(s.existingApplications))) + return nil +} + +func (s *NxiqServer) cacheExistingOrganizations() error { + s.existingOrganizations = make([]*sonatypeiq.ApiOrganizationDTO, 0) + + apiResponse, r, err := s.apiClient.OrganizationsAPI.GetOrganizations(*s.apiContext).Execute() + if err != nil { + log.Error(fmt.Sprintf("Failed to load existing Applications from Sonatype IQ: %s: %v: %v", r.Status, err, r.Body)) + return err + } + for _, a := range apiResponse.Organizations { + s.existingOrganizations = append(s.existingOrganizations, &a) + } + + log.Info(fmt.Sprintf("Loaded %d existing Organizations from Sonatype Lifecycle", len(s.existingOrganizations))) + return nil +} + +func (s *NxiqServer) ApplyOrgContents(orgContent scm.OrgContents, rootOrganization *sonatypeiq.ApiOrganizationDTO, scmConfig *scm.ScmConfiguration) error { + for _, o := range orgContent.Organizations { + org, err := s.CreateOrganization(o, *rootOrganization.Id, true, scmConfig) if err != nil { return err } - // if len(o.Applications) > 0 { - // for _, a := range o.Applications { - // app, err := s.CreateApplication(a, *org.Id) - // if err != nil { - // return err - // } - // log.Debug(fmt.Sprintf("Created Application %s - %s", a.SafeName(), *app.Id)) - // } - // } + err = s.createAppsInOrg(org, o.Applications) + if err != nil { + return err + } if len(o.SubOrganizations) > 0 { for _, so := range o.SubOrganizations { - subOrg, err := s.CreateOrganization(so, *org.Id) + subOrg, err := s.CreateOrganization(so, *org.Id, false, nil) if err != nil { return err } @@ -139,17 +179,101 @@ func (s *NxiqServer) scheduleSourceStageScan(app *sonatypeiq.ApiApplicationDTO, } } -func (s *NxiqServer) CreateOrganization(org scm.Organization, parentOrgId string) (*sonatypeiq.ApiOrganizationDTO, error) { - orgName := org.SafeName() - createdOrg, r, err := s.apiClient.OrganizationsAPI.AddOrganization(*s.apiContext).ApiOrganizationDTO(sonatypeiq.ApiOrganizationDTO{ - Name: &orgName, - ParentOrganizationId: &parentOrgId, - }).Execute() +/** + * Creates an Organization if it does not already exist. + * + * If `applyScmConfiguration` is true and the Organization already existed, SCM configuration + * will be updated. If the Organization was just created, it will be set. + * + */ +func (s *NxiqServer) CreateOrganization(org scm.Organization, parentOrgId string, applyScmConfiguration bool, scmConfig *scm.ScmConfiguration) (*sonatypeiq.ApiOrganizationDTO, error) { + existingOrg, err := s.OrganizationExists(org, parentOrgId) if err != nil { - fmt.Fprintf(os.Stderr, "Error when calling `OrganizationsAPI.AddOrganization``: %v\n", err) - fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + log.Debug(fmt.Sprintf("Failed to determine if Organization %s already exists", org.Name)) return nil, err } + + if existingOrg != nil { + if applyScmConfiguration { + err = s.UpdateOrganizationScmConfiguration(existingOrg, scmConfig) + if err != nil { + return existingOrg, err + } + log.Debug(fmt.Sprintf("Updated %s SCM Configuration for Organization %s - %s", scmConfig.Type, org.SafeName(), *existingOrg.Id)) + } + return existingOrg, nil + } + + createdOrg, err := s.createOrganization(org, parentOrgId) + if err != nil { + return createdOrg, err + } + log.Debug(fmt.Sprintf("Created Organization %s - %v", org.SafeName(), org)) + if applyScmConfiguration { + err = s.SetOrganizationScmConfiguration(createdOrg, scmConfig) + if err != nil { + return createdOrg, err + } + log.Debug(fmt.Sprintf("Applied %s SCM Configuration to Organization %s - %s", scmConfig.Type, org.SafeName(), *createdOrg.Id)) + } + + return createdOrg, nil +} + +func (s *NxiqServer) OrganizationExists(org scm.Organization, parentOrgId string) (*sonatypeiq.ApiOrganizationDTO, error) { + err := s.InitCache() + if err != nil { + log.Fatalln(err) + } + for _, existingOrg := range s.existingOrganizations { + if *existingOrg.Name == org.SafeName() && *existingOrg.ParentOrganizationId == parentOrgId { + return existingOrg, nil + } + } + return nil, nil +} + +func (s *NxiqServer) createOrganization(org scm.Organization, parentOrgId string) (*sonatypeiq.ApiOrganizationDTO, error) { + orgName := s.getUniqueOrganizationId(org.SafeName()) + + var err error + var httpResponse *http.Response + var attemptCount = 0 + var createdOrg *sonatypeiq.ApiOrganizationDTO + for httpResponse == nil || httpResponse.StatusCode != http.StatusOK { + createdOrg, httpResponse, err = s.apiClient.OrganizationsAPI.AddOrganization(*s.apiContext).ApiOrganizationDTO(sonatypeiq.ApiOrganizationDTO{ + Name: &orgName, + ParentOrganizationId: &parentOrgId, + }).Execute() + + attemptCount += 1 + + if httpResponse.StatusCode == http.StatusBadRequest { + // We possibly had a colision - check response body + defer httpResponse.Body.Close() + + b, err := io.ReadAll(httpResponse.Body) + if err != nil { + log.Fatalln(err) + } + responseBody := string(b) + log.Debug(fmt.Sprintf("Response Body: %s", responseBody)) + + if strings.HasSuffix(responseBody, "used as a name.") { + // Name had a conflict + orgName = fmt.Sprintf("%s-%d", s.getUniqueOrganizationId(org.SafeName()), attemptCount) + log.Debug(fmt.Sprintf("Bumped Organization Name to be %s", orgName)) + continue + } + } + + if attemptCount > 2 && err != nil { + log.Debug(fmt.Sprintf("Error when calling `OrganizationsAPI.AddOrganization` on attempt %d: %v\n", attemptCount, err)) + log.Debug(fmt.Sprintf("Full HTTP response: %v\n", httpResponse)) + return nil, err + } + } + return createdOrg, nil } @@ -175,22 +299,62 @@ func (s *NxiqServer) SetOrganizationScmConfiguration(org *sonatypeiq.ApiOrganiza return nil } -func (s *NxiqServer) CreateApplication(app scm.Application, parentOrgId string) (*sonatypeiq.ApiApplicationDTO, error) { - appId := app.SafeId() - appName := app.SafeName() - createdApp, r, err := s.apiClient.ApplicationsAPI.AddApplication(*s.apiContext).ApiApplicationDTO(sonatypeiq.ApiApplicationDTO{ - PublicId: &appId, - Name: &appName, - OrganizationId: &parentOrgId, +func (s *NxiqServer) UpdateOrganizationScmConfiguration(org *sonatypeiq.ApiOrganizationDTO, scmConfig *scm.ScmConfiguration) error { + // Set SCM Configuration for our top level Org(s) + t := true + f := false + _, r, err := s.apiClient.SourceControlAPI.UpdateSourceControl(*s.apiContext, "organization", *org.Id).ApiSourceControlDTO(sonatypeiq.ApiSourceControlDTO{ + Username: &scmConfig.Username, + Token: &scmConfig.Password, + Provider: &scmConfig.Type, + RemediationPullRequestsEnabled: &f, + PullRequestCommentingEnabled: &f, + SourceControlEvaluationsEnabled: &t, + SshEnabled: &f, + CommitStatusEnabled: &f, }).Execute() if err != nil { - fmt.Fprintf(os.Stderr, "Error when calling `ApplicationsAPI.AddApplication``: %v\n", err) + fmt.Fprintf(os.Stderr, "Error when calling `SourceControlAPI.UpdateSourceControl``: %v\n", err) fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + return err + } + return nil +} + +func (s *NxiqServer) CreateApplication(app scm.Application, parentOrgId string) (*sonatypeiq.ApiApplicationDTO, error) { + existingApp, err := s.ApplicationExists(app, parentOrgId) + if err != nil { + log.Debug(fmt.Sprintf("Failed to determine if Application %s already exists", app.Name)) return nil, err } + if existingApp != nil { + // Update SCM Configuration + _, r, err := s.apiClient.SourceControlAPI.UpdateSourceControl(*s.apiContext, "application", *existingApp.Id).ApiSourceControlDTO(sonatypeiq.ApiSourceControlDTO{ + RepositoryUrl: &app.RepositoryUrl, + BaseBranch: app.DefaultBranch, + EnablePullRequests: nil, + RemediationPullRequestsEnabled: nil, + PullRequestCommentingEnabled: nil, + SourceControlEvaluationsEnabled: nil, + SshEnabled: nil, + }).Execute() + if err != nil { + fmt.Fprintf(os.Stderr, "Error when calling `SourceControlAPI.UpdateSourceControl``: %v\n", err) + fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) + return nil, err + } + return existingApp, nil + } + + createdApp, err := s.createApplication(app, parentOrgId) + if err != nil { + return nil, err + } + log.Debug(fmt.Sprintf("Created App: %v", createdApp)) + // Set SCM Configuration - _, r, err = s.apiClient.SourceControlAPI.AddSourceControl(*s.apiContext, "application", *createdApp.Id).ApiSourceControlDTO(sonatypeiq.ApiSourceControlDTO{ + _, r, err := s.apiClient.SourceControlAPI.AddSourceControl(*s.apiContext, "application", *createdApp.Id).ApiSourceControlDTO(sonatypeiq.ApiSourceControlDTO{ RepositoryUrl: &app.RepositoryUrl, BaseBranch: app.DefaultBranch, EnablePullRequests: nil, @@ -208,20 +372,95 @@ func (s *NxiqServer) CreateApplication(app scm.Application, parentOrgId string) return createdApp, nil } -func (s *NxiqServer) ValidateOrganizationByName(organizationName string) (*sonatypeiq.ApiOrganizationDTO, error) { - request := s.apiClient.OrganizationsAPI.GetOrganizations(*s.apiContext) - request = request.OrganizationName([]string{organizationName}) - orgList, r, err := request.Execute() +func (s *NxiqServer) ApplicationExists(app scm.Application, parentOrgId string) (*sonatypeiq.ApiApplicationDTO, error) { + err := s.InitCache() if err != nil { - fmt.Fprintf(os.Stderr, "Error when calling `OrganizationsAPI.GetOrganizations``: %v\n", err) - fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r) - return nil, err + log.Fatalln(err) + } + for _, existingApp := range s.existingApplications { + if *existingApp.Name == app.SafeName() && *existingApp.OrganizationId == parentOrgId { + return existingApp, nil + } + } + return nil, nil +} + +func (s *NxiqServer) createApplication(app scm.Application, parentOrgId string) (*sonatypeiq.ApiApplicationDTO, error) { + appId := s.getUniqueSafeApplicationId(app.SafeId()) + appName := app.SafeName() + + var err error + var httpResponse *http.Response + var attemptCount = 0 + var createdApp *sonatypeiq.ApiApplicationDTO + for httpResponse == nil || httpResponse.StatusCode != http.StatusOK { + createdApp, httpResponse, err = s.apiClient.ApplicationsAPI.AddApplication(*s.apiContext).ApiApplicationDTO(sonatypeiq.ApiApplicationDTO{ + PublicId: &appId, + Name: &appName, + OrganizationId: &parentOrgId, + }).Execute() + + attemptCount += 1 + + if httpResponse.StatusCode == http.StatusBadRequest { + // We possibly had a colision - check response body + defer httpResponse.Body.Close() + + b, err := io.ReadAll(httpResponse.Body) + if err != nil { + log.Fatalln(err) + } + responseBody := string(b) + log.Debug(fmt.Sprintf("Response Body: %s", responseBody)) + + if strings.HasSuffix(responseBody, "as an ID.") || strings.HasSuffix(responseBody, "as a name.") { + // ID or Name had a conflict + appId = fmt.Sprintf("%s-%d", app.SafeId(), attemptCount) + appName = fmt.Sprintf("%s-%d", app.SafeName(), attemptCount) + log.Debug(fmt.Sprintf("Bumped Application ID and Name to be %s, %s", appId, appName)) + continue + } + } + + if attemptCount > 2 && err != nil { + log.Debug(fmt.Sprintf("Error when calling `ApplicationsAPI.AddApplication` on attempt %d: %v\n", attemptCount, err)) + log.Debug(fmt.Sprintf("Full HTTP response: %v\n", httpResponse)) + return nil, err + } } - if len(orgList.Organizations) == 1 { - org := &orgList.Organizations[0] - return org, nil + return createdApp, nil +} + +func (s *NxiqServer) getUniqueSafeApplicationId(id string) string { + for _, existinApp := range s.existingApplications { + if *existinApp.Id == id { + return fmt.Sprintf("%s-1", id) + } } + return id +} - return nil, fmt.Errorf("%d Organizations returned for Name '%s'", len(orgList.Organizations), organizationName) +func (s *NxiqServer) getUniqueOrganizationId(id string) string { + for _, existingOrg := range s.existingOrganizations { + if *existingOrg.Id == id { + return fmt.Sprintf("%s-1", id) + } + } + return id +} + +func (s *NxiqServer) ValidateOrganizationByName(organizationName string) *sonatypeiq.ApiOrganizationDTO { + err := s.InitCache() + if err != nil { + log.Fatalln(err) + } + + for _, o := range s.existingOrganizations { + if *o.Name == organizationName { + return o + } + } + + return nil } diff --git a/main.go b/main.go index fea1518..7edb717 100644 --- a/main.go +++ b/main.go @@ -105,13 +105,14 @@ func main() { println(strings.Repeat("⬢⬡", 42)) println("") - // Connect to IQ + // Connect to IQ and load cache nxiqServer := iq.NewNxiqServer(nxiqUrl, nxiqUsername, nxiqPassword) - iqTargetOrganization, err := nxiqServer.ValidateOrganizationByName(nxiqOrgNameToImportTo) + err = nxiqServer.InitCache() if err != nil { println(fmt.Sprintf("Error: %v", err)) os.Exit(1) } + iqTargetOrganization := nxiqServer.ValidateOrganizationByName(nxiqOrgNameToImportTo) if iqTargetOrganization == nil { println(fmt.Sprintf("Could not find requested Organization %s", nxiqOrgNameToImportTo)) os.Exit(1)