diff --git a/backend/plugins/ae/impl/impl.go b/backend/plugins/ae/impl/impl.go index 907975e4827..325146bef15 100644 --- a/backend/plugins/ae/impl/impl.go +++ b/backend/plugins/ae/impl/impl.go @@ -150,6 +150,8 @@ func (p AE) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index b863490657e..f82db88e1bc 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -263,6 +263,8 @@ func (p Azuredevops) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/bamboo/impl/impl.go b/backend/plugins/bamboo/impl/impl.go index f3ffc26503a..bc722c1b980 100644 --- a/backend/plugins/bamboo/impl/impl.go +++ b/backend/plugins/bamboo/impl/impl.go @@ -260,6 +260,8 @@ func (p Bamboo) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/bitbucket/impl/impl.go b/backend/plugins/bitbucket/impl/impl.go index 90ed9171241..8d512362081 100644 --- a/backend/plugins/bitbucket/impl/impl.go +++ b/backend/plugins/bitbucket/impl/impl.go @@ -251,7 +251,9 @@ func (p Bitbucket) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/bitbucket_server/impl/impl.go b/backend/plugins/bitbucket_server/impl/impl.go index 9b041f4bd00..d59c9a9cb00 100644 --- a/backend/plugins/bitbucket_server/impl/impl.go +++ b/backend/plugins/bitbucket_server/impl/impl.go @@ -214,7 +214,9 @@ func (p BitbucketServer) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/circleci/impl/impl.go b/backend/plugins/circleci/impl/impl.go index e3728aeced6..d9a1bd86516 100644 --- a/backend/plugins/circleci/impl/impl.go +++ b/backend/plugins/circleci/impl/impl.go @@ -219,6 +219,8 @@ func (p Circleci) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/feishu/impl/impl.go b/backend/plugins/feishu/impl/impl.go index eb321a12bc4..cfe3e1affa8 100644 --- a/backend/plugins/feishu/impl/impl.go +++ b/backend/plugins/feishu/impl/impl.go @@ -152,6 +152,8 @@ func (p Feishu) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/gitee/impl/impl.go b/backend/plugins/gitee/impl/impl.go index b31ec5bb7a0..59572fe5716 100644 --- a/backend/plugins/gitee/impl/impl.go +++ b/backend/plugins/gitee/impl/impl.go @@ -207,6 +207,8 @@ func (p Gitee) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/gitlab/impl/impl.go b/backend/plugins/gitlab/impl/impl.go index 3fdb8622018..2d4dbb4fc49 100644 --- a/backend/plugins/gitlab/impl/impl.go +++ b/backend/plugins/gitlab/impl/impl.go @@ -147,9 +147,14 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i return nil, errors.BadInput.Wrap(err, "connection not found") } - apiClient, err := tasks.NewGitlabApiClient(taskCtx, connection) - if err != nil { - return nil, err + var apiClient *helper.ApiAsyncClient + syncPolicy := taskCtx.SyncPolicy() + if syncPolicy != nil && !syncPolicy.SkipCollectors { + newApiClient, err := tasks.NewGitlabApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + apiClient = newApiClient } if op.ProjectId != 0 { var scope *models.GitlabProject @@ -163,17 +168,19 @@ func (p Gitlab) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]i } } if err != nil && db.IsErrorNotFound(err) { - var project *models.GitlabApiProject - project, err = api.GetApiProject(op, apiClient) - if err != nil { - return nil, err - } - logger.Debug(fmt.Sprintf("Current project: %d", project.GitlabId)) - scope := project.ConvertApiScope() - scope.ConnectionId = op.ConnectionId - err = taskCtx.GetDal().CreateIfNotExist(scope) - if err != nil { - return nil, err + if apiClient != nil { + var project *models.GitlabApiProject + project, err = api.GetApiProject(op, apiClient) + if err != nil { + return nil, err + } + logger.Debug(fmt.Sprintf("Current project: %d", project.GitlabId)) + scope := project.ConvertApiScope() + scope.ConnectionId = op.ConnectionId + err = taskCtx.GetDal().CreateIfNotExist(scope) + if err != nil { + return nil, err + } } } if err != nil { @@ -284,6 +291,8 @@ func (p Gitlab) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/gitlab/tasks/account_extractor.go b/backend/plugins/gitlab/tasks/account_extractor.go index d2d24b48c9e..d271a1d5db4 100644 --- a/backend/plugins/gitlab/tasks/account_extractor.go +++ b/backend/plugins/gitlab/tasks/account_extractor.go @@ -19,12 +19,11 @@ package tasks import ( "encoding/json" - "strings" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/gitlab/models" + "strings" ) func init() { @@ -44,7 +43,7 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_USER_TABLE) // Do not extract createdUserAt if we are not using /users API - var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") + //var skipCreatedUserAt = strings.HasPrefix(data.ApiClient.GetEndpoint(), "https://gitlab.com") extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, @@ -55,10 +54,14 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { return nil, err } + // Do not extract createdUserAt if we are not using /users API + // Why? I don't know, just refact this code, make it doesn't need api client. + var skipCreatedUserAt = strings.HasPrefix(row.Url, "https://gitlab.com") + results := make([]interface{}, 0) - var GitlabAccount *models.GitlabAccount + var gitlabAccount *models.GitlabAccount if skipCreatedUserAt { - GitlabAccount = &models.GitlabAccount{ + gitlabAccount = &models.GitlabAccount{ ConnectionId: data.Options.ConnectionId, GitlabId: userRes.GitlabId, Username: userRes.Username, @@ -70,7 +73,7 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { Email: userRes.Email, } } else { - GitlabAccount = &models.GitlabAccount{ + gitlabAccount = &models.GitlabAccount{ ConnectionId: data.Options.ConnectionId, GitlabId: userRes.GitlabId, Username: userRes.Username, @@ -84,7 +87,7 @@ func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { } } - results = append(results, GitlabAccount) + results = append(results, gitlabAccount) return results, nil }, diff --git a/backend/plugins/icla/impl/impl.go b/backend/plugins/icla/impl/impl.go index 1ebd1ea2db6..83be4492fdd 100644 --- a/backend/plugins/icla/impl/impl.go +++ b/backend/plugins/icla/impl/impl.go @@ -104,6 +104,8 @@ func (p Icla) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go index 0b6ff4b8c37..297d9e4b346 100644 --- a/backend/plugins/jenkins/impl/impl.go +++ b/backend/plugins/jenkins/impl/impl.go @@ -230,7 +230,9 @@ func (p Jenkins) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index f86c73ff323..8a01b7b1128 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -345,6 +345,8 @@ func (p Jira) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/slack/impl/impl.go b/backend/plugins/slack/impl/impl.go index e526d0642ab..1b8e4094021 100644 --- a/backend/plugins/slack/impl/impl.go +++ b/backend/plugins/slack/impl/impl.go @@ -151,6 +151,8 @@ func (p Slack) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go index a6c8dbbabda..a364435b137 100644 --- a/backend/plugins/sonarqube/impl/impl.go +++ b/backend/plugins/sonarqube/impl/impl.go @@ -217,6 +217,8 @@ func (p Sonarqube) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go index 231e2fd4a03..0cd8236012c 100644 --- a/backend/plugins/tapd/impl/impl.go +++ b/backend/plugins/tapd/impl/impl.go @@ -315,6 +315,8 @@ func (p Tapd) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/teambition/impl/impl.go b/backend/plugins/teambition/impl/impl.go index b203d376513..d31a587368e 100644 --- a/backend/plugins/teambition/impl/impl.go +++ b/backend/plugins/teambition/impl/impl.go @@ -189,6 +189,8 @@ func (p Teambition) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/trello/impl/impl.go b/backend/plugins/trello/impl/impl.go index 96d166a98b5..35adaa2c158 100644 --- a/backend/plugins/trello/impl/impl.go +++ b/backend/plugins/trello/impl/impl.go @@ -208,6 +208,8 @@ func (p Trello) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil } diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go index fc674112e9e..4317fa57493 100644 --- a/backend/plugins/zentao/impl/impl.go +++ b/backend/plugins/zentao/impl/impl.go @@ -305,6 +305,8 @@ func (p Zentao) Close(taskCtx plugin.TaskContext) errors.Error { if !ok { return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) } - data.ApiClient.Release() + if data != nil && data.ApiClient != nil { + data.ApiClient.Release() + } return nil }