From b7799393cfbcf85d98e95ae53dad1b833ccd14c4 Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 10:29:53 +0200 Subject: [PATCH 1/8] Add debug --- .goreleaser.yaml | 4 ++-- cmd/mirror.go | 36 ++++++++++++++++++------------------ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index c06d34c..9590256 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -10,10 +10,10 @@ builds: - CGO_ENABLED=0 goos: - linux - - darwin + # - darwin goarch: - amd64 - - arm64 + # - arm64 ldflags: - -s -w - -X go.szostok.io/version.version={{.Version}} diff --git a/cmd/mirror.go b/cmd/mirror.go index 45f52ef..83ff1b2 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -88,18 +88,18 @@ func ValidateRepositories(repositories []RepositoryPair) { } } -func ListRemote(remote *git.Remote, listOptions *git.ListOptions) ([]*gitplumbing.Reference, error) { +func ListRemote(remote *git.Remote, listOptions *git.ListOptions, repository string) ([]*gitplumbing.Reference, error) { refList, err := remote.List(listOptions) if err == gittransport.ErrAuthenticationRequired { return nil, backoff.Permanent(err) } else if err != nil { - log.Warn("Retrying listing remote...") + log.Warn("[", repository, "] Retrying listing remote because the following error occurred: ", err) } return refList, err } // GetBranchesAndTagsFromRemote returns list of branches and tags present in remoteName of repository. -func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listOptions *git.ListOptions) ([]string, []string, error) { +func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listOptions *git.ListOptions, sourceRepository string) ([]string, []string, error) { var branchList []string var tagList []string var err error @@ -112,7 +112,7 @@ func GetBranchesAndTagsFromRemote(repository *git.Repository, remoteName string, listRemoteBackoff := backoff.NewExponentialBackOff() listRemoteBackoff.MaxElapsedTime = time.Minute refList, err := backoff.RetryWithData( - func() ([]*gitplumbing.Reference, error) { return ListRemote(remote, listOptions) }, + func() ([]*gitplumbing.Reference, error) { return ListRemote(remote, listOptions, sourceRepository) }, listRemoteBackoff, ) if err != nil { @@ -229,32 +229,32 @@ func GetDestinationAuth(destAuth Authentication) *githttp.BasicAuth { } // GitPlainClone clones git repository and is retried in case of error. -func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions) (*git.Repository, error) { +func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions, repositoryName string) (*git.Repository, error) { repository, err := git.PlainClone(gitDirectory, false, cloneOptions) if err == gittransport.ErrAuthenticationRequired { // Terminate backoff. return nil, backoff.Permanent(err) } else if err != nil { - log.Warn("Retrying cloning repository...") + log.Warn("[", repositoryName, "] Retrying cloning repository because the following error occurred: ", err) } return repository, err } // GitFetchBranches fetches all branches and is retried in case of error. -func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication) error { +func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication, repositoryName string) error { gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) err := sourceRemote.Fetch(gitFetchOptions) if err == gittransport.ErrAuthenticationRequired { // Terminate backoff. return backoff.Permanent(err) } else if err != nil { - log.Warn("Retrying fetching branches...") + log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) } return err } // PushRefs pushes refs defined in refSpecString to destination remote and is retried in case of error. -func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString string) error { +func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString string, repositoryName string) error { err := repository.Push(&git.PushOptions{ RemoteName: "destination", RefSpecs: []gitconfig.RefSpec{gitconfig.RefSpec(refSpecString)}, @@ -264,7 +264,7 @@ func PushRefs(repository *git.Repository, auth *githttp.BasicAuth, refSpecString // Terminate backoff. return backoff.Permanent(err) } else if err != nil { - log.Warn("Retrying pushing refs...") + log.Warn("[", repositoryName, "] Retrying pushing refs because the following error occurred: ", err) } return err } @@ -283,7 +283,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so cloneBackoff := backoff.NewExponentialBackOff() cloneBackoff.MaxElapsedTime = 2 * time.Minute repository, err := backoff.RetryWithData( - func() (*git.Repository, error) { return GitPlainClone(gitDirectory, gitCloneOptions) }, + func() (*git.Repository, error) { return GitPlainClone(gitDirectory, gitCloneOptions, source) }, cloneBackoff, ) if err != nil { @@ -293,7 +293,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so } gitListOptions := GetListOptions(sourceAuthentication) - sourceBranchList, sourceTagList, err := GetBranchesAndTagsFromRemote(repository, "origin", gitListOptions) + sourceBranchList, sourceTagList, err := GetBranchesAndTagsFromRemote(repository, "origin", gitListOptions, source) if err != nil { ProcessError(err, "getting branches and tags from ", source, &allErrors) messages <- MirrorStatus{allErrors, time.Now(), 0, 0} @@ -313,7 +313,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so fetchBranchesBackoff := backoff.NewExponentialBackOff() fetchBranchesBackoff.MaxElapsedTime = time.Minute err = backoff.Retry( - func() error { return GitFetchBranches(sourceRemote, sourceAuthentication) }, + func() error { return GitFetchBranches(sourceRemote, sourceAuthentication, source) }, fetchBranchesBackoff, ) if err != nil { @@ -337,7 +337,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so destinationAuth := GetDestinationAuth(destinationAuthentication) - destinationBranchList, destinationTagList, err := GetBranchesAndTagsFromRemote(repository, "destination", &git.ListOptions{Auth: destinationAuth}) + destinationBranchList, destinationTagList, err := GetBranchesAndTagsFromRemote(repository, "destination", &git.ListOptions{Auth: destinationAuth}, destination) if err != nil { ProcessError(err, "getting branches and tags from ", destination, &allErrors) } @@ -351,7 +351,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so pushBranchesBackoff.MaxElapsedTime = 2 * time.Minute err = backoff.Retry( func() error { - return PushRefs(repository, destinationAuth, "+"+refBranchPrefix+branch+":"+refBranchPrefix+branch) + return PushRefs(repository, destinationAuth, "+"+refBranchPrefix+branch+":"+refBranchPrefix+branch, destination) }, pushBranchesBackoff, ) @@ -365,7 +365,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so removeBranchesBackoff := backoff.NewExponentialBackOff() removeBranchesBackoff.MaxElapsedTime = time.Minute err = backoff.Retry( - func() error { return PushRefs(repository, destinationAuth, ":"+refBranchPrefix+branch) }, + func() error { return PushRefs(repository, destinationAuth, ":"+refBranchPrefix+branch, destination) }, removeBranchesBackoff, ) ProcessError(err, "removing branch "+branch+" from ", destination, &allErrors) @@ -376,7 +376,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so pushTagsBackoff := backoff.NewExponentialBackOff() pushTagsBackoff.MaxElapsedTime = time.Minute err = backoff.Retry( - func() error { return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*") }, + func() error { return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*", destination) }, pushTagsBackoff, ) ProcessError(err, "pushing all tags to ", destination, &allErrors) @@ -388,7 +388,7 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so removeTagsBackoff := backoff.NewExponentialBackOff() removeTagsBackoff.MaxElapsedTime = time.Minute err = backoff.Retry( - func() error { return PushRefs(repository, destinationAuth, ":"+refTagPrefix+tag) }, + func() error { return PushRefs(repository, destinationAuth, ":"+refTagPrefix+tag, destination) }, removeTagsBackoff, ) ProcessError(err, "removing tag "+tag+" from ", destination, &allErrors) From 3e4dd432f46e6629ee5306c3bd5a4e036a3f15ac Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 12:42:54 +0200 Subject: [PATCH 2/8] Update --- cmd/mirror.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index 83ff1b2..1c3e6b4 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -244,8 +244,9 @@ func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions, reposito func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication, repositoryName string) error { gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) err := sourceRemote.Fetch(gitFetchOptions) - if err == gittransport.ErrAuthenticationRequired { - // Terminate backoff. + if err == gittransport.ErrAuthenticationRequired || git.NoErrAlreadyUpToDate { + // Terminate backoff in case authentication is required or the branch is already up-to-date. + // The second case can occur if source or destination repository has only one branch. return backoff.Permanent(err) } else if err != nil { log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) From 527a61e30cf47cf5eedece42324146660d05efd2 Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 12:44:45 +0200 Subject: [PATCH 3/8] Update --- cmd/mirror.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index 1c3e6b4..b65d3c0 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -244,7 +244,7 @@ func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions, reposito func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication, repositoryName string) error { gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) err := sourceRemote.Fetch(gitFetchOptions) - if err == gittransport.ErrAuthenticationRequired || git.NoErrAlreadyUpToDate { + if err == gittransport.ErrAuthenticationRequired || err == git.NoErrAlreadyUpToDate { // Terminate backoff in case authentication is required or the branch is already up-to-date. // The second case can occur if source or destination repository has only one branch. return backoff.Permanent(err) From 218f88db95e529b96a37f632a68dd4d98a5d2c96 Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 12:54:17 +0200 Subject: [PATCH 4/8] Update --- cmd/mirror.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index b65d3c0..92778b7 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -244,10 +244,12 @@ func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions, reposito func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication, repositoryName string) error { gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) err := sourceRemote.Fetch(gitFetchOptions) - if err == gittransport.ErrAuthenticationRequired || err == git.NoErrAlreadyUpToDate { - // Terminate backoff in case authentication is required or the branch is already up-to-date. - // The second case can occur if source or destination repository has only one branch. + if err == gittransport.ErrAuthenticationRequired { return backoff.Permanent(err) + } else if err == git.NoErrAlreadyUpToDate { + // Terminate backoff with no error in case the branch is already up-to-date. + // This can occur if source or destination repository has only one branch. + return nil } else if err != nil { log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) } From 168e1b2e40f50be3a6af248d46218acb181895ec Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 13:12:16 +0200 Subject: [PATCH 5/8] Update --- .goreleaser.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 9590256..c06d34c 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -10,10 +10,10 @@ builds: - CGO_ENABLED=0 goos: - linux - # - darwin + - darwin goarch: - amd64 - # - arm64 + - arm64 ldflags: - -s -w - -X go.szostok.io/version.version={{.Version}} From 136d94dcbc4b9e49e88f0ae87120953df49635ff Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 13:23:01 +0200 Subject: [PATCH 6/8] Lint --- cmd/mirror.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index 92778b7..65fdc89 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -244,16 +244,19 @@ func GitPlainClone(gitDirectory string, cloneOptions *git.CloneOptions, reposito func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authentication, repositoryName string) error { gitFetchOptions := GetFetchOptions("refs/heads/*:refs/heads/*", sourceAuthentication) err := sourceRemote.Fetch(gitFetchOptions) - if err == gittransport.ErrAuthenticationRequired { + switch err { + case gittransport.ErrAuthenticationRequired: + log.Error("[", repositoryName, "] Authentication required.") return backoff.Permanent(err) - } else if err == git.NoErrAlreadyUpToDate { + case git.NoErrAlreadyUpToDate: // Terminate backoff with no error in case the branch is already up-to-date. // This can occur if source or destination repository has only one branch. + log.Info("[", repositoryName, "] Repository up-to-date.") return nil - } else if err != nil { + default: log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) + return err } - return err } // PushRefs pushes refs defined in refSpecString to destination remote and is retried in case of error. @@ -379,7 +382,9 @@ func MirrorRepository(messages chan MirrorStatus, source, destination string, so pushTagsBackoff := backoff.NewExponentialBackOff() pushTagsBackoff.MaxElapsedTime = time.Minute err = backoff.Retry( - func() error { return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*", destination) }, + func() error { + return PushRefs(repository, destinationAuth, "+"+refTagPrefix+"*:"+refTagPrefix+"*", destination) + }, pushTagsBackoff, ) ProcessError(err, "pushing all tags to ", destination, &allErrors) From a4be1f65ad200a4d0d2c3944699b1bd043f44ef2 Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 13:34:58 +0200 Subject: [PATCH 7/8] Fix linter settings --- .golangci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 5319a28..2dbfa22 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -22,8 +22,8 @@ linters-settings: # default is false: such cases aren't reported by default. check-blank: true govet: - # report about shadowed variables - check-shadowing: true + enable: + - shadow gocyclo: # minimal code complexity to report, 30 by default min-complexity: 15 From 35b802d181b2f86194f0cd98caa615288aa0afa3 Mon Sep 17 00:00:00 2001 From: Franciszek Walkowiak Date: Wed, 24 Apr 2024 13:45:14 +0200 Subject: [PATCH 8/8] Update --- cmd/mirror.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/mirror.go b/cmd/mirror.go index 65fdc89..3e28418 100644 --- a/cmd/mirror.go +++ b/cmd/mirror.go @@ -254,7 +254,9 @@ func GitFetchBranches(sourceRemote *git.Remote, sourceAuthentication Authenticat log.Info("[", repositoryName, "] Repository up-to-date.") return nil default: - log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) + if err != nil { + log.Warn("[", repositoryName, "] Retrying fetching branches because the following error occurred: ", err) + } return err } }