From c6495ce1b583f92367f3c016622045db667b2c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Tue, 5 Jun 2018 03:33:43 +0300 Subject: [PATCH 1/7] Add push webhook support for mirrored repositories --- models/action.go | 67 ++++++++++++++ models/repo_mirror.go | 144 +++++++++++++++++++++++++++-- vendor/code.gitea.io/git/commit.go | 16 ++++ 3 files changed, 217 insertions(+), 10 deletions(-) diff --git a/models/action.go b/models/action.go index c3ed9c7c024e1..e850aea17fd29 100644 --- a/models/action.go +++ b/models/action.go @@ -47,6 +47,9 @@ const ( ActionReopenPullRequest // 15 ActionDeleteTag // 16 ActionDeleteBranch // 17 + ActionMirrorSyncPush // 20 + ActionMirrorSyncCreate // 21 + ActionMirrorSyncDelete // 22 ) var ( @@ -726,6 +729,70 @@ func MergePullRequestAction(actUser *User, repo *Repository, pull *Issue) error return mergePullRequestAction(x, actUser, repo, pull) } +func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName string, data []byte) error { + if err := notifyWatchers(e, &Action{ + ActUserID: repo.OwnerID, + ActUser: repo.MustOwner(), + OpType: opType, + RepoID: repo.ID, + Repo: repo, + IsPrivate: repo.IsPrivate, + RefName: refName, + Content: string(data), + }); err != nil { + return fmt.Errorf("notifyWatchers: %v", err) + } + return nil +} + +type MirrorSyncPushActionOptions struct { + RefName string + OldCommitID string + NewCommitID string + Commits *PushCommits +} + +// MirrorSyncPushAction adds new action for mirror synchronization of pushed commits. +func MirrorSyncPushAction(repo *Repository, opts MirrorSyncPushActionOptions) error { + if len(opts.Commits.Commits) > setting.UI.FeedMaxCommitNum { + opts.Commits.Commits = opts.Commits.Commits[:setting.UI.FeedMaxCommitNum] + } + + apiCommits := opts.Commits.ToAPIPayloadCommits(repo.HTMLURL()) + + opts.Commits.CompareURL = repo.ComposeCompareURL(opts.OldCommitID, opts.NewCommitID) + apiPusher := repo.MustOwner().APIFormat() + if err := PrepareWebhooks(repo, HookEventPush, &api.PushPayload{ + Ref: opts.RefName, + Before: opts.OldCommitID, + After: opts.NewCommitID, + CompareURL: setting.AppURL + opts.Commits.CompareURL, + Commits: apiCommits, + Repo: repo.APIFormat(AccessModeOwner), + Pusher: apiPusher, + Sender: apiPusher, + }); err != nil { + return fmt.Errorf("PrepareWebhooks: %v", err) + } + + data, err := json.Marshal(opts.Commits) + if err != nil { + return err + } + + return mirrorSyncAction(x, ActionMirrorSyncPush, repo, opts.RefName, data) +} + +// MirrorSyncCreateAction adds new action for mirror synchronization of new reference. +func MirrorSyncCreateAction(repo *Repository, refName string) error { + return mirrorSyncAction(x, ActionMirrorSyncCreate, repo, refName, nil) +} + +// MirrorSyncDeleteAction adds new action for mirror synchronization of delete reference. +func MirrorSyncDeleteAction(repo *Repository, refName string) error { + return mirrorSyncAction(x, ActionMirrorSyncDelete, repo, refName, nil) +} + // GetFeedsOptions options for retrieving feeds type GetFeedsOptions struct { RequestedUser *User diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 97fe2406c65ff..b031d40e716eb 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -6,6 +6,7 @@ package models import ( "fmt" + "strings" "time" "code.gitea.io/git" @@ -118,8 +119,68 @@ func (m *Mirror) SaveAddress(addr string) error { return cfg.SaveToIndent(configPath, "\t") } +// gitShortEmptySha Git short empty SHA +const gitShortEmptySha = "0000000" + +// mirrorSyncResult contains information of a updated reference. +// If the oldCommitID is "0000000", it means a new reference, the value of newCommitID is empty. +// If the newCommitID is "0000000", it means the reference is deleted, the value of oldCommitID is empty. +type mirrorSyncResult struct { + refName string + oldCommitID string + newCommitID string +} + +// parseRemoteUpdateOutput detects create, update and delete operations of references from upstream. +func parseRemoteUpdateOutput(output string) []*mirrorSyncResult { + results := make([]*mirrorSyncResult, 0, 3) + lines := strings.Split(output, "\n") + for i := range lines { + // Make sure reference name is presented before continue + idx := strings.Index(lines[i], "-> ") + if idx == -1 { + continue + } + + refName := lines[i][idx+3:] + + switch { + case strings.HasPrefix(lines[i], " * "): // New reference + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " - "): // Delete reference + results = append(results, &mirrorSyncResult{ + refName: refName, + newCommitID: gitShortEmptySha, + }) + case strings.HasPrefix(lines[i], " "): // New commits of a reference + delimIdx := strings.Index(lines[i][3:], " ") + if delimIdx == -1 { + log.Error(2, "SHA delimiter not found: %q", lines[i]) + continue + } + shas := strings.Split(lines[i][3:delimIdx+3], "..") + if len(shas) != 2 { + log.Error(2, "Expect two SHAs but not what found: %q", lines[i]) + continue + } + results = append(results, &mirrorSyncResult{ + refName: refName, + oldCommitID: shas[0], + newCommitID: shas[1], + }) + + default: + log.Warn("parseRemoteUpdateOutput: unexpected update line %q", lines[i]) + } + } + return results +} + // runSync returns true if sync finished without error. -func (m *Mirror) runSync() bool { +func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) { repoPath := m.Repo.RepoPath() wikiPath := m.Repo.WikiPath() timeout := time.Duration(setting.Git.Timeout.Mirror) * time.Second @@ -129,28 +190,30 @@ func (m *Mirror) runSync() bool { gitArgs = append(gitArgs, "--prune") } - if _, stderr, err := process.GetManager().ExecDir( + _, stderr, err := process.GetManager().ExecDir( timeout, repoPath, fmt.Sprintf("Mirror.runSync: %s", repoPath), - "git", gitArgs...); err != nil { + "git", gitArgs...) + if err != nil { // sanitize the output, since it may contain the remote address, which may // contain a password message, err := sanitizeOutput(stderr, repoPath) if err != nil { log.Error(4, "sanitizeOutput: %v", err) - return false + return nil, false } desc := fmt.Sprintf("Failed to update mirror repository '%s': %s", repoPath, message) log.Error(4, desc) if err = CreateRepositoryNotice(desc); err != nil { log.Error(4, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } + output := stderr gitRepo, err := git.OpenRepository(repoPath) if err != nil { log.Error(4, "OpenRepository: %v", err) - return false + return nil, false } if err = SyncReleasesWithTags(m.Repo, gitRepo); err != nil { log.Error(4, "Failed to synchronize tags to releases for repository: %v", err) @@ -169,19 +232,19 @@ func (m *Mirror) runSync() bool { message, err := sanitizeOutput(stderr, wikiPath) if err != nil { log.Error(4, "sanitizeOutput: %v", err) - return false + return nil, false } desc := fmt.Sprintf("Failed to update mirror wiki repository '%s': %s", wikiPath, message) log.Error(4, desc) if err = CreateRepositoryNotice(desc); err != nil { log.Error(4, "CreateRepositoryNotice: %v", err) } - return false + return nil, false } } m.UpdatedUnix = util.TimeStampNow() - return true + return parseRemoteUpdateOutput(output), true } func getMirrorByRepoID(e Engine, repoID int64) (*Mirror, error) { @@ -257,7 +320,8 @@ func SyncMirrors() { continue } - if !m.runSync() { + results, ok := m.runSync() + if !ok { continue } @@ -267,6 +331,66 @@ func SyncMirrors() { continue } + var gitRepo *git.Repository + if len(results) == 0 { + log.Trace("SyncMirrors [repo_id: %d]: no commits fetched", m.RepoID) + } else { + gitRepo, err = git.OpenRepository(m.Repo.RepoPath()) + if err != nil { + log.Error(2, "OpenRepository [%d]: %v", m.RepoID, err) + continue + } + } + + for _, result := range results { + // Discard GitHub pull requests, i.e. refs/pull/* + if strings.HasPrefix(result.refName, "refs/pull/") { + continue + } + + // Create reference + if result.oldCommitID == gitShortEmptySha { + if err = MirrorSyncCreateAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncCreateAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Delete reference + if result.newCommitID == gitShortEmptySha { + if err = MirrorSyncDeleteAction(m.Repo, result.refName); err != nil { + log.Error(2, "MirrorSyncDeleteAction [repo_id: %d]: %v", m.RepoID, err) + } + continue + } + + // Push commits + oldCommitID, err := git.GetFullCommitID(gitRepo.Path, result.oldCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + newCommitID, err := git.GetFullCommitID(gitRepo.Path, result.newCommitID) + if err != nil { + log.Error(2, "GetFullCommitID [%d]: %v", m.RepoID, err) + continue + } + commits, err := gitRepo.CommitsBetweenIDs(newCommitID, oldCommitID) + if err != nil { + log.Error(2, "CommitsBetweenIDs [repo_id: %d, new_commit_id: %s, old_commit_id: %s]: %v", m.RepoID, newCommitID, oldCommitID, err) + continue + } + if err = MirrorSyncPushAction(m.Repo, MirrorSyncPushActionOptions{ + RefName: result.refName, + OldCommitID: oldCommitID, + NewCommitID: newCommitID, + Commits: ListToPushCommits(commits), + }); err != nil { + log.Error(2, "MirrorSyncPushAction [repo_id: %d]: %v", m.RepoID, err) + continue + } + } + // Get latest commit date and update to current repository updated time commitDate, err := git.GetLatestCommitTime(m.Repo.RepoPath()) if err != nil { diff --git a/vendor/code.gitea.io/git/commit.go b/vendor/code.gitea.io/git/commit.go index 299a2381b65b6..4e91d3c29e7ba 100644 --- a/vendor/code.gitea.io/git/commit.go +++ b/vendor/code.gitea.io/git/commit.go @@ -274,3 +274,19 @@ func (c *Commit) GetSubModule(entryname string) (*SubModule, error) { } return nil, nil } + +// GetFullCommitID returns full length (40) of commit ID by given short SHA in a repository. +func GetFullCommitID(repoPath, shortID string) (string, error) { + if len(shortID) >= 40 { + return shortID, nil + } + + commitID, err := NewCommand("rev-parse", shortID).RunInDir(repoPath) + if err != nil { + if strings.Contains(err.Error(), "exit status 128") { + return "", ErrNotExist{shortID, ""} + } + return "", err + } + return strings.TrimSpace(commitID), nil +} From 22271a827077aeeb93b0404a8c0853972fca6382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Tue, 5 Jun 2018 03:42:47 +0300 Subject: [PATCH 2/7] Fix action icon for mirror actions --- modules/templates/helper.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/templates/helper.go b/modules/templates/helper.go index bf5c0130b643b..8261efdc3b5b3 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -345,7 +345,8 @@ func ActionIcon(opType models.ActionType) string { switch opType { case models.ActionCreateRepo, models.ActionTransferRepo: return "repo" - case models.ActionCommitRepo, models.ActionPushTag, models.ActionDeleteTag, models.ActionDeleteBranch: + case models.ActionCommitRepo, models.ActionPushTag, models.ActionDeleteTag, models.ActionDeleteBranch, + models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete: return "git-commit" case models.ActionCreateIssue: return "issue-opened" From f9f9c20810f2b72c7d249cb7fcac5f8249e8be61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Mon, 11 Jun 2018 10:19:52 +0300 Subject: [PATCH 3/7] Fix mirror sync event display in dashboard --- models/action.go | 6 +++--- modules/templates/helper.go | 5 +++-- options/locale/locale_en-US.ini | 3 +++ templates/user/dashboard/feeds.tmpl | 11 +++++++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/models/action.go b/models/action.go index e850aea17fd29..52a0365f4ec40 100644 --- a/models/action.go +++ b/models/action.go @@ -47,9 +47,9 @@ const ( ActionReopenPullRequest // 15 ActionDeleteTag // 16 ActionDeleteBranch // 17 - ActionMirrorSyncPush // 20 - ActionMirrorSyncCreate // 21 - ActionMirrorSyncDelete // 22 + ActionMirrorSyncPush // 18 + ActionMirrorSyncCreate // 19 + ActionMirrorSyncDelete // 20 ) var ( diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 8261efdc3b5b3..031f8125abd4a 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -345,8 +345,7 @@ func ActionIcon(opType models.ActionType) string { switch opType { case models.ActionCreateRepo, models.ActionTransferRepo: return "repo" - case models.ActionCommitRepo, models.ActionPushTag, models.ActionDeleteTag, models.ActionDeleteBranch, - models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete: + case models.ActionCommitRepo, models.ActionPushTag, models.ActionDeleteTag, models.ActionDeleteBranch: return "git-commit" case models.ActionCreateIssue: return "issue-opened" @@ -360,6 +359,8 @@ func ActionIcon(opType models.ActionType) string { return "issue-closed" case models.ActionReopenIssue, models.ActionReopenPullRequest: return "issue-reopened" + case models.ActionMirrorSyncPush, models.ActionMirrorSyncCreate, models.ActionMirrorSyncDelete: + return "repo-clone" default: return "invalid type" } diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 8cf6111c6de24..659256a8b4195 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1595,6 +1595,9 @@ push_tag = pushed tag %[2]s to %[3]s delete_tag = deleted tag %[2]s from %[3]s delete_branch = deleted branch %[2]s from %[3]s compare_commits = Compare %d commits +mirror_sync_push = synced commits to %[3]s at %[4]s from mirror +mirror_sync_create = synced new reference %[2]s to %[3]s from mirror +mirror_sync_delete = synced and deleted reference %[2]s at %[3]s from mirror [tool] ago = %s ago diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index bdd6c34820d04..7707c4a05ceb1 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -5,7 +5,7 @@
-
+

{{.ShortActUserName}} {{if eq .GetOpType 1}} @@ -49,9 +49,16 @@ {{else if eq .GetOpType 17}} {{ $index := index .GetIssueInfos 0}} {{$.i18n.Tr "action.delete_branch" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 18}} + {{ $branchLink := .GetBranch | EscapePound}} + {{$.i18n.Tr "action.mirror_sync_push" .GetRepoLink $branchLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 19}} + {{$.i18n.Tr "action.mirror_sync_create" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} + {{else if eq .GetOpType 20}} + {{$.i18n.Tr "action.mirror_sync_delete" .GetRepoLink .GetBranch .ShortRepoPath | Str2html}} {{end}}

- {{if eq .GetOpType 5}} + {{if or (eq .GetOpType 5) (eq .GetOpType 18)}}
    {{ $push := ActionContent2Commits .}} From 69f0253c375dcc71f963ddb42c55bf08046497e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Mon, 11 Jun 2018 10:37:37 +0300 Subject: [PATCH 4/7] Add missing comment --- models/action.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/action.go b/models/action.go index 52a0365f4ec40..429a98b84d080 100644 --- a/models/action.go +++ b/models/action.go @@ -745,6 +745,7 @@ func mirrorSyncAction(e Engine, opType ActionType, repo *Repository, refName str return nil } +// MirrorSyncPushActionOptions mirror synchronization action options. type MirrorSyncPushActionOptions struct { RefName string OldCommitID string From 17b8d0934077447d5619b50d172b339f1d18182b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauris=20Buk=C5=A1is-Haberkorns?= Date: Tue, 12 Jun 2018 13:33:15 +0300 Subject: [PATCH 5/7] Update go dep --- Gopkg.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 6551354a0948e..e258c215f099f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -5,7 +5,7 @@ branch = "master" name = "code.gitea.io/git" packages = ["."] - revision = "31f4b8e8c805438ac6d8914b38accb1d8aaf695e" + revision = "466ac2480dba1e78ae19d7753620b4205db76325" [[projects]] branch = "master" From 46f9cea0ad28f089b836c2ca5c4206969127d3ee Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Wed, 5 Sep 2018 23:04:51 -0400 Subject: [PATCH 6/7] fix build errors and add copyright --- models/repo_mirror.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/models/repo_mirror.go b/models/repo_mirror.go index 2070763646bf5..447d05530774d 100644 --- a/models/repo_mirror.go +++ b/models/repo_mirror.go @@ -1,4 +1,5 @@ // Copyright 2016 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -247,7 +248,7 @@ func (m *Mirror) runSync() ([]*mirrorSyncResult, bool) { branches, err := m.Repo.GetBranches() if err != nil { log.Error(4, "GetBranches: %v", err) - return false + return nil, false } for i := range branches { From f41d673618f8a569c9c8f70828cb4415178fb5ad Mon Sep 17 00:00:00 2001 From: Matti Ranta Date: Wed, 5 Sep 2018 23:53:26 -0400 Subject: [PATCH 7/7] update deps --- Gopkg.lock | 4 ++-- vendor/code.gitea.io/git/commit.go | 8 ++++++-- vendor/code.gitea.io/git/repo_commit.go | 17 +++++++++++++++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 04c7d18334ae8..7126f73642b2c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,11 +3,11 @@ [[projects]] branch = "master" - digest = "1:42f77a668e3bd06812ef254f334d0d0a62346969fbcd3fa3a613e75067343751" + digest = "1:835585f8450b4ec12252d032b0f13e6571ecf846e49076f69067f2503a7c1e07" name = "code.gitea.io/git" packages = ["."] pruneopts = "NUT" - revision = "31f4b8e8c805438ac6d8914b38accb1d8aaf695e" + revision = "6ef79e80b3b06ca13a1f3a7b940903ebc73b44cb" [[projects]] branch = "master" diff --git a/vendor/code.gitea.io/git/commit.go b/vendor/code.gitea.io/git/commit.go index 4e91d3c29e7ba..5e8c91d30343b 100644 --- a/vendor/code.gitea.io/git/commit.go +++ b/vendor/code.gitea.io/git/commit.go @@ -34,14 +34,18 @@ type CommitGPGSignature struct { } // similar to https://github.com/git/git/blob/3bc53220cb2dcf709f7a027a3f526befd021d858/commit.c#L1128 -func newGPGSignatureFromCommitline(data []byte, signatureStart int) (*CommitGPGSignature, error) { +func newGPGSignatureFromCommitline(data []byte, signatureStart int, tag bool) (*CommitGPGSignature, error) { sig := new(CommitGPGSignature) signatureEnd := bytes.LastIndex(data, []byte("-----END PGP SIGNATURE-----")) if signatureEnd == -1 { return nil, fmt.Errorf("end of commit signature not found") } sig.Signature = strings.Replace(string(data[signatureStart:signatureEnd+27]), "\n ", "\n", -1) - sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:]) + if tag { + sig.Payload = string(data[:signatureStart-1]) + } else { + sig.Payload = string(data[:signatureStart-8]) + string(data[signatureEnd+27:]) + } return sig, nil } diff --git a/vendor/code.gitea.io/git/repo_commit.go b/vendor/code.gitea.io/git/repo_commit.go index 1acdfffb34738..d5cab8f8736e7 100644 --- a/vendor/code.gitea.io/git/repo_commit.go +++ b/vendor/code.gitea.io/git/repo_commit.go @@ -78,7 +78,7 @@ l: } commit.Committer = sig case "gpgsig": - sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1) + sig, err := newGPGSignatureFromCommitline(data, nextline+spacepos+1, false) if err != nil { return nil, err } @@ -86,7 +86,20 @@ l: } nextline += eol + 1 case eol == 0: - commit.CommitMessage = string(data[nextline+1:]) + cm := string(data[nextline+1:]) + + // Tag GPG signatures are stored below the commit message + sigindex := strings.Index(cm, "-----BEGIN PGP SIGNATURE-----") + if sigindex != -1 { + sig, err := newGPGSignatureFromCommitline(data, (nextline+1)+sigindex, true) + if err == nil && sig != nil { + // remove signature from commit message + cm = cm[:sigindex-1] + commit.Signature = sig + } + } + + commit.CommitMessage = cm break l default: break l