Skip to content

Commit

Permalink
Merge remote-tracking branch 'giteaofficial/main'
Browse files Browse the repository at this point in the history
* giteaofficial/main:
  Fix flex layout for repo list icons (go-gitea#21896)
  Add index for access_token (go-gitea#21908)
  Fix vertical align of committer avatar rendered by email address (go-gitea#21884)
  Improve docker rootless documentation (go-gitea#21913)
  Ensure that Webhook tasks are not double delivered (go-gitea#21558)
  Move migration test fixtures to the correct directories (go-gitea#21901)
  feat: add button to quickly clear merge message (go-gitea#21548)
  • Loading branch information
zjjhot committed Nov 24, 2022
2 parents 0906600 + 09b7d81 commit 490381a
Show file tree
Hide file tree
Showing 34 changed files with 166 additions and 44 deletions.
24 changes: 20 additions & 4 deletions docs/content/doc/installation/with-docker-rootless.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,16 @@ the official [install instructions](https://docs.docker.com/compose/install/).

The most simple setup just creates a volume and a network and starts the `gitea/gitea:latest-rootless`
image as a service. Since there is no database available, one can be initialized using SQLite3.
Create a directory for `data` and `config` then paste the following content into a file named `docker-compose.yml`.
Note that the volume should be owned by the user/group with the UID/GID specified in the config file. By default Gitea in docker will use uid:1000 gid:1000. If needed you can set ownership on those folders with the command: `sudo chown 1000:1000 config/ data/`
If you don't give the volume correct permissions, the container may not start.
For a stable release you could use `:latest-rootless`, `:1-rootless` or specify a certain release like `:{{< version >}}-rootless`, but if you'd like to use the latest development version then `:dev-rootless` would be an appropriate tag. If you'd like to run the latest commit from a release branch you can use the `:1.x-dev-rootless` tag, where x is the minor version of Gitea. (e.g. `:1.16-dev-rootless`)

Create a directory for `data` and `config`:

```sh
mkdir -p gitea/{data,config}
cd gitea
touch docker-compose.yml
```

Then paste the following content into a file named `docker-compose.yml`:

```yaml
version: "2"
Expand All @@ -51,6 +57,16 @@ services:
- "2222:2222"
```
Note that the volume should be owned by the user/group with the UID/GID specified in the config file. By default Gitea in docker will use uid:1000 gid:1000. If needed you can set ownership on those folders with the command:
```sh
sudo chown 1000:1000 config/ data/
```

> If you don't give the volume correct permissions, the container may not start.
For a stable release you could use `:latest-rootless`, `:1-rootless` or specify a certain release like `:{{< version >}}-rootless`, but if you'd like to use the latest development version then `:dev-rootless` would be an appropriate tag. If you'd like to run the latest commit from a release branch you can use the `:1.x-dev-rootless` tag, where x is the minor version of Gitea. (e.g. `:1.16-dev-rootless`)

## Custom port

To bind the integrated ssh and the webserver on a different port, adjust
Expand Down
2 changes: 1 addition & 1 deletion models/auth/token.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ type AccessToken struct {
Token string `xorm:"-"`
TokenHash string `xorm:"UNIQUE"` // sha256 of token
TokenSalt string
TokenLastEight string `xorm:"token_last_eight"`
TokenLastEight string `xorm:"INDEX token_last_eight"`

CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
Expand Down
8 changes: 6 additions & 2 deletions models/avatars/avatar.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ import (
"code.gitea.io/gitea/modules/setting"
)

// DefaultAvatarPixelSize is the default size in pixels of a rendered avatar
const DefaultAvatarPixelSize = 28
const (
// DefaultAvatarClass is the default class of a rendered avatar
DefaultAvatarClass = "ui avatar vm"
// DefaultAvatarPixelSize is the default size in pixels of a rendered avatar
DefaultAvatarPixelSize = 28
)

// EmailHash represents a pre-generated hash map (mainly used by LibravatarURL, it queries email server's DNS records)
type EmailHash struct {
Expand Down
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,8 @@ var migrations = []Migration{
NewMigration("Add header_authorization_encrypted column to webhook table", v1_19.AddHeaderAuthorizationEncryptedColWebhook),
// v234 -> v235
NewMigration("Add package cleanup rule table", v1_19.CreatePackageCleanupRuleTable),
// v235 -> v236
NewMigration("Add index for access_token", v1_19.AddIndexForAccessToken),
}

// GetCurrentDBVersion returns the current db version
Expand Down
17 changes: 17 additions & 0 deletions models/migrations/v1_19/v235.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package v1_19 //nolint

import (
"xorm.io/xorm"
)

func AddIndexForAccessToken(x *xorm.Engine) error {
type AccessToken struct {
TokenLastEight string `xorm:"INDEX token_last_eight"`
}

return x.Sync(new(AccessToken))
}
22 changes: 19 additions & 3 deletions models/webhook/hooktask.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,14 +233,30 @@ func ReplayHookTask(ctx context.Context, hookID int64, uuid string) (*HookTask,
return newTask, db.Insert(ctx, newTask)
}

// FindUndeliveredHookTasks represents find the undelivered hook tasks
func FindUndeliveredHookTasks(ctx context.Context) ([]*HookTask, error) {
tasks := make([]*HookTask, 0, 10)
// FindUndeliveredHookTaskIDs will find the next 100 undelivered hook tasks with ID greater than the provided lowerID
func FindUndeliveredHookTaskIDs(ctx context.Context, lowerID int64) ([]int64, error) {
const batchSize = 100

tasks := make([]int64, 0, batchSize)
return tasks, db.GetEngine(ctx).
Select("id").
Table(new(HookTask)).
Where("is_delivered=?", false).
And("id > ?", lowerID).
Asc("id").
Limit(batchSize).
Find(&tasks)
}

func MarkTaskDelivered(ctx context.Context, task *HookTask) (bool, error) {
count, err := db.GetEngine(ctx).ID(task.ID).Where("is_delivered = ?", false).Cols("is_delivered").Update(&HookTask{
ID: task.ID,
IsDelivered: true,
})

return count != 0, err
}

// CleanupHookTaskTable deletes rows from hook_task as needed.
func CleanupHookTaskTable(ctx context.Context, cleanupType HookTaskCleanupType, olderThan time.Duration, numberToKeep int) error {
log.Trace("Doing: CleanupHookTaskTable")
Expand Down
6 changes: 3 additions & 3 deletions modules/templates/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -607,7 +607,7 @@ func AvatarHTML(src string, size int, class, name string) template.HTML {

// Avatar renders user avatars. args: user, size (int), class (string)
func Avatar(item interface{}, others ...interface{}) template.HTML {
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, "ui avatar vm", others...)
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...)

switch t := item.(type) {
case *user_model.User:
Expand Down Expand Up @@ -638,7 +638,7 @@ func AvatarByAction(action *activities_model.Action, others ...interface{}) temp

// RepoAvatar renders repo avatars. args: repo, size(int), class (string)
func RepoAvatar(repo *repo_model.Repository, others ...interface{}) template.HTML {
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, "ui avatar", others...)
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...)

src := repo.RelAvatarLink()
if src != "" {
Expand All @@ -649,7 +649,7 @@ func RepoAvatar(repo *repo_model.Repository, others ...interface{}) template.HTM

// AvatarByEmail renders avatars by email address. args: email, name, size (int), class (string)
func AvatarByEmail(email, name string, others ...interface{}) template.HTML {
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, "ui avatar", others...)
size, class := gitea_html.ParseSizeAndClass(avatars.DefaultAvatarPixelSize, avatars.DefaultAvatarClass, others...)
src := avatars.GenerateEmailAvatarFastLink(email, size*setting.Avatar.RenderedSizeFactor)

if src != "" {
Expand Down
2 changes: 2 additions & 0 deletions options/locale/locale_en-US.ini
Original file line number Diff line number Diff line change
Expand Up @@ -1630,6 +1630,8 @@ pulls.reopened_at = `reopened this pull request <a id="%[1]s" href="#%[1]s">%[2]
pulls.merge_instruction_hint = `You can also view <a class="show-instruction">command line instructions</a>.`
pulls.merge_instruction_step1_desc = From your project repository, check out a new branch and test the changes.
pulls.merge_instruction_step2_desc = Merge the changes and update on Gitea.
pulls.clear_merge_message = Clear merge message
pulls.clear_merge_message_hint = Clearing the merge message will only remove the commit message content and keep generated git trailers such as "Co-Authored-By …".

pulls.auto_merge_button_when_succeed = (When checks succeed)
pulls.auto_merge_when_succeed = Auto merge when all checks succeed
Expand Down
74 changes: 55 additions & 19 deletions services/webhook/deliver.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/hostmatcher"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/process"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/setting"
Expand All @@ -43,7 +44,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
return
}
// There was a panic whilst delivering a hook...
log.Error("PANIC whilst trying to deliver webhook[%d] to %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
log.Error("PANIC whilst trying to deliver webhook task[%d] to webhook %s Panic: %v\nStacktrace: %s", t.ID, w.URL, err, log.Stack(2))
}()

t.IsDelivered = true
Expand All @@ -52,7 +53,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {

switch w.HTTPMethod {
case "":
log.Info("HTTP Method for webhook %d empty, setting to POST as default", t.ID)
log.Info("HTTP Method for webhook %s empty, setting to POST as default", w.URL)
fallthrough
case http.MethodPost:
switch w.ContentType {
Expand All @@ -78,14 +79,14 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
case http.MethodGet:
u, err := url.Parse(w.URL)
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as cannot parse webhook url %s: %w", t.ID, w.URL, err)
}
vals := u.Query()
vals["payload"] = []string{t.PayloadContent}
u.RawQuery = vals.Encode()
req, err = http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as unable to create HTTP request for webhook url %s: %w", t.ID, w.URL, err)
}
case http.MethodPut:
switch w.Type {
Expand All @@ -97,13 +98,13 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
url := fmt.Sprintf("%s/%s", w.URL, url.PathEscape(txnID))
req, err = http.NewRequest("PUT", url, strings.NewReader(t.PayloadContent))
if err != nil {
return err
return fmt.Errorf("unable to deliver webhook task[%d] as cannot create matrix request for webhook url %s: %w", t.ID, w.URL, err)
}
default:
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}
default:
return fmt.Errorf("invalid http method for webhook: [%d] %v", t.ID, w.HTTPMethod)
return fmt.Errorf("invalid http method for webhook task[%d] in webhook %s: %v", t.ID, w.URL, w.HTTPMethod)
}

var signatureSHA1 string
Expand Down Expand Up @@ -159,6 +160,20 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
Headers: map[string]string{},
}

// OK We're now ready to attempt to deliver the task - we must double check that it
// has not been delivered in the meantime
updated, err := webhook_model.MarkTaskDelivered(ctx, t)
if err != nil {
log.Error("MarkTaskDelivered[%d]: %v", t.ID, err)
return fmt.Errorf("unable to mark task[%d] delivered in the db: %w", t.ID, err)
}
if !updated {
// This webhook task has already been attempted to be delivered or is in the process of being delivered
log.Trace("Webhook Task[%d] already delivered", t.ID)
return nil
}

// All code from this point will update the hook task
defer func() {
t.Delivered = time.Now().UnixNano()
if t.IsSucceed {
Expand Down Expand Up @@ -190,13 +205,14 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
}

if !w.IsActive {
log.Trace("Webhook %s in Webhook Task[%d] is not active", w.URL, t.ID)
return nil
}

resp, err := webhookHTTPClient.Do(req.WithContext(ctx))
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("Delivery: %v", err)
return err
return fmt.Errorf("unable to deliver webhook task[%d] in %s due to error in http client: %w", t.ID, w.URL, err)
}
defer resp.Body.Close()

Expand All @@ -210,7 +226,7 @@ func Deliver(ctx context.Context, t *webhook_model.HookTask) error {
p, err := io.ReadAll(resp.Body)
if err != nil {
t.ResponseInfo.Body = fmt.Sprintf("read body: %s", err)
return err
return fmt.Errorf("unable to deliver webhook task[%d] in %s as unable to read response body: %w", t.ID, w.URL, err)
}
t.ResponseInfo.Body = string(p)
return nil
Expand Down Expand Up @@ -272,17 +288,37 @@ func Init() error {
}
go graceful.GetManager().RunWithShutdownFns(hookQueue.Run)

tasks, err := webhook_model.FindUndeliveredHookTasks(graceful.GetManager().HammerContext())
if err != nil {
log.Error("FindUndeliveredHookTasks failed: %v", err)
return err
}
go graceful.GetManager().RunWithShutdownContext(populateWebhookSendingQueue)

return nil
}

func populateWebhookSendingQueue(ctx context.Context) {
ctx, _, finished := process.GetManager().AddContext(ctx, "Webhook: Populate sending queue")
defer finished()

for _, task := range tasks {
if err := enqueueHookTask(task); err != nil {
log.Error("enqueueHookTask failed: %v", err)
lowerID := int64(0)
for {
taskIDs, err := webhook_model.FindUndeliveredHookTaskIDs(ctx, lowerID)
if err != nil {
log.Error("Unable to populate webhook queue as FindUndeliveredHookTaskIDs failed: %v", err)
return
}
if len(taskIDs) == 0 {
return
}
lowerID = taskIDs[len(taskIDs)-1]

for _, taskID := range taskIDs {
select {
case <-ctx.Done():
log.Warn("Shutdown before Webhook Sending queue finishing being populated")
return
default:
}
if err := enqueueHookTask(taskID); err != nil {
log.Error("Unable to push HookTask[%d] to the Webhook Sending queue: %v", taskID, err)
}
}
}

return nil
}
10 changes: 9 additions & 1 deletion services/webhook/deliver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/models/unittest"
webhook_model "code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"

"github.com/stretchr/testify/assert"
)
Expand Down Expand Up @@ -67,8 +68,15 @@ func TestWebhookDeliverAuthorizationHeader(t *testing.T) {
err := hook.SetHeaderAuthorization("Bearer s3cr3t-t0ken")
assert.NoError(t, err)
assert.NoError(t, webhook_model.CreateWebhook(db.DefaultContext, hook))
db.GetEngine(db.DefaultContext).NoAutoTime().DB().Logger.ShowSQL(true)

hookTask := &webhook_model.HookTask{HookID: hook.ID, EventType: webhook_model.HookEventPush}
hookTask := &webhook_model.HookTask{HookID: hook.ID, EventType: webhook_model.HookEventPush, Payloader: &api.PushPayload{}}

hookTask, err = webhook_model.CreateHookTask(db.DefaultContext, hookTask)
assert.NoError(t, err)
if !assert.NotNil(t, hookTask) {
return
}

assert.NoError(t, Deliver(context.Background(), hookTask))
select {
Expand Down
25 changes: 16 additions & 9 deletions services/webhook/webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,19 +116,26 @@ func handle(data ...queue.Data) []queue.Data {
for _, taskID := range data {
task, err := webhook_model.GetHookTaskByID(ctx, taskID.(int64))
if err != nil {
log.Error("GetHookTaskByID failed: %v", err)
} else {
if err := Deliver(ctx, task); err != nil {
log.Error("webhook.Deliver failed: %v", err)
}
log.Error("GetHookTaskByID[%d] failed: %v", taskID.(int64), err)
continue
}

if task.IsDelivered {
// Already delivered in the meantime
log.Trace("Task[%d] has already been delivered", task.ID)
continue
}

if err := Deliver(ctx, task); err != nil {
log.Error("Unable to deliver webhook task[%d]: %v", task.ID, err)
}
}

return nil
}

func enqueueHookTask(task *webhook_model.HookTask) error {
err := hookQueue.PushFunc(task.ID, nil)
func enqueueHookTask(taskID int64) error {
err := hookQueue.Push(taskID)
if err != nil && err != queue.ErrAlreadyInQueue {
return err
}
Expand Down Expand Up @@ -205,7 +212,7 @@ func PrepareWebhook(ctx context.Context, w *webhook_model.Webhook, event webhook
return fmt.Errorf("CreateHookTask: %w", err)
}

return enqueueHookTask(task)
return enqueueHookTask(task.ID)
}

// PrepareWebhooks adds new webhooks to task queue for given payload.
Expand Down Expand Up @@ -265,5 +272,5 @@ func ReplayHookTask(ctx context.Context, w *webhook_model.Webhook, uuid string)
return err
}

return enqueueHookTask(task)
return enqueueHookTask(task.ID)
}
4 changes: 2 additions & 2 deletions templates/explore/repo_list.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@
{{end}}
{{end}}
{{if .IsFork}}
<span class="tooltip" data-content="{{$.locale.Tr "repo.fork"}}" data-position="bottom center">{{svg "octicon-repo-forked"}}</span>
<span class="tooltip df" data-content="{{$.locale.Tr "repo.fork"}}" data-position="bottom center">{{svg "octicon-repo-forked"}}</span>
{{else if .IsMirror}}
<span class="tooltip" data-content="{{$.locale.Tr "mirror"}}" data-position="bottom center">{{svg "octicon-mirror"}}</span>
<span class="tooltip df" data-content="{{$.locale.Tr "mirror"}}" data-position="bottom center">{{svg "octicon-mirror"}}</span>
{{end}}
</div>
</div>
Expand Down
Loading

0 comments on commit 490381a

Please sign in to comment.