Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Calculate MAX_WORKERS default value by CPU number #26177

Merged
merged 3 commits into from
Jul 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions custom/conf/app.example.ini
Original file line number Diff line number Diff line change
Expand Up @@ -1420,8 +1420,8 @@ LEVEL = Info
;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections.
;SET_NAME = "_unique"
;;
;; Dynamically scale the worker pool to at this many workers
;MAX_WORKERS = 10
;; Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.
;MAX_WORKERS = ; (dynamic)

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
Expand Down
2 changes: 1 addition & 1 deletion docs/content/administration/config-cheat-sheet.en-us.md
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ Configuration at `[queue]` will set defaults for queues with overrides for indiv
- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. For `redis-cluster` use `redis+cluster://127.0.0.1:6379/0`. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR`
- `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section.
- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section.
- `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue.
- `MAX_WORKERS`: **(dynamic)**: Maximum number of worker go-routines for the queue. Default value is "CpuNum/2" clipped to between 1 and 10.

Gitea creates the following non-unique queues:

Expand Down
8 changes: 4 additions & 4 deletions modules/queue/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ CONN_STR = redis://
assert.Equal(t, "", q.baseConfig.ConnStr)
assert.Equal(t, "default_queue", q.baseConfig.QueueFullName)
assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName)
assert.Equal(t, 10, q.GetWorkerMaxNumber())
assert.NotZero(t, q.GetWorkerMaxNumber())
assert.Equal(t, 0, q.GetWorkerNumber())
assert.Equal(t, 0, q.GetWorkerActiveNumber())
assert.Equal(t, 0, q.GetQueueItemNumber())
Expand All @@ -75,7 +75,7 @@ BATCH_LENGTH = 22
CONN_STR =
QUEUE_NAME = _q2
SET_NAME = _u2
MAX_WORKERS = 2
MAX_WORKERS = 123
`)

assert.NoError(t, err)
Expand All @@ -89,7 +89,7 @@ MAX_WORKERS = 2
assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr)
assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName)
assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName)
assert.Equal(t, 10, q1.GetWorkerMaxNumber())
assert.NotZero(t, q1.GetWorkerMaxNumber())
assert.Equal(t, 0, q1.GetWorkerNumber())
assert.Equal(t, 0, q1.GetWorkerActiveNumber())
assert.Equal(t, 0, q1.GetQueueItemNumber())
Expand All @@ -105,7 +105,7 @@ MAX_WORKERS = 2
assert.Equal(t, "", q2.baseConfig.ConnStr)
assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName)
assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName)
assert.Equal(t, 2, q2.GetWorkerMaxNumber())
assert.Equal(t, 123, q2.GetWorkerMaxNumber())
assert.Equal(t, 0, q2.GetWorkerNumber())
assert.Equal(t, 0, q2.GetWorkerActiveNumber())
assert.Equal(t, 0, q2.GetQueueItemNumber())
Expand Down
29 changes: 18 additions & 11 deletions modules/setting/queue.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package setting

import (
"path/filepath"
"runtime"

"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
Expand All @@ -25,18 +26,24 @@ type QueueSettings struct {
MaxWorkers int
}

var queueSettingsDefault = QueueSettings{
Type: "level", // dummy, channel, level, redis
Datadir: "queues/common", // relative to AppDataPath
Length: 100, // queue length before a channel queue will block

QueueName: "_queue",
SetName: "_unique",
BatchLength: 20,
MaxWorkers: 10,
}

func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) {
queueSettingsDefault := QueueSettings{
Type: "level", // dummy, channel, level, redis
Datadir: "queues/common", // relative to AppDataPath
Length: 100, // queue length before a channel queue will block

QueueName: "_queue",
SetName: "_unique",
BatchLength: 20,
MaxWorkers: runtime.NumCPU() / 2,
}
if queueSettingsDefault.MaxWorkers < 1 {
queueSettingsDefault.MaxWorkers = 1
}
if queueSettingsDefault.MaxWorkers > 10 {
queueSettingsDefault.MaxWorkers = 10
}

// deep copy default settings
cfg := QueueSettings{}
if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil {
Expand Down