-
Notifications
You must be signed in to change notification settings - Fork 11
/
config.jobqueue.wikimedia.yaml
153 lines (153 loc) · 4.79 KB
/
config.jobqueue.wikimedia.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
spec: &spec
x-sub-request-filters:
- type: default
name: http
options:
allow:
- pattern: /^https?:\/\//
forward_headers:
user-agent: true
title: The Change Propagation root
paths:
/sys/dedupe:
x-modules:
- path: sys/deduplicator.js
options:
redis_prefix: CPJQ
redis:
host: localhost
port: 6379
/sys/queue:
x-modules:
- path: sys/kafka.js
options:
metadata_broker_list: 127.0.0.1:9092
dc_name: test_dc
startup_delay: 0
consumer:
# These options should not be copied to puppet config.
# We're using this config for testing, so need to configure
# for minimal latency
fetch.wait.max.ms: "1"
fetch.min.bytes: "1"
queue.buffering.max.ms: "1"
producer:
queue.buffering.max.messages: "10"
compression.codec: snappy
# Redis-mock does not support evalsha that rate limiting depend on.
disable_ratelimit: '{env(MOCK_SERVICES)}'
concurrency: 250
reenqueue_delay: 5
templates:
normal_job_rule:
topics:
- 'mediawiki.job.updateBetaFeaturesUserCounts'
- 'mediawiki.job.cdnPurge'
exec:
method: post
uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
headers:
content-type: application/json
host: '{{message.meta.domain}}'
body: '{{globals.message}}'
htmlCacheUpdate_partitioner:
topic: 'mediawiki.job.htmlCacheUpdate'
exec:
method: 'post'
uri: '/sys/partition/'
headers:
content-type: application/json
body: '{{globals.message}}'
htmlCacheUpdate:
topic: 'cpjobqueue.partitioned.mediawiki.job.htmlCacheUpdate'
exec:
method: post
uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
headers:
content-type: application/json
host: '{{message.meta.domain}}'
body: '{{globals.message}}'
refreshLinks_partitioner:
topic: 'mediawiki.job.refreshLinks'
exec:
method: 'post'
uri: '/sys/partition/'
headers:
content-type: application/json
body: '{{globals.message}}'
refreshLinks:
topic: 'cpjobqueue.partitioned.mediawiki.job.refreshLinks'
exec:
method: post
uri: 'http://jobrunner.wikipedia.org/wiki/Special:RunSingleJob'
headers:
content-type: application/json
host: '{{message.meta.domain}}'
body: '{{globals.message}}'
/sys/partition:
x-modules:
- path: sys/partitioner.js
options:
templates:
partition_stream: 'cpjobqueue.partitioned.{{message.meta.stream}}'
partition_key: database
partition_default: 2
partition_map:
# s1: enwiki
enwiki: 0
# s2: large wikis
bgwiki: 1
bgwiktionary: 1
cswiki: 1
enwikiquote: 1
enwiktionary: 1
eowiki: 1
fiwiki: 1
idwiki: 1
itwiki: 1
nlwiki: 1
nowiki: 1
plwiki: 1
ptwiki: 1
svwiki: 1
thwiki: 1
trwiki: 1
zhwiki: 1
# s3 (default)
# s4: commons
commonswiki: 3
# s5: dewiki
dewiki: 4
# s6: large wikis
frwiki: 5
jawiki: 5
ruwiki: 5
# s7: large wikis centralauth
eswiki: 6
huwiki: 6
hewiki: 6
ukwiki: 6
frwiktionary: 6
metawiki: 6
arwiki: 6
centralauth: 6
cawiki: 6
viwiki: 6
fawiki: 6
rowiki: 6
kowiki: 6
# s8: wikidata
wikidatawiki: 7
num_workers: 0
logging:
name: cpjobqueue
level: fatal
streams:
- type: stdout
services:
- name: changeprop
module: hyperswitch
conf:
port: 7272
user_agent: SampleChangePropInstance
spec: *spec