Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add new flags and memcached support to query frontend #166

Merged
merged 3 commits into from
Nov 4, 2020
Merged

Add new flags and memcached support to query frontend #166

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bfd2450
add new flags and memcached support to query frontend
yeya24 Nov 3, 2020
35c7841
remove unused comment
yeya24 Nov 3, 2020
2216a61
add memcached support and new flags to query frontend
yeya24 Nov 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 21 additions & 3 deletions all.jsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,28 @@ local qf = t.queryFrontend(commonConfig {
q.service.metadata.namespace,
q.service.spec.ports[1].port,
],
splitInterval: '12h',
maxRetries: 10,
logQueriesLongerThan: '10s',
splitInterval: '24h',
maxRetries: 5,
logQueriesLongerThan: '5s',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These were changed on purposed to different values than the defaults, to show that overwriting the values actually works, which is kind of the entire point of the all.jsonnet in total 😉 😊
Happy to add this as a comment to the top of the file. Wdyt?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, got it. Thanks for the clarification!
I was wondering why this changed yesterday 🤣 I can update these values back.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that would be great and maybe even change values in all.jsonnet to different ones that the defaults, haha. :) Doesn't need to be too crazy though.

serviceMonitor: true,
queryRangeCache: {
type: 'memcached',
config+: {
// NOTICE: <MEMCACHED_SERCIVE> is a placeholder to generate examples.
// List of memcached addresses, that will get resolved with the DNS service discovery provider.
// For DNS service discovery reference https://thanos.io/service-discovery.md/#dns-service-discovery
addresses: ['dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.%s.svc.cluster.local' % commonConfig.namespace],
},
},
labelsCache: {
type: 'memcached',
config+: {
// NOTICE: <MEMCACHED_SERCIVE> is a placeholder to generate examples.
// List of memcached addresses, that will get resolved with the DNS service discovery provider.
// For DNS service discovery reference https://thanos.io/service-discovery.md/#dns-service-discovery
addresses: ['dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.%s.svc.cluster.local' % commonConfig.namespace],
},
},
});

{ ['thanos-bucket-' + name]: b[name] for name in std.objectFields(b) } +
Expand Down
34 changes: 27 additions & 7 deletions examples/all/manifests/thanos-query-frontend-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,35 @@ spec:
- --query-frontend.compress-responses
- --http-address=0.0.0.0:9090
- --query-frontend.downstream-url=http://thanos-query.thanos.svc.cluster.local.:9090
- --query-range.split-interval=12h
- --query-range.max-retries-per-request=10
- --query-frontend.log-queries-longer-than=10s
- --query-range.split-interval=24h
- --labels.split-interval=24h
- --query-range.max-retries-per-request=5
- --labels.max-retries-per-request=5
- --query-frontend.log-queries-longer-than=5s
- |-
--query-range.response-cache-config="config":
"max_size": "0"
"max_size_items": 2048
"validity": "6h"
"type": "in-memory"
"addresses":
- "dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.thanos.svc.cluster.local"
"dns_provider_update_interval": "10s"
"max_async_buffer_size": 10000
"max_async_concurrency": 20
"max_get_multi_batch_size": 0
"max_get_multi_concurrency": 100
"max_idle_connections": 100
"timeout": "500ms"
"type": "memcached"
- |-
--labels.response-cache-config="config":
"addresses":
- "dnssrv+_client._tcp.<MEMCACHED_SERCIVE>.thanos.svc.cluster.local"
"dns_provider_update_interval": "10s"
"max_async_buffer_size": 10000
"max_async_concurrency": 20
"max_get_multi_batch_size": 0
"max_get_multi_concurrency": 100
"max_idle_connections": 100
"timeout": "500ms"
"type": "memcached"
image: quay.io/thanos/thanos:v0.16.0
livenessProbe:
failureThreshold: 4
Expand Down
61 changes: 50 additions & 11 deletions jsonnet/kube-thanos/kube-thanos-query-frontend.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,37 @@ local defaults = {
splitInterval: '24h',
maxRetries: 5,
logQueriesLongerThan: '0',
fifoCache: {
max_size: '0', // Don't limit maximum item size.
max_size_items: 2048,
validity: '6h',
fifoCache+:: {
config+: {
max_size: '0', // Don't limit maximum item size.
max_size_items: 2048,
validity: '6h',
}
},
queryRangeCache: {},
labelsCache: {},
logLevel: 'info',
resources: {},
serviceMonitor: false,
ports: {
http: 9090,
},

memcachedDefaults+:: {
config+: {
// List of memcached addresses, that will get resolved with the DNS service discovery provider.
// For DNS service discovery reference https://thanos.io/service-discovery.md/#dns-service-discovery
addresses+: error 'must provide memcached addresses',
timeout: '500ms',
max_idle_connections: 100,
max_async_concurrency: 20,
max_async_buffer_size: 10000,
max_get_multi_concurrency: 100,
max_get_multi_batch_size: 0,
dns_provider_update_interval: '10s',
},
},

commonLabels:: {
'app.kubernetes.io/name': 'thanos-query-frontend',
'app.kubernetes.io/instance': defaults.name,
Expand All @@ -42,12 +61,25 @@ function(params) {
local tqf = self,

// Combine the defaults and the passed params to make the component's config.
config:: defaults + params,
config:: defaults + params + {
queryRangeCache+:
if std.objectHas(params, 'queryRangeCache') && params.queryRangeCache.type == 'memcached' then
defaults.memcachedDefaults + params.queryRangeCache
else if std.objectHas(params, 'queryRangeCache') && params.queryRangeCache.type == 'in-memory' then
defaults.fifoCache + params.queryRangeCache
else {},
labelsCache+:
if std.objectHas(params, 'labelsCache') && params.labelsCache.type == 'memcached' then
defaults.memcachedDefaults + params.labelsCache
else if std.objectHas(params, 'labelsCache') && params.labelsCache.type == 'in-memory' then
defaults.fifoCache + params.labelsCache
else {},
},
// Safety checks for combined config of defaults and params
assert std.isNumber(tqf.config.replicas) && tqf.config.replicas >= 0 : 'thanos query frontend replicas has to be number >= 0',
assert std.isObject(tqf.config.resources),
assert std.isBoolean(tqf.config.serviceMonitor),
assert std.isNumber(tqf.config.maxRetries),
assert std.isNumber(tqf.config.maxRetries) && tqf.config.maxRetries >= 0 : 'thanos query frontend maxRetries has to be number >= 0',

service:
{
Expand Down Expand Up @@ -84,14 +116,21 @@ function(params) {
'--http-address=0.0.0.0:%d' % tqf.config.ports.http,
'--query-frontend.downstream-url=%s' % tqf.config.downstreamURL,
'--query-range.split-interval=%s' % tqf.config.splitInterval,
'--labels.split-interval=%s' % tqf.config.splitInterval,
'--query-range.max-retries-per-request=%d' % tqf.config.maxRetries,
'--labels.max-retries-per-request=%d' % tqf.config.maxRetries,
'--query-frontend.log-queries-longer-than=%s' % tqf.config.logQueriesLongerThan,
] + (
if std.length(tqf.config.fifoCache) > 0 then [
'--query-range.response-cache-config=' + std.manifestYamlDoc({
type: 'in-memory',
config: tqf.config.fifoCache,
}),
if std.length(tqf.config.queryRangeCache) > 0 then [
'--query-range.response-cache-config=' + std.manifestYamlDoc(
tqf.config.queryRangeCache
),
] else []
) + (
if std.length(tqf.config.labelsCache) > 0 then [
'--labels.response-cache-config=' + std.manifestYamlDoc(
tqf.config.labelsCache
),
] else []
),
ports: [
Expand Down