From f6fc446732c997a6e5e4a82938763e9603e79b85 Mon Sep 17 00:00:00 2001 From: Gareth Anderson Date: Mon, 9 Sep 2024 13:16:40 +1000 Subject: [PATCH] Update and rename savedsearches.conf to splunkd.log --- default/{savedsearches.conf => splunkd.log} | 34 ++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) rename default/{savedsearches.conf => splunkd.log} (99%) diff --git a/default/savedsearches.conf b/default/splunkd.log similarity index 99% rename from default/savedsearches.conf rename to default/splunkd.log index 339e565..cbf48ee 100644 --- a/default/savedsearches.conf +++ b/default/splunkd.log @@ -8193,7 +8193,7 @@ quantity = 0 relation = greater than request.ui_dispatch_app = SplunkAdmins request.ui_dispatch_view = search -search = index=_internal sourcetype=splunkd source=*splunkd.log* `searchheadhosts` "Connect Timeout" OR "Unable to get authentication token" component IN (DistributedPeer, GetRemoteAuthToken) \ +search = index=_internal sourcetype=splunkd `splunkadmins_splunkd_source` `searchheadhosts` "Connect Timeout" OR "Unable to get authentication token" component IN (DistributedPeer, GetRemoteAuthToken) \ | search ```Exclude time periods where shutdowns were occurring including 10 minutes after shutdown to handle any reboot time``` NOT \ [ `splunkadmins_shutdown_time(indexerhosts,60,600)`] \ | bin _time span=5m \ @@ -8704,3 +8704,35 @@ search = index=_configtracker host=example \ | transaction maxspan=5s data.path stanza name \ | fields - _raw field_match_sum linecount closed_txn duration \ | where NOT new_value=old_value + +[IndexerLevel - indexes with unflushed buckets] +action.keyindicator.invert = 0 +alert.suppress = 0 +alert.track = 1 +alert.digest_mode = 1 +alert.severity = 4 +alert_condition = where statusCode>500 AND count>5 +counttype = number of events +cron_schedule = 4 * * * * +description = Chance the alert requires action? Moderate. The first setting to check is the splitByIndexKeys setting on the index, such as, metric.splitByIndexKeys = metric_name\ +This can result in an issue when there are many unique metric names\ +\ +Example message:\ +09-09-2024 01:27:21.990 +0000 INFO HotDBManager [41851 indexerPipe] - idx=index-name Flushing bucket id=42. maxHotOpen=10, have count=11 unflushed buckets +dispatch.earliest_time = -60m@m +dispatch.latest_time = now +display.events.fields = ["host","source","sourcetype"] +display.general.type = statistics +display.page.search.tab = statistics +display.visualizations.charting.chart = area +enableSched = 1 +quantity = 0 +relation = greater than +request.ui_dispatch_app = SplunkAdmins +request.ui_dispatch_view = search +search = ``` unflushed buckets may relate to excessive bucket rolling. If using a setting similar to metric.splitByIndexKeys then this may require a review to prevent performance issues```\ +index=_internal sourcetype=splunkd bucket `splunkadmins_splunkd_source` "unflushed buckets" `indexerhosts` \ +| eval indexer_cluster=`indexer_cluster_name(splunk_server)`\ +| stats count, min(_time) AS firstseen, max(_time) AS lastseen by idx, indexer_cluster\ +| eval firstseen=strftime(firstseen, "%+"), lastseen=strftime(lastseen, "%+") +disabled = 1