Update and rename savedsearches.conf to splunkd.log

default/savedsearches.conf → default/splunkd.log

@@ -8193,7 +8193,7 @@ quantity = 0
 relation = greater than
 request.ui_dispatch_app = SplunkAdmins
 request.ui_dispatch_view = search 
-search = index=_internal sourcetype=splunkd source=*splunkd.log* `searchheadhosts` "Connect Timeout" OR "Unable to get authentication token" component IN (DistributedPeer, GetRemoteAuthToken) \
+search = index=_internal sourcetype=splunkd `splunkadmins_splunkd_source` `searchheadhosts` "Connect Timeout" OR "Unable to get authentication token" component IN (DistributedPeer, GetRemoteAuthToken) \
 | search ```Exclude time periods where shutdowns were occurring including 10 minutes after shutdown to handle any reboot time``` NOT \
  [ `splunkadmins_shutdown_time(indexerhosts,60,600)`] \
 | bin _time span=5m \
@@ -8704,3 +8704,35 @@ search = index=_configtracker host=example \
 | transaction maxspan=5s data.path stanza name \
 | fields - _raw field_match_sum linecount closed_txn duration \
 | where NOT new_value=old_value
+
+[IndexerLevel - indexes with unflushed buckets]
+action.keyindicator.invert = 0
+alert.suppress = 0
+alert.track = 1
+alert.digest_mode = 1
+alert.severity = 4
+alert_condition = where statusCode>500 AND count>5
+counttype = number of events
+cron_schedule = 4 * * * *
+description = Chance the alert requires action? Moderate. The first setting to check is the splitByIndexKeys setting on the index, such as, metric.splitByIndexKeys = metric_name\
+This can result in an issue when there are many unique metric names\
+\
+Example message:\
+09-09-2024 01:27:21.990 +0000 INFO  HotDBManager [41851 indexerPipe] - idx=index-name Flushing bucket id=42. maxHotOpen=10, have count=11 unflushed buckets
+dispatch.earliest_time = -60m@m
+dispatch.latest_time = now
+display.events.fields = ["host","source","sourcetype"]
+display.general.type = statistics
+display.page.search.tab = statistics
+display.visualizations.charting.chart = area
+enableSched = 1
+quantity = 0
+relation = greater than
+request.ui_dispatch_app = SplunkAdmins
+request.ui_dispatch_view = search
+search = ``` unflushed buckets may relate to excessive bucket rolling. If using a setting similar to metric.splitByIndexKeys then this may require a review to prevent performance issues```\
+index=_internal sourcetype=splunkd bucket `splunkadmins_splunkd_source` "unflushed buckets" `indexerhosts` \
+| eval indexer_cluster=`indexer_cluster_name(splunk_server)`\
+| stats count, min(_time) AS firstseen, max(_time) AS lastseen by idx, indexer_cluster\
+| eval firstseen=strftime(firstseen, "%+"), lastseen=strftime(lastseen, "%+")
+disabled = 1