From cefbd2cfc8f595c60d83ed69adf9faf2467719ca Mon Sep 17 00:00:00 2001 From: TheWitness Date: Fri, 27 Dec 2024 21:02:04 -0500 Subject: [PATCH] Fix: #6001 - Extend SYSTEM STATS * Provides more visibility into Device collection errors. --- CHANGELOG | 1 + cacti.sql | 48 +++++++++++++++------ cmd.php | 32 ++++++++------ data_sources.php | 13 ++++++ host.php | 13 ++++++ install/upgrades/1_3_0.php | 38 ++++++++++++++++- lib/poller.php | 18 ++++++++ poller.php | 86 +++++++++++++++++++++++++++++++++++++- 8 files changed, 221 insertions(+), 28 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 6591545894..c8740d4d9a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -169,6 +169,7 @@ Cacti CHANGELOG -feature#5971: Add a method to mark missing Data in all graphs globally -feature#5993: Allow Automation Rules to Search through Site Information -feature#5997: Add filter for devices in maintenance plan, add icon to device list +-feature#6001: Extend SYSTEM STATS log entry -feature: Allow messages to be popup notifications -feature: Upgrade billboard.js to version 3.14.1 -feature: Upgrade jQueryUI to 1.14.1 diff --git a/cacti.sql b/cacti.sql index a9f7ddac49..8881064639 100644 --- a/cacti.sql +++ b/cacti.sql @@ -1295,19 +1295,22 @@ INSERT INTO `data_input_fields` VALUES (46,'3a33d4fc65b8329ab2ac46a36da26b72',2, -- Table structure for table `data_local` -- -CREATE TABLE data_local ( - id int(10) unsigned NOT NULL auto_increment, - data_template_id mediumint(8) unsigned NOT NULL default '0', - host_id mediumint(8) unsigned NOT NULL default '0', - snmp_query_id mediumint(8) NOT NULL default '0', - snmp_index varchar(255) NOT NULL default '', - orphan tinyint(3) unsigned NOT NULL default '0', - PRIMARY KEY (id), - KEY data_template_id (data_template_id), - KEY snmp_query_id (snmp_query_id), - KEY snmp_index (snmp_index), - KEY host_id_snmp_query_id (host_id, snmp_query_id) -) ENGINE=InnoDB ROW_FORMAT=Dynamic; +CREATE TABLE `data_local` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `data_template_id` mediumint(8) unsigned NOT NULL DEFAULT 0, + `host_id` mediumint(8) unsigned NOT NULL DEFAULT 0, + `snmp_query_id` mediumint(8) NOT NULL DEFAULT 0, + `snmp_index` varchar(255) NOT NULL DEFAULT '', + `orphan` tinyint(3) unsigned NOT NULL DEFAULT 0, + `errored` tinyint(3) unsigned NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `host_id_snmp_query_id` (`host_id`,`snmp_query_id`), + KEY `snmp_index` (`snmp_index`), + KEY `data_template_id` (`data_template_id`), + KEY `snmp_query_id` (`snmp_query_id`), + KEY `orphans` (`orphans`), + KEY `errored` (`errored`) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; -- -- Dumping data for table `data_local` @@ -2036,6 +2039,7 @@ CREATE TABLE host ( cur_time decimal(10,5) default '0.00000', avg_time decimal(10,5) default '0.00000', polling_time DOUBLE default '0', + current_errors int(10) unsigned default '0', total_polls int(10) unsigned default '0', failed_polls int(10) unsigned default '0', availability decimal(8,5) NOT NULL default '100.00000', @@ -2046,6 +2050,7 @@ CREATE TABLE host ( KEY external_id (external_id), KEY disabled (disabled), KEY status (status), + KEY current_errors (current_errors), KEY site_id_location (site_id, location), KEY hostname (hostname), KEY poller_id_last_updated (poller_id, last_updated) @@ -2055,6 +2060,23 @@ CREATE TABLE host ( -- Dumping data for table `host` -- +-- +-- Table structure for table `host_errors` +-- + +CREATE TABLE `host_errors` ( + `host_id` mediumint(8) unsigned NOT NULL DEFAULT 0, + `poller_id` int(10) unsigned NOT NULL DEFAULT 1, + `errors` mediumint(8) unsigned NOT NULL DEFAULT 0, + `local_data_ids` text DEFAULT NULL, + PRIMARY KEY (`host_id`), + KEY `poller_id` (`poller_id`) +) ENGINE=InnoDB ROW_FORMAT=DYNAMIC COMMENT='Holds Device Error buffer for Spine'; + +-- +-- Dumping data for table `host_errors` +-- + -- -- Table structure for table `host_graph` -- diff --git a/cmd.php b/cmd.php index d765371bec..e8f918d7a8 100755 --- a/cmd.php +++ b/cmd.php @@ -387,16 +387,15 @@ $output_count = 0; } - db_execute_prepared( - 'UPDATE host - SET polling_time = ? + $errors = cacti_sizeof($error_ds); + + db_execute_prepared('UPDATE host + SET polling_time = ?, current_errors = ? WHERE id = ? AND deleted = ""', - array(($host_end - $host_start), $last_host) + array(($host_end - $host_start), $errors, $last_host) ); - $errors = cacti_sizeof($error_ds); - cacti_log(sprintf('Device[%d] Time[%3.2f] Items[%d] Errors[%d]', $last_host, $host_end - $host_start, $itemcnt, $errors), $print_data_to_stdout, 'POLLER', $hmedium); if ($errors > 0) { @@ -405,6 +404,11 @@ } $tot_errors += $errors; + + db_execute_prepared('INSERT INTO host_errors + (host_id, poller_id, errors, local_data_ids) + VALUES (?, ?, ?, ?)', + array($last_host, $poller_id, $errors, implode(',', $error_ds))); } } @@ -487,16 +491,15 @@ } } - db_execute_prepared( - 'UPDATE host - SET polling_time = ? + $errors = cacti_sizeof($error_ds); + + db_execute_prepared('UPDATE host + SET polling_time = ?, current_errors = ? WHERE id = ? AND deleted = ""', - array(($host_end - $host_start), $host_id) + array(($host_end - $host_start), $errors, $host_id) ); - $errors = cacti_sizeof($error_ds); - cacti_log(sprintf('Device[%d] Time[%3.2f] Items[%d] Errors[%d]', $last_host, $host_end - $host_start, $itemcnt, $errors), $print_data_to_stdout, 'POLLER', $hmedium); if ($errors > 0) { @@ -505,6 +508,11 @@ } $tot_errors += $errors; + + db_execute_prepared('INSERT INTO host_errors + (host_id, poller_id, errors, local_data_ids) + VALUES (?, ?, ?, ?)', + array($last_host, $poller_id, $errors, implode(',', $error_ds))); } if (cacti_sizeof($width_dses)) { diff --git a/data_sources.php b/data_sources.php index 1f8b437aa7..3bca1084b6 100644 --- a/data_sources.php +++ b/data_sources.php @@ -1277,6 +1277,10 @@ function data_sources() { $sql_params2[] = get_request_var('host_id'); } + if (isset_request_var('errored') && get_request_var('errored') == 'true') { + $sql_where1 .= ($sql_where1 != '' ? ' AND':'WHERE') . ' dl.errored = 1'; + } + if (isempty_request_var('site_id')) { $sql_where1 .= ($sql_where1 != '' ? ' AND':'WHERE') . ' (h.site_id=0 OR h.site_id IS NULL)'; $sql_where2 .= ' AND (h.site_id=0 OR h.site_id IS NULL)'; @@ -1700,6 +1704,15 @@ function create_filter() { 'default' => '', 'pageset' => true, 'value' => get_nfilter_request_var('orphans') + ), + 'errored' => array( + 'method' => 'filter_checkbox', + 'friendly_name' => __('Errored'), + 'filter' => FILTER_VALIDATE_REGEXP, + 'filter_options' => array('options' => array('regexp' => '(true|false)')), + 'default' => '', + 'pageset' => true, + 'value' => get_nfilter_request_var('errored') ) ), array( diff --git a/host.php b/host.php index 25ae6bbb31..7c17a5cd07 100644 --- a/host.php +++ b/host.php @@ -1659,6 +1659,12 @@ function host() { 'sort' => 'DESC', 'tip' => __('The total number of Data Sources generated from this Device.') ), + 'current_errors' => array( + 'display' => __('Errors'), + 'align' => 'right', + 'sort' => 'DESC', + 'tip' => __('The total number current data collection errors on this Device. Enable Device Debug to track them.') + ), 'status' => array( 'display' => __('Status'), 'align' => 'center', @@ -1760,6 +1766,7 @@ function host() { $sites_url = CACTI_PATH_URL . 'host.php?site_id=' . $host['site_id']; $graphs_url = CACTI_PATH_URL . 'graphs.php?reset=1&host_id=' . $host['id']; $data_source_url = CACTI_PATH_URL . 'data_sources.php?reset=1&host_id=' . $host['id']; + $errors_url = CACTI_PATH_URL . 'data_sources.php?reset=1&errored=true&host_id=' . $host['id']; if (empty($host['graphs'])) { $host['graphs'] = 0; @@ -1769,6 +1776,10 @@ function host() { $host['data_sources'] = 0; } + if (empty($host['current_errors'])) { + $host['current_errors'] = 0; + } + form_alternate_row('line' . $host['id'], true); $maint = ($host['maint'] == 1 ? '' : ''); @@ -1782,6 +1793,8 @@ function host() { form_selectable_cell(filter_value(number_format_i18n($host['data_sources'], '-1'), '', $data_source_url), $host['id'], '', 'right'); + form_selectable_cell(filter_value(number_format_i18n($host['current_errors'], '-1'), '', $errors_url), $host['id'], '', 'right'); + form_selectable_cell($host_status, $host['id'], '', 'center'); if ($host['site_disabled'] == 'on') { diff --git a/install/upgrades/1_3_0.php b/install/upgrades/1_3_0.php index 814ede9432..8d1be7254e 100644 --- a/install/upgrades/1_3_0.php +++ b/install/upgrades/1_3_0.php @@ -38,8 +38,11 @@ function upgrade_to_1_3_0() { db_install_add_column('host', array('name' => 'created', 'type' => 'timestamp', 'default' => 'CURRENT_TIMESTAMP')); db_install_add_column('host', array('name' => 'snmp_options', 'type' => 'tinyint(3)', 'unsigned' => true, 'NULL' => false, 'default' => '0', 'after' => 'external_id')); db_install_add_column('host', array('name' => 'status_options_date', 'type' => 'timestamp', 'NULL' => false, 'default' => '0000-00-00', 'after' => 'status_rec_date')); - db_install_add_column('host', array('name' => 'snmp_retries', 'type' => 'tinyint(3) unsigned', 'NULL' => false, 'default' => '3', 'after' => 'snmp_timeout')); + db_install_add_column('host', array('name' => 'current_errors', 'type' => 'int(10)', 'unsigned' => true, 'default' => '0', 'after' => 'polling_time')); + + db_add_index('host', 'INDEX', 'current_errors', array('current_errors')); + db_install_add_column('poller_item', array('name' => 'snmp_retries', 'type' => 'tinyint(3) unsigned', 'NULL' => false, 'default' => '3', 'after' => 'snmp_timeout')); db_execute_prepared('UPDATE host SET snmp_retries = ?', array(read_config_option('snmp_retries'))); @@ -517,6 +520,39 @@ function upgrade_to_1_3_0() { $data['row_format'] = 'Dynamic'; db_update_table('reports_queued', $data); + $data = array(); + $data['columns'][] = array('name' => 'host_id', 'unsigned' => true, 'type' => 'mediumint(8)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'poller_id', 'unsigned' => true, 'type' => 'int(10)', 'NULL' => false, 'default' => '1'); + $data['columns'][] = array('name' => 'errors', 'unsigned' => true, 'type' => 'mediumint(8)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'local_data_ids', 'type' => 'text', 'NULL' => true); + $data['primary'] = 'host_id'; + $data['keys'][] = array('name' => 'poller_id', 'columns' => array('poller_id')); + $data['type'] = 'InnoDB'; + $data['charset'] = 'utf8mb4'; + $data['comment'] = 'Holds Device Error buffer for Spine'; + $data['row_format'] = 'Dynamic'; + db_update_table('host_errors', $data); + + $data = array(); + $data['columns'][] = array('name' => 'id', 'unsigned' => true, 'type' => 'int(10)', 'NULL' => false, 'auto_increment' => true); + $data['columns'][] = array('name' => 'data_template_id', 'unsigned' => true, 'type' => 'mediumint(8)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'host_id', 'unsigned' => true, 'type' => 'mediumint(8)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'snmp_query_id', 'type' => 'mediumint(8)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'snmp_index', 'type' => 'varchar(255)', 'NULL' => false, 'default' => ''); + $data['columns'][] = array('name' => 'orphan', 'unsigned' => true, 'type' => 'tinyint(3)', 'NULL' => false, 'default' => '0'); + $data['columns'][] = array('name' => 'errored', 'unsigned' => true, 'type' => 'tinyint(3)', 'NULL' => false, 'default' => '0'); + $data['primary'] = 'id'; + $data['keys'][] = array('name' => 'host_id_snmp_query_id', 'columns' => array('host_id','snmp_query_id')); + $data['keys'][] = array('name' => 'snmp_index', 'columns' => array('snmp_index')); + $data['keys'][] = array('name' => 'data_template_id', 'columns' => array('data_template_id')); + $data['keys'][] = array('name' => 'snmp_query_id', 'columns' => array('snmp_query_id')); + $data['keys'][] = array('name' => 'orphan', 'columns' => array('orphan')); + $data['keys'][] = array('name' => 'errored', 'columns' => array('errored')); + $data['type'] = 'InnoDB'; + $data['charset'] = 'utf8mb4'; + $data['row_format'] = 'Dynamic'; + db_update_table('data_local', $data); + /* clear up setting change */ $exists = db_fetch_cell_prepared('SELECT name FROM settings WHERE name = "business_hours_hideWeekends"'); if ($exists != '') { diff --git a/lib/poller.php b/lib/poller.php index a4b787ea9d..562ef7ac0f 100644 --- a/lib/poller.php +++ b/lib/poller.php @@ -1666,6 +1666,12 @@ function replicate_out($remote_poller_id = 1, $class = 'all') { array($remote_poller_id)); replicate_out_table($rcnn_id, $data, 'host', $remote_poller_id); + $data = db_fetch_assoc_prepared('SELECT h.* + FROM host_errors AS h + WHERE h.poller_id = ?', + array($remote_poller_id)); + replicate_out_table($rcnn_id, $data, 'host_errors', $remote_poller_id); + $data = db_fetch_assoc_prepared('SELECT hsc.* FROM host_snmp_cache AS hsc INNER JOIN host AS h @@ -2236,6 +2242,18 @@ function poller_push_data_to_main() { ); poller_push_table($remote_db_cnn_id, $poller_item_records, 'poller_item', true, $updates); + + $host_errors = db_fetch_assoc_prepared('SELECT * + FROM host_errors + WHERE poller_id = ?', + array($config['poller_id'])); + + $updates = array( + 'errors', + 'local_data_ids' + ); + + poller_push_table($remote_db_cnn_id, $host_errors, 'host_errors', true, $updates); } } diff --git a/poller.php b/poller.php index bb26010de1..3df4c39e4d 100755 --- a/poller.php +++ b/poller.php @@ -657,6 +657,13 @@ function sig_handler($signo) { } } + /** + * Empty the host_errors table for the current poller. + * This table is required due to spine using multiple + * threads per device. + */ + db_execute_prepared('DELETE FROM host_errors WHERE poller_id = ?', array($poller_id)); + // mainline if (read_config_option('poller_enabled') == 'on') { // determine the number of hosts to process per file @@ -796,6 +803,7 @@ function sig_handler($signo) { log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threads, ($poller_id == '1' ? $total_polling_hosts - 1 : $total_polling_hosts), $hosts_per_process, $num_polling_items, $rrds_processed); + poller_run_stats($loop_start); break; @@ -822,8 +830,10 @@ function sig_handler($signo) { snmpagent_poller_exiting(); api_plugin_hook_function('poller_exiting'); + log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threads, ($poller_id == '1' ? $total_polling_hosts - 1 : $total_polling_hosts), $hosts_per_process, $num_polling_items, $rrds_processed); + poller_run_stats($loop_start); break; @@ -842,6 +852,7 @@ function sig_handler($signo) { if ($poller_id > 1) { log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threads, ($poller_id == '1' ? $total_polling_hosts - 1 : $total_polling_hosts), $hosts_per_process, $num_polling_items, $rrds_processed); + poller_run_stats($loop_start); // Mark the poller done immediately due to lack of devices @@ -947,6 +958,7 @@ function sig_handler($signo) { if (!$logged) { log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threads, ($poller_id == '1' ? $total_polling_hosts - 1 : $total_polling_hosts), $hosts_per_process, $num_polling_items, $rrds_processed); + poller_run_stats($loop_start); } } @@ -1154,6 +1166,74 @@ function log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threa WHERE id = ?', array($poller_id), true, $poller_db_cnn_id); + /* update the host table error count first */ + db_execute_prepared('UPDATE host AS h + LEFT JOIN host_errors AS e + ON h.id = e.host_id + SET errors = e.errors + WHERE h.poller_id = ?', + array($poller_id)); + + if ($poller_id == 1) { + $error_lines = db_fetch_assoc("SELECT * FROM host_errors"); + + db_execute('CREATE TEMPORARY TABLE IF NOT EXISTS host_errors_normalized ( + `host_id` mediumint(8) unsigned NOT NULL DEFAULT 0, + `local_data_id` int(10) unsigned NOT NULL DEFAULT 0, + PRIMARY KEY (`host_id`,`local_data_id`)) + ENGINE=InnoDB ROW_FORMAT=Dynamic'); + + $sql = 'INSERT INTO host_errors_normalized (host_id, local_data_id) VALUES '; + + $i = 0; + $params = array(); + if (cacti_sizeof($error_lines)) { + foreach($error_lines as $l) { + $local_data_ids = array_unique(preg_split('/\s+/', $l['local_data_ids']), SORT_NUMERIC); + + foreach($local_data_ids as $ldi) { + $sql .= ($i == 0 ? '':',') . '(?, ?)'; + $params[] = $l['host_id']; + $params[] = $ldi; + + $i++; + } + } + + if ($i > 0) { + db_execute_prepared($sql, $params); + } + } + + db_execute_prepared('UPDATE data_local AS dl + LEFT JOIN host_errors_normalized AS e + ON dl.host_id = e.host_id + AND dl.id = e.local_data_id + SET dl.errored = IF(e.local_data_id IS NULL, 0, 1)'); + + db_execute('DROP TEMPORARY TABLE host_errors_normalized'); + } + + set_config_option('time_last_change_data_source', time()); + + $errors = db_fetch_row_prepared('SELECT COUNT(DISTINCT host_id) AS errorHosts, SUM(errors) AS totalErrors + FROM host_errors + WHERE poller_id = ?', + array($poller_id)); + + if (cacti_sizeof($errors)) { + $errorHosts = $errors['errorHosts'] ?? 0; + + if ($errors['totalErrors'] > 0) { + $totalErrors = $errors['totalErrors']; + } else { + $totalErrors = 0; + } + } else { + $errorHosts = 0; + $totalErrors = 0; + } + // take time and log performance data $loop_end = microtime(true); @@ -1165,10 +1245,12 @@ function log_cacti_stats($loop_start, $method, $concurrent_processes, $max_threa $num_hosts, $hosts_per_process, $num_polling_items, - $rrds_processed + $rrds_processed, + $errorHosts, + $totalErrors ); - $cacti_stats = vsprintf('Time:%01.4f Method:%s Processes:%s Threads:%s Hosts:%s HostsPerProcess:%s DataSources:%s RRDsProcessed:%s', $perf_data); + $cacti_stats = vsprintf('Time:%01.4f Method:%s Processes:%s Threads:%s Hosts:%s HostsPerProcess:%s DataSources:%s RRDsProcessed:%s ErrorHosts:%s TotalErrors:%s', $perf_data); cacti_log('STATS: ' . $cacti_stats , true, 'SYSTEM'); // insert poller stats into the settings table