diff --git a/CHANGELOG b/CHANGELOG index fea74ee04e..47f6a4c7e6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,10 @@ Cacti CHANGELOG +1.2.23 +-issue#4892: Recache Loop after many devices go offline +-feature#3131: Button to reindex bad indexes +-feature#4890: Add multi threading for Poller rechace script + 1.2.22 -security#4834: When creating new graphs, cross site injection is possible -issue#4768: When creating user from template, multiple Domain FullName and Mail are not propagated diff --git a/include/global_settings.php b/include/global_settings.php index 9cf04f3de6..1249a03561 100644 --- a/include/global_settings.php +++ b/include/global_settings.php @@ -1246,7 +1246,7 @@ 'size' => '5' ), 'timeouts_header' => array( - 'friendly_name' => __('Background Timeout Settings'), + 'friendly_name' => __('Background Timeout and Concurrent Process Settings'), 'collapsible' => 'true', 'method' => 'spacer', ), @@ -1279,8 +1279,8 @@ ) ), 'commands_timeout' => array( - 'friendly_name' => __('Background Commands Timeout'), - 'description' => __('The maximum amount of time Cacti\'s Background Commands script can run without generating a timeout error and being killed.'), + 'friendly_name' => __('Poller Commands Timeout'), + 'description' => __('The maximum amount of time Cacti\'s Background Commands script can run without generating a timeout error and being killed. This script will perform tasks such as re-indexing Devices and pruning devices from Remote Data Collectors.'), 'method' => 'drop_array', 'default' => '300', 'array' => array( @@ -1291,6 +1291,34 @@ '1200' => __('%s Minutes', 20) ) ), + 'commands_processes' => array( + 'friendly_name' => __('Poller Command Concurrent Processes'), + 'description' => __('The number of concurrent Poller Command processes. The will be at most one concurrent command per host within the Poller Command pool.'), + 'default' => '1', + 'method' => 'drop_array', + 'array' => array( + 1 => __('1 Process'), + 2 => __('%d Processes', 2), + 3 => __('%d Processes', 3), + 4 => __('%d Processes', 4), + 5 => __('%d Processes', 5), + 6 => __('%d Processes', 6), + 7 => __('%d Processes', 7), + 8 => __('%d Processes', 8), + 9 => __('%d Processes', 9), + 10 => __('%d Processes', 10), + 11 => __('%d Processes', 11), + 12 => __('%d Processes', 12), + 13 => __('%d Processes', 13), + 14 => __('%d Processes', 14), + 15 => __('%d Processes', 15), + 16 => __('%d Processes', 16), + 17 => __('%d Processes', 17), + 18 => __('%d Processes', 18), + 19 => __('%d Processes', 19), + 20 => __('%d Processes', 20), + ) + ), 'maintenance_timeout' => array( 'friendly_name' => __('Maintenance Background Generation Timeout'), 'description' => __('The maximum amount of time a Cacti\'s Maintenance script can run without generating a timeout error and being killed.'), diff --git a/poller_commands.php b/poller_commands.php index 04132db635..a10cb487a8 100755 --- a/poller_commands.php +++ b/poller_commands.php @@ -23,12 +23,19 @@ +-------------------------------------------------------------------------+ */ -/* we are not talking to the browser */ -define('MAX_RECACHE_RUNTIME', 1800); +if (function_exists('pcntl_async_signals')) { + pcntl_async_signals(true); +} else { + declare(ticks = 100); +} +ini_set('output_buffering', 'Off'); ini_set('max_runtime', '-1'); ini_set('memory_limit', '-1'); +/* we are not talking to the browser */ +define('MAX_RECACHE_RUNTIME', 1800); + require(__DIR__ . '/include/cli_check.php'); require_once($config['base_path'] . '/lib/api_device.php'); require_once($config['base_path'] . '/lib/api_data_source.php'); @@ -45,10 +52,13 @@ require_once($config['base_path'] . '/lib/utility.php'); $poller_id = $config['poller_id']; +$debug = false; +$host_id = false; +$forcerun = false; +$type = 'master'; +$threads = read_config_option('commands_processes'); -$debug = false; - -global $poller_db_cnn_id, $remote_db_cnn_id; +global $poller_db_cnn_id, $remote_db_cnn_id, $type, $host_id, $poller_id; if ($config['poller_id'] > 1 && $config['connection'] == 'online') { $poller_db_cnn_id = $remote_db_cnn_id; @@ -82,126 +92,354 @@ case '-p': $poller_id = $value; break; + case '--child': + case '-c': + $host_id = $value; + $type = 'child'; + break; + case '-t': + case '--threads': + $threads = $value; + break; case '--debug': case '-d': $debug = true; break; default: - print "ERROR: Invalid Argument: ($arg)\n\n"; + print "ERROR: Invalid Argument: ($arg)" . PHP_EOL . PHP_EOL; display_help(); exit(1); } } } +if ($debug) { + $verbosity = POLLER_VERBOSITY_LOW; +} else { + $verbosity = POLLER_VERBOSITY_MEDIUM; +} + +/** + * Types include + * + * master - the main process launched from the Cacti main poller and will launch child processes + * child - a child of the master process from the 'master' + * + */ + +/* install signal handlers for UNIX only */ +if (function_exists('pcntl_signal')) { + pcntl_signal(SIGTERM, 'sig_handler'); + pcntl_signal(SIGINT, 'sig_handler'); +} + /* Record Start Time */ $start = microtime(true); -$max_updated = db_fetch_cell_prepared('SELECT MAX(UNIX_TIMESTAMP(last_updated)) - FROM poller_command - WHERE poller_id = ?', - array($poller_id), '', true, $poller_db_cnn_id); +/* send a gentle message to the log and stdout */ +commands_debug('Polling Starting'); + +if ($host_id === false) { + $hosts = array_rekey( + db_fetch_assoc_prepared('SELECT DISTINCT SUBSTRING_INDEX(command, ":", 1) AS host_id + FROM poller_command + WHERE poller_id = ?', + array($poller_id), true, $poller_db_cnn_id), + 'host_id', 'host_id' + ); + + if (cacti_sizeof($hosts)) { + /** + * Register the master process + */ + if (!register_process_start('commands', 'master', $poller_id, read_config_option('commands_timeout'))) { + exit(0); + } -$poller_commands = db_fetch_assoc_prepared('SELECT action, command - FROM poller_command - WHERE poller_id = ?', - array($poller_id), true, $poller_db_cnn_id); + // Master processing + commands_master_handler($forcerun, $hosts, $threads); -$last_host_id = 0; -$first_host = true; -$recached_hosts = 0; + /* take time to log performance data */ + $recache = microtime(true); -if ($debug) { - $verbosity = POLLER_VERBOSITY_LOW; + $recache_stats = sprintf('Poller:%s RecacheTime:%01.4f DevicesRecached:%s', $poller_id, round($recache - $start, 4), cacti_sizeof($hosts)); + + if (cacti_sizeof($hosts)) { + cacti_log('STATS: ' . $recache_stats, true, 'RECACHE'); + } + + /* insert poller stats into the settings table */ + db_execute_prepared('REPLACE INTO settings (name, value) VALUES (?, ?)', + array('stats_recache_' . $poller_id, $recache_stats), true, $poller_db_cnn_id); + + unregister_process('commands', 'master', $poller_id); + } else { + cacti_log('NOTE: No Poller Commands found for processing', true, 'PCOMMAND', $verbosity); + } } else { - $verbosity = POLLER_VERBOSITY_MEDIUM; -} + /** + * Register the child process + */ + if (!register_process_start('commands', 'child', $host_id + 1000, read_config_option('commands_timeout'))) { + exit(0); + } -/* silently end if the registered process is still running, or process table missing */ -if (!register_process_start('commands', 'master', $poller_id, read_config_option('commands_timeout'))) { - exit(0); -} + $last_host_id = 0; + $first_host = true; -if (cacti_sizeof($poller_commands)) { - foreach ($poller_commands as $command) { - switch ($command['action']) { - case POLLER_COMMAND_REINDEX: - list($device_id, $data_query_id) = explode(':', $command['command']); + /** + * We will only remove records earlier than this date + */ + $max_updated = db_fetch_cell_prepared('SELECT MAX(UNIX_TIMESTAMP(last_updated)) + FROM poller_command + WHERE poller_id = ? + AND SUBSTRING_INDEX(command, ":", 1) = ?', + array($poller_id, $host_id), '', true, $poller_db_cnn_id); + + /** + * Get the poller command records for the host + */ + $poller_commands = db_fetch_assoc_prepared('SELECT action, command, + SUBSTRING_INDEX(command, ":", 1) AS host_id + FROM poller_command + WHERE poller_id = ? + AND last_updated <= FROM_UNIXTIME(?) + AND SUBSTRING_INDEX(command, ":", 1) = ?', + array($poller_id, $max_updated, $host_id), true, $poller_db_cnn_id); + + if (cacti_sizeof($poller_commands)) { + foreach ($poller_commands as $command) { + switch ($command['action']) { + case POLLER_COMMAND_REINDEX: + list($device_id, $data_query_id) = explode(':', $command['command']); + + if ($last_host_id != $device_id) { + $last_host_id = $device_id; + $first_host = true; + } else { + $first_host = false; + } + + if ($first_host) { + cacti_log("Device[$device_id] NOTE: Recache Event Detected for Device", true, 'PCOMMAND'); + } + + cacti_log("Device[$device_id] DQ[$data_query_id] RECACHE: Recache for Device started.", true, 'PCOMMAND', $verbosity); + run_data_query($device_id, $data_query_id); + cacti_log("Device[$device_id] DQ[$data_query_id] RECACHE: Recached successfully.", true, 'PCOMMAND', $verbosity); - if ($last_host_id != $device_id) { - $last_host_id = $device_id; - $first_host = true; - $recached_hosts++; - } else { - $first_host = false; + break; + case POLLER_COMMAND_PURGE: + $device_id = $command['command']; + + api_device_purge_from_remote($device_id, $poller_id); + cacti_log("Device[$device_id] PURGE: Purged successfully.", true, 'PCOMMAND', $verbosity); + + break; + default: + cacti_log('ERROR: Unknown poller command issued', true, 'PCOMMAND'); } - if ($first_host) { - cacti_log("Device[$device_id] NOTE: Recache Event Detected for Device", true, 'PCOMMAND'); + /* record current_time */ + $current = microtime(true); + + /* end if runtime has been exceeded */ + if (($current-$start) > MAX_RECACHE_RUNTIME) { + cacti_log("ERROR: Poller Command processing timed out after processing '$command'", true, 'PCOMMAND'); + break; } + } - cacti_log("Device[$device_id] DQ[$data_query_id] RECACHE: Recache for Device started.", true, 'PCOMMAND', $verbosity); - run_data_query($device_id, $data_query_id); - cacti_log("Device[$device_id] DQ[$data_query_id] RECACHE: Recached successfully.", true, 'PCOMMAND', $verbosity); + db_execute_prepared('DELETE FROM poller_command + WHERE poller_id = ? + AND SUBSTRING_INDEX(command, ":", 1) = ? + AND last_updated <= FROM_UNIXTIME(?)', + array($poller_id, $host_id, $max_updated), true, $poller_db_cnn_id); + } - break; - case POLLER_COMMAND_PURGE: - $device_id = $command['command']; + unregister_process('commands', 'child', $host_id + 1000); +} - api_device_purge_from_remote($device_id, $poller_id); - cacti_log("Device[$device_id] PURGE: Purged successfully.", true, 'PCOMMAND', $verbosity); +function commands_master_handler($forcerun, &$hosts, $threads) { + commands_debug("There are " . cacti_sizeof($hosts) . " to reindex"); - break; - default: - cacti_log('ERROR: Unknown poller command issued', true, 'PCOMMAND'); + foreach($hosts as $id) { + /* run the daily stats */ + commands_debug("Launching Host ID $id"); + commands_launch_child($id); + + /* Wait for if there are 50 processes running */ + while (true) { + $running = commands_processes_running(); + + if ($running >= $threads) { + commands_debug(sprintf('%s Processes Running, Sleeping for 2 seconds.', $running)); + sleep(2); + } else { + commands_debug(sprintf('%s Processes Running, Launching more processes.', $running)); + usleep(500000); + break; + } } + } - /* record current_time */ - $current = microtime(true); + $starting = true; + + while (true) { + if ($starting) { + sleep(5); + $starting = false; + } - /* end if runtime has been exceeded */ - if (($current-$start) > MAX_RECACHE_RUNTIME) { - cacti_log("ERROR: Poller Command processing timed out after processing '$command'", true, 'PCOMMAND'); + $running = commands_processes_running(); + + if ($running > 0) { + commands_debug(sprintf('%s Processes Running, Sleeping for 2 seconds.', $running)); + sleep(2); + } else { break; } } +} - db_execute_prepared('DELETE FROM poller_command - WHERE poller_id = ? - AND last_updated <= FROM_UNIXTIME(?)', - array($poller_id, $max_updated), true, $poller_db_cnn_id); -} else { - cacti_log('NOTE: No Poller Commands found for processing', true, 'PCOMMAND', $verbosity); +/** + * commands_launch_child - this function will launch collector children based upon + * the maximum number of threads and the process type + * + * @param (int) $host_id - The Cacti host_id + * + * @return (void) + */ +function commands_launch_child($host_id) { + global $config, $seebug; + + $php_binary = read_config_option('path_php_binary'); + + commands_debug(sprintf('Launching Commands Process Number %s for Type %s', $host_id, 'child')); + + cacti_log(sprintf('NOTE: Launching Commands Process Number %s for Type %s', $host_id, 'child'), false, 'CLEANUP', POLLER_VERBOSITY_MEDIUM); + + exec_background($php_binary, $config['base_path'] . "/poller_commands.php --child=$host_id" . ($seebug ? ' --debug':'')); } -/* take time to log performance data */ -$recache = microtime(true); +/** + * commands_processes_running - given a type, determine the number + * of sub-type or children that are currently running + * + * @return (int) The number of running processes + */ +function commands_processes_running() { + $running = db_fetch_cell('SELECT COUNT(*) + FROM processes + WHERE tasktype = "commands" + AND taskname = "child"'); + + if ($running == 0) { + return 0; + } -$recache_stats = sprintf('Poller:%s RecacheTime:%01.4f DevicesRecached:%s', $poller_id, round($recache - $start, 4), $recached_hosts); + return $running; +} -if ($recached_hosts > 0) { - cacti_log('STATS: ' . $recache_stats, true, 'RECACHE'); +/** + * commands_debug - this simple routine prints a standard message to the console + * when running in debug mode. + * + * @param (string) $message - The message to display + * + * @return (void) + */ +function commands_debug($message) { + global $seebug; + + if ($seebug) { + print 'COMMANDS: ' . $message . PHP_EOL; + } } -/* insert poller stats into the settings table */ -db_execute_prepared('REPLACE INTO settings (name, value) VALUES (?, ?)', - array('stats_recache_' . $poller_id, $recache_stats), true, $poller_db_cnn_id); +/** + * sig_handler - provides a generic means to catch exceptions to the Cacti log. + * + * @param (int) $signo - the signal that was thrown by the interface. + * + * @return (void) + */ +function sig_handler($signo) { + global $type, $host_id, $poller_id; + + switch ($signo) { + case SIGTERM: + case SIGINT: + cacti_log('WARNING: RRDfile Cleanup Poller terminated by user', false, 'CLEANUP'); + + if (strpos($type, 'master') !== false) { + commands_kill_running_processes(); + } -unregister_process('commands', 'master', $poller_id); + if ($type == 'master') { + unregister_process('commands', $type, $poller_id, getmypid()); + } else { + unregister_process('commands', $type, $host_id + 1000, getmypid()); + } -/* display_version - displays version information */ + exit(1); + break; + default: + /* ignore all other signals */ + } +} + +/** + * commands_kill_running_processes - this function is part of an interrupt + * handler to kill children processes when the parent is killed + * + * @return (void) + */ +function commands_kill_running_processes() { + global $type; + + $processes = db_fetch_assoc_prepared('SELECT * + FROM processes + WHERE tasktype = "commands" + AND taskname IN ("child") + AND pid != ?', + array(getmypid())); + + if (cacti_sizeof($processes)) { + foreach($processes as $p) { + cacti_log(sprintf('WARNING: Killing Commands %s PID %d due to another due to signal or overrun.', ucfirst($p['taskname']), $p['pid']), false, 'CLEANUP'); + posix_kill($p['pid'], SIGTERM); + + unregister_process($p['tasktype'], $p['taskname'], $p['taskid'], $p['pid']); + } + } +} + +/** + * display_version - displays version information + * + * @return (void) + */ function display_version() { $version = get_cacti_version(); - print "Cacti Poller Commands Poller, Version $version " . COPYRIGHT_YEARS . "\n"; + print "Cacti Poller Commands Poller, Version $version " . COPYRIGHT_YEARS . PHP_EOL; } +/** + * display_help - displays help information + * + * @return (void) + */ function display_help () { display_version(); - print "\nusage: poller_commands.php [--poller=ID] [--debug]\n\n"; - print "Cacti's commands poller. This poller can receive specifically crafted commands from\n"; - print "either the Cacti UI, or from the main poller, and then run them in the background.\n\n"; - print "Optional:\n"; - print " --poller=ID - The poller to run as. Defaults to the system poller\n"; - print " --debug - Display verbose output during execution\n\n"; + print PHP_EOL; + print 'usage: poller_commands.php [--poller=ID] [--debug]' . PHP_EOL . PHP_EOL; + print 'Cacti\'s Commands Poller. This poller can receive specifically crafted commands from' . PHP_EOL; + print 'either the Cacti UI, or from the main poller, and then run them in the background.' . PHP_EOL . PHP_EOL; + print 'Optional:' . PHP_EOL; + print ' --poller=ID - The poller to run as. Defaults to the system poller' . PHP_EOL; + print ' --threads=N - Override the System Processes setting and use N processes' . PHP_EOL; + print ' --debug - Display verbose output during execution' . PHP_EOL . PHP_EOL; } +