Fixing #4811 - Large Scale Performance Optimizations

First phase of this change. Second phase will be to provide various drilldowns.
Cacti · Oct 1, 2023 · e859b31 · e859b31
1 parent b5263fe
commit e859b31
Show file tree

Hide file tree

Showing 18 changed files with 1,300 additions and 775 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -59,6 +59,7 @@ Cacti CHANGELOG
 -feature#4300: Allow GPrint/CDef/VDef to be visibile in graph template item lists
 -feature#4339: Change CLI progress output for rebuilding poller cache
 -feature#4778: Add devices sysName and sysLocation name as a Device description
+-feature#4811: Performance Issues on Template and Presets Pages on Large Installs and Drilldowns
 -feature#5020: Introduce script timeout for cmd.php
 -feature#5021: Allow form input validation to be reported to GUI
 -feature#5025: Add Least Squares Special Data Types to Cacti

diff --git a/aggregate_templates.php b/aggregate_templates.php
@@ -679,35 +679,22 @@ function aggregate_template() {
 	}
 
 	if (get_request_var('has_graphs') == 'true') {
-		$sql_where .= ($sql_where != '' ? ' AND ' : 'WHERE ') . 'graphs.graphs>0';
+		$sql_where .= ($sql_where != '' ? ' AND ' : 'WHERE ') . 'graphs > 0';
 	}
 
-	$total_rows = db_fetch_cell("SELECT
-		COUNT(pgt.id)
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
 		FROM aggregate_graph_templates AS pgt
-		LEFT JOIN (
-			SELECT aggregate_template_id, COUNT(*) AS graphs
-			FROM aggregate_graphs
-			GROUP BY aggregate_template_id
-		) AS graphs
-		ON pgt.id=graphs.aggregate_template_id
 		LEFT JOIN graph_templates AS gt
-		ON gt.id=pgt.graph_template_id
+		ON pgt.graph_template_id = gt.id
 		$sql_where");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$template_list = db_fetch_assoc("SELECT pgt.*, graphs.graphs, gt.name AS graph_template_name
+	$template_list = db_fetch_assoc("SELECT pgt.*, gt.name AS graph_template_name
 		FROM aggregate_graph_templates AS pgt
-		LEFT JOIN (
-			SELECT aggregate_template_id, COUNT(*) AS graphs
-			FROM aggregate_graphs
-			GROUP BY aggregate_template_id
-		) AS graphs
-		ON pgt.id=graphs.aggregate_template_id
 		LEFT JOIN graph_templates AS gt
-		ON gt.id=pgt.graph_template_id
+		ON pgt.graph_template_id = gt.id
 		$sql_where
 		$sql_order
 		$sql_limit");

diff --git a/cacti.sql b/cacti.sql
diff --git a/cdef.php b/cdef.php
@@ -876,41 +876,17 @@ function clearFilter() {
 		$sql_having = '';
 	}
 
-	$total_rows = db_fetch_cell("SELECT
-		COUNT(`rows`)
-		FROM (
-			SELECT cd.id AS `rows`,
-			SUM(CASE WHEN local_graph_id>0 THEN 1 ELSE 0 END) AS graphs
-			FROM cdef AS cd
-			LEFT JOIN (
-				SELECT DISTINCT cdef_id, local_graph_id, graph_template_id
-				FROM graph_templates_item
-			) AS gti
-			ON gti.cdef_id=cd.id
-			$sql_where
-			GROUP BY cd.id
-			$sql_having
-		) AS rs");
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
+		FROM cdef
+		$sql_where
+		$sql_having");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$cdef_list = db_fetch_assoc("SELECT rs.*,
-		SUM(CASE WHEN local_graph_id=0 THEN 1 ELSE 0 END) AS templates,
-		SUM(CASE WHEN local_graph_id>0 THEN 1 ELSE 0 END) AS graphs
-		FROM (
-			SELECT cd.*, gti.local_graph_id
-			FROM cdef AS cd
-			LEFT JOIN (
-				SELECT DISTINCT cdef_id, local_graph_id, graph_template_id
-				FROM graph_templates_item
-			) AS gti
-			ON gti.cdef_id=cd.id
-			WHERE `system` = 0
-			GROUP BY cd.id, gti.graph_template_id, gti.local_graph_id
-		) AS rs
+	$cdef_list = db_fetch_assoc("SELECT *
+		FROM cdef
 		$sql_where
-		GROUP BY rs.id
 		$sql_having
 		$sql_order
 		$sql_limit");

diff --git a/color.php b/color.php
@@ -659,43 +659,17 @@ function clearFilter() {
 		$sql_having = '';
 	}
 
-	$total_rows = db_fetch_cell("SELECT
-		COUNT(color)
-		FROM (
-			SELECT
-			c.id AS color,
-			SUM(CASE WHEN local_graph_id>0 THEN 1 ELSE 0 END) AS graphs,
-			SUM(CASE WHEN local_graph_id=0 THEN 1 ELSE 0 END) AS templates
-			FROM colors AS c
-			LEFT JOIN (
-				SELECT DISTINCT color_id, graph_template_id, local_graph_id
-				FROM graph_templates_item
-				WHERE color_id>0
-			) AS gti
-			ON gti.color_id=c.id
-			$sql_where
-			GROUP BY c.id
-			$sql_having
-		) AS rs");
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
+		FROM colors
+		$sql_where
+		$sql_having");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$colors = db_fetch_assoc("SELECT *,
-        SUM(CASE WHEN local_graph_id>0 THEN 1 ELSE 0 END) AS graphs,
-        SUM(CASE WHEN local_graph_id=0 THEN 1 ELSE 0 END) AS templates
-        FROM (
-			SELECT c.*, local_graph_id
-			FROM colors AS c
-			LEFT JOIN (
-				SELECT DISTINCT color_id, graph_template_id, local_graph_id
-				FROM graph_templates_item
-				WHERE color_id>0
-			) AS gti
-			ON c.id=gti.color_id
-		) AS rs
+	$colors = db_fetch_assoc("SELECT *
+		FROM colors
 		$sql_where
-		GROUP BY rs.id
 		$sql_having
 		$sql_order
 		$sql_limit");
@@ -709,13 +683,48 @@ function clearFilter() {
 	html_start_box('', '100%', '', '3', 'center', '');
 
 	$display_text = array(
-		'hex'       => array('display' => __('Hex'), 'align' => 'left', 'sort' => 'DESC', 'tip' => __('The Hex Value for this Color.')),
-		'name'      => array('display' => __('Color Name'), 'align' => 'left', 'sort' => 'ASC', 'tip' => __('The name of this Color definition.')),
-		'read_only' => array('display' => __('Named Color'), 'align' => 'left', 'sort' => 'ASC', 'tip' => __('Is this color a named color which are read only.')),
-		'nosort1'   => array('display' => __('Color'), 'align' => 'center', 'sort' => 'DESC', 'tip' => __('The Color as shown on the screen.')),
-		'nosort'    => array('display' => __('Deletable'), 'align' => 'right', 'sort' => '', 'tip' => __('Colors in use cannot be Deleted.  In use is defined as being referenced either by a Graph or a Graph Template.')),
-		'graphs'    => array('display' => __('Graphs Using'), 'align' => 'right', 'sort' => 'DESC', 'tip' => __('The number of Graph using this Color.')),
-		'templates' => array('display' => __('Templates Using'), 'align' => 'right', 'sort' => 'DESC', 'tip' => __('The number of Graph Templates using this Color.'))
+		'hex' => array(
+			'display' => __('Hex'),
+			'align' => 'left',
+			'sort' => 'DESC',
+			'tip' => __('The Hex Value for this Color.')
+		),
+		'name' => array(
+			'display' => __('Color Name'),
+			'align' => 'left',
+			'sort' => 'ASC',
+			'tip' => __('The name of this Color definition.')
+		),
+		'read_only' => array(
+			'display' => __('Named Color'),
+			'align' => 'left',
+			'sort' => 'ASC',
+			'tip' => __('Is this color a named color which are read only.')
+		),
+		'nosort1' => array(
+			'display' => __('Color'),
+			'align' => 'center',
+			'sort' => 'DESC',
+			'tip' => __('The Color as shown on the screen.')
+		),
+		'nosort' => array(
+			'display' => __('Deletable'),
+			'align' => 'right',
+			'sort' => '',
+			'tip' => __('Colors in use cannot be Deleted.  In use is defined as being referenced either by a Graph or a Graph Template.')
+		),
+		'graphs' => array(
+			'display' => __('Graphs Using'),
+			'align' => 'right',
+			'sort' => 'DESC',
+			'tip' => __('The number of Graph using this Color.')
+		),
+		'templates' => array(
+			'display' => __('Templates Using'),
+			'align' => 'right',
+			'sort' => 'DESC',
+			'tip' => __('The number of Graph Templates using this Color.')
+		)
 	);
 
 	html_header_sort_checkbox($display_text, get_request_var('sort_column'), get_request_var('sort_direction'), false);

diff --git a/color_templates.php b/color_templates.php
@@ -908,41 +908,15 @@ function color_template() {
 		$sql_where .= ($sql_where != '' ? ' AND ' : 'WHERE ') . ' (templates>0 OR graphs>0)';
 	}
 
-	$total_rows = db_fetch_cell("SELECT
-		COUNT(ct.color_template_id)
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
 		FROM color_templates AS ct
-		LEFT JOIN (
-			SELECT color_template, COUNT(*) AS templates
-			FROM aggregate_graph_templates_item
-			GROUP BY color_template
-		) AS templates
-		ON ct.color_template_id=templates.color_template
-		LEFT JOIN (
-			SELECT color_template, COUNT(*) AS graphs
-			FROM aggregate_graphs_graph_item
-			GROUP BY color_template
-		) AS graphs
-		ON ct.color_template_id=graphs.color_template
 		$sql_where");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$template_list = db_fetch_assoc("SELECT
-		ct.color_template_id, ct.name, templates.templates, graphs.graphs
+	$template_list = db_fetch_assoc("SELECT *
 		FROM color_templates AS ct
-		LEFT JOIN (
-			SELECT color_template, COUNT(*) AS templates
-			FROM aggregate_graph_templates_item
-			GROUP BY color_template
-		) AS templates
-		ON ct.color_template_id=templates.color_template
-		LEFT JOIN (
-			SELECT color_template, COUNT(*) AS graphs
-			FROM aggregate_graphs_graph_item
-			GROUP BY color_template
-		) AS graphs
-		ON ct.color_template_id=graphs.color_template
 		$sql_where
 		$sql_order
 		$sql_limit");

diff --git a/data_input.php b/data_input.php
@@ -862,21 +862,16 @@ function clearFilter() {
 
 	$sql_where  = api_plugin_hook_function('data_input_sql_where', $sql_where);
 
-	$total_rows = db_fetch_cell("SELECT count(*)
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
 		FROM data_input AS di
 		$sql_where");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$data_inputs = db_fetch_assoc("SELECT di.*,
-		SUM(CASE WHEN dtd.local_data_id=0 THEN 1 ELSE 0 END) AS templates,
-		SUM(CASE WHEN dtd.local_data_id>0 THEN 1 ELSE 0 END) AS data_sources
+	$data_inputs = db_fetch_assoc("SELECT di.*
 		FROM data_input AS di
-		LEFT JOIN data_template_data AS dtd
-		ON di.id=dtd.data_input_id
 		$sql_where
-		GROUP BY di.id
 		$sql_order
 		$sql_limit");
 

diff --git a/data_queries.php b/data_queries.php
@@ -1407,25 +1407,18 @@ function clearFilter() {
 		COUNT(*)
 		FROM snmp_query AS sq
 		INNER JOIN data_input AS di
-		ON (sq.data_input_id=di.id)
+		ON sq.data_input_id=di.id
 		$sql_where");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$snmp_queries = db_fetch_assoc("SELECT sq.id, sq.name,
-		di.name AS data_input_method,
-		COUNT(DISTINCT gl.id) AS graphs,
-		COUNT(DISTINCT sqg.graph_template_id) AS templates
+	$snmp_queries = db_fetch_assoc("SELECT sq.id, sq.name, sq.graphs, sq.templates,
+		di.name AS data_input_method
 		FROM snmp_query AS sq
-		LEFT JOIN snmp_query_graph AS sqg
-		ON sq.id=sqg.snmp_query_id
 		LEFT JOIN data_input AS di
-		ON (sq.data_input_id=di.id)
-		LEFT JOIN graph_local AS gl
-		ON gl.snmp_query_id=sq.id
+		ON sq.data_input_id=di.id
 		$sql_where
-		GROUP BY sq.id
 		$sql_order
 		$sql_limit");
 
@@ -1477,8 +1470,6 @@ function clearFilter() {
 
 	html_header_sort_checkbox($display_text, get_request_var('sort_column'), get_request_var('sort_direction'), false);
 
-	$i = 0;
-
 	if (cacti_sizeof($snmp_queries)) {
 		foreach ($snmp_queries as $snmp_query) {
 			if ($snmp_query['graphs'] == 0) {

diff --git a/data_source_profiles.php b/data_source_profiles.php
@@ -1038,34 +1038,17 @@ function clearFilter() {
 		$sql_having = '';
 	}
 
-	$total_rows = db_fetch_cell("SELECT
-		COUNT(`rows`)
-		FROM (
-			SELECT dsp.id AS `rows`,
-			SUM(CASE WHEN local_data_id>0 THEN 1 ELSE 0 END) AS data_sources
-			FROM data_source_profiles AS dsp
-			LEFT JOIN data_template_data AS dtd
-			ON dsp.id=dtd.data_source_profile_id
-			$sql_where
-			GROUP BY dsp.id
-			$sql_having
-		) AS rs");
+	$total_rows = db_fetch_cell("SELECT COUNT(*)
+		FROM data_source_profiles
+		$sql_where
+		$sql_having");
 
 	$sql_order = get_order_string();
 	$sql_limit = ' LIMIT ' . ($rows * (get_request_var('page') - 1)) . ',' . $rows;
 
-	$profile_list = db_fetch_assoc("SELECT rs.*,
-		SUM(CASE WHEN local_data_id=0 THEN 1 ELSE 0 END) AS templates,
-		SUM(CASE WHEN local_data_id>0 THEN 1 ELSE 0 END) AS data_sources
-		FROM (
-			SELECT dsp.*, dtd.local_data_id
-			FROM data_source_profiles AS dsp
-			LEFT JOIN data_template_data AS dtd
-			ON dsp.id=dtd.data_source_profile_id
-			$sql_where
-			GROUP BY dsp.id, dtd.data_template_id, dtd.local_data_id
-		) AS rs
-		GROUP BY rs.id
+	$profile_list = db_fetch_assoc("SELECT *
+		FROM data_source_profiles
+		$sql_where
 		$sql_having
 		$sql_order
 		$sql_limit");