Traffic Stats: reuse http influx client (#7022)

Instead of creating a new http influx client with each request, reuse
the existing client. This should prevent connection leaks when an old
client is discarded for a new one but connections are still active on
the old one (and therefore not closed by the `Close` function).

CHANGELOG.md

@@ -7,6 +7,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 ### Added
 - [#6033](https://github.com/apache/trafficcontrol/issues/6033) Added ability to assign multiple server capabilities to a server.
 
+### Fixed
+- Traffic Stats: Reuse InfluxDB client handle to prevent potential connection leaks
+
 ### Changed
 - Traffic Portal now obscures sensitive text in Delivery Service "Raw Remap" fields, private SSL keys, "Header Rewrite" rules, and ILO interface passwords by default.
 

traffic_stats/traffic_stats.go

@@ -1041,6 +1041,18 @@ func influxConnect(config StartupConfig) (influx.Client, error) {
 		host := hosts[n]
 		hosts = append(hosts[:n], hosts[n+1:]...)
 		parsedURL, _ := url.Parse(host.URL)
+		if host.InfluxClient != nil && parsedURL.Scheme == "http" {
+			// NOTE: closing an http client just closes idle connections -- the client can still make new requests
+			if err := host.InfluxClient.Close(); err != nil {
+				errorf("closing http influx client: %s", err)
+			}
+			_, _, err := host.InfluxClient.Ping(10)
+			if err != nil {
+				warnf("pinging InfluxDB: %v", err)
+				continue
+			}
+			return host.InfluxClient, nil
+		}
 		if parsedURL.Scheme == "udp" {
 			conf := influx.UDPConfig{
 				Addr: parsedURL.Host,
@@ -1063,12 +1075,6 @@ func influxConnect(config StartupConfig) (influx.Client, error) {
 			errorf("An error occurred creating InfluxDB HTTP client: %v", err)
 			continue
 		}
-		//Close old connections explicitly
-		if host.InfluxClient != nil {
-			if err := host.InfluxClient.Close(); err != nil {
-				errorf("closing influx client: %s", err)
-			}
-		}
 		host.InfluxClient = con
 		_, _, err = con.Ping(10)
 		if err != nil {
@@ -1077,7 +1083,7 @@ func influxConnect(config StartupConfig) (influx.Client, error) {
 		}
 		return con, nil
 	}
-	err := errors.New("Could not connect to any of the InfluxDb servers defined in the influxUrls config.")
+	err := errors.New("could not connect to any of the InfluxDb servers defined in the influxUrls config")
 	return nil, err
 }