Add Transport config parameter for MaxConnsPerHost

config/consul_test.go

@@ -327,6 +327,7 @@ func TestConsulConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 			},

config/convert_test.go

@@ -216,7 +216,7 @@ func TestFileModePresent(t *testing.T) {
 		},
 		{
 			"present",
-			FileMode(0644),
+			FileMode(0o644),
 			true,
 		},
 		{

config/nomad_test.go

@@ -247,6 +247,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				Retry: &RetryConfig{
@@ -286,6 +287,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				Retry: &RetryConfig{
@@ -332,6 +334,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				Retry: &RetryConfig{
@@ -374,6 +377,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					CustomDialer:        mockDialer{},
 				},
 				Retry: &RetryConfig{
@@ -418,6 +422,7 @@ func TestNomadConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				Retry: &RetryConfig{

config/transport.go

@@ -30,6 +30,14 @@ const (
 	// per host.
 	DefaultMaxIdleConnsPerHost = 100
 
+	// DefaultMaxConnsPerHost is the default number of maximum connections to use
+	// per host. The associated HTTP Transport MaxConnsPerHost is used to limit
+	// the total number of connections per host, including connections in the
+	// dialing, active, and idle states. On limit violation, dials will block.
+	//
+	// Zero means no limit.
+	DefaultMaxConnsPerHost = 0
+
 	// DefaultTLSHandshakeTimeout is the amount of time to negotiate the TLS
 	// handshake.
 	DefaultTLSHandshakeTimeout = 10 * time.Second
@@ -63,6 +71,9 @@ type TransportConfig struct {
 	// host.
 	MaxIdleConnsPerHost *int `mapstructure:"max_idle_conns_per_host"`
 
+	// MaxConns is the maximum number of total connections.
+	MaxConnsPerHost *int `mapstructure:"max_conns_per_host"`
+
 	// TLSHandshakeTimeout is the amount of time to wait to complete the TLS
 	// handshake.
 	TLSHandshakeTimeout *time.Duration `mapstructure:"tls_handshake_timeout"`
@@ -88,6 +99,7 @@ func (c *TransportConfig) Copy() *TransportConfig {
 	o.DisableKeepAlives = c.DisableKeepAlives
 	o.IdleConnTimeout = c.IdleConnTimeout
 	o.MaxIdleConns = c.MaxIdleConns
+	o.MaxConnsPerHost = c.MaxConnsPerHost
 	o.MaxIdleConnsPerHost = c.MaxIdleConnsPerHost
 	o.TLSHandshakeTimeout = c.TLSHandshakeTimeout
 
@@ -140,6 +152,10 @@ func (c *TransportConfig) Merge(o *TransportConfig) *TransportConfig {
 		r.MaxIdleConnsPerHost = o.MaxIdleConnsPerHost
 	}
 
+	if o.MaxConnsPerHost != nil {
+		r.MaxConnsPerHost = o.MaxConnsPerHost
+	}
+
 	if o.TLSHandshakeTimeout != nil {
 		r.TLSHandshakeTimeout = o.TLSHandshakeTimeout
 	}
@@ -173,6 +189,10 @@ func (c *TransportConfig) Finalize() {
 		c.MaxIdleConnsPerHost = Int(DefaultMaxIdleConnsPerHost)
 	}
 
+	if c.MaxConnsPerHost == nil {
+		c.MaxConnsPerHost = Int(DefaultMaxConnsPerHost)
+	}
+
 	if c.TLSHandshakeTimeout == nil {
 		c.TLSHandshakeTimeout = TimeDuration(DefaultTLSHandshakeTimeout)
 	}

config/transport_test.go

@@ -33,6 +33,7 @@ func TestTransportConfig_Copy(t *testing.T) {
 				IdleConnTimeout:     TimeDuration(40 * time.Second),
 				MaxIdleConns:        Int(150),
 				MaxIdleConnsPerHost: Int(15),
+				MaxConnsPerHost:     Int(10),
 				TLSHandshakeTimeout: TimeDuration(30 * time.Second),
 			},
 		},
@@ -44,6 +45,7 @@ func TestTransportConfig_Copy(t *testing.T) {
 				IdleConnTimeout:     TimeDuration(40 * time.Second),
 				MaxIdleConns:        Int(150),
 				MaxIdleConnsPerHost: Int(15),
+				MaxConnsPerHost:     Int(10),
 				TLSHandshakeTimeout: TimeDuration(30 * time.Second),
 			},
 		},
@@ -234,6 +236,30 @@ func TestTransportConfig_Merge(t *testing.T) {
 			&TransportConfig{MaxIdleConnsPerHost: Int(10)},
 			&TransportConfig{MaxIdleConnsPerHost: Int(10)},
 		},
+		{
+			"max_conns_overrides",
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+			&TransportConfig{MaxConnsPerHost: Int(20)},
+			&TransportConfig{MaxConnsPerHost: Int(20)},
+		},
+		{
+			"max_conns_empty_one",
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+			&TransportConfig{},
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+		},
+		{
+			"max_conns_empty_two",
+			&TransportConfig{},
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+		},
+		{
+			"max_conns_same",
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+			&TransportConfig{MaxConnsPerHost: Int(10)},
+		},
 		{
 			"tls_handshake_timeout_overrides",
 			&TransportConfig{TLSHandshakeTimeout: TimeDuration(10 * time.Second)},
@@ -309,6 +335,7 @@ func TestTransportConfig_Finalize(t *testing.T) {
 				DisableKeepAlives:   Bool(false),
 				IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 				MaxIdleConns:        Int(DefaultMaxIdleConns),
+				MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 				MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
 				TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 			},

config/vault_test.go

@@ -510,6 +510,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -556,6 +557,59 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
+					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
+				},
+				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
+				DefaultLeaseDuration:       TimeDuration(DefaultVaultLeaseDuration),
+				LeaseRenewalThreshold:      Float64(DefaultLeaseRenewalThreshold),
+				K8SAuthRoleName:            String(""),
+				K8SServiceAccountTokenPath: String(DefaultK8SServiceAccountTokenPath),
+				K8SServiceAccountToken:     String(""),
+				K8SServiceMountPath:        String(DefaultK8SServiceMountPath),
+			},
+		},
+		{
+			"with_max_conns",
+			nil,
+			&VaultConfig{
+				Address: String("address"),
+				Transport: &TransportConfig{
+					MaxIdleConns:        Int(20),
+					MaxIdleConnsPerHost: Int(5),
+					MaxConnsPerHost:     Int(100),
+				},
+			},
+			&VaultConfig{
+				Address:    String("address"),
+				Enabled:    Bool(true),
+				Namespace:  String(""),
+				RenewToken: Bool(false),
+				Retry: &RetryConfig{
+					Backoff:    TimeDuration(DefaultRetryBackoff),
+					MaxBackoff: TimeDuration(DefaultRetryMaxBackoff),
+					Enabled:    Bool(true),
+					Attempts:   Int(DefaultRetryAttempts),
+				},
+				SSL: &SSLConfig{
+					CaCert:      String(""),
+					CaCertBytes: String(""),
+					CaPath:      String(""),
+					Cert:        String(""),
+					Enabled:     Bool(true),
+					Key:         String(""),
+					ServerName:  String(""),
+					Verify:      Bool(true),
+				},
+				Token: String(""),
+				Transport: &TransportConfig{
+					DialKeepAlive:       TimeDuration(DefaultDialKeepAlive),
+					DialTimeout:         TimeDuration(DefaultDialTimeout),
+					DisableKeepAlives:   Bool(false),
+					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
+					MaxIdleConns:        Int(20),
+					MaxIdleConnsPerHost: Int(5),
+					MaxConnsPerHost:     Int(100),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -612,6 +666,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -666,6 +721,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -713,6 +769,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -760,6 +817,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),
@@ -809,6 +867,7 @@ func TestVaultConfig_Finalize(t *testing.T) {
 					IdleConnTimeout:     TimeDuration(DefaultIdleConnTimeout),
 					MaxIdleConns:        Int(DefaultMaxIdleConns),
 					MaxIdleConnsPerHost: Int(DefaultMaxIdleConnsPerHost),
+					MaxConnsPerHost:     Int(DefaultMaxConnsPerHost),
 					TLSHandshakeTimeout: TimeDuration(DefaultTLSHandshakeTimeout),
 				},
 				UnwrapToken:                Bool(DefaultVaultUnwrapToken),

dependency/client_set.go

@@ -112,6 +112,7 @@ type CreateVaultClientInput struct {
 	TransportIdleConnTimeout     time.Duration
 	TransportMaxIdleConns        int
 	TransportMaxIdleConnsPerHost int
+	TransportMaxConnsPerHost     int
 	TransportTLSHandshakeTimeout time.Duration
 }
 
@@ -280,6 +281,7 @@ func (c *ClientSet) CreateVaultClient(i *CreateVaultClientInput) error {
 		MaxIdleConns:        i.TransportMaxIdleConns,
 		IdleConnTimeout:     i.TransportIdleConnTimeout,
 		MaxIdleConnsPerHost: i.TransportMaxIdleConnsPerHost,
+		MaxConnsPerHost:     i.TransportMaxConnsPerHost,
 		TLSHandshakeTimeout: i.TransportTLSHandshakeTimeout,
 	}
 

dependency/nomad_var_get_test.go

@@ -14,7 +14,6 @@ import (
 )
 
 func TestNewNVGetQuery(t *testing.T) {
-
 	cases := []struct {
 		name string
 		i    string
@@ -126,7 +125,6 @@ func TestNewNVGetQuery(t *testing.T) {
 }
 
 func TestNVGetQuery_Fetch(t *testing.T) {
-
 	type nvmap map[string]string
 	_ = testNomad.CreateVariable("test-kv-get/path", nvmap{"bar": "barp"}, nil)
 	_ = testNomad.CreateNamespace("test", nil)
@@ -274,7 +272,6 @@ func TestNVGetQuery_Fetch(t *testing.T) {
 }
 
 func TestNVGetQuery_String(t *testing.T) {
-
 	cases := []struct {
 		name string
 		i    string

dependency/nomad_var_list_test.go

@@ -13,7 +13,6 @@ import (
 )
 
 func TestNewNVListQuery(t *testing.T) {
-
 	cases := []struct {
 		name string
 		i    string
@@ -177,7 +176,6 @@ func TestNewNVListQuery(t *testing.T) {
 }
 
 func TestNVListQuery_Fetch(t *testing.T) {
-
 	type nvmap map[string]string
 	_ = testNomad.CreateVariable("test-kv-list/prefix/foo", nvmap{"bar": "barp"}, nil)
 	_ = testNomad.CreateVariable("test-kv-list/prefix/zip", nvmap{"zap": "zapp"}, nil)
@@ -346,7 +344,6 @@ func TestNVListQuery_Fetch(t *testing.T) {
 }
 
 func TestNVListQuery_String(t *testing.T) {
-
 	cases := []struct {
 		name string
 		i    string

manager/example_extfuncmap_test.go

@@ -22,7 +22,6 @@ import (
 // It is not comprehensive and does not demonstrate the dependencies, polling,
 // and rerendering features available in the manager
 func Example_customFuncMap() {
-
 	// Consul-template uses the standard logger, which needs to be silenced
 	// in this example
 	log.SetOutput(io.Discard)

manager/example_manager_test.go

@@ -19,7 +19,6 @@ import (
 // demonstrate the dependencies, polling, and rerendering features available in
 // the manager
 func Example() {
-
 	// Consul-template uses the standard logger, which needs to be silenced
 	// in this example
 	log.SetOutput(io.Discard)

manager/runner.go

@@ -1363,6 +1363,7 @@ func NewClientSet(c *config.Config) (*dep.ClientSet, error) {
 		TransportIdleConnTimeout:     config.TimeDurationVal(c.Vault.Transport.IdleConnTimeout),
 		TransportMaxIdleConns:        config.IntVal(c.Vault.Transport.MaxIdleConns),
 		TransportMaxIdleConnsPerHost: config.IntVal(c.Vault.Transport.MaxIdleConnsPerHost),
+		TransportMaxConnsPerHost:     config.IntVal(c.Vault.Transport.MaxConnsPerHost),
 		TransportTLSHandshakeTimeout: config.TimeDurationVal(c.Vault.Transport.TLSHandshakeTimeout),
 		K8SAuthRoleName:              config.StringVal(c.Vault.K8SAuthRoleName),
 		K8SServiceAccountTokenPath:   config.StringVal(c.Vault.K8SServiceAccountTokenPath),