[chore] set max open / idle conns + conn max lifetime for both postgr…

…es and sqlite (#1369)

* [chore] set max open / idle conns + conn max lifetime for both postgres and sqlite

* reduce cache size default to 8MiB, reduce connections to 2 * cpu

* introduce max open conns multiplier, tune sqlite and pg separately

* go fmt

docs/configuration/database.md

@@ -108,6 +108,30 @@ db-tls-mode: "disable"
 # Default: ""
 db-tls-ca-cert: ""
 
+# Int. Number to multiply by CPU count to set permitted total of open database connections (in-use and idle).
+# You can use this setting to tune your database connection behavior, though most admins won't need to touch it.
+#
+# Example values for multiplier 8:
+#
+# 1 cpu = 08 open connections
+# 2 cpu = 16 open connections
+# 4 cpu = 32 open connections
+#
+# Example values for multiplier 4:
+#
+# 1 cpu = 04 open connections
+# 2 cpu = 08 open connections
+# 4 cpu = 16 open connections
+#
+# A multiplier of 8 is a sensible default, but you may wish to increase this for instances
+# running on very performant hardware, or decrease it for instances using v. slow CPUs.
+#
+# If you set this to 0 or less, it will be adjusted to 1.
+#
+# Examples: [16, 8, 10, 2]
+# Default: 8
+db-max-open-conns-multiplier: 8
+
 # String. SQLite journaling mode.
 # SQLite only -- unused otherwise.
 # If set to empty string, the sqlite default will be used.
@@ -126,11 +150,11 @@ db-sqlite-synchronous: "NORMAL"
 
 # Byte size. SQlite cache size.
 # SQLite only -- unused otherwise.
-# If set to empty string or zero, the sqlite default will be used.
+# If set to empty string or zero, the sqlite default (2MiB) will be used.
 # See: https://www.sqlite.org/pragma.html#pragma_cache_size
-# Examples: ["32MiB", "0", "64MiB"]
-# Default: "64MiB"
-db-sqlite-cache-size: "64MiB"
+# Examples: ["0", "2MiB", "8MiB", "64MiB"]
+# Default: "8MiB"
+db-sqlite-cache-size: "8MiB"
 
 # Duration. SQlite busy timeout.
 # SQLite only -- unused otherwise.

example/config.yaml

@@ -164,6 +164,30 @@ db-tls-mode: "disable"
 # Default: ""
 db-tls-ca-cert: ""
 
+# Int. Number to multiply by CPU count to set permitted total of open database connections (in-use and idle).
+# You can use this setting to tune your database connection behavior, though most admins won't need to touch it.
+#
+# Example values for multiplier 8:
+#
+# 1 cpu = 08 open connections
+# 2 cpu = 16 open connections
+# 4 cpu = 32 open connections
+#
+# Example values for multiplier 4:
+#
+# 1 cpu = 04 open connections
+# 2 cpu = 08 open connections
+# 4 cpu = 16 open connections
+#
+# A multiplier of 8 is a sensible default, but you may wish to increase this for instances
+# running on very performant hardware, or decrease it for instances using v. slow CPUs.
+#
+# If you set this to 0 or less, it will be adjusted to 1.
+#
+# Examples: [16, 8, 10, 2]
+# Default: 8
+db-max-open-conns-multiplier: 8
+
 # String. SQLite journaling mode.
 # SQLite only -- unused otherwise.
 # If set to empty string, the sqlite default will be used.
@@ -182,11 +206,11 @@ db-sqlite-synchronous: "NORMAL"
 
 # Byte size. SQlite cache size.
 # SQLite only -- unused otherwise.
-# If set to empty string or zero, the sqlite default will be used.
+# If set to empty string or zero, the sqlite default (2MiB) will be used.
 # See: https://www.sqlite.org/pragma.html#pragma_cache_size
-# Examples: ["32MiB", "0", "64MiB"]
-# Default: "64MiB"
-db-sqlite-cache-size: "64MiB"
+# Examples: ["0", "2MiB", "8MiB", "64MiB"]
+# Default: "8MiB"
+db-sqlite-cache-size: "8MiB"
 
 # Duration. SQlite busy timeout.
 # SQLite only -- unused otherwise.

internal/config/config.go

@@ -58,18 +58,19 @@ type Configuration struct {
 	TrustedProxies  []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
 	SoftwareVersion string   `name:"software-version" usage:""`
 
-	DbType              string        `name:"db-type" usage:"Database type: eg., postgres"`
-	DbAddress           string        `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
-	DbPort              int           `name:"db-port" usage:"Database port"`
-	DbUser              string        `name:"db-user" usage:"Database username"`
-	DbPassword          string        `name:"db-password" usage:"Database password"`
-	DbDatabase          string        `name:"db-database" usage:"Database name"`
-	DbTLSMode           string        `name:"db-tls-mode" usage:"Database tls mode"`
-	DbTLSCACert         string        `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
-	DbSqliteJournalMode string        `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
-	DbSqliteSynchronous string        `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
-	DbSqliteCacheSize   bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
-	DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
+	DbType                   string        `name:"db-type" usage:"Database type: eg., postgres"`
+	DbAddress                string        `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
+	DbPort                   int           `name:"db-port" usage:"Database port"`
+	DbUser                   string        `name:"db-user" usage:"Database username"`
+	DbPassword               string        `name:"db-password" usage:"Database password"`
+	DbDatabase               string        `name:"db-database" usage:"Database name"`
+	DbTLSMode                string        `name:"db-tls-mode" usage:"Database tls mode"`
+	DbTLSCACert              string        `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+	DbMaxOpenConnsMultiplier int           `name:"db-max-open-conns-multiplier" usage:"Multiplier to use per cpu for max open database connections. 0 or less is normalized to 1."`
+	DbSqliteJournalMode      string        `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
+	DbSqliteSynchronous      string        `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
+	DbSqliteCacheSize        bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
+	DbSqliteBusyTimeout      time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
 
 	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`

internal/config/defaults.go

@@ -40,18 +40,19 @@ var Defaults = Configuration{
 	Port:            8080,
 	TrustedProxies:  []string{"127.0.0.1/32", "::1"}, // localhost
 
-	DbType:              "postgres",
-	DbAddress:           "",
-	DbPort:              5432,
-	DbUser:              "",
-	DbPassword:          "",
-	DbDatabase:          "gotosocial",
-	DbTLSMode:           "disable",
-	DbTLSCACert:         "",
-	DbSqliteJournalMode: "WAL",
-	DbSqliteSynchronous: "NORMAL",
-	DbSqliteCacheSize:   64 * bytesize.MiB,
-	DbSqliteBusyTimeout: time.Minute * 5,
+	DbType:                   "postgres",
+	DbAddress:                "",
+	DbPort:                   5432,
+	DbUser:                   "",
+	DbPassword:               "",
+	DbDatabase:               "gotosocial",
+	DbTLSMode:                "disable",
+	DbTLSCACert:              "",
+	DbMaxOpenConnsMultiplier: 8,
+	DbSqliteJournalMode:      "WAL",
+	DbSqliteSynchronous:      "NORMAL",
+	DbSqliteCacheSize:        8 * bytesize.MiB,
+	DbSqliteBusyTimeout:      time.Minute * 5,
 
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",

internal/config/flags.go

@@ -51,6 +51,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
 		cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage"))
 		cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage"))
 		cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage"))
+		cmd.PersistentFlags().Int(DbMaxOpenConnsMultiplierFlag(), cfg.DbMaxOpenConnsMultiplier, fieldtag("DbMaxOpenConnsMultiplier", "usage"))
 		cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage"))
 		cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage"))
 		cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage"))

internal/config/helpers.gen.go

@@ -524,6 +524,31 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() }
 // SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field
 func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) }
 
+// GetDbMaxOpenConnsMultiplier safely fetches the Configuration value for state's 'DbMaxOpenConnsMultiplier' field
+func (st *ConfigState) GetDbMaxOpenConnsMultiplier() (v int) {
+	st.mutex.Lock()
+	v = st.config.DbMaxOpenConnsMultiplier
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbMaxOpenConnsMultiplier safely sets the Configuration value for state's 'DbMaxOpenConnsMultiplier' field
+func (st *ConfigState) SetDbMaxOpenConnsMultiplier(v int) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbMaxOpenConnsMultiplier = v
+	st.reloadToViper()
+}
+
+// DbMaxOpenConnsMultiplierFlag returns the flag name for the 'DbMaxOpenConnsMultiplier' field
+func DbMaxOpenConnsMultiplierFlag() string { return "db-max-open-conns-multiplier" }
+
+// GetDbMaxOpenConnsMultiplier safely fetches the value for global configuration 'DbMaxOpenConnsMultiplier' field
+func GetDbMaxOpenConnsMultiplier() int { return global.GetDbMaxOpenConnsMultiplier() }
+
+// SetDbMaxOpenConnsMultiplier safely sets the value for global configuration 'DbMaxOpenConnsMultiplier' field
+func SetDbMaxOpenConnsMultiplier(v int) { global.SetDbMaxOpenConnsMultiplier(v) }
+
 // GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field
 func (st *ConfigState) GetDbSqliteJournalMode() (v string) {
 	st.mutex.Lock()