[feature] Tune sqlite pragmas (#1349)

* sqlite pragma tuning

* use formatuint

* add sqlite busy timeout

* fix incorrect cache size format

* update envparsing test

* add sqlite tuning flags to cli

* set sqlite timeout to 30s default

docs/configuration/database.md

@@ -107,4 +107,36 @@ db-tls-mode: "disable"
 # Examples: ["/path/to/some/cert.crt"]
 # Default: ""
 db-tls-ca-cert: ""
+
+# String. SQLite journaling mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
+# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
+# Default: "WAL"
+db-sqlite-journal-mode: "WAL"
+
+# String. SQLite synchronous mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_synchronous
+# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
+# Default: "NORMAL"
+db-sqlite-synchronous: "NORMAL"
+
+# Byte size. SQlite cache size.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_cache_size
+# Examples: ["32MiB", "0", "64MiB"]
+# Default: "64MiB"
+db-sqlite-cache-size: "64MiB"
+
+# Duration. SQlite busy timeout.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
+# Examples: ["0s", "1s", "30s", "1m", "5m"]
+# Default: "5s"
+db-sqlite-busy-timeout: "30s"
 ```

example/config.yaml

@@ -164,6 +164,38 @@ db-tls-mode: "disable"
 # Default: ""
 db-tls-ca-cert: ""
 
+# String. SQLite journaling mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_journal_mode
+# Examples: ["DELETE", "TRUNCATE", "PERSIST", "MEMORY", "WAL", "OFF"]
+# Default: "WAL"
+db-sqlite-journal-mode: "WAL"
+
+# String. SQLite synchronous mode.
+# SQLite only -- unused otherwise.
+# If set to empty string, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_synchronous
+# Examples: ["OFF", "NORMAL", "FULL", "EXTRA"]
+# Default: "NORMAL"
+db-sqlite-synchronous: "NORMAL"
+
+# Byte size. SQlite cache size.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_cache_size
+# Examples: ["32MiB", "0", "64MiB"]
+# Default: "64MiB"
+db-sqlite-cache-size: "64MiB"
+
+# Duration. SQlite busy timeout.
+# SQLite only -- unused otherwise.
+# If set to empty string or zero, the sqlite default will be used.
+# See: https://www.sqlite.org/pragma.html#pragma_busy_timeout
+# Examples: ["0s", "1s", "30s", "1m", "5m"]
+# Default: "5s"
+db-sqlite-busy-timeout: "30s"
+
 cache:
   gts:
     ###########################

internal/config/config.go

@@ -58,14 +58,18 @@ type Configuration struct {
 	TrustedProxies  []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."`
 	SoftwareVersion string   `name:"software-version" usage:""`
 
-	DbType      string `name:"db-type" usage:"Database type: eg., postgres"`
-	DbAddress   string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
-	DbPort      int    `name:"db-port" usage:"Database port"`
-	DbUser      string `name:"db-user" usage:"Database username"`
-	DbPassword  string `name:"db-password" usage:"Database password"`
-	DbDatabase  string `name:"db-database" usage:"Database name"`
-	DbTLSMode   string `name:"db-tls-mode" usage:"Database tls mode"`
-	DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+	DbType              string        `name:"db-type" usage:"Database type: eg., postgres"`
+	DbAddress           string        `name:"db-address" usage:"Database ipv4 address, hostname, or filename"`
+	DbPort              int           `name:"db-port" usage:"Database port"`
+	DbUser              string        `name:"db-user" usage:"Database username"`
+	DbPassword          string        `name:"db-password" usage:"Database password"`
+	DbDatabase          string        `name:"db-database" usage:"Database name"`
+	DbTLSMode           string        `name:"db-tls-mode" usage:"Database tls mode"`
+	DbTLSCACert         string        `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"`
+	DbSqliteJournalMode string        `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"`
+	DbSqliteSynchronous string        `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"`
+	DbSqliteCacheSize   bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"`
+	DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"`
 
 	WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."`
 	WebAssetBaseDir    string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"`

internal/config/defaults.go

@@ -40,14 +40,18 @@ var Defaults = Configuration{
 	Port:            8080,
 	TrustedProxies:  []string{"127.0.0.1/32", "::1"}, // localhost
 
-	DbType:      "postgres",
-	DbAddress:   "",
-	DbPort:      5432,
-	DbUser:      "",
-	DbPassword:  "",
-	DbDatabase:  "gotosocial",
-	DbTLSMode:   "disable",
-	DbTLSCACert: "",
+	DbType:              "postgres",
+	DbAddress:           "",
+	DbPort:              5432,
+	DbUser:              "",
+	DbPassword:          "",
+	DbDatabase:          "gotosocial",
+	DbTLSMode:           "disable",
+	DbTLSCACert:         "",
+	DbSqliteJournalMode: "WAL",
+	DbSqliteSynchronous: "NORMAL",
+	DbSqliteCacheSize:   64 * bytesize.MiB,
+	DbSqliteBusyTimeout: time.Second * 30,
 
 	WebTemplateBaseDir: "./web/template/",
 	WebAssetBaseDir:    "./web/assets/",

internal/config/flags.go

@@ -51,6 +51,10 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) {
 		cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage"))
 		cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage"))
 		cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage"))
+		cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage"))
+		cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage"))
+		cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage"))
+		cmd.PersistentFlags().Duration(DbSqliteBusyTimeoutFlag(), cfg.DbSqliteBusyTimeout, fieldtag("DbSqliteBusyTimeout", "usage"))
 	})
 }
 

internal/config/helpers.gen.go

@@ -524,6 +524,106 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() }
 // SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field
 func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) }
 
+// GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field
+func (st *ConfigState) GetDbSqliteJournalMode() (v string) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteJournalMode
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteJournalMode safely sets the Configuration value for state's 'DbSqliteJournalMode' field
+func (st *ConfigState) SetDbSqliteJournalMode(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteJournalMode = v
+	st.reloadToViper()
+}
+
+// DbSqliteJournalModeFlag returns the flag name for the 'DbSqliteJournalMode' field
+func DbSqliteJournalModeFlag() string { return "db-sqlite-journal-mode" }
+
+// GetDbSqliteJournalMode safely fetches the value for global configuration 'DbSqliteJournalMode' field
+func GetDbSqliteJournalMode() string { return global.GetDbSqliteJournalMode() }
+
+// SetDbSqliteJournalMode safely sets the value for global configuration 'DbSqliteJournalMode' field
+func SetDbSqliteJournalMode(v string) { global.SetDbSqliteJournalMode(v) }
+
+// GetDbSqliteSynchronous safely fetches the Configuration value for state's 'DbSqliteSynchronous' field
+func (st *ConfigState) GetDbSqliteSynchronous() (v string) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteSynchronous
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteSynchronous safely sets the Configuration value for state's 'DbSqliteSynchronous' field
+func (st *ConfigState) SetDbSqliteSynchronous(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteSynchronous = v
+	st.reloadToViper()
+}
+
+// DbSqliteSynchronousFlag returns the flag name for the 'DbSqliteSynchronous' field
+func DbSqliteSynchronousFlag() string { return "db-sqlite-synchronous" }
+
+// GetDbSqliteSynchronous safely fetches the value for global configuration 'DbSqliteSynchronous' field
+func GetDbSqliteSynchronous() string { return global.GetDbSqliteSynchronous() }
+
+// SetDbSqliteSynchronous safely sets the value for global configuration 'DbSqliteSynchronous' field
+func SetDbSqliteSynchronous(v string) { global.SetDbSqliteSynchronous(v) }
+
+// GetDbSqliteCacheSize safely fetches the Configuration value for state's 'DbSqliteCacheSize' field
+func (st *ConfigState) GetDbSqliteCacheSize() (v bytesize.Size) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteCacheSize
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteCacheSize safely sets the Configuration value for state's 'DbSqliteCacheSize' field
+func (st *ConfigState) SetDbSqliteCacheSize(v bytesize.Size) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteCacheSize = v
+	st.reloadToViper()
+}
+
+// DbSqliteCacheSizeFlag returns the flag name for the 'DbSqliteCacheSize' field
+func DbSqliteCacheSizeFlag() string { return "db-sqlite-cache-size" }
+
+// GetDbSqliteCacheSize safely fetches the value for global configuration 'DbSqliteCacheSize' field
+func GetDbSqliteCacheSize() bytesize.Size { return global.GetDbSqliteCacheSize() }
+
+// SetDbSqliteCacheSize safely sets the value for global configuration 'DbSqliteCacheSize' field
+func SetDbSqliteCacheSize(v bytesize.Size) { global.SetDbSqliteCacheSize(v) }
+
+// GetDbSqliteBusyTimeout safely fetches the Configuration value for state's 'DbSqliteBusyTimeout' field
+func (st *ConfigState) GetDbSqliteBusyTimeout() (v time.Duration) {
+	st.mutex.Lock()
+	v = st.config.DbSqliteBusyTimeout
+	st.mutex.Unlock()
+	return
+}
+
+// SetDbSqliteBusyTimeout safely sets the Configuration value for state's 'DbSqliteBusyTimeout' field
+func (st *ConfigState) SetDbSqliteBusyTimeout(v time.Duration) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.DbSqliteBusyTimeout = v
+	st.reloadToViper()
+}
+
+// DbSqliteBusyTimeoutFlag returns the flag name for the 'DbSqliteBusyTimeout' field
+func DbSqliteBusyTimeoutFlag() string { return "db-sqlite-busy-timeout" }
+
+// GetDbSqliteBusyTimeout safely fetches the value for global configuration 'DbSqliteBusyTimeout' field
+func GetDbSqliteBusyTimeout() time.Duration { return global.GetDbSqliteBusyTimeout() }
+
+// SetDbSqliteBusyTimeout safely sets the value for global configuration 'DbSqliteBusyTimeout' field
+func SetDbSqliteBusyTimeout(v time.Duration) { global.SetDbSqliteBusyTimeout(v) }
+
 // GetWebTemplateBaseDir safely fetches the Configuration value for state's 'WebTemplateBaseDir' field
 func (st *ConfigState) GetWebTemplateBaseDir() (v string) {
 	st.mutex.Lock()