[feature] Make rate limit requests amount configurable (#966)

* update rate limit documentation

* regenerate landingpage config helpers

* make rate limit rate configurable

docs/api/ratelimiting.md

@@ -0,0 +1,27 @@
+# Rate Limit
+
+To mitigate abuse + scraping of your instance, an IP-based HTTP rate limit is in place.
+
+This rate limit applies not just to the API, but to all requests (web, federation, etc).
+
+By default, a maximum of 1000 requests in a 5 minute time window are allowed.
+
+Every response will include the current status of the rate limit with the following headers:
+
+- `X-Ratelimit-Limit`: maximum number of requests allowed per time period.
+- `X-Ratelimit-Remaining`: number of remaining requests that can still be performed within.
+- `X-Ratelimit-Reset`: unix timestamp indicating when the rate limit will reset.
+
+In case the rate limit is exceeded, an [HTTP 429 Too Many Requests](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429) error is returned to the caller.
+
+## Rate Limiting FAQs
+
+### My rate limit keeps being exceeded! Why?
+
+If you find that your rate limit is regularly being exceeded (both for yourself and other callers) during normal use of your instance, it's possible that your `trusted-proxies` setting is not configured correctly. This can result in your instance seeing all incoming IP addresses as the same address: namely, the IP address of your reverse proxy. This means that all incoming requests are *sharing the same rate limit*, rather than being split correctly per IP.
+
+You can investigate this by viewing the logs of your instance. If (almost) all logged IP addresses appear to be the same IP address (something like `172.x.x.x`), then it's likely that your `trusted-proxies` is not correctly configured. If this is the case, try adding the IP address of your reverse proxy to the list of `trusted-proxies`, and restarting your instance.
+
+### Can I configure the rate limit? Can I just turn it off?
+
+Yes! See the config setting `advanced-rate-limit-requests`.

docs/api/swagger.md

@@ -1,16 +1,5 @@
 # API Documentation
 
-## Rate limit
-
-To prevent abuse of the API an IP-based HTTP rate limit is in place, a maximum of 1000 requests in a 5 minutes time window are allowed, every response will include the current status of the rate limit with the following headers:
-
-- `x-ratelimit-limit` maximum number of requests allowed per time period (fixed)
-- `x-ratelimit-remaining` number of remaining requests that can still be performed
-- `x-ratelimit-reset` unix timestamp when the rate limit will reset
-
-In case the rate limit is exceeded an HTTP 429 error is returned to the caller.
-
-
 GoToSocial uses [go-swagger](https://github.com/go-swagger/go-swagger) to generate a V2 [OpenAPI specification](https://swagger.io/specification/v2/) document from code annotations.
 
 The resulting API documentation is rendered below, for quick reference.

docs/configuration/advanced.md

@@ -35,4 +35,21 @@ These are set to sensible defaults, so most server admins won't need to touch th
 # Options: ["lax", "strict"]
 # Default: "lax"
 advanced-cookies-samesite: "lax"
+
+# Int. Amount of requests to permit from a single IP address within a span of 5 minutes.
+# If this amount is exceeded, a 429 HTTP error code will be returned.
+# See https://docs.gotosocial.org/en/latest/api/swagger/#rate-limit.
+#
+# If you find yourself adjusting this limit because it's regularly being exceeded,
+# you should first verify that your settings for `trusted-proxies` (above) are correct.
+# In many cases, when the rate limit is exceeded it is because your instance sees all
+# incoming requests as coming from the *same IP address* (you can verify this by looking
+# at the client IPs in your instance logs). If this is the case, try adding that IP
+# address to your `trusted-proxies` *BEFORE* you go adjusting this rate limit setting!
+#
+# If you set this to 0 or less, rate limiting will be disabled entirely.
+#
+# Examples: [1000, 500, 0]
+# Default: 1000
+advanced-rate-limit-requests: 1000
 ```

example/config.yaml

@@ -557,3 +557,20 @@ syslog-address: "localhost:514"
 # Options: ["lax", "strict"]
 # Default: "lax"
 advanced-cookies-samesite: "lax"
+
+# Int. Amount of requests to permit from a single IP address within a span of 5 minutes.
+# If this amount is exceeded, a 429 HTTP error code will be returned.
+# See https://docs.gotosocial.org/en/latest/api/swagger/#rate-limit.
+#
+# If you find yourself adjusting this limit because it's regularly being exceeded,
+# you should first verify that your settings for `trusted-proxies` (above) are correct.
+# In many cases, when the rate limit is exceeded it is because your instance sees all
+# incoming requests as coming from the *same IP address* (you can verify this by looking
+# at the client IPs in your instance logs). If this is the case, try adding that IP
+# address to your `trusted-proxies` *BEFORE* you go adjusting this rate limit setting!
+#
+# If you set this to 0 or less, rate limiting will be disabled entirely.
+#
+# Examples: [1000, 500, 0]
+# Default: 1000
+advanced-rate-limit-requests: 1000

internal/api/security/security.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/superseriousbusiness/gotosocial/internal/api"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/router"
@@ -46,11 +47,14 @@ func New(db db.DB, server oauth.Server) api.ClientModule {
 
 // Route attaches security middleware to the given router
 func (m *Module) Route(s router.Router) error {
-	s.AttachMiddleware(m.RateLimit(RateLimitOptions{
-		// accept a maximum of 1000 requests in 5 minutes window
-		Period: 5 * time.Minute,
-		Limit:  1000,
-	}))
+	// only enable rate limit middleware if configured
+	// advanced-rate-limit-requests is greater than 0
+	if rateLimitRequests := config.GetAdvancedRateLimitRequests(); rateLimitRequests > 0 {
+		s.AttachMiddleware(m.RateLimit(RateLimitOptions{
+			Period: 5 * time.Minute,
+			Limit:  int64(rateLimitRequests),
+		}))
+	}
 	s.AttachMiddleware(m.SignatureCheck)
 	s.AttachMiddleware(m.FlocBlock)
 	s.AttachMiddleware(m.ExtraHeaders)

internal/config/config.go

@@ -129,7 +129,8 @@ type Configuration struct {
 	AdminAccountPassword string `name:"password" usage:"the password to set for this account"`
 	AdminTransPath       string `name:"path" usage:"the path of the file to import from/export to"`
 
-	AdvancedCookiesSamesite string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
+	AdvancedCookiesSamesite   string `name:"advanced-cookies-samesite" usage:"'strict' or 'lax', see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie/SameSite"`
+	AdvancedRateLimitRequests int    `name:"advanced-rate-limit-requests" usage:"Amount of HTTP requests to permit within a 5 minute window. 0 or less turns rate limiting off."`
 }
 
 // MarshalMap will marshal current Configuration into a map structure (useful for JSON).

internal/config/defaults.go

@@ -97,5 +97,6 @@ var Defaults = Configuration{
 	SyslogProtocol: "udp",
 	SyslogAddress:  "localhost:514",
 
-	AdvancedCookiesSamesite: "lax",
+	AdvancedCookiesSamesite:   "lax",
+	AdvancedRateLimitRequests: 1000, // per 5 minutes
 }

internal/config/flags.go

@@ -121,6 +121,7 @@ func AddServerFlags(cmd *cobra.Command) {
 
 		// Advanced flags
 		cmd.Flags().String(AdvancedCookiesSamesiteFlag(), cfg.AdvancedCookiesSamesite, fieldtag("AdvancedCookiesSamesite", "usage"))
+		cmd.Flags().Int(AdvancedRateLimitRequestsFlag(), cfg.AdvancedRateLimitRequests, fieldtag("AdvancedRateLimitRequests", "usage"))
 	})
 }
 

internal/config/helpers.gen.go

@@ -95,6 +95,31 @@ func GetApplicationName() string { return global.GetApplicationName() }
 // SetApplicationName safely sets the value for global configuration 'ApplicationName' field
 func SetApplicationName(v string) { global.SetApplicationName(v) }
 
+// GetLandingPageUser safely fetches the Configuration value for state's 'LandingPageUser' field
+func (st *ConfigState) GetLandingPageUser() (v string) {
+	st.mutex.Lock()
+	v = st.config.LandingPageUser
+	st.mutex.Unlock()
+	return
+}
+
+// SetLandingPageUser safely sets the Configuration value for state's 'LandingPageUser' field
+func (st *ConfigState) SetLandingPageUser(v string) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.LandingPageUser = v
+	st.reloadToViper()
+}
+
+// LandingPageUserFlag returns the flag name for the 'LandingPageUser' field
+func LandingPageUserFlag() string { return "landing-page-user" }
+
+// GetLandingPageUser safely fetches the value for global configuration 'LandingPageUser' field
+func GetLandingPageUser() string { return global.GetLandingPageUser() }
+
+// SetLandingPageUser safely sets the value for global configuration 'LandingPageUser' field
+func SetLandingPageUser(v string) { global.SetLandingPageUser(v) }
+
 // GetConfigPath safely fetches the Configuration value for state's 'ConfigPath' field
 func (st *ConfigState) GetConfigPath() (v string) {
 	st.mutex.Lock()
@@ -1795,28 +1820,27 @@ func GetAdvancedCookiesSamesite() string { return global.GetAdvancedCookiesSames
 // SetAdvancedCookiesSamesite safely sets the value for global configuration 'AdvancedCookiesSamesite' field
 func SetAdvancedCookiesSamesite(v string) { global.SetAdvancedCookiesSamesite(v) }
 
-// GetLandingPageUser safely fetches the Configuration value for state's 'LandingPageUser' field
-func (st *ConfigState) GetLandingPageUser() (v string) {
+// GetAdvancedRateLimitRequests safely fetches the Configuration value for state's 'AdvancedRateLimitRequests' field
+func (st *ConfigState) GetAdvancedRateLimitRequests() (v int) {
 	st.mutex.Lock()
-	v = st.config.LandingPageUser
+	v = st.config.AdvancedRateLimitRequests
 	st.mutex.Unlock()
 	return
 }
 
-// SetLandingPageUser safely sets the Configuration value for state's 'LandingPageUser' field
-func (st *ConfigState) SetLandingPageUser(v string) {
+// SetAdvancedRateLimitRequests safely sets the Configuration value for state's 'AdvancedRateLimitRequests' field
+func (st *ConfigState) SetAdvancedRateLimitRequests(v int) {
 	st.mutex.Lock()
 	defer st.mutex.Unlock()
-	st.config.LandingPageUser = v
+	st.config.AdvancedRateLimitRequests = v
 	st.reloadToViper()
 }
 
-// LandingPageUserFlag returns the flag name for the 'LandingPageUser' field
-func LandingPageUserFlag() string { return "landing-page-user" }
+// AdvancedRateLimitRequestsFlag returns the flag name for the 'AdvancedRateLimitRequests' field
+func AdvancedRateLimitRequestsFlag() string { return "advanced-rate-limit-requests" }
 
-// GetLandingPageUser safely fetches the value for global configuration 'LandingPageUser' field
-func GetLandingPageUser() string { return global.GetLandingPageUser() }
-
-// SetLandingPageUser safely sets the value for global configuration 'LandingPageUser' field
-func SetLandingPageUser(v string) { global.SetLandingPageUser(v) }
+// GetAdvancedRateLimitRequests safely fetches the value for global configuration 'AdvancedRateLimitRequests' field
+func GetAdvancedRateLimitRequests() int { return global.GetAdvancedRateLimitRequests() }
 
+// SetAdvancedRateLimitRequests safely sets the value for global configuration 'AdvancedRateLimitRequests' field
+func SetAdvancedRateLimitRequests(v int) { global.SetAdvancedRateLimitRequests(v) }

mkdocs.yml

@@ -55,3 +55,4 @@ nav:
     - "federation/behaviors/conversation_threads.md"
   - "API Documentation":
     - "api/swagger.md"
+    - "api/ratelimiting.md"