User params

Makefile

@@ -1,4 +1,6 @@
 pkgs = $(shell go list ./...)
+gofiles := $(shell find . -name "*.go" -type f -not -path "./vendor/*")
+
 
 BUILD_TAG = $(shell git tag --points-at HEAD)
 
@@ -13,9 +15,8 @@ update:
 	dep ensure -update
 
 format:
-	dep ensure
 	go fmt $(pkgs)
-	gofmt -w -s .
+	gofmt -w -s $(gofiles)
 
 build:
 	go build

README.md

@@ -327,7 +327,7 @@ an instant cache flush may be built on top of cache namespaces - just switch to
 to flush the cache.
 
 ### Security
-`Chproxy` removes all the query params from input requests (except the `query`, `database`, `default_format`, `compress`, `decompress`, `enable_http_compression`)
+`Chproxy` removes all the query params from input requests (except the user's [params](https://github.com/Vertamedia/chproxy/blob/master/config#param_groups_config) and the `query`, `database`, `default_format`, `compress`, `decompress`, `enable_http_compression`)
 before proxying them to `ClickHouse` nodes. This prevents from unsafe overriding
 of various `ClickHouse` [settings](http://clickhouse-docs.readthedocs.io/en/latest/interfaces/http_interface.html).
 
@@ -391,6 +391,29 @@ network_groups:
   - name: "reporting-apps"
     networks: ["10.10.10.0/24"]
 
+# Optional list of GET params to send with each request
+param_groups:
+    # Group name, which may be passed into `params` option on the `user` level.
+  - name: "cron-job"
+    # List of key-value params to send
+    params:
+      - key: "max_memory_usage"
+        value: "40000000000"
+
+      - key: "max_bytes_before_external_group_by"
+        value: "20000000000"
+
+  - name: "web"
+    params:
+      - key: "max_memory_usage"
+        value: "5000000000"
+
+      - key: "max_columns_to_read"
+        value: "30"
+
+      - key: "max_execution_time"
+        value: "30"
+
 # Settings for `chproxy` input interfaces.
 server:
   # Configs for input http interface.
@@ -420,7 +443,6 @@ server:
     # Certificates are automatically issued and renewed if this section
     # is present.
     # There is no need in cert_file and key_file if this section is present.
-    # Autocert requires application to listen on :80 port for certificate generation
     autocert:
       # Path to the directory where autocert certs are cached.
       cache_dir: "certs_dir"
@@ -467,6 +489,11 @@ users:
     # By default responses aren't cached.
     cache: "longterm"
 
+    # ParamGroup name to use
+    #
+    # By default no additional params are used
+    params: "web"
+
     # The maximum number of requests that may wait for their chance
     # to be executed because they cannot run now due to the current limits.
     #
@@ -488,6 +515,8 @@ users:
     to_user: "default"
     allowed_networks: ["office", "1.2.3.0/24"]
 
+    params: "cron-job"
+
     # The maximum number of concurrently running queries for the user.
     #
     # By default there is no limit on the number of concurrently

cache/cache.go

@@ -79,12 +79,15 @@ type Key struct {
 
 	// Namespace is an optional cache namespace.
 	Namespace string
+
+	// UserParamsHash must contain hashed value of users params
+	UserParamsHash uint32
 }
 
 // String returns string representation of the key.
 func (k *Key) String() string {
-	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q",
-		cacheVersion, k.Query, k.AcceptEncoding, k.DefaultFormat, k.Database, k.Compress, k.EnableHTTPCompression, k.Namespace)
+	s := fmt.Sprintf("V%d; Query=%q; AcceptEncoding=%q; DefaultFormat=%q; Database=%q; Compress=%q; EnableHTTPCompression=%q; Namespace=%q; UserParams=%d",
+		cacheVersion, k.Query, k.AcceptEncoding, k.DefaultFormat, k.Database, k.Compress, k.EnableHTTPCompression, k.Namespace, k.UserParamsHash)
 	h := sha256.Sum256([]byte(s))
 
 	// The first 16 bytes of the hash should be enough

cache/cache_test.go

@@ -48,22 +48,22 @@ func TestKeyString(t *testing.T) {
 			key: &Key{
 				Query: []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 			},
-			expected: "cd8ba039ef5df8287ff91a3907e21fcf",
+			expected: "b84443ea3b7651f8eed84ad70cc17d55",
 		},
 		{
 			key: &Key{
 				Query:          []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				AcceptEncoding: "gzip",
 			},
-			expected: "e30d9db1d8f6a0844b7c6047d14a6c8b",
+			expected: "baece3cc15d1aa1516e2729409ece703",
 		},
 		{
 			key: &Key{
 				Query:          []byte("SELECT 1 FROM system.numbers LIMIT 10"),
 				AcceptEncoding: "gzip",
 				DefaultFormat:  "JSON",
 			},
-			expected: "70bd83ab8def6df2c65214b37410050c",
+			expected: "c238bde938f93419e93b5b7b0341f1ef",
 		},
 		{
 			key: &Key{
@@ -72,7 +72,7 @@ func TestKeyString(t *testing.T) {
 				DefaultFormat:  "JSON",
 				Database:       "foobar",
 			},
-			expected: "3fb481313cd5a0061832de383183ac3a",
+			expected: "e55de3951f08688a34e589caaeed437f",
 		},
 		{
 			key: &Key{
@@ -82,7 +82,7 @@ func TestKeyString(t *testing.T) {
 				Database:       "foobar",
 				Namespace:      "ns123",
 			},
-			expected: "556f114207e2f8bc4cbd574b5035b25e",
+			expected: "a8676b65119982a1fa135005e0583a07",
 		},
 		{
 			key: &Key{
@@ -93,7 +93,7 @@ func TestKeyString(t *testing.T) {
 				Compress:       "1",
 				Namespace:      "ns123",
 			},
-			expected: "5fe3d904e7e76c5fe7ba617567a4cb7a",
+			expected: "9a2ad211524d5c8983d43784fd59677d",
 		},
 	}
 

config/README.md

@@ -20,6 +20,10 @@ hack_me_please <bool> | default = false [optional]
 caches:
   - <cache_config> ...
 
+# Named list of parameters to apply to each query
+param_groups:
+  - <param_groups_config> ...
+
 # Named network lists
 network_groups: <network_groups_config> ... [optional]
 
@@ -72,6 +76,16 @@ expire: <duration>
 grace_time: <duration>
 ```
 
+### <param_groups_config>
+```yml
+# Group name, which may be passed into `params` option on the `user` level.
+- name: <string>
+# List of key-value params to send
+params:
+  - key: <string>
+    value: <string>
+```
+
 ### <server_config>
 ```yml
 # HTTP server configuration
@@ -179,9 +193,12 @@ allow_cors: <bool> | optional | default = false
 # Each list item could be IP address or subnet mask
 allowed_networks: <network_groups>, <networks> ... | optional
 
-# Optioanl reponse cache name from <cache_config>
+# Optional response cache name from <cache_config>
 # By default responses aren't cached.
 cache: <string> | optional
+
+# Optional ParamGroup name from <param_groups_config>
+params: <string> | optional
 ```
 
 ### <cluster_config>

config/config.go

@@ -41,6 +41,8 @@ type Config struct {
 
 	Caches []Cache `yaml:"caches,omitempty"`
 
+	ParamGroups []ParamGroup `yaml:"param_groups,omitempty"`
+
 	// Catches all undefined fields
 	XXX map[string]interface{} `yaml:",inline"`
 
@@ -411,6 +413,9 @@ type User struct {
 	// Name of Cache configuration to use for responses of this user
 	Cache string `yaml:"cache,omitempty"`
 
+	// Name of ParamGroup to use
+	Params string `yaml:"params,omitempty"`
+
 	// Catches all undefined fields
 	XXX map[string]interface{} `yaml:",inline"`
 }
@@ -464,7 +469,13 @@ func (ng *NetworkGroups) UnmarshalYAML(unmarshal func(interface{}) error) error
 	if err := unmarshal((*plain)(ng)); err != nil {
 		return err
 	}
-	return checkOverflow(ng.XXX, "network_groups")
+	if len(ng.Name) == 0 {
+		return fmt.Errorf("`network_group.name` must be specified")
+	}
+	if len(ng.Networks) == 0 {
+		return fmt.Errorf("`network_group.networks` must contain at least one network")
+	}
+	return checkOverflow(ng.XXX, fmt.Sprintf("network_group %q", ng.Name))
 }
 
 // NetworksOrGroups is a list of strings with names of NetworkGroups
@@ -515,6 +526,42 @@ func (c *Cache) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return checkOverflow(c.XXX, fmt.Sprintf("cache %q", c.Name))
 }
 
+// ParamGroup describes named group of GET params
+// for sending with each query
+type ParamGroup struct {
+	// Name of configuration for further assign
+	Name string `yaml:"name"`
+
+	// Params contains a list of GET params
+	Params []Param `yaml:"params"`
+
+	// Catches all undefined fields
+	XXX map[string]interface{} `yaml:",inline"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (pg *ParamGroup) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	type plain ParamGroup
+	if err := unmarshal((*plain)(pg)); err != nil {
+		return err
+	}
+	if len(pg.Name) == 0 {
+		return fmt.Errorf("`param_group.name` must be specified")
+	}
+	if len(pg.Params) == 0 {
+		return fmt.Errorf("`param_group.params` must contain at least one param")
+	}
+	return checkOverflow(pg.XXX, fmt.Sprintf("param_group %q", pg.Name))
+}
+
+// Params describes URL param value
+type Param struct {
+	// Key is a name of params
+	Key string `yaml:"key"`
+	// Value is a value of param
+	Value string `yaml:"value"`
+}
+
 // ClusterUser describes simplest <users> configuration
 type ClusterUser struct {
 	// User name in ClickHouse users.xml config

config/config_test.go

@@ -104,6 +104,40 @@ func TestLoadConfig(t *testing.T) {
 						HeartBeatInterval: Duration(5 * time.Second),
 					},
 				},
+
+				ParamGroups: []ParamGroup{
+					{
+						Name: "cron-job",
+						Params: []Param{
+							{
+								Key:   "max_memory_usage",
+								Value: "40000000000",
+							},
+							{
+								Key:   "max_bytes_before_external_group_by",
+								Value: "20000000000",
+							},
+						},
+					},
+					{
+						Name: "web",
+						Params: []Param{
+							{
+								Key:   "max_memory_usage",
+								Value: "5000000000",
+							},
+							{
+								Key:   "max_columns_to_read",
+								Value: "30",
+							},
+							{
+								Key:   "max_execution_time",
+								Value: "30",
+							},
+						},
+					},
+				},
+
 				Users: []User{
 					{
 						Name:         "web",
@@ -116,6 +150,7 @@ func TestLoadConfig(t *testing.T) {
 						MaxQueueSize: 100,
 						MaxQueueTime: Duration(35 * time.Second),
 						Cache:        "longterm",
+						Params:       "web",
 					},
 					{
 						Name:                 "default",
@@ -125,6 +160,7 @@ func TestLoadConfig(t *testing.T) {
 						MaxExecutionTime:     Duration(time.Minute),
 						DenyHTTPS:            true,
 						NetworksOrGroups:     []string{"office", "1.2.3.0/24"},
+						Params:               "cron-job",
 					},
 				},
 				NetworkGroups: []NetworkGroups{

config/testdata/full.yml

@@ -51,6 +51,29 @@ network_groups:
   - name: "reporting-apps"
     networks: ["10.10.10.0/24"]
 
+# Optional list of GET params to send with each request
+param_groups:
+    # Group name, which may be passed into `params` option on the `user` level.
+  - name: "cron-job"
+    # List of key-value params to send
+    params:
+      - key: "max_memory_usage"
+        value: "40000000000"
+
+      - key: "max_bytes_before_external_group_by"
+        value: "20000000000"
+
+  - name: "web"
+    params:
+      - key: "max_memory_usage"
+        value: "5000000000"
+
+      - key: "max_columns_to_read"
+        value: "30"
+
+      - key: "max_execution_time"
+        value: "30"
+
 # Settings for `chproxy` input interfaces.
 server:
   # Configs for input http interface.
@@ -126,6 +149,11 @@ users:
     # By default responses aren't cached.
     cache: "longterm"
 
+    # ParamGroup name to use
+    #
+    # By default no additional params are used
+    params: "web"
+
     # The maximum number of requests that may wait for their chance
     # to be executed because they cannot run now due to the current limits.
     #
@@ -147,6 +175,8 @@ users:
     to_user: "default"
     allowed_networks: ["office", "1.2.3.0/24"]
 
+    params: "cron-job"
+
     # The maximum number of concurrently running queries for the user.
     #
     # By default there is no limit on the number of concurrently