Merge remote-tracking branch 'origin/master' into split-metric-data

Signed-off-by: Bruce Guenter <bruce@timber.io>

.github/CODEOWNERS

@@ -37,6 +37,7 @@
 /docs/reference/components/sources/kubernetes_logs.cue @timberio/vector-kubernetes
 /docs/reference/components/sources/mongodb_metrics.cue @fanatid
 /docs/reference/components/sources/nginx_metrics.cue @fanatid
+/docs/reference/components/sources/postgresql_metrics.cue @fanatid @jszwedko
 /docs/reference/components/sources/prometheus_scrape.cue @bruceg
 /docs/reference/components/sources/prometheus_remote_write.cue @bruceg
 /docs/reference/components/sources/stdin.cue @bruceg
@@ -187,6 +188,7 @@
 /src/sources/journald.rs @bruceg
 /src/sources/mongodb_metrics/ @fanatid
 /src/sources/nginx/ @fanatid
+/src/sources/postgresql_metrics.rs @fanatid @jszwedko
 /src/sources/prometheus @bruceg
 /src/sources/stdin.rs @bruceg
 /src/sources/syslog.rs @lukesteensen

Cargo.toml

@@ -149,7 +149,7 @@ http = "0.2"
 typetag = "0.1.6"
 toml = "0.5.8"
 syslog = "5"
-syslog_loose = "0.7.0"
+syslog_loose = "0.8.0"
 leveldb = { version = "0.8", optional = true, default-features = false }
 db-key = "0.0.5"
 headers = "0.3"
@@ -255,14 +255,15 @@ rusty-fork = "0.3.0"
 
 [features]
 # Default features for *-unknown-linux-gnu and *-apple-darwin
-default = ["api", "api-client", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-plain"]
-default-musl = ["api", "api-client", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-cmake"]
+default = ["api", "api-client", "vrl-cli", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-plain"]
+default-musl = ["api", "api-client", "vrl-cli", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-cmake"]
 # Default features for *-unknown-linux-* which make use of `cmake` for dependencies
-default-cmake = ["api", "api-client", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-cmake"]
+default-cmake = ["api", "api-client", "vrl-cli", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-cmake"]
 # Default features for *-pc-windows-msvc
 # TODO: Enable SASL https://github.com/timberio/vector/pull/3081#issuecomment-659298042
-default-msvc = ["api", "api-client", "sources", "transforms", "sinks", "vendor-openssl", "vendor-libz", "leveldb", "rdkafka-cmake"]
-default-no-api-client = ["api", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-plain"]
+default-msvc = ["api", "api-client", "vrl-cli", "sources", "transforms", "sinks", "vendor-openssl", "vendor-libz", "leveldb", "rdkafka-cmake"]
+default-no-api-client = ["api", "sources", "vrl-cli", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-plain"]
+default-no-vrl-cli = ["api", "sources", "transforms", "sinks", "vendor-all", "unix", "leveldb", "rdkafka-plain"]
 
 # Target specific release features.
 # The `make` tasks will select this according to the appropriate triple.

benches/remap.rs

@@ -131,7 +131,7 @@ fn benchmark_remap(c: &mut Criterion) {
     c.bench_function("remap: parse JSON with remap", |b| {
         let mut tform: Box<dyn FunctionTransform> = Box::new(
             Remap::new(RemapConfig {
-                source: ".bar = parse_json(.foo)".to_owned(),
+                source: ".bar = parse_json!(.foo)".to_owned(),
                 drop_on_err: false,
             })
             .unwrap(),
@@ -195,9 +195,10 @@ fn benchmark_remap(c: &mut Criterion) {
     c.bench_function("remap: coerce with remap", |b| {
         let mut tform: Box<dyn FunctionTransform> = Box::new(
             Remap::new(RemapConfig {
-                source: r#".number = to_int(.number)
-                .bool = to_bool(.bool)
-                .timestamp = parse_timestamp(.timestamp, format: "%d/%m/%Y:%H:%M:%S %z")
+                source: r#"
+                .number = to_int!(.number)
+                .bool = to_bool!(.bool)
+                .timestamp = parse_timestamp!(.timestamp, format: "%d/%m/%Y:%H:%M:%S %z")
                 "#
                 .to_owned(),
                 drop_on_err: true,

docs/reference/components/sinks/aws_s3.cue

@@ -91,12 +91,14 @@ components: sinks: aws_s3: components._aws & {
 			type: string: {
 				default: null
 				enum: {
-					"private":            "Owner gets FULL_CONTROL. No one else has access rights (default)."
-					"public-read":        "Owner gets FULL_CONTROL. The AllUsers group gets READ access."
-					"public-read-write":  "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. Granting this on a bucket is generally not recommended."
-					"aws-exec-read":      "Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3."
-					"authenticated-read": "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access."
-					"log-delivery-write": "The LogDelivery group gets WRITE and READ_ACP permissions on the bucket. For more information about logs, see [Amazon S3 Server Access Logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html)."
+					"private":                   "Owner gets `FULL_CONTROL`. No one else has access rights (default)."
+					"public-read":               "Owner gets `FULL_CONTROL`. The AllUsers group gets `READ` access."
+					"public-read-write":         "Owner gets `FULL_CONTROL`. The AllUsers group gets `READ` and `WRITE` access. Granting this on a bucket is generally not recommended."
+					"aws-exec-read":             "Owner gets `FULL_CONTROL`. Amazon EC2 gets `READ` access to `GET` an Amazon Machine Image (AMI) bundle from Amazon S3."
+					"authenticated-read":        "Owner gets `FULL_CONTROL`. The AuthenticatedUsers group gets `READ` access."
+					"bucket-owner-read":         "Object owner gets `FULL_CONTROL`. Bucket owner gets `READ. access."
+					"bucket-owner-full-control": "Both the object owner and the bucket owner get `FULL_CONTROL` over the object."
+					"log-delivery-write":        "The LogDelivery group gets `WRITE` and `READ_ACP` permissions on the bucket. For more information about logs, see [Amazon S3 Server Access Logging](\(urls.aws_s3_server_access_logs))."
 				}
 			}
 		}

docs/reference/components/sources/host_metrics.cue

@@ -45,6 +45,24 @@ components: sources: host_metrics: {
 		platform_name: null
 	}
 
+	env_vars: {
+		PROCFS_ROOT: {
+			description: "Sets an arbitrary path to the system's Procfs root. Can be used to expose host metrics from within a container. Unset and uses system `/proc` by default."
+			type: string: {
+				default: null
+				examples: ["/mnt/host/proc"]
+			}
+		}
+
+		SYSFS_ROOT: {
+			description: "Sets an arbitrary path to the system's Sysfs root. Can be used to expose host metrics from within a container. Unset and uses system `/sys` by default."
+			type: string: {
+				default: null
+				examples: ["/mnt/host/sys"]
+			}
+		}
+	}
+
 	configuration: {
 		collectors: {
 			description: "The list of host metric collector services to use. Defaults to all collectors."

docs/reference/installation/_interfaces/helm3.cue

@@ -41,9 +41,8 @@ installation: _interfaces: "helm3": {
 				  stdout:
 				    type: console
 				    inputs: ["kubernetes_logs"]
-				    rawConfig: |
-				      target = "stdout"
-				      encoding = "json"
+				    target: "stdout"
+				    encoding: "json"
 				VALUES
 				"""#
 			install:   #"helm install --namespace \#(_namespace) --create-namespace \#(_release_name) \#(_repo_name)/\#(_chart_name) --values values.yaml"#

docs/reference/installation/_interfaces/homebrew.cue

@@ -37,7 +37,7 @@ installation: _interfaces: homebrew: {
 				},
 				{
 					title:   "Configure Vector"
-					command: "fix me"
+					command: commands.configure
 				},
 				{
 					title:   "Restart Vector"

docs/reference/installation/platforms/kubernetes.cue

@@ -9,5 +9,21 @@ installation: platforms: kubernetes: {
 		"""
 	minimum_supported_version: "1.14"
 
-	how_it_works: components.sources.kubernetes_logs.how_it_works
+	how_it_works: {
+		components.sources.kubernetes_logs.how_it_works
+
+		metrics: {
+			title: "Metrics"
+			body: """
+				Our Helm chart deployments provide quality of life around setup and maintenance of
+				metrics pipelines in Kubernetes. Each of the Helm charts provide an `internal_metrics`
+				source and `prometheus` sink out of the box. Agent deployments also expose `host_metrics`
+				via the same `prometheus` sink.
+
+				Charts come with options to enable Prometheus integration via annotations or Prometheus Operator
+				integration via PodMonitor. Thus, the Prometheus node_exporter agent is not required when the `host_metrics` source is
+				enabled.
+				"""
+		}
+	}
 }

docs/reference/remap/functions.cue

@@ -15,7 +15,7 @@ remap: {
 		name:        string
 		category:    #FunctionCategory
 		description: string
-		notices?: [string, ...string]
+		notices:     [string, ...string] | *[]
 
 		arguments: [...#Argument]
 		return: [remap.#Type, ...remap.#Type]

docs/reference/remap/functions/append.cue

@@ -24,11 +24,10 @@ remap: functions: append: {
 	examples: [
 		{
 			title: "Append to an array"
-			input: log: array: [1, 2]
 			source: """
-				.array = append(.array, [3, 4])
+				 append([1, 2], [3, 4])
 				"""
-			output: log: array: [1, 2, 3, 4]
+			return: [1, 2, 3, 4]
 		},
 	]
 }

docs/reference/remap/functions/assert.cue

@@ -28,17 +28,15 @@ remap: functions: assert: {
 	examples: [
 		{
 			title: "Assertion (true)"
-			input: log: foo: "foo"
 			source: #"""
-				assert(.foo == "foo", message: "Foo must be foo!")
+				assert("foo" == "foo", message: "Foo must be foo!")
 				"""#
-			output: input
+			return: null
 		},
 		{
 			title: "Assertion (false)"
-			input: log: foo: "bar"
 			source: #"""
-				assert(.foo == "foo", message: "Foo must be foo!")
+				assert("foo" == "bar", message: "Foo must be foo!")
 				"""#
 			raises: "Foo must be foo!"
 		},

docs/reference/remap/functions/ceil.cue

@@ -25,19 +25,17 @@ remap: functions: ceil: {
 	examples: [
 		{
 			title: "Round a number up (without precision)"
-			input: log: number: 4.345
 			source: #"""
-				.number = ceil(.number)
+				ceil(4.345)
 				"""#
-			output: log: number: 4
+			return: 4
 		},
 		{
 			title: "Round a number up (with precision)"
-			input: log: number: 4.345
 			source: #"""
-				.number = ceil(.number, precision: 2)
+				ceil(4.345, precision: 2)
 				"""#
-			output: log: number: 4.35
+			return: 4.35
 		},
 	]
 }

docs/reference/remap/functions/compact.cue

@@ -62,24 +62,17 @@ remap: functions: compact: {
 	examples: [
 		{
 			title: "Compact an array"
-			input: log: array: ["foo", "bar", "", null, [], "buzz"]
 			source: #"""
-				.log = compact(.array, string: true, array: true, null: true)
+				compact(["foo", "bar", "", null, [], "buzz"], string: true, array: true, null: true)
 				"""#
-			output: log: array: ["foo", "bar", "buzz"]
+			return: ["foo", "bar", "buzz"]
 		},
 		{
 			title: "Compact a map"
-			input: log: map: {
-				field1: 1
-				field2: ""
-				field3: []
-				field4: null
-			}
 			source: #"""
-				.map = compact(.map, string: true, array: true, null: true)
+				compact({"field1": 1, "field2": "", "field3": [], "field4": null}, string: true, array: true, null: true)
 				"""#
-			output: log: map: field1: 1
+			return: field1: 1
 		},
 	]
 }

docs/reference/remap/functions/contains.cue

@@ -31,23 +31,17 @@ remap: functions: contains: {
 	examples: [
 		{
 			title: "String contains (case sensitive)"
-			input: log: message: #"The Needle In The Haystack"#
 			source: #"""
-				.contains = contains(.message, "Needle")
+				contains("The Needle In The Haystack", "Needle")
 				"""#
-			output: input & {
-				log: contains: true
-			}
+			return: true
 		},
 		{
 			title: "String contains (case insensitive)"
-			input: log: message: #"The Needle In The Haystack"#
 			source: #"""
-				.contains = contains(.message, "needle", case_sensitive: false)
+				contains("The Needle In The Haystack", "needle", case_sensitive: false)
 				"""#
-			output: input & {
-				log: contains: true
-			}
+			return: true
 		},
 	]
 }

docs/reference/remap/functions/decode_base64.cue

@@ -20,11 +20,10 @@ remap: functions: decode_base64: {
 	examples: [
 		{
 			title: "Decode Base64 data"
-			input: log: message: "eW91IGhhdmUgc3VjY2Vzc2Z1bGx5IGRlY29kZWQgbWU="
-			source: ".decoded = decode_base64(.message)"
-			output: input & {log: {
-				decoded: "you have successfully decoded me"
-			}}
+			source: """
+				decode_base64("eW91IGhhdmUgc3VjY2Vzc2Z1bGx5IGRlY29kZWQgbWU=")
+				"""
+			return: "you have successfully decoded me"
 		},
 	]
 }