x-pack/filebeat/input/httpjson: add transaction tracer

CHANGELOG.next.asciidoc

@@ -118,6 +118,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Optimize grok patterns in system.auth module pipeline. {pull}32360[32360]
 - Checkpoint module: add authentication operation outcome enrichment. {issue}32230[32230] {pull}32431[32431]
 - add documentation for decode_xml_wineventlog processor field mappings.  {pull}32456[32456]
+- httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412]
 
 *Auditbeat*
 

NOTICE.txt

@@ -21901,6 +21901,36 @@ Contents of probable licence file $GOMODCACHE/gopkg.in/jcmturner/gokrb5.v7@v7.5.
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : gopkg.in/natefinch/lumberjack.v2
+Version: v2.0.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/gopkg.in/natefinch/lumberjack.v2@v2.0.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Nate Finch 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 --------------------------------------------------------------------------------
 Dependency : gopkg.in/yaml.v2
 Version: v2.4.0
@@ -26087,6 +26117,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/trac
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/BurntSushi/toml
+Version: v0.3.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v0.3.1/COPYING:
+
+The MIT License (MIT)
+
+Copyright (c) 2013 TOML authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/OneOfOne/xxhash
 Version: v1.2.2

go.mod

@@ -212,6 +212,7 @@ require (
 	go.elastic.co/apm/module/apmhttp/v2 v2.0.0
 	go.elastic.co/apm/v2 v2.0.0
 	go.mongodb.org/mongo-driver v1.5.1
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (

go.sum

@@ -136,6 +136,7 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
 github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -2502,6 +2503,7 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI
 gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM=
 gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU=
 gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=

x-pack/filebeat/docs/inputs/input-httpjson.asciidoc

@@ -546,6 +546,43 @@ filebeat.inputs:
         value: '[[now (parseDuration "-1h")]]'
 ----
 
+[float]
+==== `request.tracer.filename`
+
+It is possible to log httpjson requests and responses to a local file-system for debugging configurations.
+This option is enabled by setting the `request.tracer.filename` value. Additional options are available to
+tune log rotation behavior.
+
+Enabling this option compromises security and should only be used for debugging.
+
+[float]
+==== `request.tracer.maxsize`
+
+This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default
+logs are allowed to reach 1MB before rotation.
+
+[float]
+==== `request.tracer.maxage`
+
+This specifies the number days to retain rotated log files. If it is not set, log files are retained
+indefinitely.
+
+[float]
+==== `request.tracer.maxbackups`
+
+The number of old logs to retain. If it is not set all old logs are retained subject to the `request.tracer.maxage`
+setting.
+
+[float]
+==== `request.tracer.localtime`
+
+Whether to use the host's local time rather that UTC for timestamping rotated log file names.
+
+[float]
+==== `request.tracer.compress`
+
+This determines whether rotated logs should be gzip compressed.
+
 [float]
 ==== `response.decode_as`
 

x-pack/filebeat/input/httpjson/config_request.go

@@ -12,6 +12,8 @@ import (
 	"strings"
 	"time"
 
+	"gopkg.in/natefinch/lumberjack.v2"
+
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/transport/httpcommon"
 )
@@ -100,6 +102,8 @@ type requestConfig struct {
 	Transforms             transformsConfig `config:"transforms"`
 
 	Transport httpcommon.HTTPTransportSettings `config:",inline"`
+
+	Tracer *lumberjack.Logger `config:"tracer"`
 }
 
 func (c *requestConfig) Validate() error {
@@ -124,5 +128,17 @@ func (c *requestConfig) Validate() error {
 		}
 	}
 
+	if c.Tracer != nil {
+		if c.Tracer.Filename == "" {
+			return errors.New("request tracer must have a filename if used")
+		}
+		if c.Tracer.MaxSize == 0 {
+			// By default Lumberjack caps file sizes at 100MB which
+			// is excessive for a debugging logger, so default to 1MB
+			// which is the minimum.
+			c.Tracer.MaxSize = 1
+		}
+	}
+
 	return nil
 }

x-pack/filebeat/input/httpjson/input.go

@@ -179,10 +179,10 @@ func newHTTPClient(ctx context.Context, config config, log *logp.Logger) (*httpC
 		if err != nil {
 			return nil, err
 		}
-		return &httpClient{client: authClient, limiter: limiter}, nil
+		return &httpClient{client: authClient, limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil
 	}
 
-	return &httpClient{client: client.StandardClient(), limiter: limiter}, nil
+	return &httpClient{client: client.StandardClient(), limiter: limiter, tracer: newTracer(config.Request.Tracer)}, nil
 }
 
 func checkRedirect(config *requestConfig, log *logp.Logger) func(*http.Request, []*http.Request) error {

x-pack/filebeat/input/httpjson/request.go

@@ -10,11 +10,15 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"strings"
+	"time"
 
 	"github.com/PaesslerAG/jsonpath"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -32,6 +36,7 @@ func registerRequestTransforms() {
 type httpClient struct {
 	client  *http.Client
 	limiter *rateLimiter
+	tracer  *tracer
 }
 
 func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Response, error) {
@@ -42,6 +47,7 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon
 		return nil, fmt.Errorf("failed to execute http client.Do: %w", err)
 	}
 	defer resp.Body.Close()
+	c.tracer.trace(req, resp)
 
 	// Read the whole resp.Body so we can release the connection.
 	// This implementation is inspired by httputil.DumpResponse
@@ -57,6 +63,44 @@ func (c *httpClient) do(stdCtx context.Context, req *http.Request) (*http.Respon
 	return resp, nil
 }
 
+// tracer is a lightweight transaction tracer for debugging httpjson configurations.
+// It is not intended to be used for machine ingestion, but rather as a human aid.
+type tracer struct {
+	sess int64 // sess is the unix time of the start of a session.
+	txn  int   // txn is a session-unique transaction id.
+	log  *log.Logger
+}
+
+func newTracer(logger *lumberjack.Logger) *tracer {
+	if logger == nil {
+		return nil
+	}
+	return &tracer{sess: time.Now().UnixMilli(), log: log.New(logger, "", log.LstdFlags)}
+}
+
+func (t *tracer) trace(req *http.Request, resp *http.Response) {
+	if t == nil {
+		return
+	}
+	t.txn++
+	b, err := httputil.DumpRequestOut(req, true)
+	if err != nil {
+		t.log.Print(err)
+	} else {
+		t.log.Printf("request %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t")))
+	}
+	if resp == nil {
+		t.log.Printf("no response %d.%d", t.sess, t.txn)
+		return
+	}
+	b, err = httputil.DumpResponse(resp, true)
+	if err != nil {
+		t.log.Printf("response %d.%d: %v", t.sess, t.txn, err)
+	} else {
+		t.log.Printf("response %d.%d:\n\t%s", t.sess, t.txn, bytes.ReplaceAll(b, []byte{'\n'}, []byte("\n\t")))
+	}
+}
+
 func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, error) {
 	req := transformable{}
 	req.setURL(rf.url)