[splunkhecexporter] Copy payload before sending to avoid corruption on reuse

.chloggen/safe_bytes.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: splunkhecexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Copy the bytes to be placed in the request body to avoid corruption on reuse
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34357]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  This bug is a manifestation of https://github.com/golang/go/issues/51907.
+  Under high load, the pool of buffers used to send requests is reused enough
+  that the same buffer is used concurrently to process data and be sent as request body.
+  The fix is to copy the payload into a new byte array before sending it.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/splunkhecexporter/buffer.go

@@ -22,6 +22,7 @@ type buffer interface {
 	Reset()
 	Len() int
 	Empty() bool
+	Bytes() []byte
 }
 
 type cancellableBytesWriter struct {
@@ -59,6 +60,10 @@ func (c *cancellableBytesWriter) Empty() bool {
 	return c.innerWriter.Len() == 0
 }
 
+func (c *cancellableBytesWriter) Bytes() []byte {
+	return c.innerWriter.Bytes()
+}
+
 type cancellableGzipWriter struct {
 	innerBuffer *bytes.Buffer
 	innerWriter *gzip.Writer
@@ -125,6 +130,10 @@ func (c *cancellableGzipWriter) Empty() bool {
 	return c.rawLen == 0
 }
 
+func (c *cancellableGzipWriter) Bytes() []byte {
+	return c.innerBuffer.Bytes()
+}
+
 // bufferPool is a pool of buffer objects.
 type bufferPool struct {
 	pool *sync.Pool

exporter/splunkhecexporter/hec_worker.go

@@ -4,6 +4,7 @@
 package splunkhecexporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/splunkhecexporter"
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"net/http"
@@ -27,7 +28,11 @@ type defaultHecWorker struct {
 }
 
 func (hec *defaultHecWorker) send(ctx context.Context, buf buffer, headers map[string]string) error {
-	req, err := http.NewRequestWithContext(ctx, "POST", hec.url.String(), buf)
+	// We copy the bytes to a new buffer to avoid corruption. This is a workaround to avoid hitting https://github.com/golang/go/issues/51907.
+	nb := make([]byte, buf.Len())
+	copy(nb, buf.Bytes())
+	bodyBuf := bytes.NewReader(nb)
+	req, err := http.NewRequestWithContext(ctx, "POST", hec.url.String(), bodyBuf)
 	if err != nil {
 		return consumererror.NewPermanent(err)
 	}