From f7dd833eb86b79d50ab0b0b233dc7e863b76321b Mon Sep 17 00:00:00 2001 From: Yngvar Kristiansen <562343+yngvark@users.noreply.github.com> Date: Tue, 6 Apr 2021 17:47:48 +0200 Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=91=8C=20Truncate=20client=20log=20to?= =?UTF-8?q?=203000=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/client/core/api/rest/client.go | 8 +++-- pkg/middleware/logger/logger.go | 19 ++--------- pkg/middleware/logger/logger_test.go | 34 ------------------- pkg/truncate/truncate.go | 24 ++++++++++++++ pkg/truncate/truncate_test.go | 49 ++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 53 deletions(-) create mode 100644 pkg/truncate/truncate.go create mode 100644 pkg/truncate/truncate_test.go diff --git a/pkg/client/core/api/rest/client.go b/pkg/client/core/api/rest/client.go index 64dc4cdef..3df5ad195 100644 --- a/pkg/client/core/api/rest/client.go +++ b/pkg/client/core/api/rest/client.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/oslokommune/okctl/pkg/truncate" "io" "io/ioutil" "net/http" @@ -84,9 +85,11 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac err = resp.Body.Close() }() + const logLineMaxlength = 3000 if into != nil { if c.Debug { - _, err = fmt.Fprintf(c.Progress, "client (method: %s, endpoint: %s) received data: %s", method, endpoint, out) + truncatedOut := truncate.TruncateBytes(out, logLineMaxlength) + _, err = fmt.Fprintf(c.Progress, "client (method: %s, endpoint: %s) received data: %s", method, endpoint, truncatedOut) if err != nil { return fmt.Errorf("failed to write debug output: %w", err) } @@ -99,7 +102,8 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac } if c.Debug { - _, err = io.Copy(c.Progress, strings.NewReader(string(out))) + truncatedOut := truncate.TruncateBytes(out, logLineMaxlength) + _, err = io.Copy(c.Progress, strings.NewReader(string(truncatedOut))) if err != nil { return fmt.Errorf("%s: %w", pretty("failed to write progress for", method, endpoint), err) } diff --git a/pkg/middleware/logger/logger.go b/pkg/middleware/logger/logger.go index 0c5fabf43..e26835dd9 100644 --- a/pkg/middleware/logger/logger.go +++ b/pkg/middleware/logger/logger.go @@ -3,6 +3,7 @@ package logger import ( "context" + "github.com/oslokommune/okctl/pkg/truncate" "strings" "time" @@ -79,25 +80,9 @@ func (l *logging) ProcessResponse(err error, response interface{}, begin time.Ti d = litter.Sdump(response) } - truncatedDump := Truncate(&d, 5000) + truncatedDump := truncate.Truncate(&d, 5000) l.log.Trace("response: ", truncatedDump) } l.log.Debug("request completed in: ", time.Since(begin).String()) } - -// Truncate truncates a string to the minimum of its length and the given max length -func Truncate(s *string, maxLength int) string { - // This way of doing substrings assumes ASCII and doesn't support UTF-8. - // See https://stackoverflow.com/a/56129336 - truncateLength := min(maxLength, len(*s)) - return (*s)[:truncateLength] -} - -func min(x, y int) int { - if x < y { - return x - } - - return y -} diff --git a/pkg/middleware/logger/logger_test.go b/pkg/middleware/logger/logger_test.go index 35791b91c..10172b4b2 100644 --- a/pkg/middleware/logger/logger_test.go +++ b/pkg/middleware/logger/logger_test.go @@ -76,37 +76,3 @@ func TestLogging(t *testing.T) { }) } } - -func TestTruncate(t *testing.T) { - testCases := []struct { - name string - input string - maxLength int - expected string - }{ - { - name: "Should truncate string", - input: "1234567890", - maxLength: 5, - expected: "12345", - }, - { - name: "Should keep string if it's equal to maxLength", - input: "1234567890", - maxLength: 10, - expected: "1234567890", - }, - { - name: "Should keep string if it's over maxLength", - input: "1234567890", - maxLength: 11, - expected: "1234567890", - }, - } - for _, tc := range testCases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expected, logger.Truncate(&tc.input, tc.maxLength)) - }) - } -} diff --git a/pkg/truncate/truncate.go b/pkg/truncate/truncate.go new file mode 100644 index 000000000..05749d177 --- /dev/null +++ b/pkg/truncate/truncate.go @@ -0,0 +1,24 @@ +package truncate + +// Truncate truncates a string to the minimum of its length and the given max length +func Truncate(s *string, maxLength int) string { + // This way of doing substrings assumes ASCII and doesn't support UTF-8. + // See https://stackoverflow.com/a/56129336 + truncateLength := min(maxLength, len(*s)) + return (*s)[:truncateLength] +} +// Truncate truncates a byte array to the minimum of its length and the given max length +func TruncateBytes(b []byte, maxLength int) []byte { + // This way of doing substrings assumes ASCII and doesn't support UTF-8. + // See https://stackoverflow.com/a/56129336 + truncateLength := min(maxLength, len(b)) + return b[:truncateLength] +} + +func min(x, y int) int { + if x < y { + return x + } + + return y +} diff --git a/pkg/truncate/truncate_test.go b/pkg/truncate/truncate_test.go new file mode 100644 index 000000000..61baa97e8 --- /dev/null +++ b/pkg/truncate/truncate_test.go @@ -0,0 +1,49 @@ +package truncate_test + +import ( + "github.com/oslokommune/okctl/pkg/truncate" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestTruncate(t *testing.T) { + testCases := []struct { + name string + input string + maxLength int + expected string + }{ + { + name: "Should truncate string", + input: "1234567890", + maxLength: 5, + expected: "12345", + }, + { + name: "Should keep string if it's equal to maxLength", + input: "1234567890", + maxLength: 10, + expected: "1234567890", + }, + { + name: "Should keep string if it's over maxLength", + input: "1234567890", + maxLength: 11, + expected: "1234567890", + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + // Test Truncate string + truncatedString := truncate.Truncate(&tc.input, tc.maxLength) + assert.Equal(t, tc.expected, truncatedString) + + // Test Truncate bytes + inputBytes := []byte(tc.input) + truncatedBytes := truncate.TruncateBytes(inputBytes, tc.maxLength) + expectedBytes := []byte(tc.expected) + assert.Equal(t, expectedBytes, truncatedBytes) + }) + } +} From f4c0fdb208087ac19f2330d3ad772fd2f8938804 Mon Sep 17 00:00:00 2001 From: Yngvar Kristiansen <562343+yngvark@users.noreply.github.com> Date: Tue, 6 Apr 2021 17:58:58 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=91=8C=20Output=20how=20many=20bytes?= =?UTF-8?q?=20were=20truncated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/client/core/api/rest/client.go | 4 ++-- pkg/middleware/logger/logger.go | 2 +- pkg/truncate/truncate.go | 33 ++++++++++++++++++++++-------- pkg/truncate/truncate_test.go | 16 ++++++++++----- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/pkg/client/core/api/rest/client.go b/pkg/client/core/api/rest/client.go index 3df5ad195..ef9860dd5 100644 --- a/pkg/client/core/api/rest/client.go +++ b/pkg/client/core/api/rest/client.go @@ -88,7 +88,7 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac const logLineMaxlength = 3000 if into != nil { if c.Debug { - truncatedOut := truncate.TruncateBytes(out, logLineMaxlength) + truncatedOut := truncate.Bytes(out, logLineMaxlength) _, err = fmt.Fprintf(c.Progress, "client (method: %s, endpoint: %s) received data: %s", method, endpoint, truncatedOut) if err != nil { return fmt.Errorf("failed to write debug output: %w", err) @@ -102,7 +102,7 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac } if c.Debug { - truncatedOut := truncate.TruncateBytes(out, logLineMaxlength) + truncatedOut := truncate.Bytes(out, logLineMaxlength) _, err = io.Copy(c.Progress, strings.NewReader(string(truncatedOut))) if err != nil { return fmt.Errorf("%s: %w", pretty("failed to write progress for", method, endpoint), err) diff --git a/pkg/middleware/logger/logger.go b/pkg/middleware/logger/logger.go index e26835dd9..80ab39b32 100644 --- a/pkg/middleware/logger/logger.go +++ b/pkg/middleware/logger/logger.go @@ -80,7 +80,7 @@ func (l *logging) ProcessResponse(err error, response interface{}, begin time.Ti d = litter.Sdump(response) } - truncatedDump := truncate.Truncate(&d, 5000) + truncatedDump := truncate.String(&d, 5000) l.log.Trace("response: ", truncatedDump) } diff --git a/pkg/truncate/truncate.go b/pkg/truncate/truncate.go index 05749d177..62518c856 100644 --- a/pkg/truncate/truncate.go +++ b/pkg/truncate/truncate.go @@ -1,18 +1,35 @@ package truncate -// Truncate truncates a string to the minimum of its length and the given max length -func Truncate(s *string, maxLength int) string { +import "fmt" + +// String truncates a string to the minimum of its length and the given max length +func String(s *string, maxLength int) string { // This way of doing substrings assumes ASCII and doesn't support UTF-8. // See https://stackoverflow.com/a/56129336 truncateLength := min(maxLength, len(*s)) - return (*s)[:truncateLength] + truncated := (*s)[:truncateLength] + + if len(*s) > truncateLength { + bytesTruncated := len(*s) - truncateLength + truncated += fmt.Sprintf(" [truncated %d bytes]", bytesTruncated) + } + + return truncated } -// Truncate truncates a byte array to the minimum of its length and the given max length -func TruncateBytes(b []byte, maxLength int) []byte { - // This way of doing substrings assumes ASCII and doesn't support UTF-8. - // See https://stackoverflow.com/a/56129336 + +// Bytes truncates a byte array to the minimum of its length and the given max length +func Bytes(b []byte, maxLength int) []byte { truncateLength := min(maxLength, len(b)) - return b[:truncateLength] + + truncated := b[:truncateLength] + + if len(b) > truncateLength { + bytesTruncated := len(b) - truncateLength + truncateInfo := fmt.Sprintf(" [truncated %d bytes]", bytesTruncated) + truncated = append(truncated, truncateInfo...) + } + + return truncated } func min(x, y int) int { diff --git a/pkg/truncate/truncate_test.go b/pkg/truncate/truncate_test.go index 61baa97e8..8911203df 100644 --- a/pkg/truncate/truncate_test.go +++ b/pkg/truncate/truncate_test.go @@ -17,7 +17,13 @@ func TestTruncate(t *testing.T) { name: "Should truncate string", input: "1234567890", maxLength: 5, - expected: "12345", + expected: "12345 [truncated 5 bytes]", + }, + { + name: "Should truncate some other string", + input: "1234567890", + maxLength: 7, + expected: "1234567 [truncated 3 bytes]", }, { name: "Should keep string if it's equal to maxLength", @@ -35,13 +41,13 @@ func TestTruncate(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.name, func(t *testing.T) { - // Test Truncate string - truncatedString := truncate.Truncate(&tc.input, tc.maxLength) + // Test String string + truncatedString := truncate.String(&tc.input, tc.maxLength) assert.Equal(t, tc.expected, truncatedString) - // Test Truncate bytes + // Test String bytes inputBytes := []byte(tc.input) - truncatedBytes := truncate.TruncateBytes(inputBytes, tc.maxLength) + truncatedBytes := truncate.Bytes(inputBytes, tc.maxLength) expectedBytes := []byte(tc.expected) assert.Equal(t, expectedBytes, truncatedBytes) }) From fc518f56622386366e93b65f27ed0f64551365db Mon Sep 17 00:00:00 2001 From: Yngvar Kristiansen <562343+yngvark@users.noreply.github.com> Date: Wed, 7 Apr 2021 08:44:46 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=F0=9F=91=8C=20Log=20how=20many=20bytes=20w?= =?UTF-8?q?ere=20truncated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/client/core/api/rest/client.go | 8 ++++++-- pkg/middleware/logger/logger.go | 5 +++-- pkg/truncate/truncate.go | 1 + pkg/truncate/truncate_test.go | 4 +++- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/client/core/api/rest/client.go b/pkg/client/core/api/rest/client.go index ef9860dd5..2c2a6da32 100644 --- a/pkg/client/core/api/rest/client.go +++ b/pkg/client/core/api/rest/client.go @@ -4,12 +4,13 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/oslokommune/okctl/pkg/truncate" "io" "io/ioutil" "net/http" "strings" + "github.com/oslokommune/okctl/pkg/truncate" + validation "github.com/go-ozzo/ozzo-validation/v4" "github.com/mishudark/errors" @@ -85,10 +86,12 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac err = resp.Body.Close() }() - const logLineMaxlength = 3000 + const logLineMaxlength = 3500 + if into != nil { if c.Debug { truncatedOut := truncate.Bytes(out, logLineMaxlength) + _, err = fmt.Fprintf(c.Progress, "client (method: %s, endpoint: %s) received data: %s", method, endpoint, truncatedOut) if err != nil { return fmt.Errorf("failed to write debug output: %w", err) @@ -103,6 +106,7 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac if c.Debug { truncatedOut := truncate.Bytes(out, logLineMaxlength) + _, err = io.Copy(c.Progress, strings.NewReader(string(truncatedOut))) if err != nil { return fmt.Errorf("%s: %w", pretty("failed to write progress for", method, endpoint), err) diff --git a/pkg/middleware/logger/logger.go b/pkg/middleware/logger/logger.go index 80ab39b32..100f816f8 100644 --- a/pkg/middleware/logger/logger.go +++ b/pkg/middleware/logger/logger.go @@ -3,10 +3,11 @@ package logger import ( "context" - "github.com/oslokommune/okctl/pkg/truncate" "strings" "time" + "github.com/oslokommune/okctl/pkg/truncate" + "github.com/go-kit/kit/endpoint" "github.com/sanity-io/litter" "github.com/sirupsen/logrus" @@ -80,7 +81,7 @@ func (l *logging) ProcessResponse(err error, response interface{}, begin time.Ti d = litter.Sdump(response) } - truncatedDump := truncate.String(&d, 5000) + truncatedDump := truncate.String(&d, 3500) l.log.Trace("response: ", truncatedDump) } diff --git a/pkg/truncate/truncate.go b/pkg/truncate/truncate.go index 62518c856..25d50d659 100644 --- a/pkg/truncate/truncate.go +++ b/pkg/truncate/truncate.go @@ -1,3 +1,4 @@ +// Package truncate implements truncating variables package truncate import "fmt" diff --git a/pkg/truncate/truncate_test.go b/pkg/truncate/truncate_test.go index 8911203df..0ab07aaf0 100644 --- a/pkg/truncate/truncate_test.go +++ b/pkg/truncate/truncate_test.go @@ -1,9 +1,10 @@ package truncate_test import ( + "testing" + "github.com/oslokommune/okctl/pkg/truncate" "github.com/stretchr/testify/assert" - "testing" ) func TestTruncate(t *testing.T) { @@ -48,6 +49,7 @@ func TestTruncate(t *testing.T) { // Test String bytes inputBytes := []byte(tc.input) truncatedBytes := truncate.Bytes(inputBytes, tc.maxLength) + expectedBytes := []byte(tc.expected) assert.Equal(t, expectedBytes, truncatedBytes) }) From 783f55f488b5de168c4d9a781a08976d05a0c615 Mon Sep 17 00:00:00 2001 From: Yngvar Kristiansen <562343+yngvark@users.noreply.github.com> Date: Wed, 7 Apr 2021 08:50:23 +0200 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=91=8C=20Increase=20truncate=20size?= =?UTF-8?q?=20from=203500=20to=205000?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/client/core/api/rest/client.go | 2 +- pkg/middleware/logger/logger.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/client/core/api/rest/client.go b/pkg/client/core/api/rest/client.go index 2c2a6da32..ac20d7d1d 100644 --- a/pkg/client/core/api/rest/client.go +++ b/pkg/client/core/api/rest/client.go @@ -86,7 +86,7 @@ func (c *HTTPClient) Do(method, endpoint string, body interface{}, into interfac err = resp.Body.Close() }() - const logLineMaxlength = 3500 + const logLineMaxlength = 5000 if into != nil { if c.Debug { diff --git a/pkg/middleware/logger/logger.go b/pkg/middleware/logger/logger.go index 100f816f8..68b3de9e8 100644 --- a/pkg/middleware/logger/logger.go +++ b/pkg/middleware/logger/logger.go @@ -81,7 +81,7 @@ func (l *logging) ProcessResponse(err error, response interface{}, begin time.Ti d = litter.Sdump(response) } - truncatedDump := truncate.String(&d, 3500) + truncatedDump := truncate.String(&d, 5000) l.log.Trace("response: ", truncatedDump) } From 8594d036a4fceba612530b973b8e3d8a9c9e4b36 Mon Sep 17 00:00:00 2001 From: Yngvar Kristiansen <562343+yngvark@users.noreply.github.com> Date: Wed, 7 Apr 2021 15:37:11 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=91=8C=20Add=20release=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/release_notes/0.0.54.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release_notes/0.0.54.md b/docs/release_notes/0.0.54.md index 595a58405..3c1acd5cc 100644 --- a/docs/release_notes/0.0.54.md +++ b/docs/release_notes/0.0.54.md @@ -5,4 +5,4 @@ ## Bugfixes ## Other -- Truncate log lines to 5000 bytes (#436) +- Truncate log lines to 5000 bytes (#436, #444)