Support merging JSON arrays, objects

go.mod

@@ -3,6 +3,7 @@ module github.com/cli/go-gh/v2
 go 1.21
 
 require (
+	dario.cat/mergo v1.0.0
 	github.com/AlecAivazis/survey/v2 v2.3.7
 	github.com/MakeNowJust/heredoc v1.0.0
 	github.com/charmbracelet/glamour v0.6.0

go.sum

@@ -1,3 +1,5 @@
+dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
+dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/AlecAivazis/survey/v2 v2.3.7 h1:6I/u8FvytdGsgonrYsVn2t8t4QiRnh6QSTqkkhIiSjQ=
 github.com/AlecAivazis/survey/v2 v2.3.7/go.mod h1:xUTIdE4KCOIjsBAE1JYsUPoCqYdZ1reCfTwbto0Fduo=
 github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=

pkg/jsonmerge/array.go

@@ -0,0 +1,76 @@
+package jsonmerge
+
+import "io"
+
+type arrayMerger struct {
+	isFirstPage bool
+}
+
+// NewArrayMerger creates a Merger for JSON arrays.
+func NewArrayMerger() Merger {
+	return &arrayMerger{
+		isFirstPage: true,
+	}
+}
+
+func (merger *arrayMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser {
+	return &arrayMergerPage{
+		merger:     merger,
+		Reader:     r,
+		isLastPage: isLastPage,
+	}
+}
+
+func (m *arrayMerger) Close() error {
+	// arrayMerger merges when reading, so any output was already written
+	// and there's nothing to do on Close.
+	return nil
+}
+
+type arrayMergerPage struct {
+	merger *arrayMerger
+
+	io.Reader
+	isLastPage bool
+
+	isSubsequentRead bool
+	cachedByte       byte
+}
+
+func (page *arrayMergerPage) Read(p []byte) (int, error) {
+	var n int
+	var err error
+
+	if page.cachedByte != 0 && len(p) > 0 {
+		p[0] = page.cachedByte
+		n, err = page.Reader.Read(p[1:])
+		n += 1
+		page.cachedByte = 0
+	} else {
+		n, err = page.Reader.Read(p)
+	}
+
+	if !page.isSubsequentRead && !page.merger.isFirstPage && n > 0 && p[0] == '[' {
+		if n > 1 && p[1] == ']' {
+			// Empty array case.
+			p[0] = ' '
+		} else {
+			// Avoid starting a new array and continue with a comma instead.
+			p[0] = ','
+		}
+	}
+
+	if !page.isLastPage && n > 0 && p[n-1] == ']' {
+		// Avoid closing off an array in case we determine we are at EOF.
+		page.cachedByte = p[n-1]
+		n -= 1
+	}
+
+	page.isSubsequentRead = true
+	return n, err
+}
+
+func (page *arrayMergerPage) Close() error {
+	page.merger.isFirstPage = false
+	return nil
+}

pkg/jsonmerge/array_test.go

@@ -0,0 +1,86 @@
+package jsonmerge
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestArrayMerger_singleEmptyArray(t *testing.T) {
+	merger := NewArrayMerger()
+	w := &bytes.Buffer{}
+
+	r1 := bytes.NewBufferString(`[]`)
+	p1 := merger.NewPage(r1, true)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, `[]`, w.String())
+
+	require.NoError(t, merger.Close())
+}
+
+func TestArrayMerger_finalEmptyArray(t *testing.T) {
+	merger := NewArrayMerger()
+	w := &bytes.Buffer{}
+
+	r1 := bytes.NewBufferString(`["a","b"]`)
+	p1 := merger.NewPage(r1, false)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(8), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, `["a","b"`, w.String())
+
+	r2 := bytes.NewBufferString(`[]`)
+	p2 := merger.NewPage(r2, true)
+	n, err = io.Copy(w, p2)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), n)
+	assert.NoError(t, p2.Close())
+	assert.Equal(t, `["a","b" ]`, w.String())
+
+	require.NoError(t, merger.Close())
+}
+
+func TestArrayMerger_multiplePages(t *testing.T) {
+	merger := NewArrayMerger()
+	w := &bytes.Buffer{}
+
+	r1 := bytes.NewBufferString(`["a","b"]`)
+	p1 := merger.NewPage(r1, false)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(8), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, `["a","b"`, w.String())
+
+	r2 := bytes.NewBufferString(`["c","d"]`)
+	p2 := merger.NewPage(r2, true)
+	n, err = io.Copy(w, p2)
+	require.NoError(t, err)
+	assert.Equal(t, int64(9), n)
+	assert.NoError(t, p2.Close())
+	assert.Equal(t, `["a","b","c","d"]`, w.String())
+
+	require.NoError(t, merger.Close())
+}
+
+func TestArrayMerger_emptyObject(t *testing.T) {
+	merger := NewArrayMerger()
+	w := &bytes.Buffer{}
+
+	r1 := bytes.NewBufferString(`{}`)
+	p1 := merger.NewPage(r1, true)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(2), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, `{}`, w.String())
+
+	require.NoError(t, merger.Close())
+}

pkg/jsonmerge/merge.go

@@ -0,0 +1,12 @@
+// jsonmerge implements readers to merge JSON arrays or objects.
+package jsonmerge
+
+import (
+	"io"
+)
+
+// Merger is implemented to merge JSON arrays or objects.
+type Merger interface {
+	NewPage(r io.Reader, isLastPage bool) io.ReadCloser
+	Close() error
+}

pkg/jsonmerge/object.go

@@ -0,0 +1,66 @@
+package jsonmerge
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+
+	"dario.cat/mergo"
+)
+
+type objectMerger struct {
+	io.Writer
+	dst map[string]interface{}
+}
+
+// NewObjectMerger creates a Merger for JSON objects.
+func NewObjectMerger(w io.Writer) Merger {
+	return &objectMerger{
+		Writer: w,
+		dst:    make(map[string]interface{}),
+	}
+}
+
+func (merger *objectMerger) NewPage(r io.Reader, isLastPage bool) io.ReadCloser {
+	return &objectMergerPage{
+		merger: merger,
+		Reader: r,
+	}
+}
+
+func (merger *objectMerger) Close() error {
+	// Marshal to JSON and write to output.
+	buf, err := json.Marshal(merger.dst)
+	if err != nil {
+		return err
+	}
+
+	_, err = merger.Writer.Write(buf)
+	return err
+}
+
+type objectMergerPage struct {
+	merger *objectMerger
+
+	io.Reader
+	buffer bytes.Buffer
+}
+
+// Read caches the data in an internal buffer to be merged in Close.
+// No data is copied into p so it's not written to stdout.
+func (page *objectMergerPage) Read(p []byte) (int, error) {
+	_, err := io.CopyN(&page.buffer, page.Reader, int64(len(p)))
+	return 0, err
+}
+
+// Close converts the internal buffer to a JSON object and merges it with the final JSON object.
+func (page *objectMergerPage) Close() error {
+	var src map[string]interface{}
+
+	err := json.Unmarshal(page.buffer.Bytes(), &src)
+	if err != nil {
+		return err
+	}
+
+	return mergo.Merge(&page.merger.dst, src, mergo.WithAppendSlice, mergo.WithOverride)
+}

pkg/jsonmerge/object_test.go

@@ -0,0 +1,98 @@
+package jsonmerge
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestObjectMerger_singleEmptyObject(t *testing.T) {
+	w := &bytes.Buffer{}
+	merger := NewObjectMerger(w)
+
+	r1 := bytes.NewBufferString(`{}`)
+	p1 := merger.NewPage(r1, true)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, ``, w.String())
+
+	require.NoError(t, merger.Close())
+	assert.JSONEq(t, `{}`, w.String())
+}
+
+func TestObjectMerger_finalEmptyObject(t *testing.T) {
+	w := &bytes.Buffer{}
+	merger := NewObjectMerger(w)
+
+	r1 := bytes.NewBufferString(`{"a":1,"b":2}`)
+	p1 := merger.NewPage(r1, false)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, ``, w.String())
+
+	r2 := bytes.NewBufferString(`{}`)
+	p2 := merger.NewPage(r2, true)
+	n, err = io.Copy(w, p2)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.NoError(t, p2.Close())
+	assert.Equal(t, ``, w.String())
+
+	require.NoError(t, merger.Close())
+	assert.JSONEq(t, `{"a":1,"b":2}`, w.String())
+}
+
+func TestObjectMerger_multiplePages(t *testing.T) {
+	w := &bytes.Buffer{}
+	merger := NewObjectMerger(w)
+
+	r1 := bytes.NewBufferString(`{"a":1,"b":2,"arr":["a","b"]}`)
+	p1 := merger.NewPage(r1, false)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.NoError(t, p1.Close())
+	assert.Equal(t, ``, w.String())
+
+	r2 := bytes.NewBufferString(`{"b":3,"c":{"d":4},"arr":["c","d"]}`)
+	p2 := merger.NewPage(r2, true)
+	n, err = io.Copy(w, p2)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.NoError(t, p2.Close())
+	assert.Equal(t, ``, w.String())
+
+	require.NoError(t, merger.Close())
+	assert.JSONEq(t, `{"a":1,"b":3,"c":{"d":4},"arr":["a","b","c","d"]}`, w.String())
+}
+
+func TestObjectMerger_invalidJSON(t *testing.T) {
+	w := &bytes.Buffer{}
+	merger := NewObjectMerger(w)
+
+	r1 := bytes.NewBufferString(`invalid`)
+	p1 := merger.NewPage(r1, true)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.Error(t, p1.Close())
+}
+
+func TestObjectMerger_array(t *testing.T) {
+	w := &bytes.Buffer{}
+	merger := NewObjectMerger(w)
+
+	r1 := bytes.NewBufferString(`[]`)
+	p1 := merger.NewPage(r1, true)
+	n, err := io.Copy(w, p1)
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), n)
+	assert.Error(t, p1.Close())
+}