Merge JSON responses from gh api

Partly resolves cli#1268 and replaces cli#5652. Requires cli/go-gh#148 to be merged and optionally released.

pkg/cmd/api/api.go

@@ -25,6 +25,7 @@ import (
 	"github.com/cli/cli/v2/pkg/iostreams"
 	"github.com/cli/cli/v2/pkg/jsoncolor"
 	"github.com/cli/go-gh/v2/pkg/jq"
+	"github.com/cli/go-gh/v2/pkg/jsonmerge"
 	"github.com/cli/go-gh/v2/pkg/template"
 	"github.com/spf13/cobra"
 )
@@ -36,6 +37,7 @@ type ApiOptions struct {
 	Config     func() (config.Config, error)
 	HttpClient func() (*http.Client, error)
 	IO         *iostreams.IOStreams
+	Merger     jsonmerge.Merger
 
 	Hostname            string
 	RequestMethod       string
@@ -265,10 +267,30 @@ func apiRun(opts *ApiOptions) error {
 		method = "POST"
 	}
 
+	var bodyWriter io.Writer = opts.IO.Out
+	var headersWriter io.Writer = opts.IO.Out
+	if opts.Silent {
+		bodyWriter = io.Discard
+	}
+	if opts.Verbose {
+		// httpClient handles output when verbose flag is specified.
+		bodyWriter = io.Discard
+		headersWriter = io.Discard
+	}
+
 	if opts.Paginate && !isGraphQL {
 		requestPath = addPerPage(requestPath, 100, params)
 	}
 
+	// Merge JSON arrays and object if paginating without a filter or template.
+	if opts.Paginate && opts.FilterOutput == "" && opts.Template == "" {
+		if isGraphQL {
+			opts.Merger = jsonmerge.NewObjectMerger(bodyWriter)
+		} else {
+			opts.Merger = jsonmerge.NewArrayMerger()
+		}
+	}
+
 	if opts.RequestInputFile != "" {
 		file, size, err := openUserFile(opts.RequestInputFile, opts.IO.In)
 		if err != nil {
@@ -322,17 +344,6 @@ func apiRun(opts *ApiOptions) error {
 		}
 	}
 
-	var bodyWriter io.Writer = opts.IO.Out
-	var headersWriter io.Writer = opts.IO.Out
-	if opts.Silent {
-		bodyWriter = io.Discard
-	}
-	if opts.Verbose {
-		// httpClient handles output when verbose flag is specified.
-		bodyWriter = io.Discard
-		headersWriter = io.Discard
-	}
-
 	host, _ := cfg.Authentication().DefaultHost()
 
 	if opts.Hostname != "" {
@@ -380,6 +391,10 @@ func apiRun(opts *ApiOptions) error {
 		}
 	}
 
+	if opts.Merger != nil {
+		return opts.Merger.Close()
+	}
+
 	return tmpl.Flush()
 }
 
@@ -431,14 +446,18 @@ func processResponse(resp *http.Response, opts *ApiOptions, bodyWriter, headersW
 	} else if isJSON && opts.IO.ColorEnabled() {
 		err = jsoncolor.Write(bodyWriter, responseBody, "  ")
 	} else {
-		if isJSON && opts.Paginate && !isGraphQLPaginate && !opts.ShowResponseHeaders {
-			responseBody = &paginatedArrayReader{
-				Reader:      responseBody,
-				isFirstPage: isFirstPage,
-				isLastPage:  isLastPage,
-			}
+		if isJSON && opts.Merger != nil && !opts.ShowResponseHeaders {
+			responseBody = opts.Merger.NewPage(responseBody, isLastPage)
 		}
+
 		_, err = io.Copy(bodyWriter, responseBody)
+		if err != nil {
+			return
+		}
+
+		if closer, ok := responseBody.(io.ReadCloser); ok {
+			err = closer.Close()
+		}
 	}
 	if err != nil {
 		return

pkg/cmd/api/api_test.go

@@ -832,8 +832,18 @@ func Test_apiRun_paginationGraphQL(t *testing.T) {
 	err := apiRun(&options)
 	require.NoError(t, err)
 
-	assert.Contains(t, stdout.String(), `"page one"`)
-	assert.Contains(t, stdout.String(), `"page two"`)
+	assert.JSONEq(t, stdout.String(), `{
+		"data": {
+			"nodes": [
+				"page one",
+				"page two"
+			],
+			"pageInfo": {
+				"endCursor": "PAGE2_END",
+				"hasNextPage": false
+			}
+		}
+	}`)
 	assert.Equal(t, "", stderr.String(), "stderr")
 
 	var requestData struct {

pkg/cmd/api/pagination.go

@@ -106,43 +106,3 @@ func addPerPage(p string, perPage int, params map[string]interface{}) string {
 
 	return fmt.Sprintf("%s%sper_page=%d", p, sep, perPage)
 }
-
-// paginatedArrayReader wraps a Reader to omit the opening and/or the closing square bracket of a
-// JSON array in order to apply pagination context between multiple API requests.
-type paginatedArrayReader struct {
-	io.Reader
-	isFirstPage bool
-	isLastPage  bool
-
-	isSubsequentRead bool
-	cachedByte       byte
-}
-
-func (r *paginatedArrayReader) Read(p []byte) (int, error) {
-	var n int
-	var err error
-	if r.cachedByte != 0 && len(p) > 0 {
-		p[0] = r.cachedByte
-		n, err = r.Reader.Read(p[1:])
-		n += 1
-		r.cachedByte = 0
-	} else {
-		n, err = r.Reader.Read(p)
-	}
-	if !r.isSubsequentRead && !r.isFirstPage && n > 0 && p[0] == '[' {
-		if n > 1 && p[1] == ']' {
-			// empty array case
-			p[0] = ' '
-		} else {
-			// avoid starting a new array and continue with a comma instead
-			p[0] = ','
-		}
-	}
-	if !r.isLastPage && n > 0 && p[n-1] == ']' {
-		// avoid closing off an array in case we determine we are at EOF
-		r.cachedByte = p[n-1]
-		n -= 1
-	}
-	r.isSubsequentRead = true
-	return n, err
-}