Add pagination for list of releases and release assets

Co-authored-by: Marco Franssen <marco.franssen@philips.com>

lib/github/releases.go

@@ -44,19 +44,16 @@ func NewProvenanceClient(httpClient *http.Client) *ProvenanceClient {
 
 // FetchRelease get the release by its tagName
 func (p *ProvenanceClient) FetchRelease(ctx context.Context, owner, repo, tagName string) (*github.RepositoryRelease, error) {
-	client := p.Client
-
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
 	defer cancel()
 
-	// TODO: add pagination when there are tons of releases for the repo
-	releases, _, err := client.Repositories.ListReleases(ctx, owner, repo, &github.ListOptions{})
+	allReleases, err := p.ListReleases(listCtx, owner, repo, github.ListOptions{PerPage: 10})
 	if err != nil {
 		return nil, err
 	}
 
 	var rel *github.RepositoryRelease
-	for _, r := range releases {
+	for _, r := range allReleases {
 		if *r.TagName == tagName {
 			rel = r
 			break
@@ -69,14 +66,16 @@ func (p *ProvenanceClient) FetchRelease(ctx context.Context, owner, repo, tagNam
 // DownloadReleaseAssets download the assets for a release.
 // It is up to the caller to Close the ReadCloser.
 func (p *ProvenanceClient) DownloadReleaseAssets(ctx context.Context, owner, repo string, releaseID int64) ([]ReleaseAsset, error) {
-	// TODO: add pagination when there are tons of releaseAssets not fitting in a single page for the release
-	releaseAssets, _, err := p.Repositories.ListReleaseAssets(ctx, owner, repo, releaseID, &github.ListOptions{})
+	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	allAssets, err := p.ListReleaseAssets(listCtx, owner, repo, releaseID, github.ListOptions{PerPage: 10})
 	if err != nil {
-		return nil, fmt.Errorf("failed to list release assets: %w", err)
+		return nil, err
 	}
-	assets := make([]ReleaseAsset, len(releaseAssets))
+	assets := make([]ReleaseAsset, len(allAssets))
 
-	for i, releaseAsset := range releaseAssets {
+	for i, releaseAsset := range allAssets {
 		asset, _, err := p.Repositories.DownloadReleaseAsset(ctx, owner, repo, releaseAsset.GetID(), p.httpClient)
 		if err != nil {
 			return nil, err
@@ -102,3 +101,47 @@ func (p *ProvenanceClient) AddProvenanceToRelease(ctx context.Context, owner, re
 	asset, _, err := client.Repositories.UploadReleaseAsset(ctx, owner, repo, releaseID, uploadOptions, provenance)
 	return asset, err
 }
+
+// ListReleaseAssets will retrieve the list of all release assets.
+func (p *ProvenanceClient) ListReleaseAssets(ctx context.Context, owner, repo string, releaseID int64, listOptions github.ListOptions) ([]*github.ReleaseAsset, error) {
+	var allAssets []*github.ReleaseAsset
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+		assets, resp, err := p.Repositories.ListReleaseAssets(ctx, owner, repo, releaseID, &listOptions)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list release assets: %w", err)
+		}
+		allAssets = append(allAssets, assets...)
+		if resp.NextPage == 0 {
+			break
+		}
+		listOptions.Page = resp.NextPage
+	}
+	return allAssets, nil
+}
+
+// ListReleases will retrieve the list of all releases.
+func (p *ProvenanceClient) ListReleases(ctx context.Context, owner, repo string, listOptions github.ListOptions) ([]*github.RepositoryRelease, error) {
+	var allReleases []*github.RepositoryRelease
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+		releases, resp, err := p.Repositories.ListReleases(ctx, owner, repo, &listOptions)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list releases: %w", err)
+		}
+		allReleases = append(allReleases, releases...)
+		if resp.NextPage == 0 {
+			break
+		}
+		listOptions.Page = resp.NextPage
+	}
+	return allReleases, nil
+}

lib/github/releases_test.go

@@ -50,6 +50,8 @@ func TestFetchRelease(t *testing.T) {
 	}
 	release, err := client.FetchRelease(ctx, owner, repo, "v0.1.1")
 
+	assert.Greater(release, 2)
+
 	if !assert.NoError(err) && assert.Nil(release) {
 		return
 	}
@@ -153,3 +155,55 @@ func TestAddProvenanceToRelease(t *testing.T) {
 	assert.Equal(stat.Name(), asset.GetName())
 	assert.Equal("application/json; charset=utf-8", asset.GetContentType())
 }
+
+func TestListReleaseAssets(t *testing.T) {
+	assert := assert.New(t)
+	ctx := context.Background()
+
+	var client *github.ProvenanceClient
+	if githubToken != "" {
+		tc := github.NewOAuth2Client(ctx, tokenRetriever)
+		client = github.NewProvenanceClient(tc)
+	} else {
+		client = github.NewProvenanceClient(nil)
+	}
+	opt := gh.ListOptions{PerPage: 2}
+	assets, err := client.ListReleaseAssets(ctx, owner, repo, 51517953, opt)
+	if !assert.NoError(err) {
+		return
+	}
+	assert.Len(assets, 7)
+
+	opt = gh.ListOptions{PerPage: 10}
+	assets, err = client.ListReleaseAssets(ctx, owner, repo, 51517953, opt)
+	if !assert.NoError(err) {
+		return
+	}
+	assert.Len(assets, 7)
+}
+
+func TestListReleases(t *testing.T) {
+	assert := assert.New(t)
+	ctx := context.Background()
+
+	var client *github.ProvenanceClient
+	if githubToken != "" {
+		tc := github.NewOAuth2Client(ctx, tokenRetriever)
+		client = github.NewProvenanceClient(tc)
+	} else {
+		client = github.NewProvenanceClient(nil)
+	}
+	opt := gh.ListOptions{PerPage: 1}
+	releases, err := client.ListReleases(ctx, owner, repo, opt)
+	if !assert.NoError(err) {
+		return
+	}
+	assert.GreaterOrEqual(len(releases), 2)
+
+	opt = gh.ListOptions{PerPage: 2}
+	releases, err = client.ListReleases(ctx, owner, repo, opt)
+	if !assert.NoError(err) {
+		return
+	}
+	assert.GreaterOrEqual(len(releases), 2)
+}