sp cli: make sectors list much faster

filecoin-project · Feb 7, 2023 · 41bae08 · 41bae08
1 parent 8066833
commit 41bae08
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 11 deletions.
diff --git a/cmd/lotus-miner/sectors.go b/cmd/lotus-miner/sectors.go
@@ -9,6 +9,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/docker/go-units"
@@ -33,11 +34,14 @@ import (
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
+	"github.com/filecoin-project/lotus/lib/result"
 	"github.com/filecoin-project/lotus/lib/strle"
 	"github.com/filecoin-project/lotus/lib/tablewriter"
 	sealing "github.com/filecoin-project/lotus/storage/pipeline"
 )
 
+const parallelSectorChecks = 300
+
 var sectorsCmd = &cli.Command{
 	Name:  "sectors",
 	Usage: "interact with sector store",
@@ -306,9 +310,14 @@ var sectorsListCmd = &cli.Command{
 			Usage:   "only show sectors which aren't in the 'Proving' state",
 			Aliases: []string{"u"},
 		},
+		&cli.Int64Flag{
+			Name:  "check-parallelism",
+			Usage: "number of parallel requests to make for checking sector states",
+			Value: parallelSectorChecks,
+		},
 	},
 	Action: func(cctx *cli.Context) error {
-		minerApi, closer, err := lcli.GetStorageMinerAPI(cctx)
+		minerApi, closer, err := lcli.GetStorageMinerAPI(cctx, cliutil.StorageMinerUseHttp)
 		if err != nil {
 			return err
 		}
@@ -407,16 +416,38 @@ var sectorsListCmd = &cli.Command{
 
 		fast := cctx.Bool("fast")
 
+		throttle := make(chan struct{}, cctx.Int64("check-parallelism"))
+
+		resCh := make(chan result.Result[api.SectorInfo], len(list))
+		var wg sync.WaitGroup
 		for _, s := range list {
-			st, err := minerApi.SectorsStatus(ctx, s, !fast)
-			if err != nil {
+			throttle <- struct{}{}
+			wg.Add(1)
+			go func(s abi.SectorNumber) {
+				defer wg.Done()
+				defer func() { <-throttle }()
+				r := result.Wrap(minerApi.SectorsStatus(ctx, s, !fast))
+				if r.Error != nil {
+					r.Value.SectorID = s
+				}
+				resCh <- r
+			}(s)
+		}
+		wg.Wait()
+		close(resCh)
+
+		for rsn := range resCh {
+			if rsn.Error != nil {
 				tw.Write(map[string]interface{}{
-					"ID":    s,
+					"ID":    rsn.Value.SectorID,
 					"Error": err,
 				})
 				continue
 			}
 
+			st := rsn.Value
+			s := st.SectorID
+
 			if !showRemoved && st.State == api.SectorState(sealing.Removed) {
 				continue
 			}

diff --git a/documentation/en/cli-lotus-miner.md b/documentation/en/cli-lotus-miner.md
@@ -1717,13 +1717,14 @@ USAGE:
    lotus-miner sectors list [command options] [arguments...]
 
 OPTIONS:
-   --events, -e          display number of events the sector has received (default: false)
-   --fast, -f            don't show on-chain info for better performance (default: false)
-   --initial-pledge, -p  display initial pledge (default: false)
-   --seal-time, -t       display how long it took for the sector to be sealed (default: false)
-   --show-removed, -r    show removed sectors (default: false)
-   --states value        filter sectors by a comma-separated list of states
-   --unproven, -u        only show sectors which aren't in the 'Proving' state (default: false)
+   --check-parallelism value  number of parallel requests to make for checking sector states (default: 300)
+   --events, -e               display number of events the sector has received (default: false)
+   --fast, -f                 don't show on-chain info for better performance (default: false)
+   --initial-pledge, -p       display initial pledge (default: false)
+   --seal-time, -t            display how long it took for the sector to be sealed (default: false)
+   --show-removed, -r         show removed sectors (default: false)
+   --states value             filter sectors by a comma-separated list of states
+   --unproven, -u             only show sectors which aren't in the 'Proving' state (default: false)
    
 ```