diff --git a/api/host.go b/api/host.go index 19f28cb2f..f1e751a50 100644 --- a/api/host.go +++ b/api/host.go @@ -3,7 +3,6 @@ package api import ( "errors" "fmt" - "net/url" "strings" "time" @@ -51,7 +50,7 @@ type ( PriceTableUpdates []HostPriceTableUpdate `json:"priceTableUpdates"` } - // HostsRemoveRequest is the request type for the /hosts/remove endpoint. + // HostsRemoveRequest is the request type for the delete /hosts endpoint. HostsRemoveRequest struct { MaxDowntimeHours DurationH `json:"maxDowntimeHours"` MaxConsecutiveScanFailures uint64 `json:"maxConsecutiveScanFailures"` @@ -66,6 +65,7 @@ type ( UsabilityMode string `json:"usabilityMode"` AddressContains string `json:"addressContains"` KeyIn []types.PublicKey `json:"keyIn"` + MaxLastScan TimeRFC3339 `json:"maxLastScan"` } ) @@ -87,12 +87,6 @@ type ( // Option types. type ( - HostsForScanningOptions struct { - MaxLastScan TimeRFC3339 - Limit int - Offset int - } - HostOptions struct { AutopilotID string AddressContains string @@ -100,22 +94,11 @@ type ( UsabilityMode string KeyIn []types.PublicKey Limit int + MaxLastScan TimeRFC3339 Offset int } ) -func (opts HostsForScanningOptions) Apply(values url.Values) { - if opts.Offset != 0 { - values.Set("offset", fmt.Sprint(opts.Offset)) - } - if opts.Limit != 0 { - values.Set("limit", fmt.Sprint(opts.Limit)) - } - if !opts.MaxLastScan.IsZero() { - values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String()) - } -} - type ( Host struct { KnownSince time.Time `json:"knownSince"` @@ -133,11 +116,6 @@ type ( Subnets []string `json:"subnets"` } - HostAddress struct { - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` - } - HostInteractions struct { TotalScans uint64 `json:"totalScans"` LastScan time.Time `json:"lastScan"` diff --git a/api/object.go b/api/object.go index 09e33dba1..dcfdb4bac 100644 --- a/api/object.go +++ b/api/object.go @@ -214,12 +214,6 @@ type ( Substring string } - SearchObjectOptions struct { - Key string - Offset int - Limit int - } - // UploadObjectOptions is the options type for the worker client. UploadObjectOptions struct { MinShards int @@ -330,18 +324,6 @@ func (opts ListObjectOptions) Apply(values url.Values) { } } -func (opts SearchObjectOptions) Apply(values url.Values) { - if opts.Key != "" { - values.Set("key", opts.Key) - } - if opts.Offset != 0 { - values.Set("offset", fmt.Sprint(opts.Offset)) - } - if opts.Limit != 0 { - values.Set("limit", fmt.Sprint(opts.Limit)) - } -} - func FormatETag(eTag string) string { return fmt.Sprintf("%q", eTag) } diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index 12f4c809e..907007d73 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -55,7 +55,6 @@ type Bus interface { // hostdb Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error diff --git a/autopilot/scanner/scanner.go b/autopilot/scanner/scanner.go index f79147a17..c696a918c 100644 --- a/autopilot/scanner/scanner.go +++ b/autopilot/scanner/scanner.go @@ -19,7 +19,7 @@ const ( type ( HostStore interface { - HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) + Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) } @@ -164,7 +164,7 @@ func (s *scanner) fetchHosts(ctx context.Context, cutoff time.Time) chan scanJob var exhausted bool for offset := 0; !exhausted; offset += s.scanBatchSize { - hosts, err := s.hs.HostsForScanning(ctx, api.HostsForScanningOptions{ + hosts, err := s.hs.Hosts(ctx, api.HostOptions{ MaxLastScan: api.TimeRFC3339(cutoff), Offset: offset, Limit: s.scanBatchSize, diff --git a/autopilot/scanner/scanner_test.go b/autopilot/scanner/scanner_test.go index 665913ab0..020ca8932 100644 --- a/autopilot/scanner/scanner_test.go +++ b/autopilot/scanner/scanner_test.go @@ -26,7 +26,7 @@ type mockHostStore struct { removals []string } -func (hs *mockHostStore) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) { +func (hs *mockHostStore) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) { hs.mu.Lock() defer hs.mu.Unlock() hs.scans = append(hs.scans, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit)) @@ -41,14 +41,7 @@ func (hs *mockHostStore) HostsForScanning(ctx context.Context, opts api.HostsFor end = len(hs.hosts) } - var hostAddresses []api.HostAddress - for _, h := range hs.hosts[start:end] { - hostAddresses = append(hostAddresses, api.HostAddress{ - NetAddress: h.NetAddress, - PublicKey: h.PublicKey, - }) - } - return hostAddresses, nil + return hs.hosts[start:end], nil } func (hs *mockHostStore) RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) { diff --git a/bus/bus.go b/bus/bus.go index 90db3764d..8c966729d 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -195,7 +195,6 @@ type ( HostAllowlist(ctx context.Context) ([]types.PublicKey, error) HostBlocklist(ctx context.Context) ([]string, error) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) - HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) RecordHostScans(ctx context.Context, scans []api.HostScan) error RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) @@ -441,7 +440,6 @@ func (b *Bus) Handler() http.Handler { "POST /hosts/pricetables": b.hostsPricetableHandlerPOST, "POST /hosts/remove": b.hostsRemoveHandlerPOST, "POST /hosts/scans": b.hostsScanHandlerPOST, - "GET /hosts/scanning": b.hostsScanningHandlerGET, "GET /host/:hostkey": b.hostsPubkeyHandlerGET, "POST /host/:hostkey/resetlostsectors": b.hostsResetLostSectorsPOST, diff --git a/bus/client/hosts.go b/bus/client/hosts.go index d7aa5f6db..086c6f5ad 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -3,7 +3,6 @@ package client import ( "context" "fmt" - "net/url" "time" "go.sia.tech/core/types" @@ -26,6 +25,7 @@ func (c *Client) Hosts(ctx context.Context, opts api.HostOptions) (hosts []api.H UsabilityMode: opts.UsabilityMode, AddressContains: opts.AddressContains, KeyIn: opts.KeyIn, + MaxLastScan: opts.MaxLastScan, }, &hosts) return } @@ -42,15 +42,6 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err return } -// HostsForScanning returns 'limit' host addresses at given 'offset' which -// haven't been scanned after lastScan. -func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []api.HostAddress, err error) { - values := url.Values{} - opts.Apply(values) - err = c.c.WithContext(ctx).GET("/hosts/scanning?"+values.Encode(), &hosts) - return -} - // RecordHostInteraction records an interaction for the supplied host. func (c *Client) RecordHostScans(ctx context.Context, scans []api.HostScan) (err error) { err = c.c.WithContext(ctx).POST("/hosts/scans", api.HostsScanRequest{ diff --git a/bus/routes.go b/bus/routes.go index 7f7e0251d..9b1b22828 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -507,6 +507,7 @@ func (b *Bus) hostsHandlerPOST(jc jape.Context) { KeyIn: req.KeyIn, Offset: req.Offset, Limit: req.Limit, + MaxLastScan: req.MaxLastScan, }) if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil { return @@ -534,20 +535,6 @@ func (b *Bus) hostsRemoveHandlerPOST(jc jape.Context) { jc.Encode(removed) } -func (b *Bus) hostsScanningHandlerGET(jc jape.Context) { - offset := 0 - limit := -1 - maxLastScan := time.Now() - if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("lastScan", (*api.TimeRFC3339)(&maxLastScan)) != nil { - return - } - hosts, err := b.hs.HostsForScanning(jc.Request.Context(), maxLastScan, offset, limit) - if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil { - return - } - jc.Encode(hosts) -} - func (b *Bus) hostsPubkeyHandlerGET(jc jape.Context) { var hostKey types.PublicKey if jc.DecodeParam("hostkey", &hostKey) != nil { diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 1b51b2fc1..80723e7c6 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -2645,7 +2645,7 @@ func TestHostScan(t *testing.T) { // fetch hosts for scanning with maxLastScan set to now which should return // all hosts tt.Retry(100, 100*time.Millisecond, func() error { - toScan, err := b.HostsForScanning(context.Background(), api.HostsForScanningOptions{ + toScan, err := b.Hosts(context.Background(), api.HostOptions{ MaxLastScan: api.TimeRFC3339(time.Now()), }) tt.OK(err) @@ -2657,7 +2657,7 @@ func TestHostScan(t *testing.T) { // fetch hosts again with the unix epoch timestamp which should only return // 1 host since that one hasn't been scanned yet - toScan, err := b.HostsForScanning(context.Background(), api.HostsForScanningOptions{ + toScan, err := b.Hosts(context.Background(), api.HostOptions{ MaxLastScan: api.TimeRFC3339(time.UnixMilli(1)), }) tt.OK(err) diff --git a/stores/hostdb.go b/stores/hostdb.go index 1df6a30d6..a80df54dd 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -41,15 +41,6 @@ func (s *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk t }) } -// HostsForScanning returns the address of hosts for scanning. -func (s *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) (hosts []api.HostAddress, err error) { - err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - hosts, err = tx.HostsForScanning(ctx, maxLastScan, offset, limit) - return err - }) - return -} - func (s *SQLStore) ResetLostSectors(ctx context.Context, hk types.PublicKey) error { return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { return tx.ResetLostSectors(ctx, hk) diff --git a/stores/sql/database.go b/stores/sql/database.go index 651fe81f3..7cf4d929d 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -188,10 +188,6 @@ type ( // Hosts returns a list of hosts that match the provided filters Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) - // HostsForScanning returns a list of hosts to scan which haven't been - // scanned since at least maxLastScan. - HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) - // ListBuckets returns a list of all buckets in the database. ListBuckets(ctx context.Context) ([]api.Bucket, error) diff --git a/stores/sql/main.go b/stores/sql/main.go index 12b265587..38150dd13 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -758,6 +758,12 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er } } + // filter max last scan + if !opts.MaxLastScan.IsZero() { + whereExprs = append(whereExprs, "last_scan < ?") + args = append(args, UnixTimeMS(opts.MaxLastScan)) + } + // offset + limit if opts.Limit == -1 { opts.Limit = math.MaxInt64 @@ -899,31 +905,6 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er return hosts, nil } -func HostsForScanning(ctx context.Context, tx sql.Tx, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) { - if offset < 0 { - return nil, ErrNegativeOffset - } else if limit == -1 { - limit = math.MaxInt64 - } - - rows, err := tx.Query(ctx, "SELECT public_key, net_address FROM hosts WHERE last_scan < ? LIMIT ? OFFSET ?", - UnixTimeMS(maxLastScan), limit, offset) - if err != nil { - return nil, fmt.Errorf("failed to fetch hosts for scanning: %w", err) - } - defer rows.Close() - - var hosts []api.HostAddress - for rows.Next() { - var ha api.HostAddress - if err := rows.Scan((*PublicKey)(&ha.PublicKey), &ha.NetAddress); err != nil { - return nil, fmt.Errorf("failed to scan host row: %w", err) - } - hosts = append(hosts, ha) - } - return hosts, nil -} - func InsertBufferedSlab(ctx context.Context, tx sql.Tx, fileName string, contractSetID int64, ec object.EncryptionKey, minShards, totalShards uint8) (int64, error) { // insert buffered slab res, err := tx.Exec(ctx, `INSERT INTO buffered_slabs (created_at, filename) VALUES (?, ?)`, diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index 38f6bf97d..408727704 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -416,10 +416,6 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap return ssql.Hosts(ctx, tx, opts) } -func (tx *MainDatabaseTx) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) { - return ssql.HostsForScanning(ctx, tx, maxLastScan, offset, limit) -} - func (tx *MainDatabaseTx) InsertObject(ctx context.Context, bucket, key, contractSet string, dirID int64, o object.Object, mimeType, eTag string, md api.ObjectUserMetadata) error { // get bucket id var bucketID int64 diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 18c66fa28..f6a54f8f8 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -405,10 +405,6 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap return ssql.Hosts(ctx, tx, opts) } -func (tx *MainDatabaseTx) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) { - return ssql.HostsForScanning(ctx, tx, maxLastScan, offset, limit) -} - func (tx *MainDatabaseTx) InsertObject(ctx context.Context, bucket, key, contractSet string, dirID int64, o object.Object, mimeType, eTag string, md api.ObjectUserMetadata) error { // get bucket id var bucketID int64