Merge hosts endpoints

api/host.go

@@ -3,7 +3,6 @@ package api
 import (
 	"errors"
 	"fmt"
-	"net/url"
 	"strings"
 	"time"
 
@@ -51,7 +50,7 @@ type (
 		PriceTableUpdates []HostPriceTableUpdate `json:"priceTableUpdates"`
 	}
 
-	// HostsRemoveRequest is the request type for the /hosts/remove endpoint.
+	// HostsRemoveRequest is the request type for the delete /hosts endpoint.
 	HostsRemoveRequest struct {
 		MaxDowntimeHours           DurationH `json:"maxDowntimeHours"`
 		MaxConsecutiveScanFailures uint64    `json:"maxConsecutiveScanFailures"`
@@ -66,6 +65,7 @@ type (
 		UsabilityMode   string            `json:"usabilityMode"`
 		AddressContains string            `json:"addressContains"`
 		KeyIn           []types.PublicKey `json:"keyIn"`
+		MaxLastScan     TimeRFC3339       `json:"maxLastScan"`
 	}
 )
 
@@ -87,35 +87,18 @@ type (
 
 // Option types.
 type (
-	HostsForScanningOptions struct {
-		MaxLastScan TimeRFC3339
-		Limit       int
-		Offset      int
-	}
-
 	HostOptions struct {
 		AutopilotID     string
 		AddressContains string
 		FilterMode      string
 		UsabilityMode   string
 		KeyIn           []types.PublicKey
 		Limit           int
+		MaxLastScan     TimeRFC3339
 		Offset          int
 	}
 )
 
-func (opts HostsForScanningOptions) Apply(values url.Values) {
-	if opts.Offset != 0 {
-		values.Set("offset", fmt.Sprint(opts.Offset))
-	}
-	if opts.Limit != 0 {
-		values.Set("limit", fmt.Sprint(opts.Limit))
-	}
-	if !opts.MaxLastScan.IsZero() {
-		values.Set("lastScan", TimeRFC3339(opts.MaxLastScan).String())
-	}
-}
-
 type (
 	Host struct {
 		KnownSince        time.Time            `json:"knownSince"`
@@ -133,11 +116,6 @@ type (
 		Subnets           []string             `json:"subnets"`
 	}
 
-	HostAddress struct {
-		PublicKey  types.PublicKey `json:"publicKey"`
-		NetAddress string          `json:"netAddress"`
-	}
-
 	HostInteractions struct {
 		TotalScans              uint64        `json:"totalScans"`
 		LastScan                time.Time     `json:"lastScan"`

api/object.go

@@ -214,12 +214,6 @@ type (
 		Substring string
 	}
 
-	SearchObjectOptions struct {
-		Key    string
-		Offset int
-		Limit  int
-	}
-
 	// UploadObjectOptions is the options type for the worker client.
 	UploadObjectOptions struct {
 		MinShards     int
@@ -330,18 +324,6 @@ func (opts ListObjectOptions) Apply(values url.Values) {
 	}
 }
 
-func (opts SearchObjectOptions) Apply(values url.Values) {
-	if opts.Key != "" {
-		values.Set("key", opts.Key)
-	}
-	if opts.Offset != 0 {
-		values.Set("offset", fmt.Sprint(opts.Offset))
-	}
-	if opts.Limit != 0 {
-		values.Set("limit", fmt.Sprint(opts.Limit))
-	}
-}
-
 func FormatETag(eTag string) string {
 	return fmt.Sprintf("%q", eTag)
 }

autopilot/autopilot.go

@@ -55,7 +55,6 @@ type Bus interface {
 	// hostdb
 	Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error)
 	Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error)
-	HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error)
 	RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 	UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error
 

autopilot/scanner/scanner.go

@@ -19,7 +19,7 @@ const (
 
 type (
 	HostStore interface {
-		HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error)
+		Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error)
 		RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error)
 	}
 
@@ -164,7 +164,7 @@ func (s *scanner) fetchHosts(ctx context.Context, cutoff time.Time) chan scanJob
 
 		var exhausted bool
 		for offset := 0; !exhausted; offset += s.scanBatchSize {
-			hosts, err := s.hs.HostsForScanning(ctx, api.HostsForScanningOptions{
+			hosts, err := s.hs.Hosts(ctx, api.HostOptions{
 				MaxLastScan: api.TimeRFC3339(cutoff),
 				Offset:      offset,
 				Limit:       s.scanBatchSize,

autopilot/scanner/scanner_test.go

@@ -26,7 +26,7 @@ type mockHostStore struct {
 	removals []string
 }
 
-func (hs *mockHostStore) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) ([]api.HostAddress, error) {
+func (hs *mockHostStore) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) {
 	hs.mu.Lock()
 	defer hs.mu.Unlock()
 	hs.scans = append(hs.scans, fmt.Sprintf("%d-%d", opts.Offset, opts.Offset+opts.Limit))
@@ -41,14 +41,7 @@ func (hs *mockHostStore) HostsForScanning(ctx context.Context, opts api.HostsFor
 		end = len(hs.hosts)
 	}
 
-	var hostAddresses []api.HostAddress
-	for _, h := range hs.hosts[start:end] {
-		hostAddresses = append(hostAddresses, api.HostAddress{
-			NetAddress: h.NetAddress,
-			PublicKey:  h.PublicKey,
-		})
-	}
-	return hostAddresses, nil
+	return hs.hosts[start:end], nil
 }
 
 func (hs *mockHostStore) RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) {

bus/bus.go

@@ -195,7 +195,6 @@ type (
 		HostAllowlist(ctx context.Context) ([]types.PublicKey, error)
 		HostBlocklist(ctx context.Context) ([]string, error)
 		Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error)
-		HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error)
 		RecordHostScans(ctx context.Context, scans []api.HostScan) error
 		RecordPriceTables(ctx context.Context, priceTableUpdate []api.HostPriceTableUpdate) error
 		RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error)
@@ -441,7 +440,6 @@ func (b *Bus) Handler() http.Handler {
 		"POST   /hosts/pricetables":              b.hostsPricetableHandlerPOST,
 		"POST   /hosts/remove":                   b.hostsRemoveHandlerPOST,
 		"POST   /hosts/scans":                    b.hostsScanHandlerPOST,
-		"GET    /hosts/scanning":                 b.hostsScanningHandlerGET,
 		"GET    /host/:hostkey":                  b.hostsPubkeyHandlerGET,
 		"POST   /host/:hostkey/resetlostsectors": b.hostsResetLostSectorsPOST,
 

bus/client/hosts.go

@@ -3,7 +3,6 @@ package client
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"time"
 
 	"go.sia.tech/core/types"
@@ -26,6 +25,7 @@ func (c *Client) Hosts(ctx context.Context, opts api.HostOptions) (hosts []api.H
 		UsabilityMode:   opts.UsabilityMode,
 		AddressContains: opts.AddressContains,
 		KeyIn:           opts.KeyIn,
+		MaxLastScan:     opts.MaxLastScan,
 	}, &hosts)
 	return
 }
@@ -42,15 +42,6 @@ func (c *Client) HostBlocklist(ctx context.Context) (blocklist []string, err err
 	return
 }
 
-// HostsForScanning returns 'limit' host addresses at given 'offset' which
-// haven't been scanned after lastScan.
-func (c *Client) HostsForScanning(ctx context.Context, opts api.HostsForScanningOptions) (hosts []api.HostAddress, err error) {
-	values := url.Values{}
-	opts.Apply(values)
-	err = c.c.WithContext(ctx).GET("/hosts/scanning?"+values.Encode(), &hosts)
-	return
-}
-
 // RecordHostInteraction records an interaction for the supplied host.
 func (c *Client) RecordHostScans(ctx context.Context, scans []api.HostScan) (err error) {
 	err = c.c.WithContext(ctx).POST("/hosts/scans", api.HostsScanRequest{

bus/routes.go

@@ -507,6 +507,7 @@ func (b *Bus) hostsHandlerPOST(jc jape.Context) {
 		KeyIn:           req.KeyIn,
 		Offset:          req.Offset,
 		Limit:           req.Limit,
+		MaxLastScan:     req.MaxLastScan,
 	})
 	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", req.Offset, req.Offset+req.Limit), err) != nil {
 		return
@@ -534,20 +535,6 @@ func (b *Bus) hostsRemoveHandlerPOST(jc jape.Context) {
 	jc.Encode(removed)
 }
 
-func (b *Bus) hostsScanningHandlerGET(jc jape.Context) {
-	offset := 0
-	limit := -1
-	maxLastScan := time.Now()
-	if jc.DecodeForm("offset", &offset) != nil || jc.DecodeForm("limit", &limit) != nil || jc.DecodeForm("lastScan", (*api.TimeRFC3339)(&maxLastScan)) != nil {
-		return
-	}
-	hosts, err := b.hs.HostsForScanning(jc.Request.Context(), maxLastScan, offset, limit)
-	if jc.Check(fmt.Sprintf("couldn't fetch hosts %d-%d", offset, offset+limit), err) != nil {
-		return
-	}
-	jc.Encode(hosts)
-}
-
 func (b *Bus) hostsPubkeyHandlerGET(jc jape.Context) {
 	var hostKey types.PublicKey
 	if jc.DecodeParam("hostkey", &hostKey) != nil {

internal/test/e2e/cluster_test.go

@@ -2645,7 +2645,7 @@ func TestHostScan(t *testing.T) {
 	// fetch hosts for scanning with maxLastScan set to now which should return
 	// all hosts
 	tt.Retry(100, 100*time.Millisecond, func() error {
-		toScan, err := b.HostsForScanning(context.Background(), api.HostsForScanningOptions{
+		toScan, err := b.Hosts(context.Background(), api.HostOptions{
 			MaxLastScan: api.TimeRFC3339(time.Now()),
 		})
 		tt.OK(err)
@@ -2657,7 +2657,7 @@ func TestHostScan(t *testing.T) {
 
 	// fetch hosts again with the unix epoch timestamp which should only return
 	// 1 host since that one hasn't been scanned yet
-	toScan, err := b.HostsForScanning(context.Background(), api.HostsForScanningOptions{
+	toScan, err := b.Hosts(context.Background(), api.HostOptions{
 		MaxLastScan: api.TimeRFC3339(time.UnixMilli(1)),
 	})
 	tt.OK(err)

stores/hostdb.go

@@ -41,15 +41,6 @@ func (s *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk t
 	})
 }
 
-// HostsForScanning returns the address of hosts for scanning.
-func (s *SQLStore) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) (hosts []api.HostAddress, err error) {
-	err = s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
-		hosts, err = tx.HostsForScanning(ctx, maxLastScan, offset, limit)
-		return err
-	})
-	return
-}
-
 func (s *SQLStore) ResetLostSectors(ctx context.Context, hk types.PublicKey) error {
 	return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error {
 		return tx.ResetLostSectors(ctx, hk)

stores/sql/database.go

@@ -188,10 +188,6 @@ type (
 		// Hosts returns a list of hosts that match the provided filters
 		Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error)
 
-		// HostsForScanning returns a list of hosts to scan which haven't been
-		// scanned since at least maxLastScan.
-		HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error)
-
 		// ListBuckets returns a list of all buckets in the database.
 		ListBuckets(ctx context.Context) ([]api.Bucket, error)
 

stores/sql/main.go

@@ -758,6 +758,12 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er
 		}
 	}
 
+	// filter max last scan
+	if !opts.MaxLastScan.IsZero() {
+		whereExprs = append(whereExprs, "last_scan < ?")
+		args = append(args, UnixTimeMS(opts.MaxLastScan))
+	}
+
 	// offset + limit
 	if opts.Limit == -1 {
 		opts.Limit = math.MaxInt64
@@ -899,31 +905,6 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er
 	return hosts, nil
 }
 
-func HostsForScanning(ctx context.Context, tx sql.Tx, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) {
-	if offset < 0 {
-		return nil, ErrNegativeOffset
-	} else if limit == -1 {
-		limit = math.MaxInt64
-	}
-
-	rows, err := tx.Query(ctx, "SELECT public_key, net_address FROM hosts WHERE last_scan < ? LIMIT ? OFFSET ?",
-		UnixTimeMS(maxLastScan), limit, offset)
-	if err != nil {
-		return nil, fmt.Errorf("failed to fetch hosts for scanning: %w", err)
-	}
-	defer rows.Close()
-
-	var hosts []api.HostAddress
-	for rows.Next() {
-		var ha api.HostAddress
-		if err := rows.Scan((*PublicKey)(&ha.PublicKey), &ha.NetAddress); err != nil {
-			return nil, fmt.Errorf("failed to scan host row: %w", err)
-		}
-		hosts = append(hosts, ha)
-	}
-	return hosts, nil
-}
-
 func InsertBufferedSlab(ctx context.Context, tx sql.Tx, fileName string, contractSetID int64, ec object.EncryptionKey, minShards, totalShards uint8) (int64, error) {
 	// insert buffered slab
 	res, err := tx.Exec(ctx, `INSERT INTO buffered_slabs (created_at, filename) VALUES (?, ?)`,

stores/sql/mysql/main.go

@@ -416,10 +416,6 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap
 	return ssql.Hosts(ctx, tx, opts)
 }
 
-func (tx *MainDatabaseTx) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) {
-	return ssql.HostsForScanning(ctx, tx, maxLastScan, offset, limit)
-}
-
 func (tx *MainDatabaseTx) InsertObject(ctx context.Context, bucket, key, contractSet string, dirID int64, o object.Object, mimeType, eTag string, md api.ObjectUserMetadata) error {
 	// get bucket id
 	var bucketID int64

stores/sql/sqlite/main.go

@@ -405,10 +405,6 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap
 	return ssql.Hosts(ctx, tx, opts)
 }
 
-func (tx *MainDatabaseTx) HostsForScanning(ctx context.Context, maxLastScan time.Time, offset, limit int) ([]api.HostAddress, error) {
-	return ssql.HostsForScanning(ctx, tx, maxLastScan, offset, limit)
-}
-
 func (tx *MainDatabaseTx) InsertObject(ctx context.Context, bucket, key, contractSet string, dirID int64, o object.Object, mimeType, eTag string, md api.ObjectUserMetadata) error {
 	// get bucket id
 	var bucketID int64