Skip to content

Commit

Permalink
Add Tiers improvements for Bucket Lifecycle management (#3380)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cesnietor
cesnietor committed Jun 13, 2024
1 parent 56f22a4 commit fa32d78
Show file tree
Hide file tree
Showing 15 changed files with 811 additions and 104 deletions.
15 changes: 8 additions & 7 deletions api/admin_client_mock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,11 +66,12 @@ var (
deleteSiteReplicationInfoMock func(ctx context.Context, removeReq madmin.SRRemoveReq) (*madmin.ReplicateRemoveStatus, error)
getSiteReplicationStatus func(ctx context.Context, params madmin.SRStatusOptions) (*madmin.SRStatusInfo, error)

minioListTiersMock func(ctx context.Context) ([]*madmin.TierConfig, error)
minioTierStatsMock func(ctx context.Context) ([]madmin.TierInfo, error)
minioAddTiersMock func(ctx context.Context, tier *madmin.TierConfig) error
minioRemoveTierMock func(ctx context.Context, tierName string) error
minioEditTiersMock func(ctx context.Context, tierName string, creds madmin.TierCreds) error
minioListTiersMock func(ctx context.Context) ([]*madmin.TierConfig, error)
minioTierStatsMock func(ctx context.Context) ([]madmin.TierInfo, error)
minioAddTiersMock func(ctx context.Context, tier *madmin.TierConfig) error
minioRemoveTierMock func(ctx context.Context, tierName string) error
minioEditTiersMock func(ctx context.Context, tierName string, creds madmin.TierCreds) error
minioVerifyTierStatusMock func(ctx context.Context, tierName string) error

minioServiceTraceMock func(ctx context.Context, threshold int64, s3, internal, storage, os, errTrace bool) <-chan madmin.ServiceTraceInfo

Expand Down Expand Up @@ -121,8 +122,8 @@ func (ac AdminClientMock) speedtest(_ context.Context, _ madmin.SpeedtestOpts) (
return nil, nil
}

func (ac AdminClientMock) verifyTierStatus(_ context.Context, _ string) error {
return nil
func (ac AdminClientMock) verifyTierStatus(ctx context.Context, tier string) error {
return minioVerifyTierStatusMock(ctx, tier)
}

// mock function helpConfigKV()
Expand Down
103 changes: 75 additions & 28 deletions api/admin_tiers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/dustin/go-humanize"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/console/api/operations"
"github.com/minio/console/api/operations/tiering"
tieringApi "github.com/minio/console/api/operations/tiering"
"github.com/minio/console/models"
"github.com/minio/madmin-go/v3"
Expand All @@ -38,6 +39,13 @@ func registerAdminTiersHandlers(api *operations.ConsoleAPI) {
}
return tieringApi.NewTiersListOK().WithPayload(tierList)
})
api.TieringTiersListNamesHandler = tiering.TiersListNamesHandlerFunc(func(params tiering.TiersListNamesParams, session *models.Principal) middleware.Responder {
tierList, err := getTiersNameResponse(session, params)
if err != nil {
return tieringApi.NewTiersListDefault(err.Code).WithPayload(err.APIError)
}
return tieringApi.NewTiersListNamesOK().WithPayload(tierList)
})
// add a new tiers
api.TieringAddTierHandler = tieringApi.AddTierHandlerFunc(func(params tieringApi.AddTierParams, session *models.Principal) middleware.Responder {
err := getAddTierResponse(session, params)
Expand Down Expand Up @@ -72,33 +80,36 @@ func registerAdminTiersHandlers(api *operations.ConsoleAPI) {
})
}

// getNotificationEndpoints invokes admin info and returns a list of notification endpoints
// getTiers returns a list of tiers with their stats
func getTiers(ctx context.Context, client MinioAdmin) (*models.TierListResponse, error) {
tiers, err := client.listTiers(ctx)
if err != nil {
return nil, err
}
tiersInfo, err := client.tierStats(ctx)

tierStatsInfo, err := client.tierStats(ctx)
if err != nil {
return nil, err
}
tiersStatsMap := make(map[string]madmin.TierStats, len(tierStatsInfo))
for _, stat := range tierStatsInfo {
tiersStatsMap[stat.Name] = stat.Stats
}

var tiersList []*models.Tier
for _, tierData := range tiers {

// Default Tier Stats
stats := madmin.TierStats{
tierStats := madmin.TierStats{
NumObjects: 0,
NumVersions: 0,
TotalSize: 0,
}

// We look for the correct tier stats & set the values.
for _, stat := range tiersInfo {
if stat.Name == tierData.Name {
stats = stat.Stats
break
}
if stats, ok := tiersStatsMap[tierData.Name]; ok {
tierStats = stats
}

status := client.verifyTierStatus(ctx, tierData.Name) == nil

switch tierData.Type {
case madmin.S3:
tiersList = append(tiersList, &models.Tier{
Expand All @@ -112,11 +123,11 @@ func getTiers(ctx context.Context, client MinioAdmin) (*models.TierListResponse,
Region: tierData.S3.Region,
Secretkey: tierData.S3.SecretKey,
Storageclass: tierData.S3.StorageClass,
Usage: humanize.IBytes(stats.TotalSize),
Objects: strconv.Itoa(stats.NumObjects),
Versions: strconv.Itoa(stats.NumVersions),
Usage: humanize.IBytes(tierStats.TotalSize),
Objects: strconv.Itoa(tierStats.NumObjects),
Versions: strconv.Itoa(tierStats.NumVersions),
},
Status: client.verifyTierStatus(ctx, tierData.Name) == nil,
Status: status,
})
case madmin.MinIO:
tiersList = append(tiersList, &models.Tier{
Expand All @@ -129,11 +140,11 @@ func getTiers(ctx context.Context, client MinioAdmin) (*models.TierListResponse,
Prefix: tierData.MinIO.Prefix,
Region: tierData.MinIO.Region,
Secretkey: tierData.MinIO.SecretKey,
Usage: humanize.IBytes(stats.TotalSize),
Objects: strconv.Itoa(stats.NumObjects),
Versions: strconv.Itoa(stats.NumVersions),
Usage: humanize.IBytes(tierStats.TotalSize),
Objects: strconv.Itoa(tierStats.NumObjects),
Versions: strconv.Itoa(tierStats.NumVersions),
},
Status: client.verifyTierStatus(ctx, tierData.Name) == nil,
Status: status,
})
case madmin.GCS:
tiersList = append(tiersList, &models.Tier{
Expand All @@ -145,11 +156,11 @@ func getTiers(ctx context.Context, client MinioAdmin) (*models.TierListResponse,
Name: tierData.Name,
Prefix: tierData.GCS.Prefix,
Region: tierData.GCS.Region,
Usage: humanize.IBytes(stats.TotalSize),
Objects: strconv.Itoa(stats.NumObjects),
Versions: strconv.Itoa(stats.NumVersions),
Usage: humanize.IBytes(tierStats.TotalSize),
Objects: strconv.Itoa(tierStats.NumObjects),
Versions: strconv.Itoa(tierStats.NumVersions),
},
Status: client.verifyTierStatus(ctx, tierData.Name) == nil,
Status: status,
})
case madmin.Azure:
tiersList = append(tiersList, &models.Tier{
Expand All @@ -162,16 +173,16 @@ func getTiers(ctx context.Context, client MinioAdmin) (*models.TierListResponse,
Name: tierData.Name,
Prefix: tierData.Azure.Prefix,
Region: tierData.Azure.Region,
Usage: humanize.IBytes(stats.TotalSize),
Objects: strconv.Itoa(stats.NumObjects),
Versions: strconv.Itoa(stats.NumVersions),
Usage: humanize.IBytes(tierStats.TotalSize),
Objects: strconv.Itoa(tierStats.NumObjects),
Versions: strconv.Itoa(tierStats.NumVersions),
},
Status: client.verifyTierStatus(ctx, tierData.Name) == nil,
Status: status,
})
case madmin.Unsupported:
tiersList = append(tiersList, &models.Tier{
Type: models.TierTypeUnsupported,
Status: client.verifyTierStatus(ctx, tierData.Name) == nil,
Status: status,
})
}
}
Expand Down Expand Up @@ -200,6 +211,42 @@ func getTiersResponse(session *models.Principal, params tieringApi.TiersListPara
return tiersResp, nil
}

// getTiersNameResponse returns a response with a list of tiers' names
func getTiersNameResponse(session *models.Principal, params tieringApi.TiersListNamesParams) (*models.TiersNameListResponse, *CodedAPIError) {
ctx, cancel := context.WithCancel(params.HTTPRequest.Context())
defer cancel()
mAdmin, err := NewMinioAdminClient(params.HTTPRequest.Context(), session)
if err != nil {
return nil, ErrorWithContext(ctx, err)
}
// create a minioClient interface implementation
// defining the client to be used
adminClient := AdminClient{Client: mAdmin}
// serialize output
tiersResp, err := getTiersName(ctx, adminClient)
if err != nil {
return nil, ErrorWithContext(ctx, err)
}
return tiersResp, nil
}

// getTiersName fetches listTiers and returns a list of the tiers' names
func getTiersName(ctx context.Context, client MinioAdmin) (*models.TiersNameListResponse, error) {
tiers, err := client.listTiers(ctx)
if err != nil {
return nil, err
}

tiersNameList := make([]string, len(tiers))
for i, tierData := range tiers {
tiersNameList[i] = tierData.Name
}

return &models.TiersNameListResponse{
Items: tiersNameList,
}, nil
}

func addTier(ctx context.Context, client MinioAdmin, params *tieringApi.AddTierParams) error {
var cfg *madmin.TierConfig
var err error
Expand Down
Loading

0 comments on commit fa32d78

Please sign in to comment.