-
Notifications
You must be signed in to change notification settings - Fork 128
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
return internal s3 signed url if client request has header X-OPENCSG-… (
#145) * return internal s3 signed url if client request has header X-OPENCSG-S3-Internal * STARHUB_SERVER_S3_INTERNAL_ENDPOINT default to empty * set env S3_INTERNAL when deploy model or space if S3 internal endpoint is set --------- Co-authored-by: Lei Da <da.lei@opencsg.com>
- Loading branch information
Showing
15 changed files
with
162 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package common | ||
|
|
||
| import "time" | ||
|
|
||
| type DeployConfig struct { | ||
| ImageBuilderURL string | ||
| ImageRunnerURL string | ||
| MonitorInterval time.Duration | ||
| InternalRootDomain string | ||
| SpaceDeployTimeoutInMin int | ||
| ModelDeployTimeoutInMin int | ||
| ModelDownloadEndpoint string | ||
| PublicRootDomain string | ||
| SSHDomain string | ||
| //download lfs object from internal s3 address | ||
| S3Internal bool | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,64 @@ | ||
| package s3 | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "log/slog" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/minio/minio-go/v7" | ||
| "github.com/minio/minio-go/v7/pkg/credentials" | ||
| "opencsg.com/csghub-server/common/config" | ||
| ) | ||
|
|
||
| func NewMinio(cfg *config.Config) (*minio.Client, error) { | ||
| return minio.New(cfg.S3.Endpoint, &minio.Options{ | ||
| func NewMinio(cfg *config.Config) (*Client, error) { | ||
| minioClient, err := minio.New(cfg.S3.Endpoint, &minio.Options{ | ||
| Creds: credentials.NewStaticV4(cfg.S3.AccessKeyID, cfg.S3.AccessKeySecret, ""), | ||
| Secure: cfg.S3.EnableSSL, | ||
| BucketLookup: minio.BucketLookupAuto, | ||
| Region: cfg.S3.Region, | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to init s3 client, error:%w", err) | ||
| } | ||
| client := &Client{ | ||
| Client: minioClient, | ||
| } | ||
| if len(cfg.S3.InternalEndpoint) > 0 { | ||
| minioClientInternal, err := minio.New(cfg.S3.InternalEndpoint, &minio.Options{ | ||
| Creds: credentials.NewStaticV4(cfg.S3.AccessKeyID, cfg.S3.AccessKeySecret, ""), | ||
| Secure: cfg.S3.EnableSSL, | ||
| BucketLookup: minio.BucketLookupAuto, | ||
| Region: cfg.S3.Region, | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to init s3 internal client, error:%w", err) | ||
| } | ||
| client.internalClient = minioClientInternal | ||
| } | ||
| return client, nil | ||
| } | ||
|
|
||
| type Client struct { | ||
| *minio.Client | ||
| internalClient *minio.Client | ||
| } | ||
|
|
||
| // PresignedGetObject is a wrapper around minio.Client.PresignedGetObject that adds some extra customization. | ||
| func (c *Client) PresignedGetObject(ctx context.Context, bucketName, objectName string, expires time.Duration, reqParams url.Values) (*url.URL, error) { | ||
| if c.useInternalClient(ctx) && c.internalClient != nil { | ||
| slog.Info("use internal s3 client for presigned get object", slog.String("bucket_name", bucketName), slog.String("object_name", objectName)) | ||
| return c.internalClient.PresignedGetObject(ctx, bucketName, objectName, expires, reqParams) | ||
| } | ||
| return c.Client.PresignedGetObject(ctx, bucketName, objectName, expires, reqParams) | ||
| } | ||
|
|
||
| func (c *Client) useInternalClient(ctx context.Context) bool { | ||
| v, success := ctx.Value("X-OPENCSG-S3-Internal").(bool) | ||
| if !success { | ||
| return false | ||
| } | ||
|
|
||
| return v | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| package s3 | ||
|
|
||
| import ( | ||
| "context" | ||
| "testing" | ||
| ) | ||
|
|
||
| // test for useInternalClient | ||
| func Test_useInternalClient(t *testing.T) { | ||
| c := &Client{} | ||
| ctx := context.Background() | ||
| if c.useInternalClient(ctx) != false { | ||
| t.Errorf("should not use internal s3 client if context not defined") | ||
| } | ||
| ctxWrongValue := context.WithValue(ctx, "X-OPENCSG-S3-Internal", "test") | ||
| if c.useInternalClient(ctxWrongValue) != false { | ||
| t.Errorf("should not use internal s3 client if context value is not 'true'") | ||
| } | ||
| ctxWithRightValue := context.WithValue(ctx, "X-OPENCSG-S3-Internal", true) | ||
| if c.useInternalClient(ctxWithRightValue) != true { | ||
| t.Errorf("should not use internal s3 client if context value is 'true'") | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.