From c97cd45f60eca5869bc8621d77c3a88c54a020e1 Mon Sep 17 00:00:00 2001 From: zhaomei Date: Mon, 31 May 2021 16:22:52 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=8F=91=E5=B8=83v7.9.6=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=B7=B2=E7=9F=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 +++ README.md | 2 +- conf/conf.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc4848f0..c4d9bb9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 7.9.6 +* 在需要指定存储服务 host 情况下兼容了只配置域名和同时指定域名和访问 protocol 的问题 + ## 7.9.5 优化几个已知小问题 * 支持指定空间管理域名，默认是公有云地址 diff --git a/README.md b/README.md index 5604f236..ef30583a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.9.5 +require github.com/qiniu/go-sdk/v7 v7.9.6 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 diff --git a/conf/conf.go b/conf/conf.go index 0c59e985..a7e1d8cb 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -1,6 +1,6 @@ package conf -const Version = "7.9.5" +const Version = "7.9.6" const ( CONTENT_TYPE_JSON = "application/json" From 800dc3830539f69539630a0192e001c1f2cca412 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Mon, 28 Jun 2021 11:23:06 +0800 Subject: [PATCH 2/5] add GetBody for form upload request --- client/client.go | 22 ++++++++++++++++ rtc/api_test.go | 4 ++- storage/form_upload.go | 58 ++++++++++++++++++++++++++++++++---------- 3 files changed, 69 insertions(+), 15 deletions(-) diff --git a/client/client.go b/client/client.go index bab29800..680cae12 100644 --- a/client/client.go +++ b/client/client.go @@ -117,6 +117,18 @@ func (r Client) DoRequestWith64(ctx context.Context, method, reqUrl string, head return r.Do(ctx, req) } +func (r Client) DoRequestWithBodyGetter(ctx context.Context, method, reqUrl string, headers http.Header, body io.Reader, + getBody func() (io.ReadCloser, error), bodyLength int64) (resp *http.Response, err error) { + + req, err := newRequest(ctx, method, reqUrl, headers, body) + if err != nil { + return + } + req.ContentLength = bodyLength + req.GetBody = getBody + return r.Do(ctx, req) +} + func (r Client) DoRequestWithForm(ctx context.Context, method, reqUrl string, headers http.Header, data map[string][]string) (resp *http.Response, err error) { @@ -314,6 +326,16 @@ func (r Client) CallWith64(ctx context.Context, ret interface{}, method, reqUrl return CallRet(ctx, ret, resp) } +func (r Client) CallWithBodyGetter(ctx context.Context, ret interface{}, method, reqUrl string, headers http.Header, body io.Reader, + getBody func() (io.ReadCloser, error), bodyLength int64) (err error) { + + resp, err := r.DoRequestWithBodyGetter(ctx, method, reqUrl, headers, body, getBody, bodyLength) + if err != nil { + return err + } + return CallRet(ctx, ret, resp) +} + func (r Client) Call(ctx context.Context, ret interface{}, method, reqUrl string, headers http.Header) (err error) { resp, err := r.DoRequestWith(ctx, method, reqUrl, headers, nil, 0) diff --git a/rtc/api_test.go b/rtc/api_test.go index cb2ee501..3c6a7a1e 100644 --- a/rtc/api_test.go +++ b/rtc/api_test.go @@ -25,6 +25,9 @@ func init() { func TestApp(t *testing.T) { app := checkCreateApp(t) + defer func() { + checkDel(t, app.AppID) + }() checkGetApp(t, app.AppID) rooms := checkAllActiveRooms(t, app.AppID) room := "roomName" @@ -39,7 +42,6 @@ func TestApp(t *testing.T) { checkKickUser(t, app.AppID, room, userID) checkUpdate(t, app.AppID) checkRoomToken(t, app.AppID) - checkDel(t, app.AppID) } func checkApp(t *testing.T, app *App) { diff --git a/storage/form_upload.go b/storage/form_upload.go index 6f0bbb6e..6be69cd0 100644 --- a/storage/form_upload.go +++ b/storage/form_upload.go @@ -4,10 +4,10 @@ import ( "bytes" "context" "fmt" - "github.com/qiniu/go-sdk/v7/client" "hash" "hash/crc32" "io" + "io/ioutil" "mime/multipart" "net/http" "net/textproto" @@ -15,6 +15,8 @@ import ( "path" "path/filepath" "strings" + + "github.com/qiniu/go-sdk/v7/client" ) // PutExtra 为表单上传的额外可选项 @@ -175,23 +177,18 @@ func (p *FormUploader) put( return } - var b bytes.Buffer - writer := multipart.NewWriter(&b) - - if extra.OnProgress != nil { - data = &readerWithProgress{reader: data, fsize: size, onProgress: extra.OnProgress} - } - + b := new(bytes.Buffer) + writer := multipart.NewWriter(b) err = writeMultipart(writer, uptoken, key, hasKey, extra, fileName) if err != nil { return } var dataReader io.Reader - h := crc32.NewIEEE() dataReader = io.TeeReader(data, h) crcReader := newCrc32Reader(writer.Boundary(), h) + h = nil //write file head := make(textproto.MIMEHeader) head.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, @@ -204,6 +201,7 @@ func (p *FormUploader) put( if err != nil { return } + head = nil lastLine := fmt.Sprintf("\r\n--%s--\r\n", writer.Boundary()) r := strings.NewReader(lastLine) @@ -214,12 +212,41 @@ func (p *FormUploader) put( bodyLen += crcReader.length() } - mr := io.MultiReader(&b, dataReader, crcReader, r) + mr := io.MultiReader(b, dataReader, crcReader, r) + b = nil + dataReader = nil + crcReader = nil + r = nil + + formBytes, err := ioutil.ReadAll(mr) + if err != nil { + return + } + mr = nil + + getBodyReader := func() (io.Reader, error) { + var formReader io.Reader = bytes.NewReader(formBytes) + if extra.OnProgress != nil { + formReader = &readerWithProgress{reader: formReader, fsize: size, onProgress: extra.OnProgress} + } + return formReader, nil + } + getBodyReadCloser := func() (io.ReadCloser, error) { + reader, err := getBodyReader() + if err != nil { + return nil, err + } + return ioutil.NopCloser(reader), nil + } + bodyReader, err := getBodyReader() + if err != nil { + return + } contentType := writer.FormDataContentType() headers := http.Header{} headers.Add("Content-Type", contentType) - err = p.Client.CallWith64(ctx, ret, "POST", upHost, headers, mr, bodyLen) + err = p.Client.CallWithBodyGetter(ctx, ret, "POST", upHost, headers, bodyReader, getBodyReadCloser, bodyLen) if err != nil { return } @@ -244,7 +271,7 @@ type crc32Reader struct { h hash.Hash32 boundary string r io.Reader - flag bool + inited bool nlDashBoundaryNl string header string crc32PadLen int64 @@ -263,11 +290,11 @@ func newCrc32Reader(boundary string, h hash.Hash32) *crc32Reader { } func (r *crc32Reader) Read(p []byte) (int, error) { - if r.flag == false { + if !r.inited { crc32Sum := r.h.Sum32() crc32Line := r.nlDashBoundaryNl + r.header + fmt.Sprintf("%010d", crc32Sum) //padding crc32 results to 10 digits r.r = strings.NewReader(crc32Line) - r.flag = true + r.inited = true } return r.r.Read(p) } @@ -294,6 +321,9 @@ func (p *readerWithProgress) Read(b []byte) (n int, err error) { n, err = p.reader.Read(b) p.uploaded += int64(n) + if p.uploaded > p.fsize { + p.uploaded = p.fsize + } return } From 0b686974a07c53bf33250c3ecaa8de541c99f14c Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Tue, 29 Jun 2021 17:21:39 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=8F=91=E5=B8=83v7.9.7=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf/conf.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/conf.go b/conf/conf.go index a7e1d8cb..2f07b95c 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -1,6 +1,6 @@ package conf -const Version = "7.9.6" +const Version = "7.9.7" const ( CONTENT_TYPE_JSON = "application/json" From 0f2c040dbde31c38ceaa0fd0275d04306388d3c0 Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Tue, 29 Jun 2021 17:22:17 +0800 Subject: [PATCH 4/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef30583a..0ee6bfc2 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ github.com/qiniu/go-sdk 在您的项目中的 `go.mod` 文件内添加这行代码 ``` -require github.com/qiniu/go-sdk/v7 v7.9.6 +require github.com/qiniu/go-sdk/v7 v7.9.7 ``` 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。 From 7307026194afd1e1a0ecbdfd951dae0829a76f3c Mon Sep 17 00:00:00 2001 From: ZhaoMei Date: Tue, 29 Jun 2021 17:29:51 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=8F=91=E5=B8=83v7.9.7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4d9bb9f..e0982a36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog + +## 7.9.7 +* 修复了表单上传 FormUploader 在内部重试情况下的已知问题 + ## 7.9.6 * 在需要指定存储服务 host 情况下兼容了只配置域名和同时指定域名和访问 protocol 的问题