Merge branch 'master' of YangSen-qn:qiniu/go-sdk into dev_ys

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+
+## 7.9.7
+* 修复了表单上传 FormUploader 在内部重试情况下的已知问题
+
+## 7.9.6
+* 在需要指定存储服务 host 情况下兼容了只配置域名和同时指定域名和访问 protocol 的问题
+
 ## 7.9.5
 优化几个已知小问题
 * 支持指定空间管理域名，默认是公有云地址

README.md

@@ -17,7 +17,7 @@ github.com/qiniu/go-sdk
 在您的项目中的 `go.mod` 文件内添加这行代码
 
 ```
-require github.com/qiniu/go-sdk/v7 v7.9.5
+require github.com/qiniu/go-sdk/v7 v7.9.7
 ```
 
 并且在项目中使用 `"github.com/qiniu/go-sdk/v7"` 引用 Qiniu Go SDK。

client/client.go

@@ -117,6 +117,18 @@ func (r Client) DoRequestWith64(ctx context.Context, method, reqUrl string, head
 	return r.Do(ctx, req)
 }
 
+func (r Client) DoRequestWithBodyGetter(ctx context.Context, method, reqUrl string, headers http.Header, body io.Reader,
+	getBody func() (io.ReadCloser, error), bodyLength int64) (resp *http.Response, err error) {
+
+	req, err := newRequest(ctx, method, reqUrl, headers, body)
+	if err != nil {
+		return
+	}
+	req.ContentLength = bodyLength
+	req.GetBody = getBody
+	return r.Do(ctx, req)
+}
+
 func (r Client) DoRequestWithForm(ctx context.Context, method, reqUrl string, headers http.Header,
 	data map[string][]string) (resp *http.Response, err error) {
 
@@ -314,6 +326,16 @@ func (r Client) CallWith64(ctx context.Context, ret interface{}, method, reqUrl
 	return CallRet(ctx, ret, resp)
 }
 
+func (r Client) CallWithBodyGetter(ctx context.Context, ret interface{}, method, reqUrl string, headers http.Header, body io.Reader,
+	getBody func() (io.ReadCloser, error), bodyLength int64) (err error) {
+
+	resp, err := r.DoRequestWithBodyGetter(ctx, method, reqUrl, headers, body, getBody, bodyLength)
+	if err != nil {
+		return err
+	}
+	return CallRet(ctx, ret, resp)
+}
+
 func (r Client) Call(ctx context.Context, ret interface{}, method, reqUrl string, headers http.Header) (err error) {
 
 	resp, err := r.DoRequestWith(ctx, method, reqUrl, headers, nil, 0)

conf/conf.go

@@ -1,6 +1,6 @@
 package conf
 
-const Version = "7.9.5"
+const Version = "7.9.7"
 
 const (
 	CONTENT_TYPE_JSON      = "application/json"

rtc/api_test.go

@@ -25,6 +25,9 @@ func init() {
 
 func TestApp(t *testing.T) {
 	app := checkCreateApp(t)
+	defer func() {
+		checkDel(t, app.AppID)
+	}()
 	checkGetApp(t, app.AppID)
 	rooms := checkAllActiveRooms(t, app.AppID)
 	room := "roomName"
@@ -39,7 +42,6 @@ func TestApp(t *testing.T) {
 	checkKickUser(t, app.AppID, room, userID)
 	checkUpdate(t, app.AppID)
 	checkRoomToken(t, app.AppID)
-	checkDel(t, app.AppID)
 }
 
 func checkApp(t *testing.T, app *App) {

storage/form_upload.go

@@ -4,17 +4,19 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"github.com/qiniu/go-sdk/v7/client"
 	"hash"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+
+	"github.com/qiniu/go-sdk/v7/client"
 )
 
 // PutExtra 为表单上传的额外可选项
@@ -175,23 +177,18 @@ func (p *FormUploader) put(
 		return
 	}
 
-	var b bytes.Buffer
-	writer := multipart.NewWriter(&b)
-
-	if extra.OnProgress != nil {
-		data = &readerWithProgress{reader: data, fsize: size, onProgress: extra.OnProgress}
-	}
-
+	b := new(bytes.Buffer)
+	writer := multipart.NewWriter(b)
 	err = writeMultipart(writer, uptoken, key, hasKey, extra, fileName)
 	if err != nil {
 		return
 	}
 
 	var dataReader io.Reader
-
 	h := crc32.NewIEEE()
 	dataReader = io.TeeReader(data, h)
 	crcReader := newCrc32Reader(writer.Boundary(), h)
+	h = nil
 	//write file
 	head := make(textproto.MIMEHeader)
 	head.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`,
@@ -204,6 +201,7 @@ func (p *FormUploader) put(
 	if err != nil {
 		return
 	}
+	head = nil
 
 	lastLine := fmt.Sprintf("\r\n--%s--\r\n", writer.Boundary())
 	r := strings.NewReader(lastLine)
@@ -214,12 +212,41 @@ func (p *FormUploader) put(
 		bodyLen += crcReader.length()
 	}
 
-	mr := io.MultiReader(&b, dataReader, crcReader, r)
+	mr := io.MultiReader(b, dataReader, crcReader, r)
+	b = nil
+	dataReader = nil
+	crcReader = nil
+	r = nil
+
+	formBytes, err := ioutil.ReadAll(mr)
+	if err != nil {
+		return
+	}
+	mr = nil
+
+	getBodyReader := func() (io.Reader, error) {
+		var formReader io.Reader = bytes.NewReader(formBytes)
+		if extra.OnProgress != nil {
+			formReader = &readerWithProgress{reader: formReader, fsize: size, onProgress: extra.OnProgress}
+		}
+		return formReader, nil
+	}
+	getBodyReadCloser := func() (io.ReadCloser, error) {
+		reader, err := getBodyReader()
+		if err != nil {
+			return nil, err
+		}
+		return ioutil.NopCloser(reader), nil
+	}
+	bodyReader, err := getBodyReader()
+	if err != nil {
+		return
+	}
 
 	contentType := writer.FormDataContentType()
 	headers := http.Header{}
 	headers.Add("Content-Type", contentType)
-	err = p.Client.CallWith64(ctx, ret, "POST", upHost, headers, mr, bodyLen)
+	err = p.Client.CallWithBodyGetter(ctx, ret, "POST", upHost, headers, bodyReader, getBodyReadCloser, bodyLen)
 	if err != nil {
 		return
 	}
@@ -244,7 +271,7 @@ type crc32Reader struct {
 	h                hash.Hash32
 	boundary         string
 	r                io.Reader
-	flag             bool
+	inited           bool
 	nlDashBoundaryNl string
 	header           string
 	crc32PadLen      int64
@@ -263,11 +290,11 @@ func newCrc32Reader(boundary string, h hash.Hash32) *crc32Reader {
 }
 
 func (r *crc32Reader) Read(p []byte) (int, error) {
-	if r.flag == false {
+	if !r.inited {
 		crc32Sum := r.h.Sum32()
 		crc32Line := r.nlDashBoundaryNl + r.header + fmt.Sprintf("%010d", crc32Sum) //padding crc32 results to 10 digits
 		r.r = strings.NewReader(crc32Line)
-		r.flag = true
+		r.inited = true
 	}
 	return r.r.Read(p)
 }
@@ -294,6 +321,9 @@ func (p *readerWithProgress) Read(b []byte) (n int, err error) {
 
 	n, err = p.reader.Read(b)
 	p.uploaded += int64(n)
+	if p.uploaded > p.fsize {
+		p.uploaded = p.fsize
+	}
 	return
 }