diff --git a/CHANGELOG.md b/CHANGELOG.md index e8f2f30d0..7fe2be118 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog +## 7.5.1 (2021-04-29) + +## 增加 +* 上传服务基础上传数据 api 支持 InputStream +* 支持构建 DownloadUrl 功能 + ## 7.5.0 (2021-04-15) diff --git a/src/main/java/com/qiniu/common/Constants.java b/src/main/java/com/qiniu/common/Constants.java index 7f7fc92db..7c22230e0 100644 --- a/src/main/java/com/qiniu/common/Constants.java +++ b/src/main/java/com/qiniu/common/Constants.java @@ -9,7 +9,7 @@ public final class Constants { /** * 版本号 */ - public static final String VERSION = "7.5.0"; + public static final String VERSION = "7.5.1"; /** * 块大小，不能改变 */ diff --git a/src/main/java/com/qiniu/http/Client.java b/src/main/java/com/qiniu/http/Client.java index 4e172f691..8a6dc683e 100755 --- a/src/main/java/com/qiniu/http/Client.java +++ b/src/main/java/com/qiniu/http/Client.java @@ -279,12 +279,12 @@ public Response post(String url, byte[] body, int offset, int size, return post(url, rbody, headers); } - private Response post(String url, RequestBody body, StringMap headers) throws QiniuException { + public Response post(String url, RequestBody body, StringMap headers) throws QiniuException { Request.Builder requestBuilder = new Request.Builder().url(url).post(body); return send(requestBuilder, headers); } - private Response put(String url, RequestBody body, StringMap headers) throws QiniuException { + public Response put(String url, RequestBody body, StringMap headers) throws QiniuException { Request.Builder requestBuilder = new Request.Builder().url(url).put(body); return send(requestBuilder, headers); } diff --git a/src/main/java/com/qiniu/http/RequestStreamBody.java b/src/main/java/com/qiniu/http/RequestStreamBody.java new file mode 100644 index 000000000..137e5bebf --- /dev/null +++ b/src/main/java/com/qiniu/http/RequestStreamBody.java @@ -0,0 +1,94 @@ +package com.qiniu.http; + +import okhttp3.MediaType; +import okhttp3.RequestBody; +import okio.BufferedSink; +import okio.Okio; +import okio.Source; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; + +public class RequestStreamBody extends RequestBody { + + private long limitSize = -1; + private long sinkSize = 1024 * 100; + private final MediaType type; + private final InputStream stream; + + /** + * 构造函数 + * + * @param stream 请求数据流 + * @param contentType 请求数据类型 + */ + public RequestStreamBody(InputStream stream, String contentType) { + this.stream = stream; + this.type = MediaType.parse(contentType); + } + + /** + * 构造函数 + * + * @param stream 请求数据流 + * @param contentType 请求数据类型 + */ + public RequestStreamBody(InputStream stream, MediaType contentType) { + this.stream = stream; + this.type = contentType; + } + + /** + * 构造函数 + * + * @param stream 请求数据流 + * @param contentType 请求数据类型 + * @param limitSize 最大读取 stream 的大小；为 -1 时不限制读取所有 + */ + public RequestStreamBody(InputStream stream, MediaType contentType, long limitSize) { + this.stream = stream; + this.type = contentType; + this.limitSize = limitSize; + } + + /** + * 配置请求时，每次从流中读取的数据大小 + * + * @param sinkSize 每次从流中读取的数据大小 + * @return RequestStreamBody + * @see RequestStreamBody#writeTo(BufferedSink) + */ + public RequestStreamBody setSinkSize(long sinkSize) { + if (sinkSize > 0) { + this.sinkSize = sinkSize; + } + return this; + } + + @Override + public MediaType contentType() { + return type; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + try (Source source = Okio.source(stream)) { + int offset = 0; + while (limitSize < 0 || offset < limitSize) { + long byteSize = sinkSize; + if (offset < limitSize) { + byteSize = Math.min(sinkSize, limitSize - offset); + } + + try { + sink.write(source, byteSize); + sink.flush(); + offset += byteSize; + } catch (EOFException e) { + break; + } + } + } + } +} diff --git a/src/main/java/com/qiniu/http/Response.java b/src/main/java/com/qiniu/http/Response.java index 24272ce11..f7cefe27a 100644 --- a/src/main/java/com/qiniu/http/Response.java +++ b/src/main/java/com/qiniu/http/Response.java @@ -241,6 +241,10 @@ public String contentType() { return ctype(response); } + public String header(String name, String defaultValue) { + return response.header(name, defaultValue); + } + public boolean isJson() { return contentType().equals(Client.JsonMime); } diff --git a/src/main/java/com/qiniu/processing/OperationManager.java b/src/main/java/com/qiniu/processing/OperationManager.java index 73b5a3eb6..78e55f47f 100644 --- a/src/main/java/com/qiniu/processing/OperationManager.java +++ b/src/main/java/com/qiniu/processing/OperationManager.java @@ -52,6 +52,12 @@ public OperationManager(Auth auth, Client client) { this.configuration = new Configuration(); } + public OperationManager(Auth auth, Configuration cfg, Client client) { + this.auth = auth; + this.client = client; + this.configuration = cfg; + } + /** * 发送请求对空间中的文件进行持久化处理 * diff --git a/src/main/java/com/qiniu/storage/Api.java b/src/main/java/com/qiniu/storage/Api.java index 4da0cce24..66845b362 100644 --- a/src/main/java/com/qiniu/storage/Api.java +++ b/src/main/java/com/qiniu/storage/Api.java @@ -3,9 +3,14 @@ import com.qiniu.common.QiniuException; import com.qiniu.http.Client; import com.qiniu.http.MethodType; +import com.qiniu.http.RequestStreamBody; import com.qiniu.util.StringMap; import com.qiniu.util.StringUtils; +import okhttp3.MediaType; +import okhttp3.RequestBody; +import okio.BufferedSink; +import java.io.InputStream; import java.net.URL; import java.net.URLEncoder; import java.util.*; @@ -35,14 +40,11 @@ protected com.qiniu.http.Response requestByClient(Request request) throws QiniuE if (request.method == MethodType.GET) { return client.get(request.getUrl().toString(), request.getHeader()); } else if (request.method == MethodType.POST) { - return client.post(request.getUrl().toString(), request.body, request.bodyOffset, request.bodySize, - request.getHeader(), request.bodyContentType); + return client.post(request.getUrl().toString(), request.getRequestBody(), request.getHeader()); } else if (request.method == MethodType.PUT) { - return client.put(request.getUrl().toString(), request.body, request.bodyOffset, request.bodySize, - request.getHeader(), request.bodyContentType); + return client.put(request.getUrl().toString(), request.getRequestBody(), request.getHeader()); } else if (request.method == MethodType.DELETE) { - return client.delete(request.getUrl().toString(), request.body, request.bodyOffset, request.bodySize, - request.getHeader(), request.bodyContentType); + return client.delete(request.getUrl().toString(), request.getRequestBody(), request.getHeader()); } else { throw QiniuException.unrecoverable("暂不支持这种请求方式"); } @@ -91,12 +93,14 @@ public static class Request { private final Map header = new HashMap<>(); /** - * 请求数据是在 body 中，从 bodyOffset 开始，获取 bodySize 大小的数据 + * 请求 body */ - private byte[] body = new byte[0]; - private int bodySize = 0; - private int bodyOffset = 0; - private String bodyContentType = Client.DefaultMime; + private RequestBody body; + /** + * 请求时，每次从流中读取的数据大小 + * 注： body 使用 InputStream 时才有效 + */ + private long streamBodySinkSize = 1024 * 10; /** * 构造请求对象 @@ -174,7 +178,7 @@ protected void buildPath() throws QiniuException { * @param value value */ protected void addQueryPair(String key, String value) { - if (StringUtils.isNullOrEmpty(key) || value == null) { + if (StringUtils.isNullOrEmpty(key)) { return; } queryPairs.add(new Pair(key, value)); @@ -237,7 +241,7 @@ protected void setMethod(MethodType method) { * @param key key * @param value value */ - protected void addHeaderField(String key, String value) { + public void addHeaderField(String key, String value) { if (StringUtils.isNullOrEmpty(key) || StringUtils.isNullOrEmpty(value)) { return; } @@ -292,16 +296,59 @@ public URL getUrl() throws QiniuException { * @param contentType 请求数据类型 */ protected void setBody(byte[] body, int offset, int size, String contentType) { - this.body = body; - this.bodyOffset = offset; - this.bodySize = size; - if (!StringUtils.isNullOrEmpty(contentType)) { - this.bodyContentType = contentType; + if (StringUtils.isNullOrEmpty(contentType)) { + contentType = Client.DefaultMime; + } + MediaType type = MediaType.parse(contentType); + this.body = RequestBody.create(type, body, offset, size); + } + + /** + * 设置请求体 + * + * @param body 请求数据源 + * @param contentType 请求数据类型 + * @param limitSize 最大读取 body 的大小；body 有多余则被舍弃；body 不足则会上传多有 body； + * 如果提前不知道 body 大小，但想上传所有 body，limitSize 设置为 -1 即可； + */ + protected void setBody(InputStream body, String contentType, long limitSize) { + if (StringUtils.isNullOrEmpty(contentType)) { + contentType = Client.DefaultMime; } + MediaType type = MediaType.parse(contentType); + this.body = new RequestStreamBody(body, type, limitSize); } + /** + * 使用 streamBody 时，每次读取 streamBody 的大小，读取后发送 + * 默认：{@link Api.Request#streamBodySinkSize} + * 相关：{@link RequestStreamBody#writeTo(BufferedSink) sinkSize} + * + * @param streamBodySinkSize 每次读取 streamBody 的大小 + */ + public Request setStreamBodySinkSize(long streamBodySinkSize) { + this.streamBodySinkSize = streamBodySinkSize; + return this; + } + + /** + * 是否有请求体 + * + * @return 是否有请求体 + */ public boolean hasBody() { - return body != null && body.length > 0 && bodySize > 0; + return body != null; + } + + private RequestBody getRequestBody() { + if (hasBody()) { + if (body instanceof RequestStreamBody) { + ((RequestStreamBody) body).setSinkSize(streamBodySinkSize); + } + return body; + } else { + return RequestBody.create(null, new byte[0]); + } } /** @@ -310,11 +357,7 @@ public boolean hasBody() { * @throws QiniuException */ protected void buildBodyInfo() throws QiniuException { - if (body == null) { - body = new byte[0]; - bodySize = 0; - bodyOffset = 0; - } + } /** @@ -328,10 +371,7 @@ protected void prepareToRequest() throws QiniuException { buildBodyInfo(); } - void test() { - } - - private static class Pair { + protected static class Pair { /** * Key of this Pair. @@ -367,7 +407,7 @@ V getValue() { * @param key The key for this pair * @param value The value to use for this pair */ - Pair(K key, V value) { + protected Pair(K key, V value) { this.key = key; this.value = value; } diff --git a/src/main/java/com/qiniu/storage/ApiUploadV1MakeBlock.java b/src/main/java/com/qiniu/storage/ApiUploadV1MakeBlock.java index 440216aa8..6d32d92ac 100644 --- a/src/main/java/com/qiniu/storage/ApiUploadV1MakeBlock.java +++ b/src/main/java/com/qiniu/storage/ApiUploadV1MakeBlock.java @@ -4,6 +4,8 @@ import com.qiniu.http.Client; import com.qiniu.http.MethodType; +import java.io.InputStream; + /** * 分片上传 v1 版 api: 创建块 * 本接口用于为后续分片上传创建一个新的 block，同时上传该块第一个 chunk 数据。 @@ -83,8 +85,12 @@ public Request(String urlPrefix, String token, Integer blockSize) { } /** - * 配置块第一个上传片数据【必须】 + * 配置块第一个上传片数据 + * 块数据 size 必须不大于 4M，block 中所有 chunk 的 size 总和必须为 4M, SDK 内部不做 block/chunk size 检测 * 块数据：在 data 中，从 offset 开始的 size 大小的数据 + * 注： + * 必须通过 {@link ApiUploadV1MakeBlock.Request#setFirstChunkData(byte[], int, int, String)} 或 + * {@link ApiUploadV1MakeBlock.Request#setFirstChunkData(InputStream, String, long)} 配置块第一个上传片数据 * * @param data 块数据源 * @param offset 块数据在 data 中的偏移量 @@ -97,6 +103,24 @@ public Request setFirstChunkData(byte[] data, int offset, int size, String conte return this; } + /** + * 配置块第一个上传片数据 + * 块数据 size 必须不大于 4M，block 中所有 chunk 的 size 总和必须为 4M, SDK 内部不做 block/chunk size 检测 + * 注： + * 必须通过 {@link ApiUploadV1MakeBlock.Request#setFirstChunkData(byte[], int, int, String)} 或 + * {@link ApiUploadV1MakeBlock.Request#setFirstChunkData(InputStream, String, long)} 配置块第一个上传片数据 + * + * @param data 块数据源 + * @param contentType 块数据类型 + * @param limitSize 最大读取 data 的大小；data 有多余则被舍弃；data 不足则会上传多有 data； + * 如果提前不知道 data 大小，但想上传所有 data，limitSize 设置为 -1 即可； + * @return Request + */ + public Request setFirstChunkData(InputStream data, String contentType, long limitSize) { + super.setBody(data, contentType, limitSize); + return this; + } + @Override protected void buildPath() throws QiniuException { if (blockSize == null) { @@ -111,7 +135,7 @@ protected void buildPath() throws QiniuException { @Override protected void buildBodyInfo() throws QiniuException { if (!hasBody()) { - ApiUtils.throwInvalidRequestParamException("block data"); + ApiUtils.throwInvalidRequestParamException("block first chunk data"); } } } diff --git a/src/main/java/com/qiniu/storage/ApiUploadV1PutChunk.java b/src/main/java/com/qiniu/storage/ApiUploadV1PutChunk.java index 9b5dc544f..21a6e588d 100644 --- a/src/main/java/com/qiniu/storage/ApiUploadV1PutChunk.java +++ b/src/main/java/com/qiniu/storage/ApiUploadV1PutChunk.java @@ -4,6 +4,8 @@ import com.qiniu.http.Client; import com.qiniu.http.MethodType; +import java.io.InputStream; + /** * 分片上传 v1 版 api: 上传片 * 上传指定块的一片数据，具体数据量可根据现场环境调整。同一块的每片数据必须串行上传。 @@ -85,8 +87,12 @@ public Request(String urlPrefix, String token, String blockLastContext, Integer } /** - * 配置块中上传片数据【必须】 + * 配置块中上传片数据 + * 块数据 size 必须不大于 4M，block 中所有 chunk 的 size 总和必须为 4M, SDK 内部不做 block/chunk size 检测 * 块数据：在 data 中，从 offset 开始的 size 大小的数据 + * 注： + * 必须通过 {@link ApiUploadV1PutChunk.Request#setChunkData(byte[], int, int, String)} 或 + * {@link ApiUploadV1PutChunk.Request#setChunkData(InputStream, String, long)} 配置块中上传片数据 * * @param data 分片数据源 * @param offset 分片数据在 data 中的偏移量 @@ -99,6 +105,24 @@ public Request setChunkData(byte[] data, int offset, int size, String contentTyp return this; } + /** + * 配置块中上传片数据 + * 块数据 size 必须不大于 4M，block 中所有 chunk 的 size 总和必须为 4M, SDK 内部不做 block/chunk size 检测 + * 注： + * 必须通过 {@link ApiUploadV1PutChunk.Request#setChunkData(byte[], int, int, String)} 或 + * {@link ApiUploadV1PutChunk.Request#setChunkData(InputStream, String, long)} 配置块中上传片数据 + * + * @param data 块数据源 + * @param contentType 块数据类型 + * @param limitSize 最大读取 data 的大小；data 有多余则被舍弃；data 不足则会上传多有 data； + * 如果提前不知道 data 大小，但想上传所有 data，limitSize 设置为 -1 即可； + * @return Request + */ + public Request setChunkData(InputStream data, String contentType, long limitSize) { + super.setBody(data, contentType, limitSize); + return this; + } + @Override protected void buildPath() throws QiniuException { if (chunkOffset == null) { @@ -117,7 +141,7 @@ protected void buildPath() throws QiniuException { @Override protected void buildBodyInfo() throws QiniuException { if (!hasBody()) { - ApiUtils.throwInvalidRequestParamException("block data"); + ApiUtils.throwInvalidRequestParamException("block chunk data"); } } } diff --git a/src/main/java/com/qiniu/storage/ApiUploadV2UploadPart.java b/src/main/java/com/qiniu/storage/ApiUploadV2UploadPart.java index 069535b78..bb9277c04 100644 --- a/src/main/java/com/qiniu/storage/ApiUploadV2UploadPart.java +++ b/src/main/java/com/qiniu/storage/ApiUploadV2UploadPart.java @@ -5,6 +5,8 @@ import com.qiniu.http.MethodType; import com.qiniu.util.StringUtils; +import java.io.InputStream; + /** * 分片上传 v2 版 api: 分块上传数据 * 初始化一个 Multipart Upload 任务之后，可以根据指定的 EncodedObjectName 和 UploadId 来分 Part 上传数据。 @@ -102,9 +104,12 @@ public Request setKey(String key) { } /** - * 配置上传块数据【必须】 + * 配置上传块数据 * 块数据：在 data 中，从 offset 开始的 size 大小的数据 * 除最后一个 Part 外，单个 Part 大小范围 1 MB ~ 1 GB + * 注： + * 必须通过 {@link ApiUploadV2UploadPart.Request#setUploadData(byte[], int, int, String)} 或 + * {@link ApiUploadV2UploadPart.Request#setUploadData(InputStream, String, long)} 配置上传块数据 * * @param data 块数据源 * @param offset 块数据在 data 中的偏移量 @@ -117,6 +122,24 @@ public Request setUploadData(byte[] data, int offset, int size, String contentTy return this; } + /** + * 配置上传块数据 + * 除最后一个 Part 外，单个 Part 大小范围 1 MB ~ 1 GB + * 注： + * 必须通过 {@link ApiUploadV2UploadPart.Request#setUploadData(byte[], int, int, String)} 或 + * {@link ApiUploadV2UploadPart.Request#setUploadData(InputStream, String, long)} 配置上传块数据 + * + * @param data 块数据源 + * @param contentType 块数据类型 + * @param limitSize 最大读取 data 的大小；data 有多余则被舍弃；data 不足则会上传多有 data； + * 如果提前不知道 data 大小，但想上传所有 data，limitSize 设置为 -1 即可； + * @return Request + */ + public Request setUploadData(InputStream data, String contentType, long limitSize) { + super.setBody(data, contentType, limitSize); + return this; + } + @Override protected void buildPath() throws QiniuException { UploadToken token = getUploadToken(); @@ -144,7 +167,7 @@ protected void buildPath() throws QiniuException { @Override protected void buildBodyInfo() throws QiniuException { if (!hasBody()) { - ApiUtils.throwInvalidRequestParamException("block data"); + ApiUtils.throwInvalidRequestParamException("upload data"); } } } diff --git a/src/main/java/com/qiniu/storage/BucketManager.java b/src/main/java/com/qiniu/storage/BucketManager.java index c8083d25a..60ecbc0a9 100644 --- a/src/main/java/com/qiniu/storage/BucketManager.java +++ b/src/main/java/com/qiniu/storage/BucketManager.java @@ -54,6 +54,13 @@ public BucketManager(Auth auth, Client client) { this.configHelper = new ConfigHelper(new Configuration()); } + public BucketManager(Auth auth, Configuration cfg, Client client) { + this.auth = auth; + this.client = client; + Configuration c2 = cfg == null ? new Configuration() : cfg.clone(); + this.configHelper = new ConfigHelper(c2); + } + /** * EncodedEntryURI格式，其中 bucket+":"+key 称之为 entry * diff --git a/src/main/java/com/qiniu/storage/DownloadPrivateCloudUrl.java b/src/main/java/com/qiniu/storage/DownloadPrivateCloudUrl.java new file mode 100644 index 000000000..080f80ce9 --- /dev/null +++ b/src/main/java/com/qiniu/storage/DownloadPrivateCloudUrl.java @@ -0,0 +1,94 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; +import com.qiniu.util.StringUtils; + +/** + * 私有云下载 URL 类 + */ +public class DownloadPrivateCloudUrl extends DownloadUrl { + + private final Configuration cfg; + private final String bucketName; + private final String accessKey; + + /** + * 构造器 + * 如果知道下载的 domain 信息可以使用此接口 + * 如果不知道 domain 信息，可以使用 {@link DownloadPrivateCloudUrl#DownloadPrivateCloudUrl(Configuration, String, String, String)} + * + * @param domain 下载 domain, eg: qiniu.com 【必须】 + * @param useHttps 是否使用 https 【必须】 + * @param bucketName bucket 名称 【必须】 + * @param key 下载资源在七牛云存储的 key 【必须】 + * @param accessKey 七牛账户 accessKey 【必须】 + */ + public DownloadPrivateCloudUrl(String domain, boolean useHttps, String bucketName, String key, String accessKey) { + super(domain, useHttps, key); + this.cfg = null; + this.bucketName = bucketName; + this.accessKey = accessKey; + } + + /** + * 构造器 + * 如果不知道 domain 信息，可使用此接口；内部有查询 domain 逻辑 + * 查询 domain 流程： + * 1. 根据 {@link Configuration#defaultUcHost} 查找 bucketName 所在的{@link Configuration#region} + * 2. 获取 {@link Configuration#region} 中的 ioHost({@link Configuration#ioHost(String, String)} ) 作为 domain + * 注：需要配置正确的 {@link Configuration#defaultUcHost} + * + * @param cfg 查询 domain 时的Configuration 【必须】 + * @param bucketName bucket 名称【必须】 + * @param key 下载资源在七牛云存储的 key【必须】 + * @param accessKey 七牛账户 accessKey【必须】 + */ + public DownloadPrivateCloudUrl(Configuration cfg, String bucketName, String key, String accessKey) { + super(null, cfg.useHttpsDomains, key); + this.cfg = cfg; + this.bucketName = bucketName; + this.accessKey = accessKey; + } + + @Override + protected void willBuildUrl() throws QiniuException { + super.willBuildUrl(); + if (StringUtils.isNullOrEmpty(getDomain())) { + setDomain(queryDomain()); + } + } + + @Override + protected void willSetKeyForUrl(Api.Request request) throws QiniuException { + request.addPathSegment("getfile"); + request.addPathSegment(accessKey); + request.addPathSegment(bucketName); + super.willSetKeyForUrl(request); + } + + private String queryDomain() throws QiniuException { + if (cfg == null) { + ApiUtils.throwInvalidRequestParamException("configuration"); + } + if (accessKey == null) { + ApiUtils.throwInvalidRequestParamException("accessKey"); + } + if (bucketName == null) { + ApiUtils.throwInvalidRequestParamException("bucketName"); + } + + ConfigHelper configHelper = new ConfigHelper(cfg); + String host = configHelper.ioHost(accessKey, bucketName); + if (StringUtils.isNullOrEmpty(host)) { + return host; + } + if (host.contains("http://")) { + return host.replaceFirst("http://", ""); + } + if (host.contains("https://")) { + return host.replaceFirst("https://", ""); + } + return host; + } + +} diff --git a/src/main/java/com/qiniu/storage/DownloadUrl.java b/src/main/java/com/qiniu/storage/DownloadUrl.java new file mode 100644 index 000000000..165dd7501 --- /dev/null +++ b/src/main/java/com/qiniu/storage/DownloadUrl.java @@ -0,0 +1,223 @@ +package com.qiniu.storage; + +import com.qiniu.common.QiniuException; +import com.qiniu.util.Auth; +import com.qiniu.util.StringUtils; +import com.qiniu.util.UrlUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 公有云下载 URL 类 + */ +public class DownloadUrl { + + private String domain; + private boolean useHttps = false; + private String key; + private Auth auth; + private Long deadline; + private String style; + private String styleSeparator; + private String styleParam; + private String fop; + private String attname; + private List> customQuerys = new ArrayList<>(); + + /** + * 构造器 + * + * @param domain 下载 domain, eg: qiniu.com【必须】 + * @param useHttps 是否使用 https【必须】 + * @param key 下载资源在七牛云存储的 key【必须】 + */ + public DownloadUrl(String domain, boolean useHttps, String key) { + this.domain = domain; + this.useHttps = useHttps; + this.key = key; + } + + /** + * 设置下载 domain + * + * @param domain 下载 domain + */ + protected void setDomain(String domain) { + this.domain = domain; + } + + /** + * 获取下载 domain + * + * @return 下载 domain + */ + protected String getDomain() { + return domain; + } + + /** + * 浏览器访问时指定下载文件名【可选】 + * 默认情况下，如果在浏览器中访问一个资源URL，浏览器都会试图直接在浏览器中打开这个资源，例如一张图片。 + * 如果希望浏览器的动作是下载而不是打开，可以给该资源URL添加参数 attname 来指定文件名 + * + * @param attname 文件名 + * @return + */ + public DownloadUrl setAttname(String attname) { + this.attname = attname; + return this; + } + + /** + * 配置 fop【可选】 + * 开发者可以在访问资源时制定执行一个或多个数据处理指令，以直接获取经过处理后的结果。比较典型的一个场景是图片查看，客户端可以上传一 + * 张高精度的图片，然后在查看图片的时候根据屏幕规格生成一张大小适宜的缩略图。这样既可以明显降低网络流量，而且可以提高图片显示速度， + * 还能降低移动设备的内存占用. + * eg: + * 针对该原图生成一张480x320大小的缩略图:imageView2/2/w/320/h/480 + * https://developer.qiniu.com/dora/6217/directions-for-use-pfop + * + * @param fop fop + * @return DownloadUrl + */ + public DownloadUrl setFop(String fop) { + this.fop = fop; + return this; + } + + /** + * 配置 style【可选】 + * 如果觉得 fop 这样的形式够冗长，还可以为这些串行的 fop 集合定义一个友好别名。如此一来，就可以用友好URL风格进行访问，这个别名就是 style 。 + * eg: + * > 对 userBucket 的 fop（imageView2/2/w/320/h/480） 使用 style 的方式, 分隔符为 "-" + * >> 使用 qrsctl 命令定义 style: （qrsctl separator ） + * >> qrsctl separator userBucket - + * >> 定义数据处理的别名为 aliasName: (qrsctl style ) + * >> qrsctl style userBucket iphone imageView2/2/w/320/h/480 + *

+ * https://developer.qiniu.com/dora/6217/directions-for-use-pfop + * + * @param style style 名【必须】 + * @param styleSeparator url 和数据处理之间的分隔符【必须】 + * @param styleParam style 参数【可选】 + * @return DownloadUrl + */ + public DownloadUrl setStyle(String style, String styleSeparator, String styleParam) { + this.style = style; + this.styleSeparator = styleSeparator; + this.styleParam = styleParam; + return this; + } + + /** + * URL 增加 query 信息 【可选】 + * query 信息必须为七牛云支持的，否则会被视为无效 + * + * @param queryName query 名 + * @param queryValue query 值 + * @return DownloadUrl + */ + public DownloadUrl addCustomQuery(String queryName, String queryValue) { + customQuerys.add(new Api.Request.Pair(queryName, queryValue)); + return this; + } + + /** + * 构建带有有效期的下载 URL 字符串 + * 一般构建私有资源的下载 URL 字符串；公开资源可以直接使用 {@link DownloadUrl#buildURL } + * + * @param auth 凭证信息【必须】 + * @param deadline 有效期时间戳，单位：秒 【必须】 + * @return 下载 URL 字符串 + * @throws QiniuException 构建异常，一般为参数缺失 + */ + public String buildURL(Auth auth, long deadline) throws QiniuException { + this.auth = auth; + this.deadline = deadline; + return buildURL(); + } + + /** + * 构建资源下载 URL 字符串 + * + * @return 下载 URL 字符串 + * @throws QiniuException 构建异常，一般为参数缺失 + */ + public String buildURL() throws QiniuException { + willBuildUrl(); + + Api.Request request = new Api.Request(getUrlPrefix()); + + willSetKeyForUrl(request); + String keyAndStyle = null; + keyAndStyle = urlPathEncode(key); + if (!StringUtils.isNullOrEmpty(style) && !StringUtils.isNullOrEmpty(styleSeparator)) { + keyAndStyle += urlPathEncode(styleSeparator + style); + if (!StringUtils.isNullOrEmpty(styleParam)) { + keyAndStyle += "@" + urlPathEncode(styleParam); + } + } + if (!StringUtils.isNullOrEmpty(keyAndStyle)) { + request.addPathSegment(keyAndStyle); + } + didSetKeyForUrl(request); + + if (!StringUtils.isNullOrEmpty(fop)) { + request.addQueryPair(fop, null); + } + + for (Api.Request.Pair pair : customQuerys) { + request.addQueryPair(pair.getKey(), pair.getValue()); + } + + if (!StringUtils.isNullOrEmpty(attname)) { + request.addQueryPair("attname", attname); + } + + didBuildUrl(); + + String url = request.getUrl().toString(); + if (auth != null && deadline != null) { + url = auth.privateDownloadUrlWithDeadline(url, deadline); + } + return url; + } + + protected void willBuildUrl() throws QiniuException { + // key 可以为 "" + if (key == null) { + ApiUtils.throwInvalidRequestParamException("key"); + } + } + + protected void willSetKeyForUrl(Api.Request request) throws QiniuException { + if (StringUtils.isNullOrEmpty(domain)) { + ApiUtils.throwInvalidRequestParamException("domain"); + } + } + + protected void didSetKeyForUrl(Api.Request request) throws QiniuException { + } + + protected void didBuildUrl() throws QiniuException { + } + + private String getUrlPrefix() throws QiniuException { + if (useHttps) { + return "https://" + domain; + } else { + return "http://" + domain; + } + } + + /** + * 七牛 url path 特殊处理 + * + * @param path raw url path + * @return encode url path + */ + private String urlPathEncode(String path) { + return UrlUtils.urlEncode(path, "/~"); + } +} diff --git a/src/main/java/com/qiniu/util/Md5.java b/src/main/java/com/qiniu/util/Md5.java index 0c2bca30d..dfade41a9 100644 --- a/src/main/java/com/qiniu/util/Md5.java +++ b/src/main/java/com/qiniu/util/Md5.java @@ -31,7 +31,11 @@ public static String md5(byte[] data, int offset, int len) { public static String md5(File file) throws IOException { FileInputStream fis = new FileInputStream(file); - return md5(fis, file.length()); + try { + return md5(fis, file.length()); + } finally { + fis.close(); + } } diff --git a/src/test/java/test/com/qiniu/HttpTest.java b/src/test/java/test/com/qiniu/HttpTest.java index 33f21cfcf..e5f6f9c81 100644 --- a/src/test/java/test/com/qiniu/HttpTest.java +++ b/src/test/java/test/com/qiniu/HttpTest.java @@ -10,6 +10,7 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.util.Date; import java.util.concurrent.TimeUnit; public class HttpTest { @@ -136,7 +137,7 @@ public void testTimeout() throws NoSuchFieldException, IllegalAccessException { Field field = client.getClass().getDeclaredField("httpClient"); field.setAccessible(true); OkHttpClient okHttpClient = (OkHttpClient) field.get(client); - okHttpClient = okHttpClient.newBuilder().connectTimeout(3, TimeUnit.MILLISECONDS).build(); + okHttpClient = okHttpClient.newBuilder().connectTimeout(1, TimeUnit.MILLISECONDS).build(); field.set(client, okHttpClient); try { @@ -168,19 +169,22 @@ public void testTimeout() throws NoSuchFieldException, IllegalAccessException { Assert.assertTrue("https, must have port 443", e.getMessage().indexOf(":443") > 10); } + long start = new Date().getTime(); try { - client.get("http://www.qiniu.com/?v=543"); - Assert.fail("should be timeout"); + Response response = client.get("http://uc.qbox.me/?v=543"); + long end = new Date().getTime(); + Assert.fail("should be timeout," + " duration:" + (end - start) + " detail:" + response); } catch (QiniuException e) { e.printStackTrace(); - Assert.assertTrue("http, must have port 80", e.getMessage().indexOf(":80") > 10); + long end = new Date().getTime(); + Assert.assertTrue("http, must have port 80," + " duration:" + (end - start) + "detail:" + e.getMessage(), e.getMessage().indexOf(":80") > 10); } try { - client.get("https://www.qiniu.com/?v=kgd"); - Assert.fail("should be timeout"); + Response response = client.get("https://uc.qbox.me/?v=kgd"); + Assert.fail("should be timeout, detail:" + response); } catch (QiniuException e) { e.printStackTrace(); - Assert.assertTrue("https, must have port 443", e.getMessage().indexOf(":443") > 10); + Assert.assertTrue("https, must have port 443, detail:" + e.getMessage(), e.getMessage().indexOf(":443") > 10); } } } diff --git a/src/test/java/test/com/qiniu/TestConfig.java b/src/test/java/test/com/qiniu/TestConfig.java index 42bbeebc0..73857842f 100644 --- a/src/test/java/test/com/qiniu/TestConfig.java +++ b/src/test/java/test/com/qiniu/TestConfig.java @@ -41,9 +41,14 @@ public final class TestConfig { //na0 public static final String testBucket_na0 = "java-sdk-na0"; public static final String testKey_na0 = "do_not_delete/1.png"; + public static final String testChineseKey_na0 = "do_not_delete/水 果-iphone.png"; public static final String testDomain_na0 = "javasdk-na0.peterpy.cn"; public static final String testDomain_na0_timeStamp = "javasdk-na0-timestamp.peterpy.cn"; public static final String testUrl_na0 = "http://" + testDomain_na0 + "/" + testKey_na0; + public static final String testPrivateKey_na0 = "水果.png"; + public static final String testPrivateBucket_na0 = "private-na0"; + public static final String testPrivateBucketDomain_na0 = "private-na0.sdk.qiniu-solutions.com"; + //sg public static final String testBucket_as0 = "sdk-as0"; //code diff --git a/src/test/java/test/com/qiniu/storage/ApiUploadV1Test.java b/src/test/java/test/com/qiniu/storage/ApiUploadV1Test.java index 79a6a8121..083748681 100644 --- a/src/test/java/test/com/qiniu/storage/ApiUploadV1Test.java +++ b/src/test/java/test/com/qiniu/storage/ApiUploadV1Test.java @@ -10,6 +10,7 @@ import test.com.qiniu.TempFile; import test.com.qiniu.TestConfig; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -22,7 +23,16 @@ public class ApiUploadV1Test { @Test - public void testUpload() { + public void testUploadBytes() { + testUpload(true); + } + + @Test + public void testUploadStream() { + testUpload(false); + } + + public void testUpload(boolean isUploadBytes) { long fileSize = 1024 * 7 + 2341; // 单位： k File f = null; @@ -121,7 +131,12 @@ public void testUpload() { // 1.2.2.1 块中第一片，采用 make block 接口 ApiUploadV1MakeBlock makeBlockApi = new ApiUploadV1MakeBlock(client); ApiUploadV1MakeBlock.Request makeBlockRequest = new ApiUploadV1MakeBlock.Request(urlPrefix, token, (int) blockSize); - makeBlockRequest.setFirstChunkData(chunkData, 0, (int) chunkSize, null); + if (isUploadBytes) { + makeBlockRequest.setFirstChunkData(chunkData, 0, (int) chunkSize, null); + } else { + makeBlockRequest.setFirstChunkData(new ByteArrayInputStream(chunkData), null, chunkSize); + } + try { ApiUploadV1MakeBlock.Response makeBlockResponse = makeBlockApi.request(makeBlockRequest); blockLastCtx = makeBlockResponse.getCtx(); @@ -142,7 +157,11 @@ public void testUpload() { // 1.2.2.2 非块中第一片，采用 make block 接口 ApiUploadV1PutChunk putChunkApi = new ApiUploadV1PutChunk(client); ApiUploadV1PutChunk.Request putChunkRequest = new ApiUploadV1PutChunk.Request(urlPrefix, token, blockLastCtx, (int) chunkOffset); - putChunkRequest.setChunkData(chunkData, 0, (int) chunkSize, null); + if (isUploadBytes) { + putChunkRequest.setChunkData(chunkData, 0, (int) chunkSize, null); + } else { + putChunkRequest.setChunkData(new ByteArrayInputStream(chunkData), null, chunkSize); + } try { ApiUploadV1PutChunk.Response putChunkResponse = putChunkApi.request(putChunkRequest); blockLastCtx = putChunkResponse.getCtx(); diff --git a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java index ab159d8e2..54dc040dc 100644 --- a/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java +++ b/src/test/java/test/com/qiniu/storage/ApiUploadV2Test.java @@ -10,6 +10,7 @@ import test.com.qiniu.TempFile; import test.com.qiniu.TestConfig; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -22,7 +23,21 @@ public class ApiUploadV2Test { @Test - public void testUpload() { + public void testUploadBytes() { + testUpload(true, false); + } + + @Test + public void testUploadStream() { + testUpload(false, false); + } + + @Test + public void testUploadStreamWithContentLength() { + testUpload(false, true); + } + + public void testUpload(boolean isUploadBytes, boolean isSetContentLength) { long fileSize = 1024 * 7 + 2341; // 单位： k File f = null; try { @@ -119,8 +134,12 @@ public void testUpload() { // 1.2.2 上传 part 数据 ApiUploadV2UploadPart uploadPartApi = new ApiUploadV2UploadPart(client); ApiUploadV2UploadPart.Request uploadPartRequest = new ApiUploadV2UploadPart.Request(urlPrefix, token, uploadId, partNumber) - .setKey(key) - .setUploadData(partData, 0, partData.length, null); + .setKey(key); + if (isUploadBytes) { + uploadPartRequest.setUploadData(partData, 0, partData.length, null); + } else { + uploadPartRequest.setUploadData(new ByteArrayInputStream(partData), null, isSetContentLength ? partData.length + 1 : -1); + } try { ApiUploadV2UploadPart.Response uploadPartResponse = uploadPartApi.request(uploadPartRequest); String etag = uploadPartResponse.getEtag(); diff --git a/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java new file mode 100644 index 000000000..b27856208 --- /dev/null +++ b/src/test/java/test/com/qiniu/storage/DownloadUrlTest.java @@ -0,0 +1,181 @@ +package test.com.qiniu.storage; + +import com.qiniu.common.QiniuException; +import com.qiniu.http.Client; +import com.qiniu.http.Response; +import com.qiniu.storage.Configuration; +import com.qiniu.storage.DownloadPrivateCloudUrl; +import com.qiniu.storage.DownloadUrl; +import com.qiniu.util.Auth; +import org.junit.Assert; +import org.junit.Test; +import test.com.qiniu.TestConfig; + +import java.net.URLEncoder; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +public class DownloadUrlTest { + + @Test + public void testUrl() { + String key = TestConfig.testChineseKey_na0; + String domain = TestConfig.testDomain_na0; + + String attname = "download" + key; + String fop = "imageView2/2/w/320/h/480"; + String style = "iphone"; + String styleSeparator = "-"; + String styleParam = ""; + String customQueryKey = "Key"; + String customQueryValue = "Value"; + try { + String url = new DownloadUrl(domain, false, key) + .setAttname(attname).setFop(fop).setStyle(style, styleSeparator, styleParam) + .addCustomQuery(customQueryKey, customQueryValue) + .buildURL(); + System.out.println("create url:" + url); + testHasAuthority(url); + + url = new DownloadUrl(domain, true, key) + .setAttname(attname).setFop(fop).setStyle(style, styleSeparator, styleParam) + .addCustomQuery(customQueryKey, customQueryValue) + .buildURL(); + Assert.assertTrue("url:" + url, url.contains("https://")); + } catch (QiniuException e) { + Assert.assertTrue(e.error(), false); + } + } + + @Test + public void testSpecialKey() { + String domain = "abc.com:123"; + Map keys = new HashMap() {{ + put("", ""); + put("abc_def.mp4", "abc_def.mp4"); + put("/ab/cd", "/ab/cd"); + put("ab/中文/de", "ab/%E4%B8%AD%E6%96%87/de"); + put("ab+-*de f", "ab%2B-%2Ade%20f"); + put("ab:cd", "ab%3Acd"); + put("ab@cd", "ab%40cd"); + put("ab?cd=ef", "ab%3Fcd%3Def"); + put("ab#e~f", "ab%23e~f"); + put("ab//cd", "ab//cd"); + put("abc%2F%2B", "abc%252F%252B"); + put("ab cd", "ab%20cd"); + put("ab/c:d?e#f//gh汉子", "ab/c%3Ad%3Fe%23f//gh%E6%B1%89%E5%AD%90"); + }}; + + for (String key : keys.keySet()) { + String encodeKey = keys.get(key); + try { + String url = new DownloadUrl(domain, false, key).buildURL(); + String exceptUrl = "http://" + domain + "/" + encodeKey; + Assert.assertEquals("url:" + url + " exceptUrl:" + exceptUrl, exceptUrl, url); + } catch (QiniuException e) { + Assert.assertTrue(e.error(), false); + } + } + + + } + + @Test + public void testUrlWithDeadline() { + String key = TestConfig.testKey_na0; + String domain = TestConfig.testPrivateBucketDomain_na0; + Auth auth = TestConfig.testAuth; + + try { + long expire = 10; + long deadline = new Date().getTime() / 1000 + expire; + String url = new DownloadUrl(domain, false, key).buildURL(auth, deadline); + System.out.println("create url:" + url); + Client client = new Client(); + Response response = client.get(url); + Assert.assertTrue(response.toString(), response.isOK()); + + try { + Thread.sleep((expire + 5) * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + testNoAuthority(url); + } catch (QiniuException e) { + Assert.assertTrue(e.response.toString(), false); + } + } + + private void testNoAuthority(String url) { + try { + Client client = new Client(); + Response response = client.get(url); + Assert.assertFalse(url, response.isOK()); + } catch (QiniuException e) { + Assert.assertNotNull("except no authority:" + url + "\n but no response:" + e, e.response); + Assert.assertTrue("except no authority:" + url + "\n but:" + e.response, e.response.statusCode == 401); + } + } + + private void testHasAuthority(String url) { + try { + Client client = new Client(); + Response response = client.get(url); + Assert.assertTrue(url, response.isOK()); + } catch (QiniuException e) { + Assert.assertTrue("except has authority:" + url + "\n response:" + e.response, false); + } + } + + @Test + public void testPrivateCloudUrl() { + TestConfig.TestFile[] files = TestConfig.getTestFileArray(); + for (TestConfig.TestFile file : files) { + String key = file.getKey(); + String bucket = file.getBucketName(); + String domain = file.getTestDomain(); + + String attname = "test_file.jpg"; + String fop = "imageView2/2/w/320/h/480"; + String style = "iphone"; + String styleSeparator = "-"; + try { + DownloadPrivateCloudUrl downloadUrl = new DownloadPrivateCloudUrl(domain, false, bucket, key, TestConfig.testAccessKey); + String url = downloadUrl.setAttname(attname).setFop(fop).setStyle(style, styleSeparator, null).buildURL(); + String urlExpire = "http://" + domain + "/getfile/" + TestConfig.testAccessKey + "/" + bucket + "/" + key + URLEncoder.encode(styleSeparator) + URLEncoder.encode(style) + "?" + URLEncoder.encode(fop) + "&attname=" + URLEncoder.encode(attname); + System.out.println("create url:" + url + " expire url:" + urlExpire); + Assert.assertEquals("create url:" + url + " expire url:" + urlExpire, urlExpire, url); + + downloadUrl = new DownloadPrivateCloudUrl(domain, true, bucket, key, TestConfig.testAccessKey); + url = downloadUrl.setAttname(attname).setFop(fop).setStyle(style, styleSeparator, null).buildURL(); + urlExpire = "https://" + domain + "/getfile/" + TestConfig.testAccessKey + "/" + bucket + "/" + key + URLEncoder.encode(styleSeparator) + URLEncoder.encode(style) + "?" + URLEncoder.encode(fop) + "&attname=" + URLEncoder.encode(attname); + System.out.println("create url:" + url + " expire url:" + urlExpire); + Assert.assertEquals("create url:" + url + " expire url:" + urlExpire, urlExpire, url); + + + Configuration cfg = new Configuration(); + cfg.useHttpsDomains = false; + String host = cfg.ioHost(TestConfig.testAccessKey, bucket); + + + downloadUrl = new DownloadPrivateCloudUrl(cfg, bucket, key, TestConfig.testAccessKey); + url = downloadUrl.setAttname(attname).setFop(fop).setStyle(style, styleSeparator, null).buildURL(); + urlExpire = host + "/getfile/" + TestConfig.testAccessKey + "/" + bucket + "/" + key + URLEncoder.encode(styleSeparator) + URLEncoder.encode(style) + "?" + URLEncoder.encode(fop) + "&attname=" + URLEncoder.encode(attname); + System.out.println("create url:" + url + " expire url:" + urlExpire); + Assert.assertEquals("create url:" + url + " expire url:" + urlExpire, urlExpire, url); + + + cfg.useHttpsDomains = true; + host = cfg.ioHost(TestConfig.testAccessKey, bucket); + downloadUrl = new DownloadPrivateCloudUrl(cfg, bucket, key, TestConfig.testAccessKey); + url = downloadUrl.setAttname(attname).setFop(fop).setStyle(style, styleSeparator, null).buildURL(); + urlExpire = host + "/getfile/" + TestConfig.testAccessKey + "/" + bucket + "/" + key + URLEncoder.encode(styleSeparator) + URLEncoder.encode(style) + "?" + URLEncoder.encode(fop) + "&attname=" + URLEncoder.encode(attname); + System.out.println("create url:" + url + " expire url:" + urlExpire); + Assert.assertEquals("create url:" + url + " expire url:" + urlExpire, urlExpire, url); + } catch (QiniuException e) { + Assert.assertTrue(e.error(), false); + } + } + } +} diff --git a/src/test/java/test/com/qiniu/storage/ZoneTest.java b/src/test/java/test/com/qiniu/storage/ZoneTest.java index 3b5aad2cc..ad51ce800 100644 --- a/src/test/java/test/com/qiniu/storage/ZoneTest.java +++ b/src/test/java/test/com/qiniu/storage/ZoneTest.java @@ -13,8 +13,8 @@ public void zone1() { String sk = ""; Auth auth = Auth.create(ak, sk); - Configuration.defaultApiHost = "apiserver-sdfrsd-s.qiniubbo.com"; - Configuration.defaultRsHost = "rs-sdfrsd-s.qiniubbo.com"; +// Configuration.defaultApiHost = "apiserver-sdfrsd-s.qiniubbo.com"; +// Configuration.defaultRsHost = "rs-sdfrsd-s.qiniubbo.com"; Zone zone = new Zone.Builder() .upHttp("http://up-sdfrsd-s.qiniubbo.com") .upBackupHttp("http://up-sdfrsd-s.qiniubbo.com")