diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b85969f1..287b1268 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -20,7 +20,7 @@ jobs:
run: |
set -e
cmake -S . -B build
- cmake --build build
+ cmake --build build -j $(nproc)
- name: Unit test
working-directory: '${{ github.workspace }}/build/gtests'
run: |
@@ -38,7 +38,7 @@ jobs:
run: |
set -e
cmake -DOPENSSL_ROOT_DIR="$(brew --prefix)/opt/openssl/" -S . -B build
- cmake --build build
+ cmake --build build -j $(nproc)
- name: Unit test
working-directory: '${{ github.workspace }}/build/gtests'
run: |
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e01b9d3..dec93d77 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,14 @@
## CHANGE LOG
+## v7.8.0 (2024-08-27)
+
+- 支持闲时任务和 prefop 接口
+- 调整查询区域主备域名
+- 支持缓存多个 `v4/query` 查询结果
+- 上传重试支持切换上传域名
+- 支持上传加速域名
+- 增加上传进度回调支持
+
## v7.7.0 (2023-12-25)
- 支持归档直读存储类型
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18b8ac3d..eeab57fe 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,7 +8,7 @@ CMAKE_POLICY(SET CMP0074 NEW)
SET(PROJECT_NAME qiniu)
# 建立项目
-PROJECT(${PROJECT_NAME} VERSION 7.7.0 LANGUAGES C)
+PROJECT(${PROJECT_NAME} VERSION 7.8.0 LANGUAGES C)
set(CMAKE_C_STANDARD 99)
@@ -19,16 +19,17 @@ ENDIF()
AUX_SOURCE_DIRECTORY(b64 DIR_SRCS_B64)
AUX_SOURCE_DIRECTORY(cJSON DIR_SRCS_CJSON)
+AUX_SOURCE_DIRECTORY(hashmap DIR_SRCS_HASHMAP)
AUX_SOURCE_DIRECTORY(qiniu DIR_SRCS_QINIU)
-MESSAGE(STATUS "Src file: ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU}")
+MESSAGE(STATUS "Src file: ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU}")
# 编译可执行程序
# ADD_EXECUTABLE(${PROJECT_NAME} ${DIR_SRCS})
# 如果要生成静态链接库
-ADD_LIBRARY(${PROJECT_NAME}_static STATIC ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU})
+ADD_LIBRARY(${PROJECT_NAME}_static STATIC ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU})
# 如果要生成动态链接库
-ADD_LIBRARY(${PROJECT_NAME} SHARED ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU})
+ADD_LIBRARY(${PROJECT_NAME} SHARED ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU})
TARGET_INCLUDE_DIRECTORIES(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
diff --git a/docs/README.md b/docs/README.md
index acd00ac0..b3fe2f4e 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -68,6 +68,24 @@ C-SDK 使用 [cURL](http://curl.haxx.se/) 进行网络相关操作。无论是
如果在项目构建过程中出现环境相关的编译错误和链接错误，请确认这些选项是否都已经正确配置，以及所依赖的库是否都已经正确的安装。
+#### 通过 CMake 编译
+
+如果想在 CMake 项目里使用 C-SDK，可以直接在 CMake 里导入 C-SDK：
+
+```cmake
+INCLUDE(FetchContent)
+FetchContent_Declare(
+ qiniu
+ GIT_REPOSITORY https://github.com/qiniu/c-sdk.git
+ GIT_TAG v7.8.0
+)
+FetchContent_MakeAvailable(qiniu)
+FIND_PACKAGE(CURL REQUIRED)
+FIND_PACKAGE(OpenSSL REQUIRED)
+TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE qiniu m)
+TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE ${CURL_LIBRARIES})
+TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE ${OPENSSL_LIBRARIES})
+```
@@ -169,7 +187,7 @@ typedef struct _Qiniu_Error {
typedef struct _Qiniu_RS_StatRet {
const char* hash;
const char* mimeType;
- Qiniu_Int64 fsize;
+ Qiniu_Int64 fsize;
Qiniu_Int64 putTime;
} Qiniu_RS_StatRet;
```
@@ -427,7 +445,7 @@ void stat(Qiniu_Client* client, const char* bucket, const char* key)
typedef struct _Qiniu_RS_StatRet {
const char* hash;
const char* mimeType;
- Qiniu_Int64 fsize;
+ Qiniu_Int64 fsize;
Qiniu_Int64 putTime;
} Qiniu_RS_StatRet;
```
@@ -456,8 +474,8 @@ void delete(Qiniu_Client* client, const char* bucket, const char* key)
复制和移动操作，需要指定源路径和目标路径。
```{c}
-void copy(Qiniu_Client* client,
- const char* bucketSrc, const char* keySrc,
+void copy(Qiniu_Client* client,
+ const char* bucketSrc, const char* keySrc,
const char* bucketDest, const char* keyDest)
{
Qiniu_Error err = Qiniu_RS_Copy(client, bucketSrc, keySrc, bucketDest, keyDest);
@@ -470,8 +488,8 @@ void copy(Qiniu_Client* client,
```
```{c}
-void move(Qiniu_Client* client,
- const char* bucketSrc, const char* keySrc,
+void move(Qiniu_Client* client,
+ const char* bucketSrc, const char* keySrc,
const char* bucketDest, const char* keyDest)
{
Qiniu_Error err = Qiniu_RS_Move(client, bucketSrc, keySrc, bucketDest, keyDest);
@@ -494,7 +512,7 @@ void move(Qiniu_Client* client,
调用`Qiniu_RS_BatchStat`可以批量查看多个文件的属性信息。
```{c}
-void batchStat(Qiniu_Client* client,
+void batchStat(Qiniu_Client* client,
Qiniu_RS_EntryPath* entries, Qiniu_ItemCount entryCount)
{
Qiniu_RS_BatchStatRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchStatRet));
@@ -551,7 +569,7 @@ typedef struct _Qiniu_RS_BatchStatRet {
typedef struct _Qiniu_RS_StatRet {
const char* hash;
const char* mimeType;
- Qiniu_Int64 fsize;
+ Qiniu_Int64 fsize;
Qiniu_Int64 putTime;
} Qiniu_RS_StatRet;
```
@@ -563,7 +581,7 @@ typedef struct _Qiniu_RS_StatRet {
调用`Qiniu_RS_BatchDelete`可以批量删除多个文件。
```{c}
-void batchDelete(Qiniu_Client* client,
+void batchDelete(Qiniu_Client* client,
Qiniu_RS_EntryPath* entries, Qiniu_ItemCount entryCount)
{
Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet));
@@ -604,7 +622,7 @@ typedef struct _Qiniu_RS_BatchItemRet {
调用`Qiniu_RS_BatchCopy`可以批量复制多个文件。
```{c}
-void batchCopy(Qiniu_Client* client,
+void batchCopy(Qiniu_Client* client,
Qiniu_RS_EntryPathPair* entryPairs, Qiniu_ItemCount entryCount)
{
Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet));
@@ -644,7 +662,7 @@ typedef struct _Qiniu_RS_EntryPathPair {
批量移动和批量复制很类似，唯一的区别就是调用`Qiniu_RS_BatchMove`。
```{c}
-void batchMove(Qiniu_Client* client,
+void batchMove(Qiniu_Client* client,
Qiniu_RS_EntryPathPair* entryPairs, Qiniu_ItemCount entryCount)
{
Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet));
diff --git a/examples/fop_video_avthumb.c b/examples/fop_video_avthumb.c
index 36857c14..ed3da2bd 100644
--- a/examples/fop_video_avthumb.c
+++ b/examples/fop_video_avthumb.c
@@ -43,7 +43,16 @@ int main(int argc, char **argv) {
char *fops[] = {avthumbMp4Fop, vframeJpgFop};
- Qiniu_Error error = Qiniu_FOP_Pfop(&client, &pfopRet, bucket, key, fops, 2, pipeline, notifyURL, force);
+ Qiniu_FOP_PfopParams params = {
+ .bucket = bucket,
+ .key = key,
+ .pipeline = pipeline,
+ .notifyURL = notifyURL,
+ .fops = fops,
+ .fopCount = 2,
+ .force = force,
+ };
+ Qiniu_Error error = Qiniu_FOP_Pfop_v2(&client, &pfopRet, ¶ms);
if (error.code != 200) {
printf("video file pfop %s:%s error.\n", bucket, key);
debug_log(&client, error);
@@ -52,7 +61,41 @@ int main(int argc, char **argv) {
printf("video file pfop %s:%s success, persistentId: %s .\n\n", bucket, key, pfopRet.persistentId);
}
+ Qiniu_FOP_PrefopRet prefopRet;
+ Qiniu_FOP_PrefopItemRet prefopItemRet[2];
+ Qiniu_ItemCount itemsCount;
+ error = Qiniu_FOP_Prefop(&client, &prefopRet, (Qiniu_FOP_PrefopItemRet *)&prefopItemRet, &itemsCount, pfopRet.persistentId, 2);
+ if (error.code != 200)
+ {
+ debug_log(&client, error);
+ }
+ else
+ {
+ printf("ID: %s\n", prefopRet.id);
+ printf("Code: %d\n", prefopRet.code);
+ printf("Desc: %s\n", prefopRet.desc);
+ printf("InputBucket: %s\n", prefopRet.inputBucket);
+ printf("InputKey: %s\n", prefopRet.inputKey);
+ printf("Type: %d\n", prefopRet.type);
+ printf("CreationDate: %d-%d-%d %d:%d:%d +%d\n", prefopRet.creationDate.date.year, prefopRet.creationDate.date.month,
+ prefopRet.creationDate.date.day, prefopRet.creationDate.time.hour,
+ prefopRet.creationDate.time.minute, prefopRet.creationDate.time.second,
+ prefopRet.creationDate.time.offset);
+
+ for (Qiniu_ItemCount i = 0; i < itemsCount; i++)
+ {
+ printf("\tIndex: %d\n", i);
+ printf("\tCmd: %s\n", prefopItemRet[i].cmd);
+ printf("\tCode: %d\n", prefopItemRet[i].code);
+ printf("\tDesc: %s\n", prefopItemRet[i].desc);
+ printf("\tError: %s\n", prefopItemRet[i].error);
+ printf("\tHash: %s\n", prefopItemRet[i].hash);
+ printf("\tKey: %s\n", prefopItemRet[i].key);
+ printf("\tReturnOld: %d\n", prefopItemRet[i].returnOld);
+ }
+ }
+
Qiniu_Free(avthumbMp4Fop);
Qiniu_Free(vframeJpgFop);
Qiniu_Client_Cleanup(&client);
-}
\ No newline at end of file
+}
diff --git a/examples/up_form_upload.c b/examples/up_form_upload.c
index a593b308..ed874059 100644
--- a/examples/up_form_upload.c
+++ b/examples/up_form_upload.c
@@ -31,12 +31,6 @@ int main(int argc, char **argv) {
putPolicy.scope = bucket;
char *uptoken = Qiniu_RS_PutPolicy_Token(&putPolicy, &mac);
- //设置机房域名
- //Qiniu_Use_Zone_Beimei(Qiniu_False);
- //Qiniu_Use_Zone_Huabei(Qiniu_True);
- Qiniu_Use_Zone_Huadong(Qiniu_False);
- //Qiniu_Use_Zone_Huanan(Qiniu_True);
-
//put extra
//putExtra.upHost="http://nbxs-gate-up.qiniu.com";
@@ -60,6 +54,7 @@ int main(int argc, char **argv) {
//init
Qiniu_Client_InitMacAuth(&client, 1024, &mac);
+ Qiniu_Client_EnableAutoQuery(&client, Qiniu_True);
Qiniu_Error error = Qiniu_Io_PutFile(&client, &putRet, uptoken, key, localFile, &putExtra);
if (error.code != 200) {
printf("upload file %s:%s error.\n", bucket, key);
diff --git a/examples/up_multipart_sample.c b/examples/up_multipart_sample.c
index 9bd84b6a..ba4d593d 100644
--- a/examples/up_multipart_sample.c
+++ b/examples/up_multipart_sample.c
@@ -62,6 +62,7 @@ int main(int argc, char **argv)
char *uptoken = Qiniu_RS_PutPolicy_Token(&putPolicy, &mac);
Qiniu_Client_InitMacAuth(&client, 1024, &mac);
+ Qiniu_Client_EnableAutoQuery(&client, Qiniu_True);
Qiniu_RS_Delete(&client, bucket, key); //to avoid "file exist" err
error = Qiniu_Multipart_PutFile(&client, uptoken, key, localFile, &putExtra, &putRet);
diff --git a/examples/up_resumeable_upload.c b/examples/up_resumeable_upload.c
index a08a3faf..f7e2b956 100644
--- a/examples/up_resumeable_upload.c
+++ b/examples/up_resumeable_upload.c
@@ -23,6 +23,7 @@ static void resumableUploadWithKey(Qiniu_Mac *mac, const char *bucket, const cha
Qiniu_Client client;
Qiniu_Global_Init(-1);
Qiniu_Client_InitNoAuth(&client, 1024);
+ Qiniu_Client_EnableAutoQuery(&client, Qiniu_True);
Qiniu_Error error;
Qiniu_Rio_PutRet putRet;
diff --git a/gtests/region_test.cc b/gtests/region_test.cc
index 599d9ac9..102a7975 100644
--- a/gtests/region_test.cc
+++ b/gtests/region_test.cc
@@ -97,24 +97,21 @@ TEST(UnitTest, TestUseRegion)
EXPECT_STREQ(hosts[0], "https://rs-z0.qiniuapi.com");
hosts = Qiniu_Region_Get_Rs_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "https://rs-z0.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "https://rsf-z0.qiniuapi.com");
hosts = Qiniu_Region_Get_Rsf_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "https://rsf-z0.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Api_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "https://api-z0.qiniuapi.com");
hosts = Qiniu_Region_Get_Api_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "https://api-z0.qbox.me");
+ EXPECT_EQ(count, 0);
Qiniu_Region_Free(region);
@@ -126,40 +123,35 @@ TEST(UnitTest, TestUseRegion)
EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com");
hosts = Qiniu_Region_Get_Up_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "http://up-z1.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Io_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "http://iovip-z1.qiniuio.com");
hosts = Qiniu_Region_Get_Io_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "http://iovip-z1.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "http://rs-z1.qiniuapi.com");
hosts = Qiniu_Region_Get_Rs_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "http://rs-z1.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "http://rsf-z1.qiniuapi.com");
hosts = Qiniu_Region_Get_Rsf_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "http://rsf-z1.qbox.me");
+ EXPECT_EQ(count, 0);
hosts = Qiniu_Region_Get_Api_Preferred_Hosts(region, &count);
EXPECT_EQ(count, 1);
EXPECT_STREQ(hosts[0], "http://api-z1.qiniuapi.com");
hosts = Qiniu_Region_Get_Api_Alternative_Hosts(region, &count);
- EXPECT_EQ(count, 1);
- EXPECT_STREQ(hosts[0], "http://api-z1.qbox.me");
+ EXPECT_EQ(count, 0);
Qiniu_Region_Free(region);
}
@@ -242,11 +234,17 @@ TEST(IntegrationTest, TestRegionAutoQuery)
Qiniu_Client_EnableAutoQuery(&client, Qiniu_True);
const char *host;
+ const char *const *hosts;
+ size_t count;
Qiniu_Error err;
- err = _Qiniu_Region_Get_Up_Host(&client, NULL, Test_bucket, &host);
+ err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count);
EXPECT_EQ(err.code, 200);
- EXPECT_STREQ(host, "https://upload.qiniup.com");
+ EXPECT_EQ(count, 4);
+ EXPECT_STREQ(hosts[0], "https://upload.qiniup.com");
+ EXPECT_STREQ(hosts[1], "https://up.qiniup.com");
+ EXPECT_STREQ(hosts[2], "https://upload.qbox.me");
+ EXPECT_STREQ(hosts[3], "https://up.qbox.me");
err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host);
EXPECT_EQ(err.code, 200);
@@ -276,11 +274,15 @@ TEST(IntegrationTest, TestRegionSpecify)
Qiniu_Client_SpecifyRegion(&client, region);
const char *host;
+ const char *const *hosts;
+ size_t count;
Qiniu_Error err;
- err = _Qiniu_Region_Get_Up_Host(&client, NULL, Test_bucket, &host);
+ err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count);
EXPECT_EQ(err.code, 200);
- EXPECT_STREQ(host, "http://upload-z1.qiniup.com");
+ EXPECT_EQ(count, 2);
+ EXPECT_STREQ(hosts[0], "http://upload-z1.qiniup.com");
+ EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com");
err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host);
EXPECT_EQ(err.code, 200);
@@ -300,3 +302,54 @@ TEST(IntegrationTest, TestRegionSpecify)
Qiniu_Region_Free(region);
}
+
+TEST(IntegrationTest, TestRegionSpecifyAcceleration)
+{
+ Qiniu_Client client;
+ Qiniu_Client_InitMacAuth(&client, 1024, NULL);
+ Qiniu_Client_SetTimeout(&client, 5000);
+ Qiniu_Client_SetConnectTimeout(&client, 3000);
+
+ Qiniu_Region *region = Qiniu_Use_Region("z1", Qiniu_False);
+ const char *setHosts[1] = {"http://acc.upload-z1.qiniup.com"};
+ Qiniu_Region_Set_Up_Accelerated_Hosts(region, (const char *const *)&setHosts, 1, Qiniu_False);
+ Qiniu_Client_SpecifyRegion(&client, region);
+ Qiniu_Client_EnableUploadingAcceleration(&client);
+
+ const char *host;
+ const char *const *hosts;
+ size_t count;
+ Qiniu_Error err;
+
+ err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_EQ(count, 3);
+ EXPECT_STREQ(hosts[0], "http://acc.upload-z1.qiniup.com");
+ EXPECT_STREQ(hosts[1], "http://upload-z1.qiniup.com");
+ EXPECT_STREQ(hosts[2], "http://up-z1.qiniup.com");
+
+ err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_STREQ(host, "http://iovip-z1.qiniuio.com");
+
+ err = _Qiniu_Region_Get_Rs_Host(&client, NULL, Test_bucket, &host);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_STREQ(host, "http://rs-z1.qiniuapi.com");
+
+ err = _Qiniu_Region_Get_Rsf_Host(&client, NULL, Test_bucket, &host);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_STREQ(host, "http://rsf-z1.qiniuapi.com");
+
+ err = _Qiniu_Region_Get_Api_Host(&client, NULL, Test_bucket, &host);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_STREQ(host, "http://api-z1.qiniuapi.com");
+
+ Qiniu_Client_DisableUploadingAcceleration(&client);
+ err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count);
+ EXPECT_EQ(err.code, 200);
+ EXPECT_EQ(count, 2);
+ EXPECT_STREQ(hosts[0], "http://upload-z1.qiniup.com");
+ EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com");
+
+ Qiniu_Region_Free(region);
+}
diff --git a/gtests/rfc3339_test.cc b/gtests/rfc3339_test.cc
new file mode 100644
index 00000000..fa4a4897
--- /dev/null
+++ b/gtests/rfc3339_test.cc
@@ -0,0 +1,53 @@
+#include
+#include "test.h"
+#include "qiniu/rfc3339.h"
+
+#ifdef _WIN32
+#include
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+ void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt);
+#ifdef __cplusplus
+}
+#endif
+
+TEST(UnitTest, TestQiniuParseDateTime)
+{
+ Qiniu_DateTime dt;
+ Qiniu_Zero(dt);
+ _Qiniu_Parse_Date_Time((char *)"1937-02-02T12:00:27.87+00:20", &dt);
+ EXPECT_TRUE(dt.ok);
+ EXPECT_TRUE(dt.date.ok);
+ EXPECT_EQ(dt.date.year, 1937);
+ EXPECT_EQ(dt.date.month, 2);
+ EXPECT_EQ(dt.date.day, 2);
+ EXPECT_TRUE(dt.time.ok);
+ EXPECT_EQ(dt.time.hour, 12);
+ EXPECT_EQ(dt.time.minute, 0);
+ EXPECT_EQ(dt.time.second, 27);
+ EXPECT_EQ(dt.time.fraction, 870000);
+ EXPECT_EQ(dt.time.offset, 20);
+
+ _Qiniu_Parse_Date_Time((char *)"1937-02-29T12:00:27.87+00:20", &dt);
+ EXPECT_FALSE(dt.ok);
+
+ _Qiniu_Parse_Date_Time((char *)"1936-02-29T12:00:27.87Z", &dt);
+ EXPECT_TRUE(dt.ok);
+ EXPECT_TRUE(dt.date.ok);
+ EXPECT_EQ(dt.date.year, 1936);
+ EXPECT_EQ(dt.date.month, 2);
+ EXPECT_EQ(dt.date.day, 29);
+ EXPECT_TRUE(dt.time.ok);
+ EXPECT_EQ(dt.time.hour, 12);
+ EXPECT_EQ(dt.time.minute, 0);
+ EXPECT_EQ(dt.time.second, 27);
+ EXPECT_EQ(dt.time.fraction, 870000);
+ EXPECT_EQ(dt.time.offset, 0);
+
+ _Qiniu_Parse_Date_Time((char *)"1900-02-29T12:00:27.87+00:20", &dt);
+ EXPECT_FALSE(dt.ok);
+}
diff --git a/hashmap/hashmap.c b/hashmap/hashmap.c
new file mode 100644
index 00000000..d70b1627
--- /dev/null
+++ b/hashmap/hashmap.c
@@ -0,0 +1,1154 @@
+// Copyright 2020 Joshua J Baker. All rights reserved.
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+#include
+#include
+#include
+#include
+#include
+#include "hashmap.h"
+
+#define GROW_AT 0.60 /* 60% */
+#define SHRINK_AT 0.10 /* 10% */
+
+#ifndef HASHMAP_LOAD_FACTOR
+#define HASHMAP_LOAD_FACTOR GROW_AT
+#endif
+
+static void *(*__malloc)(size_t) = NULL;
+static void *(*__realloc)(void *, size_t) = NULL;
+static void (*__free)(void *) = NULL;
+
+// hashmap_set_allocator allows for configuring a custom allocator for
+// all hashmap library operations. This function, if needed, should be called
+// only once at startup and a prior to calling hashmap_new().
+void hashmap_set_allocator(void *(*malloc)(size_t), void (*free)(void*)) {
+ __malloc = malloc;
+ __free = free;
+}
+
+struct bucket {
+ uint64_t hash:48;
+ uint64_t dib:16;
+};
+
+// hashmap is an open addressed hash map using robinhood hashing.
+struct hashmap {
+ void *(*malloc)(size_t);
+ void *(*realloc)(void *, size_t);
+ void (*free)(void *);
+ size_t elsize;
+ size_t cap;
+ uint64_t seed0;
+ uint64_t seed1;
+ uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1);
+ int (*compare)(const void *a, const void *b, void *udata);
+ void (*elfree)(void *item);
+ void *udata;
+ size_t bucketsz;
+ size_t nbuckets;
+ size_t count;
+ size_t mask;
+ size_t growat;
+ size_t shrinkat;
+ uint8_t loadfactor;
+ uint8_t growpower;
+ bool oom;
+ void *buckets;
+ void *spare;
+ void *edata;
+};
+
+void hashmap_set_grow_by_power(struct hashmap *map, size_t power) {
+ map->growpower = power < 1 ? 1 : power > 16 ? 16 : power;
+}
+
+static double clamp_load_factor(double factor, double default_factor) {
+ // Check for NaN and clamp between 50% and 90%
+ return factor != factor ? default_factor :
+ factor < 0.50 ? 0.50 :
+ factor > 0.95 ? 0.95 :
+ factor;
+}
+
+void hashmap_set_load_factor(struct hashmap *map, double factor) {
+ factor = clamp_load_factor(factor, map->loadfactor / 100.0);
+ map->loadfactor = factor * 100;
+ map->growat = map->nbuckets * (map->loadfactor / 100.0);
+}
+
+static struct bucket *bucket_at0(void *buckets, size_t bucketsz, size_t i) {
+ return (struct bucket*)(((char*)buckets)+(bucketsz*i));
+}
+
+static struct bucket *bucket_at(struct hashmap *map, size_t index) {
+ return bucket_at0(map->buckets, map->bucketsz, index);
+}
+
+static void *bucket_item(struct bucket *entry) {
+ return ((char*)entry)+sizeof(struct bucket);
+}
+
+static uint64_t clip_hash(uint64_t hash) {
+ return hash & 0xFFFFFFFFFFFF;
+}
+
+static uint64_t get_hash(struct hashmap *map, const void *key) {
+ return clip_hash(map->hash(key, map->seed0, map->seed1));
+}
+
+
+// hashmap_new_with_allocator returns a new hash map using a custom allocator.
+// See hashmap_new for more information information
+struct hashmap *hashmap_new_with_allocator(void *(*_malloc)(size_t),
+ void *(*_realloc)(void*, size_t), void (*_free)(void*),
+ size_t elsize, size_t cap, uint64_t seed0, uint64_t seed1,
+ uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
+ int (*compare)(const void *a, const void *b, void *udata),
+ void (*elfree)(void *item),
+ void *udata)
+{
+ _malloc = _malloc ? _malloc : __malloc ? __malloc : malloc;
+ _realloc = _realloc ? _realloc : __realloc ? __realloc : realloc;
+ _free = _free ? _free : __free ? __free : free;
+ size_t ncap = 16;
+ if (cap < ncap) {
+ cap = ncap;
+ } else {
+ while (ncap < cap) {
+ ncap *= 2;
+ }
+ cap = ncap;
+ }
+ size_t bucketsz = sizeof(struct bucket) + elsize;
+ while (bucketsz & (sizeof(uintptr_t)-1)) {
+ bucketsz++;
+ }
+ // hashmap + spare + edata
+ size_t size = sizeof(struct hashmap)+bucketsz*2;
+ struct hashmap *map = _malloc(size);
+ if (!map) {
+ return NULL;
+ }
+ memset(map, 0, sizeof(struct hashmap));
+ map->elsize = elsize;
+ map->bucketsz = bucketsz;
+ map->seed0 = seed0;
+ map->seed1 = seed1;
+ map->hash = hash;
+ map->compare = compare;
+ map->elfree = elfree;
+ map->udata = udata;
+ map->spare = ((char*)map)+sizeof(struct hashmap);
+ map->edata = (char*)map->spare+bucketsz;
+ map->cap = cap;
+ map->nbuckets = cap;
+ map->mask = map->nbuckets-1;
+ map->buckets = _malloc(map->bucketsz*map->nbuckets);
+ if (!map->buckets) {
+ _free(map);
+ return NULL;
+ }
+ memset(map->buckets, 0, map->bucketsz*map->nbuckets);
+ map->growpower = 1;
+ map->loadfactor = clamp_load_factor(HASHMAP_LOAD_FACTOR, GROW_AT) * 100;
+ map->growat = map->nbuckets * (map->loadfactor / 100.0);
+ map->shrinkat = map->nbuckets * SHRINK_AT;
+ map->malloc = _malloc;
+ map->realloc = _realloc;
+ map->free = _free;
+ return map;
+}
+
+// hashmap_new returns a new hash map.
+// Param `elsize` is the size of each element in the tree. Every element that
+// is inserted, deleted, or retrieved will be this size.
+// Param `cap` is the default lower capacity of the hashmap. Setting this to
+// zero will default to 16.
+// Params `seed0` and `seed1` are optional seed values that are passed to the
+// following `hash` function. These can be any value you wish but it's often
+// best to use randomly generated values.
+// Param `hash` is a function that generates a hash value for an item. It's
+// important that you provide a good hash function, otherwise it will perform
+// poorly or be vulnerable to Denial-of-service attacks. This implementation
+// comes with two helper functions `hashmap_sip()` and `hashmap_murmur()`.
+// Param `compare` is a function that compares items in the tree. See the
+// qsort stdlib function for an example of how this function works.
+// The hashmap must be freed with hashmap_free().
+// Param `elfree` is a function that frees a specific item. This should be NULL
+// unless you're storing some kind of reference data in the hash.
+struct hashmap *hashmap_new(size_t elsize, size_t cap, uint64_t seed0,
+ uint64_t seed1,
+ uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
+ int (*compare)(const void *a, const void *b, void *udata),
+ void (*elfree)(void *item),
+ void *udata)
+{
+ return hashmap_new_with_allocator(NULL, NULL, NULL, elsize, cap, seed0,
+ seed1, hash, compare, elfree, udata);
+}
+
+static void free_elements(struct hashmap *map) {
+ if (map->elfree) {
+ for (size_t i = 0; i < map->nbuckets; i++) {
+ struct bucket *bucket = bucket_at(map, i);
+ if (bucket->dib) map->elfree(bucket_item(bucket));
+ }
+ }
+}
+
+// hashmap_clear quickly clears the map.
+// Every item is called with the element-freeing function given in hashmap_new,
+// if present, to free any data referenced in the elements of the hashmap.
+// When the update_cap is provided, the map's capacity will be updated to match
+// the currently number of allocated buckets. This is an optimization to ensure
+// that this operation does not perform any allocations.
+void hashmap_clear(struct hashmap *map, bool update_cap) {
+ map->count = 0;
+ free_elements(map);
+ if (update_cap) {
+ map->cap = map->nbuckets;
+ } else if (map->nbuckets != map->cap) {
+ void *new_buckets = map->malloc(map->bucketsz*map->cap);
+ if (new_buckets) {
+ map->free(map->buckets);
+ map->buckets = new_buckets;
+ }
+ map->nbuckets = map->cap;
+ }
+ memset(map->buckets, 0, map->bucketsz*map->nbuckets);
+ map->mask = map->nbuckets-1;
+ map->growat = map->nbuckets * (map->loadfactor / 100.0) ;
+ map->shrinkat = map->nbuckets * SHRINK_AT;
+}
+
+static bool resize0(struct hashmap *map, size_t new_cap) {
+ struct hashmap *map2 = hashmap_new_with_allocator(map->malloc, map->realloc,
+ map->free, map->elsize, new_cap, map->seed0, map->seed1, map->hash,
+ map->compare, map->elfree, map->udata);
+ if (!map2) return false;
+ for (size_t i = 0; i < map->nbuckets; i++) {
+ struct bucket *entry = bucket_at(map, i);
+ if (!entry->dib) {
+ continue;
+ }
+ entry->dib = 1;
+ size_t j = entry->hash & map2->mask;
+ while(1) {
+ struct bucket *bucket = bucket_at(map2, j);
+ if (bucket->dib == 0) {
+ memcpy(bucket, entry, map->bucketsz);
+ break;
+ }
+ if (bucket->dib < entry->dib) {
+ memcpy(map2->spare, bucket, map->bucketsz);
+ memcpy(bucket, entry, map->bucketsz);
+ memcpy(entry, map2->spare, map->bucketsz);
+ }
+ j = (j + 1) & map2->mask;
+ entry->dib += 1;
+ }
+ }
+ map->free(map->buckets);
+ map->buckets = map2->buckets;
+ map->nbuckets = map2->nbuckets;
+ map->mask = map2->mask;
+ map->growat = map2->growat;
+ map->shrinkat = map2->shrinkat;
+ map->free(map2);
+ return true;
+}
+
+static bool resize(struct hashmap *map, size_t new_cap) {
+ return resize0(map, new_cap);
+}
+
+// hashmap_set_with_hash works like hashmap_set but you provide your
+// own hash. The 'hash' callback provided to the hashmap_new function
+// will not be called
+const void *hashmap_set_with_hash(struct hashmap *map, const void *item,
+ uint64_t hash)
+{
+ hash = clip_hash(hash);
+ map->oom = false;
+ if (map->count >= map->growat) {
+ if (!resize(map, map->nbuckets*(1<growpower))) {
+ map->oom = true;
+ return NULL;
+ }
+ }
+
+ struct bucket *entry = map->edata;
+ entry->hash = hash;
+ entry->dib = 1;
+ void *eitem = bucket_item(entry);
+ memcpy(eitem, item, map->elsize);
+
+ void *bitem;
+ size_t i = entry->hash & map->mask;
+ while(1) {
+ struct bucket *bucket = bucket_at(map, i);
+ if (bucket->dib == 0) {
+ memcpy(bucket, entry, map->bucketsz);
+ map->count++;
+ return NULL;
+ }
+ bitem = bucket_item(bucket);
+ if (entry->hash == bucket->hash && (!map->compare ||
+ map->compare(eitem, bitem, map->udata) == 0))
+ {
+ memcpy(map->spare, bitem, map->elsize);
+ memcpy(bitem, eitem, map->elsize);
+ return map->spare;
+ }
+ if (bucket->dib < entry->dib) {
+ memcpy(map->spare, bucket, map->bucketsz);
+ memcpy(bucket, entry, map->bucketsz);
+ memcpy(entry, map->spare, map->bucketsz);
+ eitem = bucket_item(entry);
+ }
+ i = (i + 1) & map->mask;
+ entry->dib += 1;
+ }
+}
+
+// hashmap_set inserts or replaces an item in the hash map. If an item is
+// replaced then it is returned otherwise NULL is returned. This operation
+// may allocate memory. If the system is unable to allocate additional
+// memory then NULL is returned and hashmap_oom() returns true.
+const void *hashmap_set(struct hashmap *map, const void *item) {
+ return hashmap_set_with_hash(map, item, get_hash(map, item));
+}
+
+// hashmap_get_with_hash works like hashmap_get but you provide your
+// own hash. The 'hash' callback provided to the hashmap_new function
+// will not be called
+const void *hashmap_get_with_hash(struct hashmap *map, const void *key,
+ uint64_t hash)
+{
+ hash = clip_hash(hash);
+ size_t i = hash & map->mask;
+ while(1) {
+ struct bucket *bucket = bucket_at(map, i);
+ if (!bucket->dib) return NULL;
+ if (bucket->hash == hash) {
+ void *bitem = bucket_item(bucket);
+ if (!map->compare || map->compare(key, bitem, map->udata) == 0) {
+ return bitem;
+ }
+ }
+ i = (i + 1) & map->mask;
+ }
+}
+
+// hashmap_get returns the item based on the provided key. If the item is not
+// found then NULL is returned.
+const void *hashmap_get(struct hashmap *map, const void *key) {
+ return hashmap_get_with_hash(map, key, get_hash(map, key));
+}
+
+// hashmap_probe returns the item in the bucket at position or NULL if an item
+// is not set for that bucket. The position is 'moduloed' by the number of
+// buckets in the hashmap.
+const void *hashmap_probe(struct hashmap *map, uint64_t position) {
+ size_t i = position & map->mask;
+ struct bucket *bucket = bucket_at(map, i);
+ if (!bucket->dib) {
+ return NULL;
+ }
+ return bucket_item(bucket);
+}
+
+// hashmap_delete_with_hash works like hashmap_delete but you provide your
+// own hash. The 'hash' callback provided to the hashmap_new function
+// will not be called
+const void *hashmap_delete_with_hash(struct hashmap *map, const void *key,
+ uint64_t hash)
+{
+ hash = clip_hash(hash);
+ map->oom = false;
+ size_t i = hash & map->mask;
+ while(1) {
+ struct bucket *bucket = bucket_at(map, i);
+ if (!bucket->dib) {
+ return NULL;
+ }
+ void *bitem = bucket_item(bucket);
+ if (bucket->hash == hash && (!map->compare ||
+ map->compare(key, bitem, map->udata) == 0))
+ {
+ memcpy(map->spare, bitem, map->elsize);
+ bucket->dib = 0;
+ while(1) {
+ struct bucket *prev = bucket;
+ i = (i + 1) & map->mask;
+ bucket = bucket_at(map, i);
+ if (bucket->dib <= 1) {
+ prev->dib = 0;
+ break;
+ }
+ memcpy(prev, bucket, map->bucketsz);
+ prev->dib--;
+ }
+ map->count--;
+ if (map->nbuckets > map->cap && map->count <= map->shrinkat) {
+ // Ignore the return value. It's ok for the resize operation to
+ // fail to allocate enough memory because a shrink operation
+ // does not change the integrity of the data.
+ resize(map, map->nbuckets/2);
+ }
+ return map->spare;
+ }
+ i = (i + 1) & map->mask;
+ }
+}
+
+// hashmap_delete removes an item from the hash map and returns it. If the
+// item is not found then NULL is returned.
+const void *hashmap_delete(struct hashmap *map, const void *key) {
+ return hashmap_delete_with_hash(map, key, get_hash(map, key));
+}
+
+// hashmap_count returns the number of items in the hash map.
+size_t hashmap_count(struct hashmap *map) {
+ return map->count;
+}
+
+// hashmap_free frees the hash map
+// Every item is called with the element-freeing function given in hashmap_new,
+// if present, to free any data referenced in the elements of the hashmap.
+void hashmap_free(struct hashmap *map) {
+ if (!map) return;
+ free_elements(map);
+ map->free(map->buckets);
+ map->free(map);
+}
+
+// hashmap_oom returns true if the last hashmap_set() call failed due to the
+// system being out of memory.
+bool hashmap_oom(struct hashmap *map) {
+ return map->oom;
+}
+
+// hashmap_scan iterates over all items in the hash map
+// Param `iter` can return false to stop iteration early.
+// Returns false if the iteration has been stopped early.
+bool hashmap_scan(struct hashmap *map,
+ bool (*iter)(const void *item, void *udata), void *udata)
+{
+ for (size_t i = 0; i < map->nbuckets; i++) {
+ struct bucket *bucket = bucket_at(map, i);
+ if (bucket->dib && !iter(bucket_item(bucket), udata)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// hashmap_iter iterates one key at a time yielding a reference to an
+// entry at each iteration. Useful to write simple loops and avoid writing
+// dedicated callbacks and udata structures, as in hashmap_scan.
+//
+// map is a hash map handle. i is a pointer to a size_t cursor that
+// should be initialized to 0 at the beginning of the loop. item is a void
+// pointer pointer that is populated with the retrieved item. Note that this
+// is NOT a copy of the item stored in the hash map and can be directly
+// modified.
+//
+// Note that if hashmap_delete() is called on the hashmap being iterated,
+// the buckets are rearranged and the iterator must be reset to 0, otherwise
+// unexpected results may be returned after deletion.
+//
+// This function has not been tested for thread safety.
+//
+// The function returns true if an item was retrieved; false if the end of the
+// iteration has been reached.
+bool hashmap_iter(struct hashmap *map, size_t *i, void **item) {
+ struct bucket *bucket;
+ do {
+ if (*i >= map->nbuckets) return false;
+ bucket = bucket_at(map, *i);
+ (*i)++;
+ } while (!bucket->dib);
+ *item = bucket_item(bucket);
+ return true;
+}
+
+
+//-----------------------------------------------------------------------------
+// SipHash reference C implementation
+//
+// Copyright (c) 2012-2016 Jean-Philippe Aumasson
+//
+// Copyright (c) 2012-2014 Daniel J. Bernstein
+//
+// To the extent possible under law, the author(s) have dedicated all copyright
+// and related and neighboring rights to this software to the public domain
+// worldwide. This software is distributed without any warranty.
+//
+// You should have received a copy of the CC0 Public Domain Dedication along
+// with this software. If not, see
+// .
+//
+// default: SipHash-2-4
+//-----------------------------------------------------------------------------
+static uint64_t SIP64(const uint8_t *in, const size_t inlen, uint64_t seed0,
+ uint64_t seed1)
+{
+#define U8TO64_LE(p) \
+ { (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \
+ ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \
+ ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \
+ ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) }
+#define U64TO8_LE(p, v) \
+ { U32TO8_LE((p), (uint32_t)((v))); \
+ U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); }
+#define U32TO8_LE(p, v) \
+ { (p)[0] = (uint8_t)((v)); \
+ (p)[1] = (uint8_t)((v) >> 8); \
+ (p)[2] = (uint8_t)((v) >> 16); \
+ (p)[3] = (uint8_t)((v) >> 24); }
+#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b))))
+#define SIPROUND \
+ { v0 += v1; v1 = ROTL(v1, 13); \
+ v1 ^= v0; v0 = ROTL(v0, 32); \
+ v2 += v3; v3 = ROTL(v3, 16); \
+ v3 ^= v2; \
+ v0 += v3; v3 = ROTL(v3, 21); \
+ v3 ^= v0; \
+ v2 += v1; v1 = ROTL(v1, 17); \
+ v1 ^= v2; v2 = ROTL(v2, 32); }
+ uint64_t k0 = U8TO64_LE((uint8_t*)&seed0);
+ uint64_t k1 = U8TO64_LE((uint8_t*)&seed1);
+ uint64_t v3 = UINT64_C(0x7465646279746573) ^ k1;
+ uint64_t v2 = UINT64_C(0x6c7967656e657261) ^ k0;
+ uint64_t v1 = UINT64_C(0x646f72616e646f6d) ^ k1;
+ uint64_t v0 = UINT64_C(0x736f6d6570736575) ^ k0;
+ const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t));
+ for (; in != end; in += 8) {
+ uint64_t m = U8TO64_LE(in);
+ v3 ^= m;
+ SIPROUND; SIPROUND;
+ v0 ^= m;
+ }
+ const int left = inlen & 7;
+ uint64_t b = ((uint64_t)inlen) << 56;
+ switch (left) {
+ case 7: b |= ((uint64_t)in[6]) << 48; /* fall through */
+ case 6: b |= ((uint64_t)in[5]) << 40; /* fall through */
+ case 5: b |= ((uint64_t)in[4]) << 32; /* fall through */
+ case 4: b |= ((uint64_t)in[3]) << 24; /* fall through */
+ case 3: b |= ((uint64_t)in[2]) << 16; /* fall through */
+ case 2: b |= ((uint64_t)in[1]) << 8; /* fall through */
+ case 1: b |= ((uint64_t)in[0]); break;
+ case 0: break;
+ }
+ v3 ^= b;
+ SIPROUND; SIPROUND;
+ v0 ^= b;
+ v2 ^= 0xff;
+ SIPROUND; SIPROUND; SIPROUND; SIPROUND;
+ b = v0 ^ v1 ^ v2 ^ v3;
+ uint64_t out = 0;
+ U64TO8_LE((uint8_t*)&out, b);
+ return out;
+}
+
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+//
+// Murmur3_86_128
+//-----------------------------------------------------------------------------
+static uint64_t MM86128(const void *key, const int len, uint32_t seed) {
+#define ROTL32(x, r) ((x << r) | (x >> (32 - r)))
+#define FMIX32(h) h^=h>>16; h*=0x85ebca6b; h^=h>>13; h*=0xc2b2ae35; h^=h>>16;
+ const uint8_t * data = (const uint8_t*)key;
+ const int nblocks = len / 16;
+ uint32_t h1 = seed;
+ uint32_t h2 = seed;
+ uint32_t h3 = seed;
+ uint32_t h4 = seed;
+ uint32_t c1 = 0x239b961b;
+ uint32_t c2 = 0xab0e9789;
+ uint32_t c3 = 0x38b34ae5;
+ uint32_t c4 = 0xa1e38b93;
+ const uint32_t * blocks = (const uint32_t *)(data + nblocks*16);
+ for (int i = -nblocks; i; i++) {
+ uint32_t k1 = blocks[i*4+0];
+ uint32_t k2 = blocks[i*4+1];
+ uint32_t k3 = blocks[i*4+2];
+ uint32_t k4 = blocks[i*4+3];
+ k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+ h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;
+ k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+ h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;
+ k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+ h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;
+ k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+ h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;
+ }
+ const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+ uint32_t k1 = 0;
+ uint32_t k2 = 0;
+ uint32_t k3 = 0;
+ uint32_t k4 = 0;
+ switch(len & 15) {
+ case 15: k4 ^= tail[14] << 16; /* fall through */
+ case 14: k4 ^= tail[13] << 8; /* fall through */
+ case 13: k4 ^= tail[12] << 0;
+ k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4;
+ /* fall through */
+ case 12: k3 ^= tail[11] << 24; /* fall through */
+ case 11: k3 ^= tail[10] << 16; /* fall through */
+ case 10: k3 ^= tail[ 9] << 8; /* fall through */
+ case 9: k3 ^= tail[ 8] << 0;
+ k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3;
+ /* fall through */
+ case 8: k2 ^= tail[ 7] << 24; /* fall through */
+ case 7: k2 ^= tail[ 6] << 16; /* fall through */
+ case 6: k2 ^= tail[ 5] << 8; /* fall through */
+ case 5: k2 ^= tail[ 4] << 0;
+ k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2;
+ /* fall through */
+ case 4: k1 ^= tail[ 3] << 24; /* fall through */
+ case 3: k1 ^= tail[ 2] << 16; /* fall through */
+ case 2: k1 ^= tail[ 1] << 8; /* fall through */
+ case 1: k1 ^= tail[ 0] << 0;
+ k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+ /* fall through */
+ };
+ h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
+ h1 += h2; h1 += h3; h1 += h4;
+ h2 += h1; h3 += h1; h4 += h1;
+ FMIX32(h1); FMIX32(h2); FMIX32(h3); FMIX32(h4);
+ h1 += h2; h1 += h3; h1 += h4;
+ h2 += h1; h3 += h1; h4 += h1;
+ return (((uint64_t)h2)<<32)|h1;
+}
+
+//-----------------------------------------------------------------------------
+// xxHash Library
+// Copyright (c) 2012-2021 Yann Collet
+// All rights reserved.
+//
+// BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
+//
+// xxHash3
+//-----------------------------------------------------------------------------
+#define XXH_PRIME_1 11400714785074694791ULL
+#define XXH_PRIME_2 14029467366897019727ULL
+#define XXH_PRIME_3 1609587929392839161ULL
+#define XXH_PRIME_4 9650029242287828579ULL
+#define XXH_PRIME_5 2870177450012600261ULL
+
+static uint64_t XXH_read64(const void* memptr) {
+ uint64_t val;
+ memcpy(&val, memptr, sizeof(val));
+ return val;
+}
+
+static uint32_t XXH_read32(const void* memptr) {
+ uint32_t val;
+ memcpy(&val, memptr, sizeof(val));
+ return val;
+}
+
+static uint64_t XXH_rotl64(uint64_t x, int r) {
+ return (x << r) | (x >> (64 - r));
+}
+
+static uint64_t xxh3(const void* data, size_t len, uint64_t seed) {
+ const uint8_t* p = (const uint8_t*)data;
+ const uint8_t* const end = p + len;
+ uint64_t h64;
+
+ if (len >= 32) {
+ const uint8_t* const limit = end - 32;
+ uint64_t v1 = seed + XXH_PRIME_1 + XXH_PRIME_2;
+ uint64_t v2 = seed + XXH_PRIME_2;
+ uint64_t v3 = seed + 0;
+ uint64_t v4 = seed - XXH_PRIME_1;
+
+ do {
+ v1 += XXH_read64(p) * XXH_PRIME_2;
+ v1 = XXH_rotl64(v1, 31);
+ v1 *= XXH_PRIME_1;
+
+ v2 += XXH_read64(p + 8) * XXH_PRIME_2;
+ v2 = XXH_rotl64(v2, 31);
+ v2 *= XXH_PRIME_1;
+
+ v3 += XXH_read64(p + 16) * XXH_PRIME_2;
+ v3 = XXH_rotl64(v3, 31);
+ v3 *= XXH_PRIME_1;
+
+ v4 += XXH_read64(p + 24) * XXH_PRIME_2;
+ v4 = XXH_rotl64(v4, 31);
+ v4 *= XXH_PRIME_1;
+
+ p += 32;
+ } while (p <= limit);
+
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) +
+ XXH_rotl64(v4, 18);
+
+ v1 *= XXH_PRIME_2;
+ v1 = XXH_rotl64(v1, 31);
+ v1 *= XXH_PRIME_1;
+ h64 ^= v1;
+ h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4;
+
+ v2 *= XXH_PRIME_2;
+ v2 = XXH_rotl64(v2, 31);
+ v2 *= XXH_PRIME_1;
+ h64 ^= v2;
+ h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4;
+
+ v3 *= XXH_PRIME_2;
+ v3 = XXH_rotl64(v3, 31);
+ v3 *= XXH_PRIME_1;
+ h64 ^= v3;
+ h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4;
+
+ v4 *= XXH_PRIME_2;
+ v4 = XXH_rotl64(v4, 31);
+ v4 *= XXH_PRIME_1;
+ h64 ^= v4;
+ h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4;
+ }
+ else {
+ h64 = seed + XXH_PRIME_5;
+ }
+
+ h64 += (uint64_t)len;
+
+ while (p + 8 <= end) {
+ uint64_t k1 = XXH_read64(p);
+ k1 *= XXH_PRIME_2;
+ k1 = XXH_rotl64(k1, 31);
+ k1 *= XXH_PRIME_1;
+ h64 ^= k1;
+ h64 = XXH_rotl64(h64, 27) * XXH_PRIME_1 + XXH_PRIME_4;
+ p += 8;
+ }
+
+ if (p + 4 <= end) {
+ h64 ^= (uint64_t)(XXH_read32(p)) * XXH_PRIME_1;
+ h64 = XXH_rotl64(h64, 23) * XXH_PRIME_2 + XXH_PRIME_3;
+ p += 4;
+ }
+
+ while (p < end) {
+ h64 ^= (*p) * XXH_PRIME_5;
+ h64 = XXH_rotl64(h64, 11) * XXH_PRIME_1;
+ p++;
+ }
+
+ h64 ^= h64 >> 33;
+ h64 *= XXH_PRIME_2;
+ h64 ^= h64 >> 29;
+ h64 *= XXH_PRIME_3;
+ h64 ^= h64 >> 32;
+
+ return h64;
+}
+
+// hashmap_sip returns a hash value for `data` using SipHash-2-4.
+uint64_t hashmap_sip(const void *data, size_t len, uint64_t seed0,
+ uint64_t seed1)
+{
+ return SIP64((uint8_t*)data, len, seed0, seed1);
+}
+
+// hashmap_murmur returns a hash value for `data` using Murmur3_86_128.
+uint64_t hashmap_murmur(const void *data, size_t len, uint64_t seed0,
+ uint64_t seed1)
+{
+ (void)seed1;
+ return MM86128(data, len, seed0);
+}
+
+uint64_t hashmap_xxhash3(const void *data, size_t len, uint64_t seed0,
+ uint64_t seed1)
+{
+ (void)seed1;
+ return xxh3(data, len ,seed0);
+}
+
+//==============================================================================
+// TESTS AND BENCHMARKS
+// $ cc -DHASHMAP_TEST hashmap.c && ./a.out # run tests
+// $ cc -DHASHMAP_TEST -O3 hashmap.c && BENCH=1 ./a.out # run benchmarks
+//==============================================================================
+#ifdef HASHMAP_TEST
+
+static size_t deepcount(struct hashmap *map) {
+ size_t count = 0;
+ for (size_t i = 0; i < map->nbuckets; i++) {
+ if (bucket_at(map, i)->dib) {
+ count++;
+ }
+ }
+ return count;
+}
+
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wpedantic"
+#endif
+#ifdef __clang__
+#pragma GCC diagnostic ignored "-Wunknown-warning-option"
+#pragma GCC diagnostic ignored "-Wcompound-token-split-by-macro"
+#pragma GCC diagnostic ignored "-Wgnu-statement-expression-from-macro-expansion"
+#endif
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#endif
+
+#include
+#include
+#include
+#include
+#include
+#include "hashmap.h"
+
+static bool rand_alloc_fail = false;
+static int rand_alloc_fail_odds = 3; // 1 in 3 chance malloc will fail.
+static uintptr_t total_allocs = 0;
+static uintptr_t total_mem = 0;
+
+static void *xmalloc(size_t size) {
+ if (rand_alloc_fail && rand()%rand_alloc_fail_odds == 0) {
+ return NULL;
+ }
+ void *mem = malloc(sizeof(uintptr_t)+size);
+ assert(mem);
+ *(uintptr_t*)mem = size;
+ total_allocs++;
+ total_mem += size;
+ return (char*)mem+sizeof(uintptr_t);
+}
+
+static void xfree(void *ptr) {
+ if (ptr) {
+ total_mem -= *(uintptr_t*)((char*)ptr-sizeof(uintptr_t));
+ free((char*)ptr-sizeof(uintptr_t));
+ total_allocs--;
+ }
+}
+
+static void shuffle(void *array, size_t numels, size_t elsize) {
+ char tmp[elsize];
+ char *arr = array;
+ for (size_t i = 0; i < numels - 1; i++) {
+ int j = i + rand() / (RAND_MAX / (numels - i) + 1);
+ memcpy(tmp, arr + j * elsize, elsize);
+ memcpy(arr + j * elsize, arr + i * elsize, elsize);
+ memcpy(arr + i * elsize, tmp, elsize);
+ }
+}
+
+static bool iter_ints(const void *item, void *udata) {
+ int *vals = *(int**)udata;
+ vals[*(int*)item] = 1;
+ return true;
+}
+
+static int compare_ints_udata(const void *a, const void *b, void *udata) {
+ return *(int*)a - *(int*)b;
+}
+
+static int compare_strs(const void *a, const void *b, void *udata) {
+ return strcmp(*(char**)a, *(char**)b);
+}
+
+static uint64_t hash_int(const void *item, uint64_t seed0, uint64_t seed1) {
+ return hashmap_xxhash3(item, sizeof(int), seed0, seed1);
+ // return hashmap_sip(item, sizeof(int), seed0, seed1);
+ // return hashmap_murmur(item, sizeof(int), seed0, seed1);
+}
+
+static uint64_t hash_str(const void *item, uint64_t seed0, uint64_t seed1) {
+ return hashmap_xxhash3(*(char**)item, strlen(*(char**)item), seed0, seed1);
+ // return hashmap_sip(*(char**)item, strlen(*(char**)item), seed0, seed1);
+ // return hashmap_murmur(*(char**)item, strlen(*(char**)item), seed0, seed1);
+}
+
+static void free_str(void *item) {
+ xfree(*(char**)item);
+}
+
+static void all(void) {
+ int seed = getenv("SEED")?atoi(getenv("SEED")):time(NULL);
+ int N = getenv("N")?atoi(getenv("N")):2000;
+ printf("seed=%d, count=%d, item_size=%zu\n", seed, N, sizeof(int));
+ srand(seed);
+
+ rand_alloc_fail = true;
+
+ // test sip and murmur hashes
+ assert(hashmap_sip("hello", 5, 1, 2) == 2957200328589801622);
+ assert(hashmap_murmur("hello", 5, 1, 2) == 1682575153221130884);
+ assert(hashmap_xxhash3("hello", 5, 1, 2) == 2584346877953614258);
+
+ int *vals;
+ while (!(vals = xmalloc(N * sizeof(int)))) {}
+ for (int i = 0; i < N; i++) {
+ vals[i] = i;
+ }
+
+ struct hashmap *map;
+
+ while (!(map = hashmap_new(sizeof(int), 0, seed, seed,
+ hash_int, compare_ints_udata, NULL, NULL))) {}
+ shuffle(vals, N, sizeof(int));
+ for (int i = 0; i < N; i++) {
+ // // printf("== %d ==\n", vals[i]);
+ assert(map->count == (size_t)i);
+ assert(map->count == hashmap_count(map));
+ assert(map->count == deepcount(map));
+ const int *v;
+ assert(!hashmap_get(map, &vals[i]));
+ assert(!hashmap_delete(map, &vals[i]));
+ while (true) {
+ assert(!hashmap_set(map, &vals[i]));
+ if (!hashmap_oom(map)) {
+ break;
+ }
+ }
+
+ for (int j = 0; j < i; j++) {
+ v = hashmap_get(map, &vals[j]);
+ assert(v && *v == vals[j]);
+ }
+ while (true) {
+ v = hashmap_set(map, &vals[i]);
+ if (!v) {
+ assert(hashmap_oom(map));
+ continue;
+ } else {
+ assert(!hashmap_oom(map));
+ assert(v && *v == vals[i]);
+ break;
+ }
+ }
+ v = hashmap_get(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ v = hashmap_delete(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ assert(!hashmap_get(map, &vals[i]));
+ assert(!hashmap_delete(map, &vals[i]));
+ assert(!hashmap_set(map, &vals[i]));
+ assert(map->count == (size_t)(i+1));
+ assert(map->count == hashmap_count(map));
+ assert(map->count == deepcount(map));
+ }
+
+ int *vals2;
+ while (!(vals2 = xmalloc(N * sizeof(int)))) {}
+ memset(vals2, 0, N * sizeof(int));
+ assert(hashmap_scan(map, iter_ints, &vals2));
+
+ // Test hashmap_iter. This does the same as hashmap_scan above.
+ size_t iter = 0;
+ void *iter_val;
+ while (hashmap_iter (map, &iter, &iter_val)) {
+ assert (iter_ints(iter_val, &vals2));
+ }
+ for (int i = 0; i < N; i++) {
+ assert(vals2[i] == 1);
+ }
+ xfree(vals2);
+
+ shuffle(vals, N, sizeof(int));
+ for (int i = 0; i < N; i++) {
+ const int *v;
+ v = hashmap_delete(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ assert(!hashmap_get(map, &vals[i]));
+ assert(map->count == (size_t)(N-i-1));
+ assert(map->count == hashmap_count(map));
+ assert(map->count == deepcount(map));
+ for (int j = N-1; j > i; j--) {
+ v = hashmap_get(map, &vals[j]);
+ assert(v && *v == vals[j]);
+ }
+ }
+
+ for (int i = 0; i < N; i++) {
+ while (true) {
+ assert(!hashmap_set(map, &vals[i]));
+ if (!hashmap_oom(map)) {
+ break;
+ }
+ }
+ }
+
+ assert(map->count != 0);
+ size_t prev_cap = map->cap;
+ hashmap_clear(map, true);
+ assert(prev_cap < map->cap);
+ assert(map->count == 0);
+
+
+ for (int i = 0; i < N; i++) {
+ while (true) {
+ assert(!hashmap_set(map, &vals[i]));
+ if (!hashmap_oom(map)) {
+ break;
+ }
+ }
+ }
+
+ prev_cap = map->cap;
+ hashmap_clear(map, false);
+ assert(prev_cap == map->cap);
+
+ hashmap_free(map);
+
+ xfree(vals);
+
+
+ while (!(map = hashmap_new(sizeof(char*), 0, seed, seed,
+ hash_str, compare_strs, free_str, NULL)));
+
+ for (int i = 0; i < N; i++) {
+ char *str;
+ while (!(str = xmalloc(16)));
+ snprintf(str, 16, "s%i", i);
+ while(!hashmap_set(map, &str));
+ }
+
+ hashmap_clear(map, false);
+ assert(hashmap_count(map) == 0);
+
+ for (int i = 0; i < N; i++) {
+ char *str;
+ while (!(str = xmalloc(16)));
+ snprintf(str, 16, "s%i", i);
+ while(!hashmap_set(map, &str));
+ }
+
+ hashmap_free(map);
+
+ if (total_allocs != 0) {
+ fprintf(stderr, "total_allocs: expected 0, got %lu\n", total_allocs);
+ exit(1);
+ }
+}
+
+#define bench(name, N, code) {{ \
+ if (strlen(name) > 0) { \
+ printf("%-14s ", name); \
+ } \
+ size_t tmem = total_mem; \
+ size_t tallocs = total_allocs; \
+ uint64_t bytes = 0; \
+ clock_t begin = clock(); \
+ for (int i = 0; i < N; i++) { \
+ (code); \
+ } \
+ clock_t end = clock(); \
+ double elapsed_secs = (double)(end - begin) / CLOCKS_PER_SEC; \
+ double bytes_sec = (double)bytes/elapsed_secs; \
+ printf("%d ops in %.3f secs, %.0f ns/op, %.0f op/sec", \
+ N, elapsed_secs, \
+ elapsed_secs/(double)N*1e9, \
+ (double)N/elapsed_secs \
+ ); \
+ if (bytes > 0) { \
+ printf(", %.1f GB/sec", bytes_sec/1024/1024/1024); \
+ } \
+ if (total_mem > tmem) { \
+ size_t used_mem = total_mem-tmem; \
+ printf(", %.2f bytes/op", (double)used_mem/N); \
+ } \
+ if (total_allocs > tallocs) { \
+ size_t used_allocs = total_allocs-tallocs; \
+ printf(", %.2f allocs/op", (double)used_allocs/N); \
+ } \
+ printf("\n"); \
+}}
+
+static void benchmarks(void) {
+ int seed = getenv("SEED")?atoi(getenv("SEED")):time(NULL);
+ int N = getenv("N")?atoi(getenv("N")):5000000;
+ printf("seed=%d, count=%d, item_size=%zu\n", seed, N, sizeof(int));
+ srand(seed);
+
+
+ int *vals = xmalloc(N * sizeof(int));
+ for (int i = 0; i < N; i++) {
+ vals[i] = i;
+ }
+
+ shuffle(vals, N, sizeof(int));
+
+ struct hashmap *map;
+ shuffle(vals, N, sizeof(int));
+
+ map = hashmap_new(sizeof(int), 0, seed, seed, hash_int, compare_ints_udata,
+ NULL, NULL);
+ bench("set", N, {
+ const int *v = hashmap_set(map, &vals[i]);
+ assert(!v);
+ })
+ shuffle(vals, N, sizeof(int));
+ bench("get", N, {
+ const int *v = hashmap_get(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ })
+ shuffle(vals, N, sizeof(int));
+ bench("delete", N, {
+ const int *v = hashmap_delete(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ })
+ hashmap_free(map);
+
+ map = hashmap_new(sizeof(int), N, seed, seed, hash_int, compare_ints_udata,
+ NULL, NULL);
+ bench("set (cap)", N, {
+ const int *v = hashmap_set(map, &vals[i]);
+ assert(!v);
+ })
+ shuffle(vals, N, sizeof(int));
+ bench("get (cap)", N, {
+ const int *v = hashmap_get(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ })
+ shuffle(vals, N, sizeof(int));
+ bench("delete (cap)" , N, {
+ const int *v = hashmap_delete(map, &vals[i]);
+ assert(v && *v == vals[i]);
+ })
+
+ hashmap_free(map);
+
+
+ xfree(vals);
+
+ if (total_allocs != 0) {
+ fprintf(stderr, "total_allocs: expected 0, got %lu\n", total_allocs);
+ exit(1);
+ }
+}
+
+int main(void) {
+ hashmap_set_allocator(xmalloc, xfree);
+
+ if (getenv("BENCH")) {
+ printf("Running hashmap.c benchmarks...\n");
+ benchmarks();
+ } else {
+ printf("Running hashmap.c tests...\n");
+ all();
+ printf("PASSED\n");
+ }
+}
+
+
+#endif
+
+
+
diff --git a/hashmap/hashmap.h b/hashmap/hashmap.h
new file mode 100644
index 00000000..e22990e0
--- /dev/null
+++ b/hashmap/hashmap.h
@@ -0,0 +1,62 @@
+// Copyright 2020 Joshua J Baker. All rights reserved.
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+#ifndef HASHMAP_H
+#define HASHMAP_H
+
+#include
+#include
+#include
+
+#if defined(__cplusplus)
+extern "C" {
+#endif // __cplusplus
+
+struct hashmap;
+
+struct hashmap *hashmap_new(size_t elsize, size_t cap, uint64_t seed0,
+ uint64_t seed1,
+ uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
+ int (*compare)(const void *a, const void *b, void *udata),
+ void (*elfree)(void *item),
+ void *udata);
+
+struct hashmap *hashmap_new_with_allocator(void *(*malloc)(size_t),
+ void *(*realloc)(void *, size_t), void (*free)(void*), size_t elsize,
+ size_t cap, uint64_t seed0, uint64_t seed1,
+ uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1),
+ int (*compare)(const void *a, const void *b, void *udata),
+ void (*elfree)(void *item),
+ void *udata);
+
+void hashmap_free(struct hashmap *map);
+void hashmap_clear(struct hashmap *map, bool update_cap);
+size_t hashmap_count(struct hashmap *map);
+bool hashmap_oom(struct hashmap *map);
+const void *hashmap_get(struct hashmap *map, const void *item);
+const void *hashmap_set(struct hashmap *map, const void *item);
+const void *hashmap_delete(struct hashmap *map, const void *item);
+const void *hashmap_probe(struct hashmap *map, uint64_t position);
+bool hashmap_scan(struct hashmap *map, bool (*iter)(const void *item, void *udata), void *udata);
+bool hashmap_iter(struct hashmap *map, size_t *i, void **item);
+
+uint64_t hashmap_sip(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
+uint64_t hashmap_murmur(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
+uint64_t hashmap_xxhash3(const void *data, size_t len, uint64_t seed0, uint64_t seed1);
+
+const void *hashmap_get_with_hash(struct hashmap *map, const void *key, uint64_t hash);
+const void *hashmap_delete_with_hash(struct hashmap *map, const void *key, uint64_t hash);
+const void *hashmap_set_with_hash(struct hashmap *map, const void *item, uint64_t hash);
+void hashmap_set_grow_by_power(struct hashmap *map, size_t power);
+void hashmap_set_load_factor(struct hashmap *map, double load_factor);
+
+
+// DEPRECATED: use `hashmap_new_with_allocator`
+void hashmap_set_allocator(void *(*malloc)(size_t), void (*free)(void*));
+
+#if defined(__cplusplus)
+}
+#endif // __cplusplus
+
+#endif // HASHMAP_H
diff --git a/qiniu/base.c b/qiniu/base.c
index 76b1f313..00adda91 100644
--- a/qiniu/base.c
+++ b/qiniu/base.c
@@ -130,7 +130,7 @@ static char *Qiniu_escape(const char *s, escapeMode mode, Qiniu_Bool *fesc)
{
int spaceCount = 0;
int hexCount = 0;
- int i, j, len = strlen(s);
+ size_t i, j, len = strlen(s);
int c;
char *t;
@@ -674,8 +674,7 @@ Qiniu_Writer Qiniu_FILE_Writer(FILE *fp)
/*============================================================================*/
/* func Qiniu_Copy */
-Qiniu_Error Qiniu_OK = {
- 200, "OK"};
+Qiniu_Error Qiniu_OK = {200, "OK"};
Qiniu_Error Qiniu_Copy(Qiniu_Writer w, Qiniu_Reader r, void *buf, size_t n, Qiniu_Int64 *ret)
{
diff --git a/qiniu/base.h b/qiniu/base.h
index e57cac61..d3d85875 100644
--- a/qiniu/base.h
+++ b/qiniu/base.h
@@ -43,7 +43,9 @@ typedef off_t Qiniu_Off_T;
#endif
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -357,7 +359,9 @@ typedef struct stat Qiniu_FileInfo;
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/base_io.c b/qiniu/base_io.c
index 42ea25e3..572fef29 100644
--- a/qiniu/base_io.c
+++ b/qiniu/base_io.c
@@ -135,7 +135,7 @@ static ssize_t Qiniu_ReadBuf_ReadAt(Qiniu_ReadBuf *self, void *buf, size_t n, Qi
n = max & (~(size_t)0L);
}
memcpy(buf, self->buf + off, n);
- return n;
+ return (ssize_t)n;
}
Qiniu_Reader Qiniu_BufReader(Qiniu_ReadBuf *self, const char *buf, size_t bytes)
diff --git a/qiniu/cdn.c b/qiniu/cdn.c
index fa74dce9..13d932a8 100644
--- a/qiniu/cdn.c
+++ b/qiniu/cdn.c
@@ -13,7 +13,7 @@ char *Qiniu_CDN_CreateTimestampAntiLeechURL(const char *host, const char *fileNa
char *queryStrEscaped = NULL;
char expireHex[20];
- sprintf(expireHex, "%0llx", deadline);
+ snprintf(expireHex, 20, "%0llx", deadline);
if (queryStr != NULL && strcmp("", queryStr) != 0) {
queryStrEscaped = Qiniu_PathEscape(queryStr, &queryStrEscapeOk);
@@ -611,4 +611,4 @@ void Qiniu_Free_CDNLogListRet(Qiniu_CDN_LogListRet *ret) {
}
Qiniu_Free(ret->data);
}
-}
\ No newline at end of file
+}
diff --git a/qiniu/cdn.h b/qiniu/cdn.h
index efd17924..1a883333 100644
--- a/qiniu/cdn.h
+++ b/qiniu/cdn.h
@@ -14,6 +14,10 @@
#include "macro.h"
#include "http.h"
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
#ifdef __cplusplus
extern "C"
{
@@ -164,6 +168,10 @@ QINIU_DLLAPI extern void Qiniu_Free_CDNLogListRet(Qiniu_CDN_LogListRet *ret);
//=====================================================================
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/qiniu/code.c b/qiniu/code.c
new file mode 100644
index 00000000..c8d00333
--- /dev/null
+++ b/qiniu/code.c
@@ -0,0 +1,47 @@
+#include
+#include "private/code.h"
+
+#define CURLE_WEIRD_SERVER_REPLY 8
+#define CURLE_HTTP2_STREAM 92
+#define CURLE_HTTP3 95
+#define CURLE_QUIC_CONNECT_ERROR 96
+
+Qiniu_Retry_Decision _Qiniu_Should_Retry(int code)
+{
+ if (code / 100 == 4 && code != 406 && code != 429)
+ {
+ return QINIU_DONT_RETRY;
+ }
+ switch (code)
+ {
+ case CURLE_COULDNT_RESOLVE_HOST:
+ case CURLE_COULDNT_CONNECT:
+ case CURLE_WEIRD_SERVER_REPLY:
+ case CURLE_PARTIAL_FILE:
+ case CURLE_UPLOAD_FAILED:
+ case CURLE_OPERATION_TIMEDOUT:
+ case CURLE_SSL_CONNECT_ERROR:
+ case CURLE_SEND_ERROR:
+ case CURLE_RECV_ERROR:
+ case CURLE_SSL_CERTPROBLEM:
+ case CURLE_SSL_CIPHER:
+ case CURLE_PEER_FAILED_VERIFICATION:
+ case CURLE_HTTP2_STREAM:
+ case CURLE_HTTP3:
+ case CURLE_QUIC_CONNECT_ERROR:
+ case 406:
+ case 429:
+ case 500:
+ case 502:
+ case 503:
+ case 504:
+ case 509:
+ case 571:
+ case 573:
+ case 599:
+ return QINIU_TRY_NEXT_DOMAIN;
+
+ default:
+ return QINIU_DONT_RETRY;
+ }
+}
diff --git a/qiniu/conf.c b/qiniu/conf.c
index 5090a9dc..e356220b 100644
--- a/qiniu/conf.c
+++ b/qiniu/conf.c
@@ -22,8 +22,9 @@ const char *QINIU_RS_HOST = "http://rs.qbox.me";
const char *QINIU_RSF_HOST = "http://rsf.qbox.me";
const char *QINIU_API_HOST = "http://api.qiniu.com";
const char *QINIU_FUSION_HOST = "http://fusion.qiniuapi.com";
-const char *QINIU_UC_HOST = "http://kodo-config.qiniuapi.com";
-const char *QINIU_UC_HOST_BACKUP = "http://uc.qbox.me";
+const char *QINIU_UC_HOST = "http://uc.qiniuapi.com";
+const char *QINIU_UC_HOST_BACKUP = "http://kodo-config.qiniuapi.com";
+const char *QINIU_UC_HOST_BACKUP_2 = "http://uc.qbox.me";
// 默认华东机房
//
diff --git a/qiniu/conf.h b/qiniu/conf.h
index b8aaba60..f662d530 100644
--- a/qiniu/conf.h
+++ b/qiniu/conf.h
@@ -13,6 +13,10 @@
#include "macro.h"
#include "base.h"
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
#ifdef __cplusplus
extern "C"
{
@@ -64,6 +68,10 @@ extern "C"
// 不再推荐使用，建议使用 Qiniu_Use_Region("cn-east-2") 方法替代
QINIU_DLLAPI extern void Qiniu_Use_Zone_Cn_East_2(Qiniu_Bool useHttps);
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/qiniu/emu_posix.h b/qiniu/emu_posix.h
index 5580865a..fee1acff 100644
--- a/qiniu/emu_posix.h
+++ b/qiniu/emu_posix.h
@@ -14,6 +14,10 @@
#include "macro.h"
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
#ifdef __cplusplus
extern "C"
{
@@ -51,6 +55,10 @@ QINIU_DLLAPI extern char* Qiniu_Posix_strndup(const char *, size_t n);
/*============================================================================*/
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/qiniu/fop.c b/qiniu/fop.c
index 3fdefe90..dc387448 100644
--- a/qiniu/fop.c
+++ b/qiniu/fop.c
@@ -13,8 +13,18 @@
#include "private/region.h"
#include "../cJSON/cJSON.h"
-Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, const char *key,
- char *fops[], int fopCount, const char *pipeline, const char *notifyURL, int force)
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+ void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt);
+
+#ifdef __cplusplus
+}
+#endif
+
+Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_FOP_PfopParams *params)
{
Qiniu_Error err;
cJSON *root;
@@ -25,6 +35,7 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha
char *encodedNotifyURL = NULL;
char *encodedPipeline = NULL;
char *forceStr = NULL;
+ char *typeStr = NULL;
char *url = NULL;
char *body = NULL;
Qiniu_Bool escapeBucketOk;
@@ -34,29 +45,29 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha
Qiniu_Bool escapeNotifyURLOk;
// Add encoded bucket
- encodedBucket = Qiniu_QueryEscape(bucket, &escapeBucketOk);
- encodedKey = Qiniu_QueryEscape(key, &escapeKeyOk);
- fopsStr = Qiniu_String_Join(";", fops, fopCount);
+ encodedBucket = Qiniu_QueryEscape(params->bucket, &escapeBucketOk);
+ encodedKey = Qiniu_QueryEscape(params->key, &escapeKeyOk);
+ fopsStr = Qiniu_String_Join(";", params->fops, params->fopCount);
encodedFops = Qiniu_QueryEscape(fopsStr, &escapeFopsOk);
Qiniu_Free(fopsStr);
- if (pipeline)
+ if (params->pipeline)
{
- encodedPipeline = Qiniu_QueryEscape(pipeline, &escapePipelineOk);
+ encodedPipeline = Qiniu_QueryEscape(params->pipeline, &escapePipelineOk);
}
else
{
encodedPipeline = "";
}
- if (notifyURL)
+ if (params->notifyURL)
{
- encodedNotifyURL = Qiniu_QueryEscape(notifyURL, &escapeNotifyURLOk);
+ encodedNotifyURL = Qiniu_QueryEscape(params->notifyURL, &escapeNotifyURLOk);
}
else
{
encodedNotifyURL = "";
}
- if (force == 1)
+ if (params->force == 1)
{
forceStr = "1";
}
@@ -64,9 +75,18 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha
{
forceStr = "0";
}
+ if (params->type == 1)
+ {
+ typeStr = "1";
+ }
+ else
+ {
+ typeStr = "0";
+ }
body = Qiniu_String_Concat("bucket=", encodedBucket, "&key=", encodedKey, "&fops=", encodedFops,
- "&pipeline=", encodedPipeline, "¬ifyURL=", encodedNotifyURL, "&force=", forceStr, NULL);
+ "&pipeline=", encodedPipeline, "¬ifyURL=", encodedNotifyURL, "&force=", forceStr,
+ "&type=", typeStr, NULL);
if (escapeBucketOk)
{
Qiniu_Free(encodedBucket);
@@ -82,23 +102,22 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha
Qiniu_Free(encodedFops);
}
- if (pipeline && escapePipelineOk)
+ if (params->pipeline && escapePipelineOk)
{
Qiniu_Free(encodedPipeline);
}
- if (notifyURL && escapeNotifyURLOk)
+ if (params->notifyURL && escapeNotifyURLOk)
{
Qiniu_Free(encodedNotifyURL);
}
const char *apiHost;
- err = _Qiniu_Region_Get_Api_Host(self, NULL, bucket, &apiHost);
+ err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost);
if (err.code != 200)
{
- goto error;
+ return err;
}
-
url = Qiniu_String_Concat2(apiHost, "/pfop/");
err = Qiniu_Client_CallWithBuffer(
self,
@@ -107,13 +126,94 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha
body,
strlen(body),
"application/x-www-form-urlencoded");
+ Qiniu_Free((void *)body);
+ Qiniu_Free((void *)url);
if (err.code == 200)
{
ret->persistentId = Qiniu_Json_GetString(root, "persistentId", 0);
}
-error:
- Qiniu_Free(body);
- Qiniu_Free(url);
return err;
+}
+
+Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, const char *key,
+ char *fops[], int fopCount, const char *pipeline, const char *notifyURL, int force)
+{
+ Qiniu_FOP_PfopParams params;
+ Qiniu_Zero(params);
+ params.bucket = bucket;
+ params.key = key;
+ params.fops = (char **)fops;
+ params.fopCount = fopCount;
+ params.pipeline = pipeline;
+ params.notifyURL = notifyURL;
+ params.force = force;
+ return Qiniu_FOP_Pfop_v2(self, ret, ¶ms);
} // Qiniu_FOP_Pfop
+
+Qiniu_Error Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu_FOP_PrefopItemRet *itemsRet, Qiniu_ItemCount *itemsCount, const char *persistentId, Qiniu_ItemCount maxItemsCount)
+{
+ Qiniu_Error err;
+ cJSON *root, *items, *item;
+ Qiniu_ItemCount curIndex;
+ char *encodedPersistentId = NULL, *url = NULL, *creationDateStr = NULL;
+ Qiniu_ReadBuf emptyBodyBuf = {NULL, 0, 0};
+ Qiniu_Reader emptyBody = Qiniu_BufReader(&emptyBodyBuf, 0, 0);
+ Qiniu_Bool escapePersistentIdOk;
+
+ encodedPersistentId = Qiniu_QueryEscape(persistentId, &escapePersistentIdOk);
+
+ const char *apiHost;
+ err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost);
+ if (err.code != 200)
+ {
+ return err;
+ }
+ url = Qiniu_String_Concat(apiHost, "/status/get/prefop?id=", encodedPersistentId, NULL);
+ if (escapePersistentIdOk)
+ {
+ Qiniu_Free(encodedPersistentId);
+ }
+
+ err = Qiniu_Client_CallWithMethod(
+ self,
+ &root,
+ url,
+ emptyBody,
+ 0,
+ "application/x-www-form-urlencoded",
+ "GET",
+ NULL);
+ Qiniu_Free(url);
+ if (err.code != 200)
+ {
+ return err;
+ }
+
+ ret->id = Qiniu_Json_GetString(root, "id", NULL);
+ ret->code = Qiniu_Json_GetInt(root, "code", 0);
+ ret->desc = Qiniu_Json_GetString(root, "desc", NULL);
+ ret->inputBucket = Qiniu_Json_GetString(root, "inputBucket", NULL);
+ ret->inputKey = Qiniu_Json_GetString(root, "inputBucket", NULL);
+ ret->type = Qiniu_Json_GetInt(root, "type", 0);
+ creationDateStr = (char *)Qiniu_Json_GetString(root, "creationDate", NULL);
+ if (creationDateStr != NULL)
+ {
+ _Qiniu_Parse_Date_Time(creationDateStr, &ret->creationDate);
+ }
+
+ *itemsCount = Qiniu_Json_GetArraySize(root, "items", 0);
+ items = Qiniu_Json_GetObjectItem(root, "items", 0);
+ for (curIndex = 0; curIndex < *itemsCount && curIndex < maxItemsCount; curIndex++)
+ {
+ item = cJSON_GetArrayItem(items, curIndex);
+ itemsRet[curIndex].cmd = Qiniu_Json_GetString(item, "cmd", NULL);
+ itemsRet[curIndex].code = Qiniu_Json_GetInt(item, "code", 0);
+ itemsRet[curIndex].desc = Qiniu_Json_GetString(item, "desc", NULL);
+ itemsRet[curIndex].error = Qiniu_Json_GetString(item, "error", NULL);
+ itemsRet[curIndex].hash = Qiniu_Json_GetString(item, "hash", NULL);
+ itemsRet[curIndex].key = Qiniu_Json_GetString(item, "key", NULL);
+ itemsRet[curIndex].returnOld = Qiniu_Json_GetInt(item, "returnOld", 0);
+ }
+ return err;
+}
diff --git a/qiniu/fop.h b/qiniu/fop.h
index 9300671f..27d4554d 100644
--- a/qiniu/fop.h
+++ b/qiniu/fop.h
@@ -3,16 +3,19 @@
Name : fop.h
Author : Qiniu.com
Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd.
- Description :
+ Description :
============================================================================
*/
#ifndef QINIU_FOP_H
#define QINIU_FOP_H
-#include "http.h"
+#include "rs.h"
+#include "rfc3339.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -22,6 +25,22 @@ extern "C"
/*============================================================================*/
/* func Qiniu_FOP_Pfop */
+/* @gist pfop */
+
+typedef struct _Qiniu_FOP_PfopParams
+{
+ const char *bucket;
+ const char *key;
+ const char *pipeline;
+ const char *notifyURL;
+ char **fops;
+ int fopCount;
+ int force;
+ int type;
+} Qiniu_FOP_PfopParams;
+
+/* @endgist */
+
/* @gist pfopret */
typedef struct _Qiniu_FOP_PfopRet {
@@ -30,13 +49,46 @@ typedef struct _Qiniu_FOP_PfopRet {
/* @endgist */
-QINIU_DLLAPI extern Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket,
- const char *key, char *fops[], int fopCount, const char *pipeline,
- const char *notifyURL, int force);
+/* @gist prefopret */
+
+typedef struct _Qiniu_FOP_PrefopRet
+{
+ const char *id;
+ const char *desc;
+ const char *inputBucket;
+ const char *inputKey;
+ int code;
+ int type;
+ Qiniu_DateTime creationDate;
+} Qiniu_FOP_PrefopRet;
+
+typedef struct _Qiniu_FOP_PrefopItemRet
+{
+ const char *cmd;
+ const char *desc;
+ const char *error;
+ const char *hash;
+ const char *key;
+ int code;
+ int returnOld;
+} Qiniu_FOP_PrefopItemRet;
+
+/* @endgist */
+
+QINIU_DLLAPI extern Qiniu_Error
+Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket,
+ const char *key, char *fops[], int fopCount, const char *pipeline,
+ const char *notifyURL, int force);
+QINIU_DLLAPI extern Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_FOP_PfopParams *params);
+QINIU_DLLAPI extern Qiniu_Error
+Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu_FOP_PrefopItemRet *itemsRet,
+ Qiniu_ItemCount *itemsCount, const char *persistentId, Qiniu_ItemCount maxItemsCount);
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/http.c b/qiniu/http.c
index 2068ace5..ce5c31e0 100644
--- a/qiniu/http.c
+++ b/qiniu/http.c
@@ -9,6 +9,7 @@
#include "http.h"
#include "../cJSON/cJSON.h"
+#include "../hashmap/hashmap.h"
#include
Qiniu_Error Qiniu_Client_config(Qiniu_Client *self);
@@ -347,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize)
Qiniu_Zero_Ptr(self);
self->curl = curl_easy_init();
self->auth = auth;
+ self->hostsRetriesMax = 3;
Qiniu_Buffer_Init(&self->b, bufSize);
Qiniu_Buffer_Init(&self->respHeader, bufSize);
@@ -377,8 +379,12 @@ void Qiniu_Client_Cleanup(Qiniu_Client *self)
Qiniu_Buffer_Cleanup(&self->b);
Qiniu_Buffer_Cleanup(&self->respHeader);
- Qiniu_FreeV2((void **)&self->cachedRegion);
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
+ if (self->cachedRegions != NULL)
+ {
+ hashmap_free(self->cachedRegions);
+ self->cachedRegions = NULL;
+ }
+ self->enableUploadingAcceleration = Qiniu_False;
}
void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic)
@@ -392,6 +398,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long
self->lowSpeedTime = lowSpeedTime;
} // Qiniu_Client_SetLowSpeedLimit
+void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t hostsRetriesMax)
+{
+ self->hostsRetriesMax = hostsRetriesMax;
+} // Qiniu_Client_SetMaximumHostsRetries
+
void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs)
{
self->timeoutMs = timeoutMs;
@@ -408,6 +419,16 @@ void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps)
self->autoQueryHttpsRegion = useHttps;
}
+void Qiniu_Client_EnableUploadingAcceleration(Qiniu_Client *self)
+{
+ self->enableUploadingAcceleration = Qiniu_True;
+}
+
+void Qiniu_Client_DisableUploadingAcceleration(Qiniu_Client *self)
+{
+ self->enableUploadingAcceleration = Qiniu_False;
+}
+
void Qiniu_Client_SpecifyRegion(Qiniu_Client *self, Qiniu_Region *region)
{
self->specifiedRegion = region;
@@ -471,7 +492,7 @@ static Qiniu_Error Qiniu_Client_callWithBody(
Qiniu_Error err;
const char *ctxType;
char ctxLength[64], userAgent[64];
- Qiniu_Header *headers = NULL;
+ Qiniu_Header *headers;
CURL *curl = (CURL *)self->curl;
err = Qiniu_Client_config(self);
if (err.code != 200)
@@ -532,12 +553,30 @@ static Qiniu_Error Qiniu_Client_callWithBody(
Qiniu_Error Qiniu_Client_CallWithMethod(
Qiniu_Client *self, Qiniu_Json **ret, const char *url,
Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5)
+{
+ return Qiniu_Client_CallWithMethodAndProgressCallback(self, ret, url, body, bodyLen, mimeType, httpMethod, md5, NULL, NULL);
+}
+
+Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback(
+ Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+ Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5,
+ int (*callback)(void *, double, double, double, double), void *callbackData)
{
CURL *curl = Qiniu_Client_initcall_withMethod(self, url, httpMethod);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read);
curl_easy_setopt(curl, CURLOPT_READDATA, body.self);
+ if (callback != NULL)
+ {
+ curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+ }
+ else
+ {
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+ }
return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, md5);
}
@@ -545,12 +584,30 @@ Qiniu_Error Qiniu_Client_CallWithMethod(
Qiniu_Error Qiniu_Client_CallWithBinary(
Qiniu_Client *self, Qiniu_Json **ret, const char *url,
Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType)
+{
+ return Qiniu_Client_CallWithBinaryAndProgressCallback(self, ret, url, body, bodyLen, mimeType, NULL, NULL);
+}
+
+Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback(
+ Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+ Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType,
+ int (*callback)(void *, double, double, double, double), void *callbackData)
{
CURL *curl = Qiniu_Client_initcall(self, url);
curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read);
curl_easy_setopt(curl, CURLOPT_READDATA, body.self);
+ if (callback != NULL)
+ {
+ curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+ }
+ else
+ {
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+ }
return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, NULL);
}
diff --git a/qiniu/http.h b/qiniu/http.h
index 240082b1..c64f1600 100644
--- a/qiniu/http.h
+++ b/qiniu/http.h
@@ -79,7 +79,9 @@ extern "C"
/*============================================================================*/
/* type Qiniu_Auth */
+#if defined(_WIN32)
#pragma pack(1)
+#endif
typedef struct curl_slist Qiniu_Header;
@@ -106,7 +108,9 @@ extern "C"
typedef struct _Qiniu_Region Qiniu_Region;
QINIU_DLLAPI extern void Qiniu_Region_Free(Qiniu_Region *region);
- struct _Qiniu_Client
+ struct hashmap;
+
+ typedef struct _Qiniu_Client
{
void *curl;
Qiniu_Auth auth;
@@ -116,8 +120,7 @@ extern "C"
Qiniu_Bool autoQueryRegion;
Qiniu_Bool autoQueryHttpsRegion;
- const char *cachedRegionBucketName;
- Qiniu_Region *cachedRegion;
+ struct hashmap *cachedRegions;
Qiniu_Region *specifiedRegion;
// Use the following field to specify which NIC to use for sending packets.
@@ -138,16 +141,24 @@ extern "C"
// Millisecond timeout for the connection phase.
long connectTimeoutMs;
- };
- typedef struct _Qiniu_Client Qiniu_Client;
+
+ // Max retries count.
+ size_t hostsRetriesMax;
+
+ // Is uploading acceleration enabled.
+ Qiniu_Bool enableUploadingAcceleration;
+ } Qiniu_Client;
QINIU_DLLAPI extern void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize);
QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self);
QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic);
QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime);
+ QINIU_DLLAPI extern void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t retries);
QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps);
+ QINIU_DLLAPI extern void Qiniu_Client_EnableUploadingAcceleration(Qiniu_Client *self);
+ QINIU_DLLAPI extern void Qiniu_Client_DisableUploadingAcceleration(Qiniu_Client *self);
QINIU_DLLAPI extern void Qiniu_Client_SpecifyRegion(Qiniu_Client *self, Qiniu_Region *region);
QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_Call(Qiniu_Client *self, Qiniu_Json **ret, const char *url);
@@ -166,6 +177,16 @@ extern "C"
QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethod(
Qiniu_Client *self, Qiniu_Json **ret, const char *url,
Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5);
+
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback(
+ Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+ Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType,
+ int (*callback)(void *, double, double, double, double), void *callbackData);
+
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback(
+ Qiniu_Client *self, Qiniu_Json **ret, const char *url,
+ Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5,
+ int (*callback)(void *, double, double, double, double), void *callbackData);
/*============================================================================*/
/* func Qiniu_Client_InitNoAuth/InitMacAuth */
@@ -185,7 +206,9 @@ extern "C"
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/io.c b/qiniu/io.c
index 31ea14e2..5ad35b51 100644
--- a/qiniu/io.c
+++ b/qiniu/io.c
@@ -11,6 +11,7 @@
#include "reader.h"
#include "recorder_utils.h"
#include "private/region.h"
+#include "private/code.h"
#include
/*============================================================================*/
@@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self);
Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph);
-static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost)
+static Qiniu_Error _Qiniu_Get_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount)
{
if (extra && extra->ipCount != 0)
{
Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex);
- *upHost = extra->upIps[abs(oldIndex % extra->ipCount)];
+ *upHosts = &extra->upIps[labs(oldIndex % extra->ipCount)];
+ *upHostsCount = 1;
}
else if (extra && extra->upHost != NULL)
{
- *upHost = extra->upHost;
+ *upHosts = &extra->upHost;
+ *upHostsCount = 1;
+ }
+ else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0)
+ {
+ *upHosts = extra->upHosts;
+ *upHostsCount = extra->upHostsCount;
}
else
{
- Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost);
+ Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount);
if (err.code != 200)
{
return err;
@@ -144,54 +152,85 @@ Qiniu_Error Qiniu_Client_config(Qiniu_Client *self)
return err;
}
+static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+ void (*uploadingProgress)(size_t, size_t) = (void (*)(size_t, size_t))clientp;
+ uploadingProgress((size_t)ultotal, (size_t)ulnow);
+ return 0;
+}
+
static Qiniu_Error Qiniu_Io_call(
Qiniu_Client *self, const char *accessKey, const char *bucketName,
Qiniu_Io_PutRet *ret, struct curl_httppost *formpost, Qiniu_Io_PutExtra *extra)
{
int retCode = 0;
Qiniu_Error err;
- struct curl_slist *headers = NULL;
- const char *upHost = NULL;
+ struct curl_slist *headers;
+ const char *const *upHosts;
+ const char *defaultUpHosts[] = {QINIU_UP_HOST};
+ size_t upHostsCount;
- err = Qiniu_Client_config(self);
+ headers = curl_slist_append(NULL, "Expect:");
+ err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != 200)
{
return err;
}
- err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
- if (err.code != 200)
+ if (upHostsCount == 0)
{
- return err;
+ upHosts = defaultUpHosts;
+ upHostsCount = 1;
}
- headers = curl_slist_append(NULL, "Expect:");
-
- CURL *curl = Qiniu_Client_reset(self);
- curl_easy_setopt(curl, CURLOPT_URL, upHost);
- curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
-
- //// For aborting uploading file.
- if (extra->upAbortCallback)
+ for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++)
{
- curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
- } // if
+ CURL *curl = Qiniu_Client_reset(self);
+ err = Qiniu_Client_config(self);
+ if (err.code != 200)
+ {
+ return err;
+ }
- err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
- if (err.code == 200 && ret != NULL)
- {
- if (extra->callbackRetParser != NULL)
+ curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]);
+ curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+ if (extra->uploadingProgress != NULL)
{
- err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
}
else
{
- ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
- ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
- ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
}
- }
+ //// For aborting uploading file.
+ if (extra->upAbortCallback)
+ {
+ curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
+ } // if
+
+ err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
+ if (err.code == 200)
+ {
+ if (extra->callbackRetParser != NULL)
+ {
+ err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
+ }
+ else if (ret != NULL)
+ {
+ ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
+ ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
+ ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
+ }
+ break;
+ }
+ else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
+ {
+ break;
+ }
+ }
curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
@@ -313,7 +352,6 @@ Qiniu_Error Qiniu_Io_PutBuffer(
CURLFORM_BUFFER, key, CURLFORM_BUFFERPTR, buf, CURLFORM_BUFFERLENGTH, fsize, CURLFORM_END);
err = Qiniu_Io_call(self, accessKey, bucketName, ret, form.formpost, extra);
-error:
Qiniu_Free((void *)accessKey);
Qiniu_Free((void *)bucketName);
return err;
@@ -327,42 +365,67 @@ static Qiniu_Error Qiniu_Io_call_with_callback(
{
int retCode = 0;
Qiniu_Error err;
- struct curl_slist *headers = NULL;
- const char *upHost;
+ struct curl_slist *headers;
+ const char *const *upHosts;
+ const char *defaultUpHosts[] = {QINIU_UP_HOST};
+ size_t upHostsCount;
- CURL *curl = Qiniu_Client_reset(self);
- err = Qiniu_Client_config(self);
+ headers = curl_slist_append(NULL, "Expect:");
+ err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != 200)
{
return err;
}
- err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
- if (err.code != 200)
+ if (upHostsCount == 0)
{
- return err;
+ upHosts = defaultUpHosts;
+ upHostsCount = 1;
}
- headers = curl_slist_append(NULL, "Expect:");
+ for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++)
+ {
+ CURL *curl = Qiniu_Client_reset(self);
+ err = Qiniu_Client_config(self);
+ if (err.code != 200)
+ {
+ return err;
+ }
- curl_easy_setopt(curl, CURLOPT_URL, upHost);
- curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
- curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
- curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
+ curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]);
+ curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
+ curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
+ curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
- err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
- if (err.code == 200 && ret != NULL)
- {
- if (extra->callbackRetParser != NULL)
+ if (extra->uploadingProgress != NULL)
{
- err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback);
+ curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
}
else
{
- ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
- ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
+ curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
}
- }
+ err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
+ if (err.code == 200)
+ {
+ if (extra->callbackRetParser != NULL)
+ {
+ err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
+ }
+ else if (ret != NULL)
+ {
+ ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
+ ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
+ }
+ break;
+ }
+ else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
+ {
+ break;
+ }
+ }
curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
diff --git a/qiniu/io.h b/qiniu/io.h
index 2329bcaa..e5eb036b 100644
--- a/qiniu/io.h
+++ b/qiniu/io.h
@@ -3,7 +3,7 @@
Name : io.h
Author : Qiniu.com
Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd.
- Description :
+ Description :
============================================================================
*/
@@ -13,7 +13,9 @@
#include "http.h"
#include "reader.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -50,10 +52,18 @@ extern "C"
void *upAbortUserData;
Qiniu_Rd_FnAbort upAbortCallback;
+ // Deprecated fields, prefer upHosts.
const char *upHost;
const char **upIps;
Qiniu_Count ipCount;
Qiniu_Count ipIndex;
+
+ // Specify multiple upHosts
+ const char *const *upHosts;
+ size_t upHostsCount;
+
+ // Uploading file progress
+ void (*uploadingProgress)(size_t ultotal, size_t ulnow);
} Qiniu_Io_PutExtra;
/*============================================================================*/
@@ -98,7 +108,9 @@ extern "C"
void *self, Qiniu_Header **header, const char *url, const char *addition, size_t addlen);
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/multipart_upload.c b/qiniu/multipart_upload.c
index cea290bd..d217b08b 100644
--- a/qiniu/multipart_upload.c
+++ b/qiniu/multipart_upload.c
@@ -16,6 +16,7 @@
#include "../cJSON/cJSON.h"
#include "tm.h"
#include "private/region.h"
+#include "private/code.h"
/*============================================================================*/
#if defined(_WIN32)
@@ -49,6 +50,12 @@ typedef struct
int totalPartNum;
} Qiniu_UploadParts_Ret;
+struct _Qiniu_Progress_Callback_Data
+{
+ size_t base, totalSize, previousUlNow;
+ void (*callback)(size_t, size_t);
+};
+
static Qiniu_Error readMedium(struct Qiniu_Record_Medium *medium, char **uploadId, Qiniu_Uint64 *expireAt, Qiniu_UploadPartResp *ret);
static Qiniu_Error writeMedium(struct Qiniu_Record_Medium *medium, const Qiniu_InitPart_Ret *, const Qiniu_UploadPartResp *);
static Qiniu_Error initializeRecorder(Qiniu_Multipart_PutExtra *param, const char *uptoken, const char *key, const char *fileName, Qiniu_FileInfo *fi, Qiniu_Record_Medium *medium, Qiniu_Multipart_Recorder *recorder);
@@ -124,67 +131,116 @@ getTotalPartNum(Qiniu_Int64 fileSize, Qiniu_Int64 partSize, int *totalPartNum)
static Qiniu_Error init_upload(Qiniu_Client *client, const char *bucket, const char *encodedKey, Qiniu_Multipart_PutExtra *extraParam, Qiniu_InitPart_Ret *initPartRet)
{
Qiniu_Error err = Qiniu_OK;
- const char *uphost = extraParam->upHost;
- char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL);
+ const char *const *upHosts;
+ const char *upHost, *reqUrl;
+ size_t upHostsCount;
+ Qiniu_Json *callRet;
- for (int i = 0; i < extraParam->tryTimes; i++)
+ if (extraParam->upHost != NULL)
{
- Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call.
+ upHosts = &extraParam->upHost;
+ upHostsCount = 1;
+ }
+ else
+ {
+ upHosts = extraParam->upHosts;
+ upHostsCount = extraParam->upHostsCount;
+ }
+
+ for (int i = 0; i < extraParam->tryTimes && i <= client->hostsRetriesMax; i++)
+ {
+ upHost = upHosts[i % upHostsCount];
+ reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL);
+
+ callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call.
err = Qiniu_Client_Call(client, &callRet, reqUrl);
- if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx , net error will retry
+ Qiniu_Free((void *)reqUrl);
+ if (err.code == 200)
{
- Qiniu_Log_Error("init_upload retry: %E", err);
- continue;
+ if (initPartRet != NULL)
+ {
+ initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL));
+ initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0);
+ }
+ break;
}
- else if (err.code != 200)
+ else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
- Qiniu_Log_Error("init_upload: %E", err);
break;
}
-
- initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL));
- initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0);
- break;
}
- Qiniu_Free(reqUrl);
return err;
}
-Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *reqUrl, int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str, Qiniu_UploadPartResp *ret)
+static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+ struct _Qiniu_Progress_Callback_Data *data = (struct _Qiniu_Progress_Callback_Data *)clientp;
+ if (data->previousUlNow != (size_t)ulnow)
+ {
+ data->callback((size_t)data->totalSize, data->base + (size_t)ulnow);
+ data->previousUlNow = (size_t)ulnow;
+ }
+ return 0;
+}
+
+Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *path,
+ int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str,
+ Qiniu_UploadPartResp *ret, struct _Qiniu_Progress_Callback_Data *progressCallback)
{
Qiniu_Error err = Qiniu_OK;
- for (int try = 0; try < extraParam->tryTimes; try++)
+ const char *const *upHosts;
+ size_t upHostsCount;
+ int (*callback)(void *, double, double, double, double) = NULL;
+ void *callbackData = NULL;
+ if (progressCallback != NULL && progressCallback->callback != NULL)
+ {
+ callback = _Qiniu_Progress_Callback;
+ callbackData = (void *)progressCallback;
+ }
+
+ if (extraParam->upHost != NULL)
+ {
+ upHosts = &extraParam->upHost;
+ upHostsCount = 1;
+ }
+ else
+ {
+ upHosts = extraParam->upHosts;
+ upHostsCount = extraParam->upHostsCount;
+ }
+ for (int tries = 0; tries < extraParam->tryTimes && tries <= client->hostsRetriesMax; tries++)
{
Qiniu_Section section;
Qiniu_Zero(section);
Qiniu_Reader thisPartBody = Qiniu_SectionReader(§ion, reader, (Qiniu_Off_T)partOffset, partSize);
Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call.
- err = Qiniu_Client_CallWithMethod(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str);
- if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry
+ const char *upHost = upHosts[tries % upHostsCount];
+ const char *reqUrl = Qiniu_String_Concat(upHost, path, NULL);
+ err = Qiniu_Client_CallWithMethodAndProgressCallback(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str, callback, callbackData);
+ Qiniu_Free((void *)reqUrl);
+ if (err.code == 200)
{
- Qiniu_Log_Error("upload_part retry: partNum:%d, %E", partNum, err);
- continue;
+ if (ret != NULL)
+ {
+ const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL);
+ const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL);
+ ret->etag = Qiniu_String_Dup(etag);
+ ret->md5 = Qiniu_String_Dup(md5);
+ ret->partNum = partNum;
+ }
+ break;
}
- else if (err.code != 200)
+ else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
- Qiniu_Log_Error("upload_part: partNum:%d, %E", partNum, err);
break;
}
-
- const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL);
- const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL);
- ret->etag = Qiniu_String_Dup(etag);
- ret->md5 = Qiniu_String_Dup(md5);
- ret->partNum = partNum;
- // Qiniu_Log_Debug("partNum:%d,remote md5:%s ", partNum, md5);
- break;
}
// notify callback
- if ((err.code == 200) && extraParam->notify)
+ if (err.code == 200 && extraParam->notify)
{
extraParam->notify(ret);
}
- if ((err.code != 200) && extraParam->notifyErr)
+ else if (err.code != 200 && extraParam->notifyErr)
{
extraParam->notifyErr(partNum, err);
}
@@ -195,22 +251,19 @@ Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extr
static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const char *encodedKey, const Qiniu_InitPart_Ret *initParts, Qiniu_ReaderAt *reader, Qiniu_Int64 fsize, int totalPartNum, Qiniu_Multipart_PutExtra *extraParam, Qiniu_Multipart_Recorder *recorder, Qiniu_UploadParts_Ret *uploadPartsRet)
{
Qiniu_Error err = Qiniu_OK;
- const char *uphost = extraParam->upHost;
-
Qiniu_Int64 partSize = extraParam->partSize;
- int lastPart = totalPartNum - 1;
- for (int partNum = 0; partNum < totalPartNum; partNum++)
- {
- if ((uploadPartsRet->PartsRet + partNum)->etag != NULL)
- {
- continue;
- }
+ const int lastPart = totalPartNum - 1;
+ struct _Qiniu_Progress_Callback_Data progressCallbackData;
+ Qiniu_Zero(progressCallbackData);
- int partNumInReq = partNum + 1; // partNum start from 1
- char partNumStr[10]; // valid partNum ={"1"~"10000"}
- snprintf(partNumStr, 10, "%d", partNumInReq);
- char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL);
+ if (extraParam->uploadingProgress != NULL)
+ {
+ progressCallbackData.callback = extraParam->uploadingProgress;
+ progressCallbackData.totalSize = (size_t)fsize;
+ }
+ for (int partNum = 0; partNum < totalPartNum; partNum++)
+ {
Qiniu_Int64 thisPartOffset = partNum * partSize;
Qiniu_Int64 thisPartSize = partSize;
if (partNum == lastPart)
@@ -218,29 +271,36 @@ static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const
thisPartSize = fsize - (totalPartNum - 1) * partSize;
}
- const char *md5str = NULL;
- if (extraParam->enableContentMd5)
+ if ((uploadPartsRet->PartsRet + partNum)->etag == NULL)
{
- md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize);
- // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str);
- }
- err = upload_one_part(client, extraParam, reqUrl, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum]);
-
- Qiniu_Multi_Free(2, (void *)reqUrl, (void *)md5str);
+ const int partNumInReq = partNum + 1; // partNum start from 1
+ char partNumStr[10]; // valid partNum ={"1"~"10000"}
+ snprintf(partNumStr, 10, "%d", partNumInReq);
+ const char *path = Qiniu_String_Concat("/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL);
- if (err.code != 200)
- {
- return err;
- }
+ const char *md5str = NULL;
+ if (extraParam->enableContentMd5)
+ {
+ md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize);
+ // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str);
+ }
+ err = upload_one_part(client, extraParam, path, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum], &progressCallbackData);
+ Qiniu_Multi_Free(2, (void *)path, (void *)md5str);
- if (recorder != NULL && recorder->recorderMedium != NULL)
- {
- err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]);
if (err.code != 200)
{
return err;
}
+ if (recorder != NULL && recorder->recorderMedium != NULL)
+ {
+ err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]);
+ if (err.code != 200)
+ {
+ return err;
+ }
+ }
}
+ progressCallbackData.base += thisPartSize;
}
return err;
}
@@ -283,35 +343,54 @@ static Qiniu_Error complete_upload(
// step2:send req
Qiniu_Error err = Qiniu_OK;
- char *reqUrl = Qiniu_String_Concat(extraParam->upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL);
+ const char *const *upHosts;
+ size_t upHostsCount;
- for (int try = 0; try < extraParam->tryTimes; try++)
+ if (extraParam->upHost != NULL)
{
+ upHosts = &extraParam->upHost;
+ upHostsCount = 1;
+ }
+ else
+ {
+ upHosts = extraParam->upHosts;
+ upHostsCount = extraParam->upHostsCount;
+ }
+
+ for (int tries = 0; tries < extraParam->tryTimes && tries <= client->hostsRetriesMax; tries++)
+ {
+ const char *upHost = upHosts[tries % upHostsCount];
+ const char *reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL);
Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call.
err = Qiniu_Client_CallWithBuffer(client, &callRet, reqUrl, body, strlen(body), "application/json");
- if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry
- {
- Qiniu_Log_Error("cupload: retry uploadId:%s, %E", initPartsRet->uploadId, err);
- continue;
- }
- if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL)
+ Qiniu_Free((void *)reqUrl);
+
+ if (err.code == 200)
{
- extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey);
+ if (extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL)
+ {
+ extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey);
+ }
+ if (callRet != NULL)
+ {
+ const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen.
+ const char *key = Qiniu_Json_GetString(callRet, "key", NULL);
+ completeRet->hash = Qiniu_String_Dup(hash);
+ completeRet->key = Qiniu_String_Dup(key);
+ }
+ break;
}
- if (err.code != 200)
+ else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
- Qiniu_Log_Error("cupload: uploadId:%s, %E", initPartsRet->uploadId, err);
+ if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL)
+ {
+ extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey);
+ }
break;
}
-
- const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen.
- const char *key = Qiniu_Json_GetString(callRet, "key", NULL);
- completeRet->hash = Qiniu_String_Dup(hash);
- completeRet->key = Qiniu_String_Dup(key);
- break;
}
// Qiniu_Log_Debug("Upload result: uploadid:%s, hash:%s, key:%s ", uploadId, completeRet->hash, completeRet->key);
- Qiniu_Multi_Free(2, (void *)reqUrl, (void *)body);
+ Qiniu_Free((void *)body);
return err;
}
@@ -506,9 +585,9 @@ Qiniu_Error verifyParam(Qiniu_Client *client, const char *accessKey, const char
{
param->tryTimes = 3;
}
- if (param->upHost == NULL)
+ if (param->upHost == NULL && (param->upHosts == NULL || param->upHostsCount == 0))
{
- err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, ¶m->upHost);
+ err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, ¶m->upHosts, ¶m->upHostsCount);
}
return err;
diff --git a/qiniu/multipart_upload.h b/qiniu/multipart_upload.h
index 3d265ca7..4c536d7e 100644
--- a/qiniu/multipart_upload.h
+++ b/qiniu/multipart_upload.h
@@ -17,7 +17,9 @@
#include "stdio.h"
#include "recorder.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -37,7 +39,7 @@ extern "C"
typedef struct _Qiniu_Multipart_PutExtra
{
- const char *upHost; //if not set explicitly ,will use global QINIU_UP_HOST;
+ const char *upHost; // if not set explicitly, will use global upHosts;
Qiniu_Int64 partSize; //size for each part
const char *mimeType;
int tryTimes; //at least 1, default=3
@@ -58,6 +60,12 @@ extern "C"
Qiniu_Recorder *recorder;
+ // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST
+ const char *const *upHosts;
+ size_t upHostsCount;
+
+ // Uploading file progress
+ void (*uploadingProgress)(size_t ultotal, size_t ulnow);
} Qiniu_Multipart_PutExtra;
typedef struct
diff --git a/qiniu/private/code.h b/qiniu/private/code.h
new file mode 100644
index 00000000..492bc735
--- /dev/null
+++ b/qiniu/private/code.h
@@ -0,0 +1,21 @@
+#ifndef QINIU_CODE_H
+#define QINIU_CODE_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+ typedef enum
+ {
+ QINIU_DONT_RETRY,
+ QINIU_TRY_NEXT_DOMAIN
+ } Qiniu_Retry_Decision;
+
+ Qiniu_Retry_Decision _Qiniu_Should_Retry(int code);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // QINIU_CODE_H
diff --git a/qiniu/private/region.h b/qiniu/private/region.h
index 8a5cc465..1dc9f0ea 100644
--- a/qiniu/private/region.h
+++ b/qiniu/private/region.h
@@ -1,26 +1,17 @@
#ifndef QINIU_PRIVATE_REGION_H
#define QINIU_PRIVATE_REGION_H
#include
-#include "../base.h"
+#include "../http.h"
#ifdef __cplusplus
extern "C"
{
#endif
- struct _Qiniu_Client;
- typedef struct _Qiniu_Client Qiniu_Client;
-
- struct _Qiniu_Region_Hosts
- {
- const char *const *hosts;
- size_t hostsCount;
- };
-
struct _Qiniu_Region_Service
{
- struct _Qiniu_Region_Hosts *preferredHosts;
- struct _Qiniu_Region_Hosts *alternativeHosts;
+ const char *const *hosts;
+ size_t acceleratedHostsCount, preferredHostsCount, alternativeHostsCount;
};
struct _Qiniu_Region
@@ -37,20 +28,31 @@ extern "C"
struct _Qiniu_Region_Service *apiService;
};
- static struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New(const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
- static struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New_without_dup(const char *const *hosts, size_t hostsCount);
- static void _Qiniu_Region_Hosts_Free(struct _Qiniu_Region_Hosts *hosts);
+ struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(const char *const *acceleratedHosts, size_t acceleratedHostsCount,
+ const char *const *preferredHosts, size_t preferredHostsCount,
+ const char *const *alternativeHosts, size_t alternativeHostsCount,
+ Qiniu_Bool useHttps);
+ void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service);
- static struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(struct _Qiniu_Region_Hosts *preferredHosts, struct _Qiniu_Region_Hosts *alternativeHosts);
- static void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service);
-
- Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host);
Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host);
Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host);
Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host);
Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host);
+
+ Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count);
+ Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count);
+ Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count);
+ Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count);
+ Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count);
Qiniu_Error _Qiniu_Region_Query(Qiniu_Client *self, struct _Qiniu_Region **region, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps);
+ const char *const *_Qiniu_Region_Get_All_Up_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Io_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Io_Src_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Bucket_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Rs_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Rsf_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
+ const char *const *_Qiniu_Region_Get_All_Api_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled);
#ifdef __cplusplus
}
#endif
diff --git a/qiniu/qetag.c b/qiniu/qetag.c
index 9e9bcaa5..5a87823e 100644
--- a/qiniu/qetag.c
+++ b/qiniu/qetag.c
@@ -10,7 +10,9 @@ extern "C"
{
#endif
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#define BLOCK_ELEMENT_MAX_COUNT 16
#define BLOCK_MAX_SIZE (1 << 22)
@@ -60,7 +62,7 @@ static Qiniu_Error Qiniu_Qetag_mergeBlocks(struct _Qiniu_Qetag_Context * ctx) {
}
blk->done = NO;
-
+
ctx->blkBegin += 1;
if (ctx->blkBegin >= ctx->blkElementCount) {
ctx->blkBegin = 0;
@@ -266,7 +268,7 @@ Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context * ctx, char ** digest)
Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block ** blk, size_t * remainderSize)
{
Qiniu_Error err;
-
+
if (ctx->blk) {
*blk = ctx->blk;
ctx->blk = NULL;
@@ -394,7 +396,9 @@ Qiniu_Error Qiniu_Qetag_DigestBuffer(const char * buf, size_t bufSize, char ** d
return err;
} // Qiniu_Qetag_DigestBuffer
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/qetag.h b/qiniu/qetag.h
index 165b8821..ef12ed99 100644
--- a/qiniu/qetag.h
+++ b/qiniu/qetag.h
@@ -3,7 +3,7 @@
Name : qetag.h
Author : Qiniu.com
Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd.
- Description :
+ Description :
============================================================================
*/
@@ -18,27 +18,31 @@ extern "C"
{
#endif
+#if defined(_WIN32)
#pragma pack(1)
+#endif
-struct _Qiniu_Qetag_Context;
-struct _Qiniu_Qetag_Block;
+ struct _Qiniu_Qetag_Context;
+ struct _Qiniu_Qetag_Block;
-// 底层函数
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_New(struct _Qiniu_Qetag_Context ** ctx, unsigned int concurrency);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Reset(struct _Qiniu_Qetag_Context * ctx);
-QINIU_DLLAPI extern void Qiniu_Qetag_Destroy(struct _Qiniu_Qetag_Context * ctx);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Update(struct _Qiniu_Qetag_Context * ctx, const char * buf, size_t bufSize);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context * ctx, char ** digest);
+ // 底层函数
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_New(struct _Qiniu_Qetag_Context **ctx, unsigned int concurrency);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Reset(struct _Qiniu_Qetag_Context *ctx);
+ QINIU_DLLAPI extern void Qiniu_Qetag_Destroy(struct _Qiniu_Qetag_Context *ctx);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Update(struct _Qiniu_Qetag_Context *ctx, const char *buf, size_t bufSize);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context *ctx, char **digest);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block ** blk, size_t * blkCapacity);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_UpdateBlock(struct _Qiniu_Qetag_Block * blk, const char * buf, size_t bufSize, size_t * blkCapacity);
-QINIU_DLLAPI extern void Qiniu_Qetag_CommitBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block * blk);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context *ctx, struct _Qiniu_Qetag_Block **blk, size_t *blkCapacity);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_UpdateBlock(struct _Qiniu_Qetag_Block *blk, const char *buf, size_t bufSize, size_t *blkCapacity);
+ QINIU_DLLAPI extern void Qiniu_Qetag_CommitBlock(struct _Qiniu_Qetag_Context *ctx, struct _Qiniu_Qetag_Block *blk);
-// 单线程计算 QETAG
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestFile(const char * localFile, char ** digest);
-QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestBuffer(const char * buf, size_t fsize, char ** digest);
+ // 单线程计算 QETAG
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestFile(const char *localFile, char **digest);
+ QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestBuffer(const char *buf, size_t fsize, char **digest);
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/reader.h b/qiniu/reader.h
index 8515475f..2a07f5ca 100644
--- a/qiniu/reader.h
+++ b/qiniu/reader.h
@@ -3,7 +3,7 @@
Name : reader.h
Author : Qiniu.com
Copyright : 2012-2016(c) Shanghai Qiniu Information Technologies Co., Ltd.
- Description :
+ Description :
============================================================================
*/
@@ -12,7 +12,9 @@
#include "base.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -47,7 +49,9 @@ QINIU_DLLAPI extern void Qiniu_Rd_Reader_Close(Qiniu_Rd_Reader * rdr);
QINIU_DLLAPI extern size_t Qiniu_Rd_Reader_Callback(char * buffer, size_t size, size_t nitems, void * rdr);
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/recorder.c b/qiniu/recorder.c
index 3afc31dc..d786c32d 100644
--- a/qiniu/recorder.c
+++ b/qiniu/recorder.c
@@ -157,7 +157,6 @@ Qiniu_Error _Qiniu_FileSystem_RecordMedium_Read_Entry(const struct Qiniu_Record_
Qiniu_Error _Qiniu_FileSystem_RecordMedium_Has_Next_Entry(const struct Qiniu_Record_Medium *medium, Qiniu_Bool *has)
{
struct _FileSystem_Recorder_Data *data = (struct _FileSystem_Recorder_Data *)medium->data;
- size_t i;
Qiniu_Bool hasEntry = Qiniu_True;
if (data->buf == NULL || data->bufOffset >= data->bufSize)
diff --git a/qiniu/recorder.h b/qiniu/recorder.h
index 08ae4fe0..49698b32 100644
--- a/qiniu/recorder.h
+++ b/qiniu/recorder.h
@@ -14,7 +14,9 @@
#define Qiniu_Recorder_Read_Error 9801
#define Qiniu_Recorder_Write_Error 9802
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -45,7 +47,9 @@ extern "C"
QINIU_DLLAPI extern Qiniu_Error Qiniu_FileSystem_Recorder_New(const char *rootPath, struct Qiniu_Recorder *recorder);
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/recorder_key.h b/qiniu/recorder_key.h
index a7f56602..c5d10e55 100644
--- a/qiniu/recorder_key.h
+++ b/qiniu/recorder_key.h
@@ -13,7 +13,9 @@
#include "base.h"
#include "private/crypto.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -36,7 +38,9 @@ QINIU_DLLAPI extern void Qiniu_Recorder_Key_Generator_Free(Qiniu_Recorder_Key_Ge
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/recorder_utils.h b/qiniu/recorder_utils.h
index a8623805..4b717f46 100644
--- a/qiniu/recorder_utils.h
+++ b/qiniu/recorder_utils.h
@@ -13,7 +13,9 @@
#include "base.h"
#include "recorder.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -29,7 +31,9 @@ extern "C"
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/region.c b/qiniu/region.c
index 821a69b5..7d3d7871 100644
--- a/qiniu/region.c
+++ b/qiniu/region.c
@@ -11,6 +11,7 @@
#include "region.h"
#include "tm.h"
#include "../cJSON/cJSON.h"
+#include "../hashmap/hashmap.h"
/*============================================================================*/
@@ -18,8 +19,7 @@
#include
#endif
-static const char *
-duplicate_host(const char *host, Qiniu_Bool useHttps)
+static const char *_Qiniu_Duplicate_Host(const char *host, Qiniu_Bool useHttps)
{
if (host == NULL)
{
@@ -39,83 +39,82 @@ duplicate_host(const char *host, Qiniu_Bool useHttps)
}
}
-struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New(const char *const *const hosts, size_t hostsCount, Qiniu_Bool useHttps)
+static Qiniu_Bool _Qiniu_Region_Service_Copy_Hosts(const char **targetHosts, size_t copyOffset, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
{
- const char **newHosts = malloc(sizeof(const char *) * hostsCount);
- if (newHosts == NULL)
- {
- return NULL;
- }
-
- struct _Qiniu_Region_Hosts *newRegionHosts = malloc(sizeof(struct _Qiniu_Region_Hosts));
- if (newRegionHosts == NULL)
- {
- free((void *)newHosts);
- return NULL;
- }
- newRegionHosts->hosts = (const char *const *)newHosts;
-
- for (size_t i = 0; i < hostsCount; i++)
+ size_t i, j;
+ for (i = 0; i < hostsCount; i++)
{
if (hosts[i] == NULL)
{
break;
}
- newHosts[i] = duplicate_host(hosts[i], useHttps);
- if (newHosts[i] == NULL)
+ targetHosts[copyOffset + i] = _Qiniu_Duplicate_Host(hosts[i], useHttps);
+ if (targetHosts[copyOffset + i] == NULL)
{
- for (size_t j = 0; j < i; j++)
- {
- free((void *)newHosts[j]);
- }
- free((void *)newHosts);
- free((void *)newRegionHosts);
- return NULL;
+ goto handleErr;
}
- newRegionHosts->hostsCount = i + 1;
}
- return newRegionHosts;
-}
-
-struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New_without_dup(const char *const *hosts, size_t hostsCount)
-{
- struct _Qiniu_Region_Hosts *newRegionHosts = malloc(sizeof(struct _Qiniu_Region_Hosts));
- if (newRegionHosts == NULL)
+ return Qiniu_True;
+handleErr:
+ for (j = 0; j < i; j++)
{
- return NULL;
+ free((void *)targetHosts[copyOffset + j]);
}
- newRegionHosts->hosts = hosts;
- newRegionHosts->hostsCount = hostsCount;
- return newRegionHosts;
+ return Qiniu_False;
}
-void _Qiniu_Region_Hosts_Free(struct _Qiniu_Region_Hosts *hosts)
+struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(
+ const char *const *acceleratedHosts, size_t acceleratedHostsCount,
+ const char *const *preferredHosts, size_t preferredHostsCount,
+ const char *const *alternativeHosts, size_t alternativeHostsCount,
+ Qiniu_Bool useHttps)
{
- if (hosts == NULL)
+ struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service));
+ if (newService == NULL)
{
- return;
+ goto handleErr;
}
- for (int i = 0; i < hosts->hostsCount; i++)
+ size_t allHostsCount = acceleratedHostsCount + preferredHostsCount + alternativeHostsCount;
+ const char **newHosts = malloc(sizeof(const char *) * allHostsCount);
+ if (newHosts == NULL)
{
- free((void *)hosts->hosts[i]);
+ goto handleErr;
}
- free((void *)hosts->hosts);
- hosts->hosts = NULL;
- hosts->hostsCount = 0;
- free((void *)hosts);
-}
-
-struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(struct _Qiniu_Region_Hosts *preferredHosts, struct _Qiniu_Region_Hosts *alternativeHosts)
-{
- struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service));
- if (newService == NULL)
+ size_t offset = 0;
+ if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, acceleratedHosts, acceleratedHostsCount, useHttps) == Qiniu_False)
{
- return NULL;
+ goto handleErr;
}
- newService->preferredHosts = preferredHosts;
- newService->alternativeHosts = alternativeHosts;
-
+ offset += acceleratedHostsCount;
+ if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, preferredHosts, preferredHostsCount, useHttps) == Qiniu_False)
+ {
+ goto handleErr;
+ }
+ offset += preferredHostsCount;
+ if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, alternativeHosts, alternativeHostsCount, useHttps) == Qiniu_False)
+ {
+ goto handleErr;
+ }
+ offset += alternativeHostsCount;
+ newService->hosts = newHosts;
+ newService->acceleratedHostsCount = acceleratedHostsCount;
+ newService->preferredHostsCount = preferredHostsCount;
+ newService->alternativeHostsCount = alternativeHostsCount;
return newService;
+handleErr:
+ if (newHosts != NULL)
+ {
+ for (size_t j = 0; j < offset; j++)
+ {
+ free((void *)newHosts[offset + j]);
+ }
+ free((void *)newHosts);
+ }
+ if (newService != NULL)
+ {
+ free((void *)newService);
+ }
+ return NULL;
}
void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service)
@@ -124,11 +123,13 @@ void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service)
{
return;
}
- _Qiniu_Region_Hosts_Free(service->preferredHosts);
- service->preferredHosts = NULL;
- _Qiniu_Region_Hosts_Free(service->alternativeHosts);
- service->alternativeHosts = NULL;
- free(service);
+ size_t allHostsCount = service->acceleratedHostsCount + service->preferredHostsCount + service->alternativeHostsCount;
+ for (size_t i = 0; i < allHostsCount; i++)
+ {
+ free((void *)service->hosts[i]);
+ }
+ free((void *)service->hosts);
+ free((void *)service);
}
Qiniu_Region *Qiniu_Region_New(const char *regionId, Qiniu_Uint64 ttl)
@@ -182,22 +183,48 @@ void Qiniu_Region_Free(Qiniu_Region *region)
free((void *)region);
}
+void Qiniu_Region_Set_Up_Accelerated_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
+{
+ if (region == NULL)
+ {
+ return;
+ }
+ if (region->upService != NULL)
+ {
+ struct _Qiniu_Region_Service *oldUpService = region->upService;
+ region->upService = _Qiniu_Region_Service_New(
+ hosts, hostsCount,
+ oldUpService->hosts + oldUpService->acceleratedHostsCount, oldUpService->preferredHostsCount,
+ oldUpService->hosts + oldUpService->acceleratedHostsCount + oldUpService->preferredHostsCount, oldUpService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldUpService);
+ }
+ else
+ {
+ region->upService = _Qiniu_Region_Service_New(hosts, hostsCount, NULL, 0, NULL, 0, useHttps);
+ }
+}
+
void Qiniu_Region_Set_Up_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
{
if (region == NULL)
{
return;
}
- if (region->upService == NULL)
+ if (region->upService != NULL)
{
- region->upService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldUpService = region->upService;
+ region->upService = _Qiniu_Region_Service_New(
+ oldUpService->hosts, oldUpService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldUpService->hosts + oldUpService->acceleratedHostsCount + oldUpService->preferredHostsCount, oldUpService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldUpService);
}
- if (region->upService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->upService->preferredHosts);
- region->upService->preferredHosts = NULL;
+ region->upService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->upService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -206,16 +233,20 @@ void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *con
{
return;
}
- if (region->upService == NULL)
+ if (region->upService != NULL)
{
- region->upService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldUpService = region->upService;
+ region->upService = _Qiniu_Region_Service_New(
+ oldUpService->hosts, oldUpService->acceleratedHostsCount,
+ oldUpService->hosts + oldUpService->acceleratedHostsCount, oldUpService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldUpService);
}
- if (region->upService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->upService->alternativeHosts);
- region->upService->alternativeHosts = NULL;
+ region->upService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->upService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -224,16 +255,20 @@ void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const
{
return;
}
- if (region->ioService == NULL)
+ if (region->ioService != NULL)
{
- region->ioService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldIoService = region->ioService;
+ region->ioService = _Qiniu_Region_Service_New(
+ oldIoService->hosts, oldIoService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldIoService->hosts + oldIoService->acceleratedHostsCount + oldIoService->preferredHostsCount, oldIoService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldIoService);
}
- if (region->ioService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ioService->preferredHosts);
- region->ioService->preferredHosts = NULL;
+ region->ioService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->ioService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -242,16 +277,20 @@ void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *con
{
return;
}
- if (region->ioService == NULL)
+ if (region->ioService != NULL)
{
- region->ioService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldIoService = region->ioService;
+ region->ioService = _Qiniu_Region_Service_New(
+ oldIoService->hosts, oldIoService->acceleratedHostsCount,
+ oldIoService->hosts + oldIoService->acceleratedHostsCount, oldIoService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldIoService);
}
- if (region->ioService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ioService->alternativeHosts);
- region->ioService->alternativeHosts = NULL;
+ region->ioService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->ioService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -260,16 +299,20 @@ void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *c
{
return;
}
- if (region->ioSrcService == NULL)
+ if (region->ioSrcService != NULL)
{
- region->ioSrcService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldIoSrcService = region->ioSrcService;
+ region->ioSrcService = _Qiniu_Region_Service_New(
+ oldIoSrcService->hosts, oldIoSrcService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldIoSrcService->hosts + oldIoSrcService->acceleratedHostsCount + oldIoSrcService->preferredHostsCount, oldIoSrcService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldIoSrcService);
}
- if (region->ioSrcService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ioSrcService->preferredHosts);
- region->ioSrcService->preferredHosts = NULL;
+ region->ioSrcService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->ioSrcService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Io_Src_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -278,16 +321,20 @@ void Qiniu_Region_Set_Io_Src_Alternative_Hosts(Qiniu_Region *region, const char
{
return;
}
- if (region->ioSrcService == NULL)
+ if (region->ioSrcService != NULL)
{
- region->ioSrcService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldIoSrcService = region->ioSrcService;
+ region->ioSrcService = _Qiniu_Region_Service_New(
+ oldIoSrcService->hosts, oldIoSrcService->acceleratedHostsCount,
+ oldIoSrcService->hosts + oldIoSrcService->acceleratedHostsCount, oldIoSrcService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldIoSrcService);
}
- if (region->ioSrcService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ioSrcService->alternativeHosts);
- region->ioSrcService->alternativeHosts = NULL;
+ region->ioSrcService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->ioSrcService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Bucket_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -296,16 +343,20 @@ void Qiniu_Region_Set_Bucket_Preferred_Hosts(Qiniu_Region *region, const char *c
{
return;
}
- if (region->ucService == NULL)
+ if (region->ucService != NULL)
{
- region->ucService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldUcService = region->ucService;
+ region->ucService = _Qiniu_Region_Service_New(
+ oldUcService->hosts, oldUcService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldUcService->hosts + oldUcService->acceleratedHostsCount + oldUcService->preferredHostsCount, oldUcService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldUcService);
}
- if (region->ucService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ucService->preferredHosts);
- region->ucService->preferredHosts = NULL;
+ region->ucService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->ucService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Bucket_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -314,16 +365,20 @@ void Qiniu_Region_Set_Bucket_Alternative_Hosts(Qiniu_Region *region, const char
{
return;
}
- if (region->ucService == NULL)
+ if (region->ucService != NULL)
{
- region->ucService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldUcService = region->ucService;
+ region->ucService = _Qiniu_Region_Service_New(
+ oldUcService->hosts, oldUcService->acceleratedHostsCount,
+ oldUcService->hosts + oldUcService->acceleratedHostsCount, oldUcService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldUcService);
}
- if (region->ucService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->ucService->alternativeHosts);
- region->ucService->alternativeHosts = NULL;
+ region->ucService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->ucService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Rs_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -332,16 +387,20 @@ void Qiniu_Region_Set_Rs_Preferred_Hosts(Qiniu_Region *region, const char *const
{
return;
}
- if (region->rsService == NULL)
+ if (region->rsService != NULL)
{
- region->rsService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldRsService = region->rsService;
+ region->rsService = _Qiniu_Region_Service_New(
+ oldRsService->hosts, oldRsService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldRsService->hosts + oldRsService->acceleratedHostsCount + oldRsService->preferredHostsCount, oldRsService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldRsService);
}
- if (region->rsService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->rsService->preferredHosts);
- region->rsService->preferredHosts = NULL;
+ region->rsService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->rsService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Rs_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -350,16 +409,20 @@ void Qiniu_Region_Set_Rs_Alternative_Hosts(Qiniu_Region *region, const char *con
{
return;
}
- if (region->rsService == NULL)
+ if (region->rsService != NULL)
{
- region->rsService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldRsService = region->rsService;
+ region->rsService = _Qiniu_Region_Service_New(
+ oldRsService->hosts, oldRsService->acceleratedHostsCount,
+ oldRsService->hosts + oldRsService->acceleratedHostsCount, oldRsService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldRsService);
}
- if (region->rsService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->rsService->alternativeHosts);
- region->rsService->alternativeHosts = NULL;
+ region->rsService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->rsService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Rsf_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -368,16 +431,20 @@ void Qiniu_Region_Set_Rsf_Preferred_Hosts(Qiniu_Region *region, const char *cons
{
return;
}
- if (region->rsfService == NULL)
+ if (region->rsfService != NULL)
{
- region->rsfService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldRsfService = region->rsfService;
+ region->rsfService = _Qiniu_Region_Service_New(
+ oldRsfService->hosts, oldRsfService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldRsfService->hosts + oldRsfService->acceleratedHostsCount + oldRsfService->preferredHostsCount, oldRsfService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldRsfService);
}
- if (region->rsfService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->rsfService->preferredHosts);
- region->rsfService->preferredHosts = NULL;
+ region->rsfService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->rsfService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Rsf_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -386,16 +453,20 @@ void Qiniu_Region_Set_Rsf_Alternative_Hosts(Qiniu_Region *region, const char *co
{
return;
}
- if (region->rsfService == NULL)
+ if (region->rsfService != NULL)
{
- region->rsfService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldRsfService = region->rsfService;
+ region->rsfService = _Qiniu_Region_Service_New(
+ oldRsfService->hosts, oldRsfService->acceleratedHostsCount,
+ oldRsfService->hosts + oldRsfService->acceleratedHostsCount, oldRsfService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldRsfService);
}
- if (region->rsfService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->rsfService->alternativeHosts);
- region->rsfService->alternativeHosts = NULL;
+ region->rsfService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->rsfService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Api_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -404,16 +475,20 @@ void Qiniu_Region_Set_Api_Preferred_Hosts(Qiniu_Region *region, const char *cons
{
return;
}
- if (region->apiService == NULL)
+ if (region->apiService != NULL)
{
- region->apiService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldApiService = region->apiService;
+ region->apiService = _Qiniu_Region_Service_New(
+ oldApiService->hosts, oldApiService->acceleratedHostsCount,
+ hosts, hostsCount,
+ oldApiService->hosts + oldApiService->acceleratedHostsCount + oldApiService->preferredHostsCount, oldApiService->alternativeHostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldApiService);
}
- if (region->apiService->preferredHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->apiService->preferredHosts);
- region->apiService->preferredHosts = NULL;
+ region->apiService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps);
}
- region->apiService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
void Qiniu_Region_Set_Api_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps)
@@ -422,19 +497,23 @@ void Qiniu_Region_Set_Api_Alternative_Hosts(Qiniu_Region *region, const char *co
{
return;
}
- if (region->apiService == NULL)
+ if (region->apiService != NULL)
{
- region->apiService = _Qiniu_Region_Service_New(NULL, NULL);
+ struct _Qiniu_Region_Service *oldApiService = region->apiService;
+ region->apiService = _Qiniu_Region_Service_New(
+ oldApiService->hosts, oldApiService->acceleratedHostsCount,
+ oldApiService->hosts + oldApiService->acceleratedHostsCount, oldApiService->preferredHostsCount,
+ hosts, hostsCount,
+ useHttps);
+ _Qiniu_Region_Service_Free(oldApiService);
}
- if (region->apiService->alternativeHosts != NULL)
+ else
{
- _Qiniu_Region_Hosts_Free(region->apiService->alternativeHosts);
- region->apiService->alternativeHosts = NULL;
+ region->apiService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps);
}
- region->apiService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps);
}
-const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Up_Accelerated_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -448,18 +527,18 @@ const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, siz
{
return NULL;
}
- if (region->upService->preferredHosts == NULL)
+ if (region->upService->hosts == NULL || region->upService->acceleratedHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->upService->preferredHosts->hostsCount;
+ *count = region->upService->acceleratedHostsCount;
}
- return region->upService->preferredHosts->hosts;
+ return region->upService->hosts;
}
-const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -473,18 +552,18 @@ const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, s
{
return NULL;
}
- if (region->upService->alternativeHosts == NULL)
+ if (region->upService->hosts == NULL || region->upService->preferredHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->upService->alternativeHosts->hostsCount;
+ *count = region->upService->preferredHostsCount;
}
- return region->upService->alternativeHosts->hosts;
+ return region->upService->hosts + region->upService->acceleratedHostsCount;
}
-const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -494,22 +573,22 @@ const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, siz
{
return NULL;
}
- if (region->ioService == NULL)
+ if (region->upService == NULL)
{
return NULL;
}
- if (region->ioService->preferredHosts == NULL)
+ if (region->upService->hosts == NULL || region->upService->alternativeHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ioService->preferredHosts->hostsCount;
+ *count = region->upService->alternativeHostsCount;
}
- return region->ioService->preferredHosts->hosts;
+ return region->upService->hosts + region->upService->acceleratedHostsCount + region->upService->preferredHostsCount;
}
-const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *_Qiniu_Region_Get_All_Up_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
{
if (count != NULL)
{
@@ -519,22 +598,34 @@ const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, s
{
return NULL;
}
- if (region->ioService == NULL)
+ if (region->upService == NULL)
{
return NULL;
}
- if (region->ioService->alternativeHosts == NULL)
+ size_t allHostsCount = region->upService->preferredHostsCount + region->upService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->upService->acceleratedHostsCount;
+ }
+ if (region->upService->hosts == NULL || allHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ioService->alternativeHosts->hostsCount;
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->upService->hosts;
+ }
+ else
+ {
+ return region->upService->hosts + region->upService->acceleratedHostsCount;
}
- return region->ioService->alternativeHosts->hosts;
}
-const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -544,22 +635,22 @@ const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region,
{
return NULL;
}
- if (region->ioSrcService == NULL)
+ if (region->ioService == NULL)
{
return NULL;
}
- if (region->ioSrcService->preferredHosts == NULL)
+ if (region->ioService->hosts == NULL || region->ioService->preferredHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ioSrcService->preferredHosts->hostsCount;
+ *count = region->ioService->preferredHostsCount;
}
- return region->ioSrcService->preferredHosts->hosts;
+ return region->ioService->hosts + region->ioService->acceleratedHostsCount;
}
-const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -569,22 +660,22 @@ const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *regio
{
return NULL;
}
- if (region->ioSrcService == NULL)
+ if (region->ioService == NULL)
{
return NULL;
}
- if (region->ioSrcService->alternativeHosts == NULL)
+ if (region->ioService->hosts == NULL || region->ioService->alternativeHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ioSrcService->alternativeHosts->hostsCount;
+ *count = region->ioService->alternativeHostsCount;
}
- return region->ioSrcService->alternativeHosts->hosts;
+ return region->ioService->hosts + region->ioService->acceleratedHostsCount + region->ioService->preferredHostsCount;
}
-const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *_Qiniu_Region_Get_All_Io_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
{
if (count != NULL)
{
@@ -594,22 +685,34 @@ const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region,
{
return NULL;
}
- if (region->ucService == NULL)
+ if (region->ioService == NULL)
{
return NULL;
}
- if (region->ucService->preferredHosts == NULL)
+ size_t allHostsCount = region->ioService->preferredHostsCount + region->ioService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->ioService->acceleratedHostsCount;
+ }
+ if (region->ioService->hosts == NULL || allHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ucService->preferredHosts->hostsCount;
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->ioService->hosts;
+ }
+ else
+ {
+ return region->ioService->hosts + region->ioService->acceleratedHostsCount;
}
- return region->ucService->preferredHosts->hosts;
}
-const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -619,22 +722,22 @@ const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *regio
{
return NULL;
}
- if (region->ucService == NULL)
+ if (region->ioSrcService == NULL)
{
return NULL;
}
- if (region->ucService->alternativeHosts == NULL)
+ if (region->ioSrcService->hosts == NULL || region->ioSrcService->preferredHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->ucService->alternativeHosts->hostsCount;
+ *count = region->ioSrcService->preferredHostsCount;
}
- return region->ucService->alternativeHosts->hosts;
+ return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount;
}
-const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -644,22 +747,22 @@ const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, siz
{
return NULL;
}
- if (region->rsService == NULL)
+ if (region->ioSrcService == NULL)
{
return NULL;
}
- if (region->rsService->preferredHosts == NULL)
+ if (region->ioSrcService->hosts == NULL || region->ioSrcService->alternativeHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->rsService->preferredHosts->hostsCount;
+ *count = region->ioSrcService->alternativeHostsCount;
}
- return region->rsService->preferredHosts->hosts;
+ return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount + region->ioSrcService->preferredHostsCount;
}
-const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *_Qiniu_Region_Get_All_Io_Src_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
{
if (count != NULL)
{
@@ -669,22 +772,34 @@ const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, s
{
return NULL;
}
- if (region->rsService == NULL)
+ if (region->ioSrcService == NULL)
{
return NULL;
}
- if (region->rsService->alternativeHosts == NULL)
+ size_t allHostsCount = region->ioSrcService->preferredHostsCount + region->ioSrcService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->ioSrcService->acceleratedHostsCount;
+ }
+ if (region->ioSrcService->hosts == NULL || allHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->rsService->alternativeHosts->hostsCount;
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->ioSrcService->hosts;
+ }
+ else
+ {
+ return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount;
}
- return region->rsService->alternativeHosts->hosts;
}
-const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -694,22 +809,22 @@ const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, si
{
return NULL;
}
- if (region->rsfService == NULL)
+ if (region->ucService == NULL)
{
return NULL;
}
- if (region->rsfService->preferredHosts == NULL)
+ if (region->ucService->hosts == NULL || region->ucService->preferredHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->rsfService->preferredHosts->hostsCount;
+ *count = region->ucService->preferredHostsCount;
}
- return region->rsfService->preferredHosts->hosts;
+ return region->ucService->hosts + region->ucService->acceleratedHostsCount;
}
-const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -719,22 +834,22 @@ const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region,
{
return NULL;
}
- if (region->rsfService == NULL)
+ if (region->ucService == NULL)
{
return NULL;
}
- if (region->rsfService->alternativeHosts == NULL)
+ if (region->ucService->hosts == NULL || region->ucService->alternativeHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->rsfService->alternativeHosts->hostsCount;
+ *count = region->ucService->alternativeHostsCount;
}
- return region->rsfService->alternativeHosts->hosts;
+ return region->ucService->hosts + region->ucService->acceleratedHostsCount + region->ucService->preferredHostsCount;
}
-const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *_Qiniu_Region_Get_All_Bucket_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
{
if (count != NULL)
{
@@ -744,22 +859,34 @@ const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, si
{
return NULL;
}
- if (region->apiService == NULL)
+ if (region->ucService == NULL)
{
return NULL;
}
- if (region->apiService->preferredHosts == NULL)
+ size_t allHostsCount = region->ucService->preferredHostsCount + region->ucService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->ucService->acceleratedHostsCount;
+ }
+ if (region->ucService->hosts == NULL || allHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->apiService->preferredHosts->hostsCount;
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->ucService->hosts;
+ }
+ else
+ {
+ return region->ucService->hosts + region->ucService->acceleratedHostsCount;
}
- return region->apiService->preferredHosts->hosts;
}
-const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
if (count != NULL)
{
@@ -769,153 +896,485 @@ const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region,
{
return NULL;
}
- if (region->apiService == NULL)
+ if (region->rsService == NULL)
{
return NULL;
}
- if (region->apiService->alternativeHosts == NULL)
+ if (region->rsService->hosts == NULL || region->rsService->preferredHostsCount == 0)
{
return NULL;
}
if (count != NULL)
{
- *count = region->apiService->alternativeHosts->hostsCount;
+ *count = region->rsService->preferredHostsCount;
}
- return region->apiService->alternativeHosts->hosts;
+ return region->rsService->hosts + region->rsService->acceleratedHostsCount;
}
-Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps)
+const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, size_t *count)
{
- Qiniu_Region *region = Qiniu_Region_New(regionId, 0);
+ if (count != NULL)
+ {
+ *count = 0;
+ }
if (region == NULL)
{
return NULL;
}
-
- char buf[32], buf2[32];
- Qiniu_Zero(buf);
- Qiniu_Zero(buf2);
-
- if (strcmp(regionId, "z0") == 0)
+ if (region->rsService == NULL)
{
- Qiniu_Region_Set_Up_Preferred_Hosts(region, (const char *const[]){"upload.qiniup.com", "up.qiniup.com"}, 2, useHttps);
- Qiniu_Region_Set_Up_Alternative_Hosts(region, (const char *const[]){"up.qbox.me"}, 1, useHttps);
- Qiniu_Region_Set_Io_Preferred_Hosts(region, (const char *const[]){"iovip.qiniuio.com"}, 1, useHttps);
- Qiniu_Region_Set_Io_Alternative_Hosts(region, (const char *const[]){"iovip.qbox.me"}, 1, useHttps);
+ return NULL;
}
- else
+ if (region->rsService->hosts == NULL || region->rsService->alternativeHostsCount == 0)
{
- Qiniu_snprintf(buf, sizeof(buf), "upload-%s.qiniup.com", regionId);
- Qiniu_snprintf(buf2, sizeof(buf2), "up-%s.qiniup.com", regionId);
- Qiniu_Region_Set_Up_Preferred_Hosts(region, (const char *const[]){buf, buf2}, 2, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "up-%s.qbox.me", regionId);
- Qiniu_Region_Set_Up_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qiniuio.com", regionId);
- Qiniu_Region_Set_Io_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qbox.me", regionId);
- Qiniu_Region_Set_Io_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps);
+ return NULL;
}
- Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qiniuapi.com", regionId);
- Qiniu_Region_Set_Rs_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qbox.me", regionId);
- Qiniu_Region_Set_Rs_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qiniuapi.com", regionId);
- Qiniu_Region_Set_Rsf_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qbox.me", regionId);
- Qiniu_Region_Set_Rsf_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "api-%s.qiniuapi.com", regionId);
- Qiniu_Region_Set_Api_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- Qiniu_snprintf(buf, sizeof(buf), "api-%s.qbox.me", regionId);
- Qiniu_Region_Set_Api_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps);
- return region;
-}
-
-static Qiniu_Bool isRegionQueryRetryable(Qiniu_Error err)
-{
- switch (err.code)
+ if (count != NULL)
{
- case 501:
- case 509:
- case 573:
- case 579:
- case 608:
- case 612:
- case 614:
- case 616:
- case 618:
- case 630:
- case 631:
- case 632:
- case 640:
- case 701:
- return Qiniu_False;
- default:
- return err.code >= 500 || err.code < 100;
+ *count = region->rsService->alternativeHostsCount;
}
+ return region->rsService->hosts + region->rsService->acceleratedHostsCount + region->rsService->preferredHostsCount;
}
-static Qiniu_Error _Qiniu_Region_Query_call(Qiniu_Client *self, const char *accessKey, const char *const bucketName, cJSON **ret)
+const char *const *_Qiniu_Region_Get_All_Rs_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
{
- char *url;
- Qiniu_Error err;
- Qiniu_Zero(err);
- const char *hosts[3] = {NULL, NULL, NULL};
- int hostsLen = 0;
-
- if (QINIU_UC_HOST != NULL)
+ if (count != NULL)
{
- hosts[hostsLen++] = QINIU_UC_HOST;
+ *count = 0;
}
- if (QINIU_UC_HOST_BACKUP != NULL)
+ if (region == NULL)
{
- hosts[hostsLen++] = QINIU_UC_HOST_BACKUP;
+ return NULL;
}
- if (QINIU_API_HOST != NULL)
+ if (region->rsService == NULL)
{
- hosts[hostsLen++] = QINIU_API_HOST;
+ return NULL;
}
-
- for (int i = 0; i < hostsLen; i++)
+ size_t allHostsCount = region->rsService->preferredHostsCount + region->rsService->alternativeHostsCount;
+ if (accelerationEnabled)
{
- url = Qiniu_String_Concat(hosts[i], "/v4/query?ak=", accessKey, "&bucket=", bucketName, NULL);
- err = Qiniu_Client_Call(self, ret, url);
- Qiniu_Free(url);
- if (err.code == 200)
- {
- cJSON *hostsJson = Qiniu_Json_GetObjectItem(*ret, "hosts", NULL);
- if (hostsJson == NULL)
- {
- continue;
- }
- cJSON *firstHostJson = Qiniu_Json_GetArrayItem(hostsJson, 0, NULL);
- if (firstHostJson == NULL)
- {
- continue;
- }
- return err;
- }
- else if (!isRegionQueryRetryable(err))
- {
- return err;
- }
+ allHostsCount += region->rsService->acceleratedHostsCount;
}
- if (err.code == 0)
+ if (region->rsService->hosts == NULL || allHostsCount == 0)
{
- err.code = 599;
- err.message = "no uc hosts available";
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->rsService->hosts;
+ }
+ else
+ {
+ return region->rsService->hosts + region->rsService->acceleratedHostsCount;
}
- return err;
}
-Qiniu_Error
-_Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps)
+const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, size_t *count)
{
- Qiniu_Error err;
- cJSON *root;
-
- if (accessKey == NULL)
+ if (count != NULL)
{
- accessKey = self->auth.itbl->GetAccessKey(self->auth.self);
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->rsfService == NULL)
+ {
+ return NULL;
+ }
+ if (region->rsfService->hosts == NULL || region->rsfService->preferredHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = region->rsfService->preferredHostsCount;
+ }
+ return region->rsfService->hosts + region->rsfService->acceleratedHostsCount;
+}
+
+const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+{
+ if (count != NULL)
+ {
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->rsfService == NULL)
+ {
+ return NULL;
+ }
+ if (region->rsfService->hosts == NULL || region->rsfService->alternativeHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = region->rsfService->alternativeHostsCount;
+ }
+ return region->rsfService->hosts + region->rsfService->acceleratedHostsCount + region->rsfService->preferredHostsCount;
+}
+
+const char *const *_Qiniu_Region_Get_All_Rsf_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
+{
+ if (count != NULL)
+ {
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->rsfService == NULL)
+ {
+ return NULL;
+ }
+ size_t allHostsCount = region->rsfService->preferredHostsCount + region->rsfService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->rsfService->acceleratedHostsCount;
+ }
+ if (region->rsfService->hosts == NULL || allHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->rsfService->hosts;
+ }
+ else
+ {
+ return region->rsfService->hosts + region->rsfService->acceleratedHostsCount;
+ }
+}
+
+const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, size_t *count)
+{
+ if (count != NULL)
+ {
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->apiService == NULL)
+ {
+ return NULL;
+ }
+ if (region->apiService->hosts == NULL || region->apiService->preferredHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = region->apiService->preferredHostsCount;
+ }
+ return region->apiService->hosts + region->apiService->acceleratedHostsCount;
+}
+
+const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region, size_t *count)
+{
+ if (count != NULL)
+ {
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->apiService == NULL)
+ {
+ return NULL;
+ }
+ if (region->apiService->hosts == NULL || region->apiService->alternativeHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = region->apiService->alternativeHostsCount;
+ }
+ return region->apiService->hosts + region->apiService->acceleratedHostsCount + region->apiService->preferredHostsCount;
+}
+
+const char *const *_Qiniu_Region_Get_All_Api_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled)
+{
+ if (count != NULL)
+ {
+ *count = 0;
+ }
+ if (region == NULL)
+ {
+ return NULL;
+ }
+ if (region->apiService == NULL)
+ {
+ return NULL;
+ }
+ size_t allHostsCount = region->apiService->preferredHostsCount + region->apiService->alternativeHostsCount;
+ if (accelerationEnabled)
+ {
+ allHostsCount += region->apiService->acceleratedHostsCount;
+ }
+ if (region->apiService->hosts == NULL || allHostsCount == 0)
+ {
+ return NULL;
+ }
+ if (count != NULL)
+ {
+ *count = allHostsCount;
+ }
+ if (accelerationEnabled)
+ {
+ return region->apiService->hosts;
+ }
+ else
+ {
+ return region->apiService->hosts + region->apiService->acceleratedHostsCount;
+ }
+}
+
+Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps)
+{
+ Qiniu_Region *region = Qiniu_Region_New(regionId, 0);
+ if (region == NULL)
+ {
+ return NULL;
+ }
+
+ char buf[32], buf2[32];
+ Qiniu_Zero(buf);
+ Qiniu_Zero(buf2);
+
+ if (strcmp(regionId, "z0") == 0)
+ {
+ region->upService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){"upload.qiniup.com", "up.qiniup.com"}, 2, (const char *const[]){"up.qbox.me"}, 1, useHttps);
+ region->ioService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){"iovip.qiniuio.com"}, 1, (const char *const[]){"iovip.qbox.me"}, 1, useHttps);
+ }
+ else
+ {
+ Qiniu_snprintf(buf, sizeof(buf), "upload-%s.qiniup.com", regionId);
+ Qiniu_snprintf(buf2, sizeof(buf2), "up-%s.qiniup.com", regionId);
+ region->upService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf, buf2}, 2, NULL, 0, useHttps);
+ Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qiniuio.com", regionId);
+ region->ioService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps);
+ }
+ Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qiniuapi.com", regionId);
+ region->rsService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps);
+ Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qiniuapi.com", regionId);
+ region->rsfService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps);
+ Qiniu_snprintf(buf, sizeof(buf), "api-%s.qiniuapi.com", regionId);
+ region->apiService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps);
+ return region;
+}
+
+static Qiniu_Bool isRegionQueryRetryable(Qiniu_Error err)
+{
+ switch (err.code)
+ {
+ case 501:
+ case 509:
+ case 573:
+ case 579:
+ case 608:
+ case 612:
+ case 614:
+ case 616:
+ case 618:
+ case 630:
+ case 631:
+ case 632:
+ case 640:
+ case 701:
+ return Qiniu_False;
+ default:
+ return err.code >= 500 || err.code < 100;
+ }
+}
+
+QINIU_DLLAPI extern const char *QINIU_UC_HOST_BACKUP_2;
+
+static const char **_Qiniu_Get_Bucket_Hosts(Qiniu_Client *self, size_t *count, Qiniu_Bool *needToFree)
+{
+ const char **hosts = NULL;
+ *count = 0;
+ *needToFree = Qiniu_False;
+
+ if (self->specifiedRegion == NULL)
+ {
+ hosts = (const char **)malloc(sizeof(const char *) * 3);
+ *needToFree = Qiniu_True;
+ if (QINIU_UC_HOST != NULL)
+ {
+ hosts[(*count)++] = QINIU_UC_HOST;
+ }
+ if (QINIU_UC_HOST_BACKUP != NULL)
+ {
+ hosts[(*count)++] = QINIU_UC_HOST_BACKUP;
+ }
+ if (QINIU_UC_HOST_BACKUP_2 != NULL)
+ {
+ hosts[(*count)++] = QINIU_UC_HOST_BACKUP_2;
+ }
+ }
+ else
+ {
+ hosts = (const char **)_Qiniu_Region_Get_All_Bucket_Hosts(self->specifiedRegion, count, Qiniu_False);
+ }
+ return hosts;
+}
+
+static Qiniu_Error _Qiniu_Region_Query_call(Qiniu_Client *self, const char *accessKey, const char *const bucketName, cJSON **ret)
+{
+ char *url;
+ Qiniu_Error err;
+ Qiniu_Zero(err);
+ const char *const *hosts;
+ size_t hostsLen;
+ Qiniu_Bool needToFreeHosts;
+
+ hosts = _Qiniu_Get_Bucket_Hosts(self, &hostsLen, &needToFreeHosts);
+
+ for (size_t i = 0; i < hostsLen; i++)
+ {
+ url = Qiniu_String_Concat(hosts[i], "/v4/query?ak=", accessKey, "&bucket=", bucketName, NULL);
+ err = Qiniu_Client_Call(self, ret, url);
+ Qiniu_Free(url);
+ if (err.code == 200)
+ {
+ cJSON *hostsJson = Qiniu_Json_GetObjectItem(*ret, "hosts", NULL);
+ if (hostsJson == NULL)
+ {
+ continue;
+ }
+ cJSON *firstHostJson = Qiniu_Json_GetArrayItem(hostsJson, 0, NULL);
+ if (firstHostJson == NULL)
+ {
+ continue;
+ }
+ goto handleErr;
+ }
+ else if (!isRegionQueryRetryable(err))
+ {
+ goto handleErr;
+ }
+ }
+ if (err.code == 0)
+ {
+ err.code = 599;
+ err.message = "no uc hosts available";
+ }
+handleErr:
+ if (needToFreeHosts)
+ {
+ Qiniu_Free((void *)hosts);
+ }
+ return err;
+}
+
+static struct _Qiniu_Region_Service *_Qiniu_Region_New_From_JSON(cJSON *json, Qiniu_Bool useHttps)
+{
+ cJSON *accDomainsJson = Qiniu_Json_GetObjectItem(json, "acc_domains", NULL);
+ cJSON *domainsJson = Qiniu_Json_GetObjectItem(json, "domains", NULL);
+ cJSON *oldJson = Qiniu_Json_GetObjectItem(json, "old", NULL);
+ size_t accDomainsCount = 0, domainsCount = 0, oldCount = 0;
+ if (accDomainsJson != NULL)
+ {
+ accDomainsCount = cJSON_GetArraySize(accDomainsJson);
+ }
+ if (domainsJson != NULL)
+ {
+ domainsCount = cJSON_GetArraySize(domainsJson);
+ }
+ if (oldJson != NULL)
+ {
+ oldCount = cJSON_GetArraySize(oldJson);
+ }
+
+ struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service));
+ if (newService == NULL)
+ {
+ goto handleErr;
+ }
+ const char **newHosts = malloc(sizeof(const char *) * (accDomainsCount + domainsCount + oldCount));
+ if (newHosts == NULL)
+ {
+ goto handleErr;
+ }
+ size_t offset = 0, i;
+ const char *h;
+ for (i = 0; i < accDomainsCount; i++)
+ {
+ h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(accDomainsJson, i, NULL), useHttps);
+ if (h == NULL)
+ {
+ goto handleErr;
+ }
+ newHosts[offset + i] = h;
+ }
+ offset += accDomainsCount;
+ for (i = 0; i < domainsCount; i++)
+ {
+ h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(domainsJson, i, NULL), useHttps);
+ if (h == NULL)
+ {
+ goto handleErr;
+ }
+ newHosts[offset + i] = h;
+ }
+ offset += domainsCount;
+ for (i = 0; i < oldCount; i++)
+ {
+ h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(oldJson, i, NULL), useHttps);
+ if (h == NULL)
+ {
+ goto handleErr;
+ }
+ newHosts[offset + i] = h;
+ }
+ newService->hosts = newHosts;
+ newService->acceleratedHostsCount = accDomainsCount;
+ newService->preferredHostsCount = domainsCount;
+ newService->alternativeHostsCount = oldCount;
+ return newService;
+handleErr:
+ if (newHosts != NULL)
+ {
+ for (size_t j = 0; j < offset + i; j++)
+ {
+ free((void *)newHosts[j]);
+ }
+ free((void *)newHosts);
+ }
+ if (newService != NULL)
+ {
+ free((void *)newService);
+ }
+ return NULL;
+}
+
+Qiniu_Error
+_Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps)
+{
+ Qiniu_Error err;
+ cJSON *root;
+
+ if (accessKey == NULL)
+ {
+ accessKey = self->auth.itbl->GetAccessKey(self->auth.self);
}
err = _Qiniu_Region_Query_call(self, accessKey, bucketName, &root);
if (err.code == 200)
@@ -943,253 +1402,43 @@ _Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *acce
cJSON *upJson = Qiniu_Json_GetObjectItem(firstHostJson, "up", NULL);
if (upJson != NULL)
{
- struct _Qiniu_Region_Hosts *upPreferredRegionHosts = NULL, *upAlternativeRegionHosts = NULL;
- cJSON *upDomainsJson = Qiniu_Json_GetObjectItem(upJson, "domains", NULL);
- if (upDomainsJson != NULL)
- {
- int upPreferredHostsCount = cJSON_GetArraySize(upDomainsJson);
- if (upPreferredHostsCount > 0)
- {
- const char **upPreferredHosts = (const char **)malloc(sizeof(const char *) * upPreferredHostsCount);
- for (int i = 0; i < upPreferredHostsCount; i++)
- {
- upPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(upDomainsJson, i, NULL), useHttps);
- }
- upPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(upPreferredHosts, upPreferredHostsCount);
- }
- }
- cJSON *upOldJson = Qiniu_Json_GetObjectItem(upJson, "old", NULL);
- if (upOldJson != NULL)
- {
- int upAlternativeHostsCount = cJSON_GetArraySize(upOldJson);
- if (upAlternativeHostsCount > 0)
- {
- const char **upAlternativeHosts = (const char **)malloc(sizeof(const char *) * upAlternativeHostsCount);
- for (int i = 0; i < upAlternativeHostsCount; i++)
- {
- upAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(upOldJson, i, NULL), useHttps);
- }
- upAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(upAlternativeHosts, upAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *upService = _Qiniu_Region_Service_New(upPreferredRegionHosts, upAlternativeRegionHosts);
- region->upService = upService;
+ region->upService = _Qiniu_Region_New_From_JSON(upJson, useHttps);
}
cJSON *ioJson = Qiniu_Json_GetObjectItem(firstHostJson, "io", NULL);
if (ioJson != NULL)
{
- struct _Qiniu_Region_Hosts *ioPreferredRegionHosts = NULL, *ioAlternativeRegionHosts = NULL;
- cJSON *ioDomainsJson = Qiniu_Json_GetObjectItem(ioJson, "domains", NULL);
- if (ioDomainsJson != NULL)
- {
- int ioPreferredHostsCount = cJSON_GetArraySize(ioDomainsJson);
- if (ioPreferredHostsCount > 0)
- {
- const char **ioPreferredHosts = (const char **)malloc(sizeof(const char *) * ioPreferredHostsCount);
- for (int i = 0; i < ioPreferredHostsCount; i++)
- {
- ioPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioDomainsJson, i, NULL), useHttps);
- }
- ioPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioPreferredHosts, ioPreferredHostsCount);
- }
- }
- cJSON *ioOldJson = Qiniu_Json_GetObjectItem(ioJson, "old", NULL);
- if (ioOldJson != NULL)
- {
- int ioAlternativeHostsCount = cJSON_GetArraySize(ioOldJson);
- if (ioAlternativeHostsCount > 0)
- {
- const char **ioAlternativeHosts = (const char **)malloc(sizeof(const char *) * ioAlternativeHostsCount);
- for (int i = 0; i < ioAlternativeHostsCount; i++)
- {
- ioAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioOldJson, i, NULL), useHttps);
- }
- ioAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioAlternativeHosts, ioAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *ioService = _Qiniu_Region_Service_New(ioPreferredRegionHosts, ioAlternativeRegionHosts);
- region->ioService = ioService;
+ region->ioService = _Qiniu_Region_New_From_JSON(ioJson, useHttps);
}
cJSON *ioSrcJson = Qiniu_Json_GetObjectItem(firstHostJson, "io_src", NULL);
if (ioSrcJson != NULL)
{
- struct _Qiniu_Region_Hosts *ioSrcPreferredRegionHosts = NULL, *ioSrcAlternativeRegionHosts = NULL;
- cJSON *ioSrcDomainsJson = Qiniu_Json_GetObjectItem(ioSrcJson, "domains", NULL);
- if (ioSrcDomainsJson != NULL)
- {
- int ioSrcPreferredHostsCount = cJSON_GetArraySize(ioSrcDomainsJson);
- if (ioSrcPreferredHostsCount > 0)
- {
- const char **ioSrcPreferredHosts = (const char **)malloc(sizeof(const char *) * ioSrcPreferredHostsCount);
- for (int i = 0; i < ioSrcPreferredHostsCount; i++)
- {
- ioSrcPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioSrcDomainsJson, i, NULL), useHttps);
- }
- ioSrcPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioSrcPreferredHosts, ioSrcPreferredHostsCount);
- }
- }
- cJSON *ioSrcOldJson = Qiniu_Json_GetObjectItem(ioSrcJson, "old", NULL);
- if (ioSrcOldJson != NULL)
- {
- int ioSrcAlternativeHostsCount = cJSON_GetArraySize(ioSrcOldJson);
- if (ioSrcAlternativeHostsCount > 0)
- {
- const char **ioSrcAlternativeHosts = (const char **)malloc(sizeof(const char *) * ioSrcAlternativeHostsCount);
- for (int i = 0; i < ioSrcAlternativeHostsCount; i++)
- {
- ioSrcAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioSrcOldJson, i, NULL), useHttps);
- }
- ioSrcAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioSrcAlternativeHosts, ioSrcAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *ioSrcService = _Qiniu_Region_Service_New(ioSrcPreferredRegionHosts, ioSrcAlternativeRegionHosts);
- region->ioSrcService = ioSrcService;
+ region->ioSrcService = _Qiniu_Region_New_From_JSON(ioSrcJson, useHttps);
}
cJSON *ucJson = Qiniu_Json_GetObjectItem(firstHostJson, "uc", NULL);
if (ucJson != NULL)
{
- struct _Qiniu_Region_Hosts *ucPreferredRegionHosts = NULL, *ucAlternativeRegionHosts = NULL;
- cJSON *ucDomainsJson = Qiniu_Json_GetObjectItem(ucJson, "domains", NULL);
- if (ucDomainsJson != NULL)
- {
- int ucPreferredHostsCount = cJSON_GetArraySize(ucDomainsJson);
- if (ucPreferredHostsCount > 0)
- {
- const char **ucPreferredHosts = (const char **)malloc(sizeof(const char *) * ucPreferredHostsCount);
- for (int i = 0; i < ucPreferredHostsCount; i++)
- {
- ucPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ucDomainsJson, i, NULL), useHttps);
- }
- ucPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ucPreferredHosts, ucPreferredHostsCount);
- }
- }
- cJSON *ucOldJson = Qiniu_Json_GetObjectItem(ucJson, "old", NULL);
- if (ucOldJson != NULL)
- {
- int ucAlternativeHostsCount = cJSON_GetArraySize(ucOldJson);
- if (ucAlternativeHostsCount > 0)
- {
- const char **ucAlternativeHosts = (const char **)malloc(sizeof(const char *) * ucAlternativeHostsCount);
- for (int i = 0; i < ucAlternativeHostsCount; i++)
- {
- ucAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ucOldJson, i, NULL), useHttps);
- }
- ucAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ucAlternativeHosts, ucAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *ucService = _Qiniu_Region_Service_New(ucPreferredRegionHosts, ucAlternativeRegionHosts);
- region->ucService = ucService;
+ region->ucService = _Qiniu_Region_New_From_JSON(ucJson, useHttps);
}
cJSON *rsJson = Qiniu_Json_GetObjectItem(firstHostJson, "rs", NULL);
if (rsJson != NULL)
{
- struct _Qiniu_Region_Hosts *rsPreferredRegionHosts = NULL, *rsAlternativeRegionHosts = NULL;
- cJSON *rsDomainsJson = Qiniu_Json_GetObjectItem(rsJson, "domains", NULL);
- if (rsDomainsJson != NULL)
- {
- int rsPreferredHostsCount = cJSON_GetArraySize(rsDomainsJson);
- if (rsPreferredHostsCount > 0)
- {
- const char **rsPreferredHosts = (const char **)malloc(sizeof(const char *) * rsPreferredHostsCount);
- for (int i = 0; i < rsPreferredHostsCount; i++)
- {
- rsPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsDomainsJson, i, NULL), useHttps);
- }
- rsPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsPreferredHosts, rsPreferredHostsCount);
- }
- }
- cJSON *rsOldJson = Qiniu_Json_GetObjectItem(rsJson, "old", NULL);
- if (rsOldJson != NULL)
- {
- int rsAlternativeHostsCount = cJSON_GetArraySize(rsOldJson);
- if (rsAlternativeHostsCount > 0)
- {
- const char **rsAlternativeHosts = (const char **)malloc(sizeof(const char *) * rsAlternativeHostsCount);
- for (int i = 0; i < rsAlternativeHostsCount; i++)
- {
- rsAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsOldJson, i, NULL), useHttps);
- }
- rsAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsAlternativeHosts, rsAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *rsService = _Qiniu_Region_Service_New(rsPreferredRegionHosts, rsAlternativeRegionHosts);
- region->rsService = rsService;
+ region->rsService = _Qiniu_Region_New_From_JSON(rsJson, useHttps);
}
cJSON *rsfJson = Qiniu_Json_GetObjectItem(firstHostJson, "rsf", NULL);
if (rsfJson != NULL)
{
- struct _Qiniu_Region_Hosts *rsfPreferredRegionHosts = NULL, *rsfAlternativeRegionHosts = NULL;
- cJSON *rsfDomainsJson = Qiniu_Json_GetObjectItem(rsfJson, "domains", NULL);
- if (rsfDomainsJson != NULL)
- {
- int rsfPreferredHostsCount = cJSON_GetArraySize(rsfDomainsJson);
- if (rsfPreferredHostsCount > 0)
- {
- const char **rsfPreferredHosts = (const char **)malloc(sizeof(const char *) * rsfPreferredHostsCount);
- for (int i = 0; i < rsfPreferredHostsCount; i++)
- {
- rsfPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsfDomainsJson, i, NULL), useHttps);
- }
- rsfPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsfPreferredHosts, rsfPreferredHostsCount);
- }
- }
- cJSON *rsfOldJson = Qiniu_Json_GetObjectItem(rsfJson, "old", NULL);
- if (rsfOldJson != NULL)
- {
- int rsfAlternativeHostsCount = cJSON_GetArraySize(rsfOldJson);
- if (rsfAlternativeHostsCount > 0)
- {
- const char **rsfAlternativeHosts = (const char **)malloc(sizeof(const char *) * rsfAlternativeHostsCount);
- for (int i = 0; i < rsfAlternativeHostsCount; i++)
- {
- rsfAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsfOldJson, i, NULL), useHttps);
- }
- rsfAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsfAlternativeHosts, rsfAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *rsfService = _Qiniu_Region_Service_New(rsfPreferredRegionHosts, rsfAlternativeRegionHosts);
- region->rsfService = rsfService;
+ region->rsfService = _Qiniu_Region_New_From_JSON(rsfJson, useHttps);
}
cJSON *apiJson = Qiniu_Json_GetObjectItem(firstHostJson, "api", NULL);
if (apiJson != NULL)
{
- struct _Qiniu_Region_Hosts *apiPreferredRegionHosts = NULL, *apiAlternativeRegionHosts = NULL;
- cJSON *apiDomainsJson = Qiniu_Json_GetObjectItem(apiJson, "domains", NULL);
- if (apiDomainsJson != NULL)
- {
- int apiPreferredHostsCount = cJSON_GetArraySize(apiDomainsJson);
- if (apiPreferredHostsCount > 0)
- {
- const char **apiPreferredHosts = (const char **)malloc(sizeof(const char *) * apiPreferredHostsCount);
- for (int i = 0; i < apiPreferredHostsCount; i++)
- {
- apiPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(apiDomainsJson, i, NULL), useHttps);
- }
- apiPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(apiPreferredHosts, apiPreferredHostsCount);
- }
- }
- cJSON *apiOldJson = Qiniu_Json_GetObjectItem(apiJson, "old", NULL);
- if (apiOldJson != NULL)
- {
- int apiAlternativeHostsCount = cJSON_GetArraySize(apiOldJson);
- if (apiAlternativeHostsCount > 0)
- {
- const char **apiAlternativeHosts = (const char **)malloc(sizeof(const char *) * apiAlternativeHostsCount);
- for (int i = 0; i < apiAlternativeHostsCount; i++)
- {
- apiAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(apiOldJson, i, NULL), useHttps);
- }
- apiAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(apiAlternativeHosts, apiAlternativeHostsCount);
- }
- }
- struct _Qiniu_Region_Service *apiService = _Qiniu_Region_Service_New(apiPreferredRegionHosts, apiAlternativeRegionHosts);
- region->apiService = apiService;
+ region->apiService = _Qiniu_Region_New_From_JSON(apiJson, useHttps);
}
*pRegion = region;
@@ -1204,99 +1453,237 @@ Qiniu_Error Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const
return _Qiniu_Region_Query(self, pRegion, NULL, bucketName, useHttps);
}
-Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+struct _Qiniu_Region_Cache
{
- const char *const *hosts;
- size_t count;
- Qiniu_Error err = Qiniu_OK;
- Qiniu_Region **foundRegion = &self->cachedRegion;
+ Qiniu_Region *region;
+ const char *accessKey, *bucketName, *const *bucketHosts;
+ size_t bucketHostsLen;
+ Qiniu_Bool needToFreeBucketHosts, uploadingAccelerationEnabled;
+};
- if (self->specifiedRegion)
+static int _Qiniu_Compare_Str(const char *a, const char *b)
+{
+ if (a == NULL && b == NULL)
{
- foundRegion = &self->specifiedRegion;
- goto foundCache;
+ return 0;
+ }
+ else if (a == NULL)
+ {
+ return -1;
+ }
+ else if (b == NULL)
+ {
+ return 1;
+ }
+ else
+ {
+ return strcmp(a, b);
+ }
+}
+
+static int _Qiniu_Region_Cache_Compare(const void *a, const void *b, void *user_data)
+{
+ int result;
+ struct _Qiniu_Region_Cache *cacheA = (struct _Qiniu_Region_Cache *)a;
+ struct _Qiniu_Region_Cache *cacheB = (struct _Qiniu_Region_Cache *)b;
+ result = _Qiniu_Compare_Str(cacheA->accessKey, cacheB->accessKey);
+ if (!result)
+ {
+ return result;
}
- else if (self->autoQueryRegion)
+ result = _Qiniu_Compare_Str(cacheA->bucketName, cacheB->bucketName);
+ if (!result)
{
- if (self->cachedRegion != NULL)
+ return result;
+ }
+ if (cacheA->bucketHostsLen == cacheB->bucketHostsLen)
+ {
+ for (size_t i = 0; i < cacheA->bucketHostsLen; i++)
{
- if (strcmp(self->cachedRegionBucketName, bucketName) == 0)
- {
- goto foundCache;
- }
- else
+ result = _Qiniu_Compare_Str(cacheA->bucketHosts[i], cacheB->bucketHosts[i]);
+ if (!result)
{
- Qiniu_Region_Free(self->cachedRegion);
- self->cachedRegion = NULL;
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
+ return result;
}
}
- err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
- if (err.code != 200)
- {
- goto useDefault;
- }
- self->cachedRegionBucketName = Qiniu_String_Dup(bucketName);
- goto foundCache;
+ return result;
}
-useDefault:
- *host = QINIU_UP_HOST;
- return err;
+ else
+ {
+ return cacheA->bucketHostsLen - cacheB->bucketHostsLen;
+ }
+}
-foundCache:
- hosts = Qiniu_Region_Get_Up_Preferred_Hosts(*foundRegion, &count);
- if (count == 0)
+static void _Qiniu_Region_Cache_Free(void *r)
+{
+ struct _Qiniu_Region_Cache *cache = (struct _Qiniu_Region_Cache *)r;
+ Qiniu_Free((void *)cache->accessKey);
+ Qiniu_Free((void *)cache->bucketName);
+ if (cache->needToFreeBucketHosts)
{
- *host = QINIU_UP_HOST;
+ Qiniu_Free((void *)cache->bucketHosts);
}
- else
+ Qiniu_Region_Free(cache->region);
+ Qiniu_Zero_Ptr(cache);
+}
+
+static uint64_t _Qiniu_Region_Cache_Hash(const void *r, uint64_t seed0, uint64_t seed1)
+{
+ struct _Qiniu_Region_Cache *cache = (struct _Qiniu_Region_Cache *)r;
+ uint64_t hash = 0;
+ if (cache->accessKey != NULL)
{
- *host = hosts[0];
+ hash ^= hashmap_sip(cache->accessKey, strlen(cache->accessKey), seed0, seed1);
}
- return err;
+ if (cache->bucketName != NULL)
+ {
+ hash ^= hashmap_sip(cache->bucketName, strlen(cache->bucketName), seed0, seed1);
+ }
+ hash ^= hashmap_sip(&cache->bucketHostsLen, sizeof(cache->bucketHostsLen), seed0, seed1);
+ for (size_t i = 0; i < cache->bucketHostsLen; i++)
+ {
+ hash ^= hashmap_sip(cache->bucketHosts[i], strlen(cache->bucketHosts[i]), seed0, seed1);
+ }
+ hash ^= hashmap_sip(&cache->uploadingAccelerationEnabled, sizeof(cache->uploadingAccelerationEnabled), seed0, seed1);
+ return hash;
}
-Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+static Qiniu_Error _Qiniu_Region_Auto_Query_With_Cache(Qiniu_Client *self, const char *accessKey, const char *bucketName, Qiniu_Region **foundRegion)
{
const char *const *hosts;
- size_t count;
+ size_t hostsLen;
+ Qiniu_Bool needToFreeHosts;
Qiniu_Error err = Qiniu_OK;
- Qiniu_Region **foundRegion = &self->cachedRegion;
- if (self->specifiedRegion)
+ if (accessKey == NULL)
{
- foundRegion = &self->specifiedRegion;
- goto foundCache;
+ accessKey = QINIU_ACCESS_KEY;
}
- else if (self->autoQueryRegion)
+
+ hosts = _Qiniu_Get_Bucket_Hosts(self, &hostsLen, &needToFreeHosts);
+ const struct _Qiniu_Region_Cache cacheKey = {
+ .accessKey = accessKey,
+ .bucketName = bucketName,
+ .bucketHosts = hosts,
+ .bucketHostsLen = hostsLen,
+ .needToFreeBucketHosts = needToFreeHosts,
+ .uploadingAccelerationEnabled = self->enableUploadingAcceleration,
+ };
+ struct _Qiniu_Region_Cache *cache = NULL;
+ if (self->cachedRegions != NULL)
{
- if (self->cachedRegion != NULL)
+ cache = (struct _Qiniu_Region_Cache *)hashmap_get(self->cachedRegions, &cacheKey);
+ if (cache != NULL && !Qiniu_Region_Is_Expired(cache->region))
{
- if (strcmp(self->cachedRegionBucketName, bucketName) == 0 && !Qiniu_Region_Is_Expired(self->cachedRegion))
- {
- goto foundCache;
- }
- else
+ *foundRegion = cache->region;
+ err = Qiniu_OK;
+ goto handleErr;
+ }
+ }
+ err = _Qiniu_Region_Query(self, foundRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
+ if (err.code != 200)
+ {
+ if (cache != NULL)
+ { // 有已经过期的缓存区域可以使用
+ *foundRegion = cache->region;
+ err = Qiniu_OK;
+ }
+ goto handleErr;
+ }
+ if (self->cachedRegions == NULL)
+ {
+ self->cachedRegions = hashmap_new(sizeof(struct _Qiniu_Region_Cache), 0, rand(), rand(), _Qiniu_Region_Cache_Hash, _Qiniu_Region_Cache_Compare, _Qiniu_Region_Cache_Free, NULL);
+ }
+ if (self->cachedRegions != NULL)
+ {
+ if (cache != NULL)
+ { // 复用前面已经过期的缓存的内存
+ Qiniu_Region_Free(cache->region);
+ cache->region = *foundRegion;
+ }
+ else
+ {
+ struct _Qiniu_Region_Cache *newCache = malloc(sizeof(struct _Qiniu_Region_Cache));
+ if (newCache != NULL)
{
- Qiniu_Region_Free(self->cachedRegion);
- self->cachedRegion = NULL;
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
+ *newCache = (struct _Qiniu_Region_Cache){
+ .region = *foundRegion,
+ .accessKey = Qiniu_String_Dup(accessKey),
+ .bucketName = Qiniu_String_Dup(bucketName),
+ .bucketHosts = hosts,
+ .bucketHostsLen = hostsLen,
+ .needToFreeBucketHosts = needToFreeHosts,
+ };
+ hashmap_set(self->cachedRegions, newCache);
+ needToFreeHosts = Qiniu_False;
}
}
- err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
- if (err.code != 200)
+ }
+handleErr:
+ if (needToFreeHosts)
+ {
+ Qiniu_Free((void *)hosts);
+ }
+ return err;
+}
+
+Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count)
+{
+ Qiniu_Error err = Qiniu_OK;
+ Qiniu_Region *foundRegion = NULL;
+
+ if (self->specifiedRegion)
+ {
+ foundRegion = self->specifiedRegion;
+ goto foundCache;
+ }
+ else if (self->autoQueryRegion && bucketName != NULL)
+ {
+ err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion);
+ if (err.code == 200)
{
- goto useDefault;
+ goto foundCache;
}
- self->cachedRegionBucketName = Qiniu_String_Dup(bucketName);
- goto foundCache;
+ return err;
}
-useDefault:
- *host = QINIU_IOVIP_HOST;
+foundCache:
+ *hosts = _Qiniu_Region_Get_All_Up_Hosts(foundRegion, count, self->enableUploadingAcceleration);
return err;
+}
+Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count)
+{
+ Qiniu_Error err = Qiniu_OK;
+ Qiniu_Region *foundRegion = NULL;
+
+ if (self->specifiedRegion)
+ {
+ foundRegion = self->specifiedRegion;
+ goto foundCache;
+ }
+ else if (self->autoQueryRegion && bucketName != NULL)
+ {
+ err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion);
+ if (err.code == 200)
+ {
+ goto foundCache;
+ }
+ return err;
+ }
foundCache:
- hosts = Qiniu_Region_Get_Io_Preferred_Hosts(*foundRegion, &count);
+ *hosts = _Qiniu_Region_Get_All_Io_Hosts(foundRegion, count, Qiniu_False);
+ return err;
+}
+
+Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+{
+ const char *const *hosts;
+ size_t count;
+ Qiniu_Error err = _Qiniu_Region_Get_Io_Hosts(self, accessKey, bucketName, &hosts, &count);
+ if (err.code != 200)
+ {
+ return err;
+ }
if (count == 0)
{
*host = QINIU_IOVIP_HOST;
@@ -1308,47 +1695,39 @@ Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey,
return err;
}
-Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count)
{
- const char *const *hosts;
- size_t count;
Qiniu_Error err = Qiniu_OK;
- Qiniu_Region **foundRegion = &self->cachedRegion;
+ Qiniu_Region *foundRegion = NULL;
if (self->specifiedRegion)
{
- foundRegion = &self->specifiedRegion;
+ foundRegion = self->specifiedRegion;
goto foundCache;
}
- else if (self->autoQueryRegion)
+ else if (self->autoQueryRegion && bucketName != NULL)
{
- if (self->cachedRegion != NULL)
- {
- if (strcmp(self->cachedRegionBucketName, bucketName) == 0)
- {
- goto foundCache;
- }
- else
- {
- Qiniu_Region_Free(self->cachedRegion);
- self->cachedRegion = NULL;
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
- }
- }
- err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
- if (err.code != 200)
+ err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion);
+ if (err.code == 200)
{
- goto useDefault;
+ goto foundCache;
}
- self->cachedRegionBucketName = Qiniu_String_Dup(bucketName);
- goto foundCache;
+ return err;
}
-useDefault:
- *host = QINIU_RS_HOST;
+foundCache:
+ *hosts = _Qiniu_Region_Get_All_Rs_Hosts(foundRegion, count, Qiniu_False);
return err;
+}
-foundCache:
- hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(*foundRegion, &count);
+Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+{
+ const char *const *hosts;
+ size_t count;
+ Qiniu_Error err = _Qiniu_Region_Get_Rs_Hosts(self, accessKey, bucketName, &hosts, &count);
+ if (err.code != 200)
+ {
+ return err;
+ }
if (count == 0)
{
*host = QINIU_RS_HOST;
@@ -1360,47 +1739,39 @@ Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey,
return err;
}
-Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count)
{
- const char *const *hosts;
- size_t count;
Qiniu_Error err = Qiniu_OK;
- Qiniu_Region **foundRegion = &self->cachedRegion;
+ Qiniu_Region *foundRegion = NULL;
if (self->specifiedRegion)
{
- foundRegion = &self->specifiedRegion;
+ foundRegion = self->specifiedRegion;
goto foundCache;
}
- else if (self->autoQueryRegion)
+ else if (self->autoQueryRegion && bucketName != NULL)
{
- if (self->cachedRegion != NULL)
- {
- if (strcmp(self->cachedRegionBucketName, bucketName) == 0)
- {
- goto foundCache;
- }
- else
- {
- Qiniu_Region_Free(self->cachedRegion);
- self->cachedRegion = NULL;
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
- }
- }
- err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
- if (err.code != 200)
+ err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion);
+ if (err.code == 200)
{
- goto useDefault;
+ goto foundCache;
}
- self->cachedRegionBucketName = Qiniu_String_Dup(bucketName);
- goto foundCache;
+ return err;
}
-useDefault:
- *host = QINIU_RSF_HOST;
+foundCache:
+ *hosts = _Qiniu_Region_Get_All_Rsf_Hosts(foundRegion, count, Qiniu_False);
return err;
+}
-foundCache:
- hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(*foundRegion, &count);
+Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+{
+ const char *const *hosts;
+ size_t count;
+ Qiniu_Error err = _Qiniu_Region_Get_Rsf_Hosts(self, accessKey, bucketName, &hosts, &count);
+ if (err.code != 200)
+ {
+ return err;
+ }
if (count == 0)
{
*host = QINIU_RSF_HOST;
@@ -1412,47 +1783,39 @@ Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey
return err;
}
-Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count)
{
- const char *const *hosts;
- size_t count;
Qiniu_Error err = Qiniu_OK;
- Qiniu_Region **foundRegion = &self->cachedRegion;
+ Qiniu_Region *foundRegion = NULL;
if (self->specifiedRegion)
{
- foundRegion = &self->specifiedRegion;
+ foundRegion = self->specifiedRegion;
goto foundCache;
}
- else if (self->autoQueryRegion)
+ else if (self->autoQueryRegion && bucketName != NULL)
{
- if (self->cachedRegion != NULL)
- {
- if (strcmp(self->cachedRegionBucketName, bucketName) == 0)
- {
- goto foundCache;
- }
- else
- {
- Qiniu_Region_Free(self->cachedRegion);
- self->cachedRegion = NULL;
- Qiniu_FreeV2((void **)&self->cachedRegionBucketName);
- }
- }
- err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion);
- if (err.code != 200)
+ err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion);
+ if (err.code == 200)
{
- goto useDefault;
+ goto foundCache;
}
- self->cachedRegionBucketName = Qiniu_String_Dup(bucketName);
- goto foundCache;
+ return err;
}
-useDefault:
- *host = QINIU_API_HOST;
+foundCache:
+ *hosts = _Qiniu_Region_Get_All_Api_Hosts(foundRegion, count, Qiniu_False);
return err;
+}
-foundCache:
- hosts = Qiniu_Region_Get_Api_Preferred_Hosts(*foundRegion, &count);
+Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host)
+{
+ const char *const *hosts;
+ size_t count;
+ Qiniu_Error err = _Qiniu_Region_Get_Api_Hosts(self, accessKey, bucketName, &hosts, &count);
+ if (err.code != 200)
+ {
+ return err;
+ }
if (count == 0)
{
*host = QINIU_API_HOST;
diff --git a/qiniu/region.h b/qiniu/region.h
index 9d3d0194..b9d54b23 100644
--- a/qiniu/region.h
+++ b/qiniu/region.h
@@ -11,19 +11,18 @@
#define QINIU_REGION_H
#include "http.h"
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
#ifdef __cplusplus
extern "C"
{
#endif
- struct _Qiniu_Client;
- typedef struct _Qiniu_Client Qiniu_Client;
-
- struct _Qiniu_Region;
- typedef struct _Qiniu_Region Qiniu_Region;
-
QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count);
QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count);
+ QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Accelerated_Hosts(Qiniu_Region *region, size_t *count);
QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count);
QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count);
QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count);
@@ -44,6 +43,7 @@ extern "C"
QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
+ QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Accelerated_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps);
@@ -60,6 +60,10 @@ extern "C"
QINIU_DLLAPI extern Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps);
QINIU_DLLAPI extern Qiniu_Error Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **region, const char *const bucketName, Qiniu_Bool useHttps);
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/qiniu/resumable_io.c b/qiniu/resumable_io.c
index e916c458..11315547 100644
--- a/qiniu/resumable_io.c
+++ b/qiniu/resumable_io.c
@@ -16,7 +16,9 @@
#include "recorder_key.h"
#include "recorder_utils.h"
#include "../cJSON/cJSON.h"
+#include "../hashmap/hashmap.h"
#include "private/region.h"
+#include "private/code.h"
#define blockBits 22
#define blockMask ((1 << blockBits) - 1)
@@ -277,7 +279,7 @@ static Qiniu_Error Qiniu_Rio_PutExtra_Init(
}
else
{
- memset(self, 0, sizeof(Qiniu_Rio_PutExtra));
+ Qiniu_Zero_Ptr(self);
}
cbprog = sizeof(Qiniu_Rio_BlkputRet) * blockCnt;
@@ -349,19 +351,133 @@ static void Qiniu_Io_PutExtra_initFrom(Qiniu_Io_PutExtra *self, Qiniu_Rio_PutExt
}
else
{
- memset(self, 0, sizeof(*self));
+ Qiniu_Zero_Ptr(self);
}
}
/*============================================================================*/
-static Qiniu_Error Qiniu_Rio_bput(
- Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength, const char *url)
+struct _Qiniu_Uploading_Parts_Progress
+{
+ size_t uploaded;
+ struct hashmap *uploading;
+ Qiniu_Mutex mutex;
+};
+
+struct _Qiniu_Uploading_Parts_Progress_Pair
+{
+ int blkIdx;
+ size_t uploaded;
+};
+
+static uint64_t _Qiniu_Uploading_Parts_Progress_Hash(const void *item, uint64_t seed0, uint64_t seed1)
+{
+ const struct _Qiniu_Uploading_Parts_Progress_Pair *pair = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)item;
+ return hashmap_sip(&pair->blkIdx, sizeof(pair->blkIdx), seed0, seed1);
+}
+
+static int _Qiniu_Uploading_Parts_Progress_Compare(const void *a, const void *b, void *udata)
+{
+ const struct _Qiniu_Uploading_Parts_Progress_Pair *left = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)a;
+ const struct _Qiniu_Uploading_Parts_Progress_Pair *right = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)b;
+ return left->blkIdx - right->blkIdx;
+}
+
+static struct _Qiniu_Uploading_Parts_Progress *_Qiniu_Uploading_Parts_Progress_New()
+{
+ struct _Qiniu_Uploading_Parts_Progress *data = (struct _Qiniu_Uploading_Parts_Progress *)malloc(sizeof(struct _Qiniu_Uploading_Parts_Progress));
+ Qiniu_Zero_Ptr(data);
+ Qiniu_Mutex_Init(&data->mutex);
+ data->uploading = hashmap_new(
+ sizeof(struct _Qiniu_Uploading_Parts_Progress_Pair), 0, rand(), rand(),
+ _Qiniu_Uploading_Parts_Progress_Hash, _Qiniu_Uploading_Parts_Progress_Compare,
+ free, NULL);
+ return data;
+}
+
+static void _Qiniu_Uploading_Parts_Progress_Free(struct _Qiniu_Uploading_Parts_Progress *data)
+{
+ hashmap_free(data->uploading);
+ Qiniu_Mutex_Cleanup(&data->mutex);
+ free((void *)data);
+}
+
+static void _Qiniu_Uploading_Parts_Progress_Set_Progress(struct _Qiniu_Uploading_Parts_Progress *progress, int blkIdx, size_t uploaded)
+{
+ Qiniu_Mutex_Lock(&progress->mutex);
+ const struct _Qiniu_Uploading_Parts_Progress_Pair pair = {.blkIdx = blkIdx};
+ struct _Qiniu_Uploading_Parts_Progress_Pair *pPair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)hashmap_get(progress->uploading, &pair);
+ if (pPair == NULL)
+ {
+ pPair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)malloc(sizeof(struct _Qiniu_Uploading_Parts_Progress_Pair));
+ pPair->blkIdx = blkIdx;
+ pPair->uploaded = uploaded;
+ hashmap_set(progress->uploading, pPair);
+ }
+ else
+ {
+ pPair->uploaded = uploaded;
+ }
+
+ Qiniu_Mutex_Unlock(&progress->mutex);
+}
+
+static void _Qiniu_Uploading_Parts_Progress_Part_Uploaded(struct _Qiniu_Uploading_Parts_Progress *progress, int blkIdx, size_t partSize)
+{
+ Qiniu_Mutex_Lock(&progress->mutex);
+ const struct _Qiniu_Uploading_Parts_Progress_Pair pair = {.blkIdx = blkIdx};
+ hashmap_delete(progress->uploading, (const void *)&pair);
+ progress->uploaded += partSize;
+ Qiniu_Mutex_Unlock(&progress->mutex);
+}
+
+static size_t _Qiniu_Uploading_Parts_Progress_Get_Total_Size(struct _Qiniu_Uploading_Parts_Progress *progress)
+{
+ Qiniu_Mutex_Lock(&progress->mutex);
+ size_t total = progress->uploaded, iter = 0;
+ void *iter_val;
+ while (hashmap_iter(progress->uploading, &iter, &iter_val))
+ {
+ struct _Qiniu_Uploading_Parts_Progress_Pair *pair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)iter_val;
+ total += pair->uploaded;
+ }
+ Qiniu_Mutex_Unlock(&progress->mutex);
+ return total;
+}
+
+struct _Qiniu_Progress_Callback_Data
+{
+ size_t totalSize, previousUlNow;
+ int blkIdx;
+ struct _Qiniu_Uploading_Parts_Progress *progress;
+ void (*callback)(size_t, size_t);
+};
+
+static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
+{
+ struct _Qiniu_Progress_Callback_Data *data = (struct _Qiniu_Progress_Callback_Data *)clientp;
+ if (data->previousUlNow != (size_t)ulnow)
+ {
+ _Qiniu_Uploading_Parts_Progress_Set_Progress(data->progress, data->blkIdx, (size_t)ulnow);
+ data->callback((size_t)data->totalSize, _Qiniu_Uploading_Parts_Progress_Get_Total_Size(data->progress));
+ data->previousUlNow = (size_t)ulnow;
+ }
+ return 0;
+}
+
+static Qiniu_Error Qiniu_Rio_bput(Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength, const char *url, struct _Qiniu_Progress_Callback_Data *progressCallback)
{
Qiniu_Rio_BlkputRet retFromResp;
Qiniu_Json *root;
+ int (*callback)(void *, double, double, double, double) = NULL;
+ void *callbackData = NULL;
+ if (progressCallback != NULL && progressCallback->callback != NULL)
+ {
+ callback = _Qiniu_Progress_Callback;
+ callbackData = (void *)progressCallback;
+ }
- Qiniu_Error err = Qiniu_Client_CallWithBinary(self, &root, url, body, bodyLength, NULL);
+ Qiniu_Error err = Qiniu_Client_CallWithBinaryAndProgressCallback(self, &root, url, body, bodyLength, NULL, callback, callbackData);
if (err.code == 200)
{
retFromResp.ctx = Qiniu_Json_GetString(root, "ctx", NULL);
@@ -385,22 +501,21 @@ static Qiniu_Error Qiniu_Rio_bput(
}
static Qiniu_Error Qiniu_Rio_Mkblock(
- Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, Qiniu_Reader body, int bodyLength,
- Qiniu_Rio_PutExtra *extra)
+ Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, const char *upHost, Qiniu_Reader body, int bodyLength, struct _Qiniu_Progress_Callback_Data *progressCallback)
{
- char *url = Qiniu_String_Format(128, "%s/mkblk/%d", extra->upHost, blkSize);
- Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url);
- Qiniu_Free(url);
+ const char *url = Qiniu_String_Format(128, "%s/mkblk/%d", upHost, blkSize);
+ Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url, progressCallback);
+ Qiniu_Free((void *)url);
return err;
}
static Qiniu_Error Qiniu_Rio_Blockput(
- Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength)
+ Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, const char *upHost, Qiniu_Reader body, int bodyLength, struct _Qiniu_Progress_Callback_Data *progressCallback)
{
- char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", ret->host, ret->ctx, (int)ret->offset);
- Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url);
- Qiniu_Free(url);
+ const char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", upHost, ret->ctx, (int)ret->offset);
+ Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url, progressCallback);
+ Qiniu_Free((void *)url);
return err;
}
@@ -409,19 +524,18 @@ static Qiniu_Error Qiniu_Rio_Blockput(
static Qiniu_Error ErrUnmatchedChecksum = {
Qiniu_Rio_UnmatchedChecksum, "unmatched checksum"};
-static int Qiniu_TemporaryError(int code)
-{
- return code / 100 != 4;
-}
-
-static Qiniu_Error Qiniu_Rio_ResumableBlockput(
- Qiniu_Client *c, Qiniu_Rio_BlkputRet *ret, Qiniu_ReaderAt f, int blkIdx, int blkSize,
- Qiniu_Rio_PutExtra *extra, size_t *chunksUploaded)
+static Qiniu_Error
+Qiniu_Rio_ResumableBlockput(
+ Qiniu_Client *c, Qiniu_Rio_BlkputRet *ret, Qiniu_ReaderAt f, int blkIdx, int blkSize, int fsize,
+ struct _Qiniu_Uploading_Parts_Progress *uploadingPartsProgress, Qiniu_Rio_PutExtra *extra, size_t *pChunksUploaded)
{
Qiniu_Error err = Qiniu_OK;
Qiniu_Tee tee;
Qiniu_Section section;
Qiniu_Reader body, body1;
+ const char *const *upHosts;
+ const char *upHost;
+ size_t upHostsCount;
Qiniu_Crc32 crc32;
Qiniu_Writer h = Qiniu_Crc32Writer(&crc32, 0);
@@ -431,11 +545,32 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
int bodyLength;
int tryTimes;
int notifyRet = 0;
- size_t httpCalled = 0;
+ size_t chunksUploaded = 0;
- if (ret->ctx == NULL)
+ struct _Qiniu_Progress_Callback_Data progressCallbackData;
+ Qiniu_Zero(progressCallbackData);
+
+ if (extra->uploadingProgress != NULL)
{
+ progressCallbackData.callback = extra->uploadingProgress;
+ progressCallbackData.totalSize = (size_t)fsize;
+ progressCallbackData.progress = uploadingPartsProgress;
+ progressCallbackData.blkIdx = blkIdx;
+ }
+ if (extra->upHost != NULL)
+ {
+ upHosts = &extra->upHost;
+ upHostsCount = 1;
+ }
+ else
+ {
+ upHosts = extra->upHosts;
+ upHostsCount = extra->upHostsCount;
+ }
+
+ if (ret->ctx == NULL)
+ {
if (chunkSize < blkSize)
{
bodyLength = chunkSize;
@@ -445,19 +580,42 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
bodyLength = blkSize;
}
- body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength);
- body = Qiniu_TeeReader(&tee, body1, h);
+ for (int tries = 0; tries < extra->tryTimes && tries <= c->hostsRetriesMax; tries++)
+ {
+ body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength);
+ body = Qiniu_TeeReader(&tee, body1, h);
- err = Qiniu_Rio_Mkblock(c, ret, blkSize, body, bodyLength, extra);
+ upHost = upHosts[tries % upHostsCount];
+ err = Qiniu_Rio_Mkblock(c, ret, blkSize, upHost, body, bodyLength, &progressCallbackData);
+ if (err.code == 200)
+ {
+ if (extra->uploadingProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, bodyLength);
+ }
+ break;
+ }
+ else
+ {
+ if (extra->uploadingProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0);
+ }
+ if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
+ {
+ goto handleErr;
+ }
+ }
+ }
if (err.code != 200)
{
- return err;
+ goto handleErr;
}
-
- httpCalled++;
+ chunksUploaded++;
if (ret->crc32 != crc32.val || (int)(ret->offset) != bodyLength)
{
- return ErrUnmatchedChecksum;
+ err = ErrUnmatchedChecksum;
+ goto handleErr;
}
notifyRet = extra->notify(extra->notifyRecvr, blkIdx, blkSize, ret);
if (notifyRet == QINIU_RIO_NOTIFY_EXIT)
@@ -465,9 +623,13 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
// Terminate the upload process if the caller requests
err.code = Qiniu_Rio_PutInterrupted;
err.message = "Interrupted by the caller";
- return err;
+ goto handleErr;
}
}
+ else if (extra->uploadingProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, ret->offset);
+ }
while ((int)(ret->offset) < blkSize)
{
@@ -482,16 +644,21 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
}
tryTimes = extra->tryTimes;
+ if (tryTimes > c->hostsRetriesMax + 1)
+ {
+ tryTimes = c->hostsRetriesMax + 1;
+ }
lzRetry:
crc32.val = 0;
body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase + (ret->offset), bodyLength);
body = Qiniu_TeeReader(&tee, body1, h);
+ upHost = upHosts[tryTimes % upHostsCount];
- err = Qiniu_Rio_Blockput(c, ret, body, bodyLength);
+ err = Qiniu_Rio_Blockput(c, ret, upHost, body, bodyLength, &progressCallbackData);
if (err.code == 200)
{
- httpCalled++;
+ chunksUploaded++;
if (ret->crc32 == crc32.val)
{
notifyRet = extra->notify(extra->notifyRecvr, blkIdx, blkSize, ret);
@@ -500,25 +667,39 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
// Terminate the upload process if the caller requests
err.code = Qiniu_Rio_PutInterrupted;
err.message = "Interrupted by the caller";
- return err;
+ goto handleErr;
+ }
+ if (extra->uploadingProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, bodyLength);
}
-
continue;
}
- Qiniu_Log_Warn("ResumableBlockput: invalid checksum, retry");
- err = ErrUnmatchedChecksum;
+ else
+ {
+ Qiniu_Log_Warn("ResumableBlockput: invalid checksum, retry");
+ err = ErrUnmatchedChecksum;
+ if (extra->uploadingProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0);
+ }
+ }
+ }
+ else if (err.code == Qiniu_Rio_InvalidCtx)
+ {
+ Qiniu_Rio_BlkputRet_Cleanup(ret); // reset
+ Qiniu_Log_Warn("ResumableBlockput: invalid ctx, please retry");
+ goto handleErr;
}
else
{
- if (err.code == Qiniu_Rio_InvalidCtx)
+ Qiniu_Log_Warn("ResumableBlockput %d off:%d failed - %E", blkIdx, (int)ret->offset, err);
+ if (extra->uploadingProgress != NULL)
{
- Qiniu_Rio_BlkputRet_Cleanup(ret); // reset
- Qiniu_Log_Warn("ResumableBlockput: invalid ctx, please retry");
- return err;
+ _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0);
}
- Qiniu_Log_Warn("ResumableBlockput %d off:%d failed - %E", blkIdx, (int)ret->offset, err);
}
- if (tryTimes > 1 && Qiniu_TemporaryError(err.code))
+ if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY)
{
tryTimes--;
Qiniu_Log_Info("ResumableBlockput %E, retrying ...", err);
@@ -527,9 +708,10 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
break;
}
- if (chunksUploaded != NULL)
+handleErr:
+ if (pChunksUploaded != NULL)
{
- *chunksUploaded = httpCalled;
+ *pChunksUploaded = chunksUploaded;
}
return err;
@@ -538,7 +720,7 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput(
/*============================================================================*/
static Qiniu_Error Qiniu_Rio_Mkfile(
- Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra)
+ Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *upHost, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra)
{
size_t i, blkCount = extra->blockCnt;
Qiniu_Json *root;
@@ -547,7 +729,7 @@ static Qiniu_Error Qiniu_Rio_Mkfile(
int j = 0;
Qiniu_Buffer_Init(&url, 2048);
- Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", extra->upHost, fsize);
+ Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", upHost, fsize);
if (key != NULL)
{
@@ -674,6 +856,8 @@ typedef struct _Qiniu_Rio_task
Qiniu_Count *ninterrupts;
int blkIdx;
int blkSize1;
+ int fsize;
+ struct _Qiniu_Uploading_Parts_Progress *progress;
} Qiniu_Rio_task;
static void Qiniu_Rio_doTask(void *params)
@@ -701,7 +885,7 @@ static void Qiniu_Rio_doTask(void *params)
lzRetry:
Qiniu_Rio_BlkputRet_Assign(&ret, &extra->progresses[blkIdx]);
- Qiniu_Error err = Qiniu_Rio_ResumableBlockput(c, &ret, task->f, blkIdx, task->blkSize1, extra, &chunksUploaded);
+ Qiniu_Error err = Qiniu_Rio_ResumableBlockput(c, &ret, task->f, blkIdx, task->blkSize1, task->fsize, task->progress, extra, &chunksUploaded);
if (err.code != 200)
{
if (err.code == Qiniu_Rio_PutInterrupted)
@@ -714,7 +898,7 @@ static void Qiniu_Rio_doTask(void *params)
return;
}
- if (tryTimes > 1 && Qiniu_TemporaryError(err.code))
+ if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY)
{
tryTimes--;
Qiniu_Log_Info("resumable.Put %E, retrying ...", err);
@@ -740,10 +924,8 @@ static void Qiniu_Rio_doTask(void *params)
/*============================================================================*/
/* func Qiniu_Rio_PutXXX */
-static Qiniu_Error ErrPutFailed = {
- Qiniu_Rio_PutFailed, "resumable put failed"};
-static Qiniu_Error ErrPutInterrupted = {
- Qiniu_Rio_PutInterrupted, "resumable put interrupted"};
+static Qiniu_Error ErrPutFailed;
+static Qiniu_Error ErrPutInterrupted;
static Qiniu_Error Qiniu_Rio_loadProgresses(Qiniu_Rio_PutRet *ret, Qiniu_Rio_PutExtra *extra, Qiniu_Rio_Recorder *recorder)
{
@@ -807,12 +989,13 @@ static Qiniu_Error _Qiniu_Rio_Put(
int nfails;
int retCode;
Qiniu_Count ninterrupts;
+ struct _Qiniu_Uploading_Parts_Progress *uploadingPartProgress = NULL;
Qiniu_Error err = Qiniu_Rio_PutExtra_Init(&extra, fsize, extra1);
if (err.code != 200)
{
return err;
}
- if (extra.upHost == NULL)
+ if (extra.upHost == NULL && (extra.upHosts == NULL || extra.upHostsCount == 0))
{
if (!Qiniu_Utils_Extract_Bucket(uptoken, &accessKey, &bucketName))
{
@@ -820,7 +1003,7 @@ static Qiniu_Error _Qiniu_Rio_Put(
err.message = "parse uptoken failed";
return err;
}
- err = _Qiniu_Region_Get_Up_Host(self, accessKey, bucketName, &extra.upHost);
+ err = _Qiniu_Region_Get_Up_Hosts(self, accessKey, bucketName, &extra.upHosts, &extra.upHostsCount);
if (err.code != 200)
{
Qiniu_Free((void *)accessKey);
@@ -828,6 +1011,10 @@ static Qiniu_Error _Qiniu_Rio_Put(
return err;
}
}
+ if (extra.uploadingProgress != NULL)
+ {
+ uploadingPartProgress = _Qiniu_Uploading_Parts_Progress_New();
+ }
tm = extra.threadModel;
wg = tm.itbl->WaitGroup(tm.self);
@@ -853,6 +1040,8 @@ static Qiniu_Error _Qiniu_Rio_Put(
task->ninterrupts = &ninterrupts;
task->blkIdx = i;
task->blkSize1 = blkSize;
+ task->fsize = fsize;
+ task->progress = uploadingPartProgress;
if (i == last)
{
offbase = (Qiniu_Int64)(i) << blockBits;
@@ -865,7 +1054,7 @@ static Qiniu_Error _Qiniu_Rio_Put(
{
wg.itbl->Done(wg.self);
Qiniu_Count_Inc(&ninterrupts);
- free(task);
+ Qiniu_Free((void *)task);
}
if (ninterrupts > 0)
@@ -885,19 +1074,45 @@ static Qiniu_Error _Qiniu_Rio_Put(
}
else
{
- err = Qiniu_Rio_Mkfile(self, ret, key, fsize, &extra);
- if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL)
+ const char *const *upHosts;
+ size_t upHostsCount;
+
+ if (extra.upHost != NULL)
{
- Qiniu_Rio_PutExtra_Clear(&extra);
- reinitializeRecorder(&extra, fi, recorder);
- goto reinit;
+ upHosts = &extra.upHost;
+ upHostsCount = 1;
}
- if (err.code / 100 == 2 || err.code / 100 == 4 || err.code == Qiniu_Rio_InvalidCtx)
+ else
{
- Qiniu_Rio_clearProgresses(&extra, recorder);
+ upHosts = extra.upHosts;
+ upHostsCount = extra.upHostsCount;
+ }
+
+ for (int tries = 0; tries < extra.tryTimes && tries <= self->hostsRetriesMax; tries++)
+ {
+ const char *upHost = upHosts[tries % upHostsCount];
+ err = Qiniu_Rio_Mkfile(self, ret, upHost, key, fsize, &extra);
+ if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL)
+ {
+ Qiniu_Rio_PutExtra_Clear(&extra);
+ reinitializeRecorder(&extra, fi, recorder);
+ goto reinit;
+ }
+ else if (err.code == 200 || _Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
+ {
+ if (err.code / 100 != 4 || err.code == Qiniu_Rio_InvalidCtx)
+ {
+ Qiniu_Rio_clearProgresses(&extra, recorder);
+ }
+ break;
+ }
}
}
+ if (uploadingPartProgress != NULL)
+ {
+ _Qiniu_Uploading_Parts_Progress_Free(uploadingPartProgress);
+ }
Qiniu_Rio_PutExtra_Cleanup(&extra);
Qiniu_Free((void *)accessKey);
Qiniu_Free((void *)bucketName);
diff --git a/qiniu/resumable_io.h b/qiniu/resumable_io.h
index a1efacf7..091da120 100644
--- a/qiniu/resumable_io.h
+++ b/qiniu/resumable_io.h
@@ -14,7 +14,9 @@
#include "io.h"
#include "recorder.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -137,6 +139,13 @@ extern "C"
// For those who want to send request to specific host.
const char *upHost;
+
+ // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST
+ const char *const *upHosts;
+ size_t upHostsCount;
+
+ // Uploading file progress
+ void (*uploadingProgress)(size_t ultotal, size_t ulnow);
} Qiniu_Rio_PutExtra;
/*============================================================================*/
@@ -166,7 +175,9 @@ extern "C"
/*============================================================================*/
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/rfc3339.c b/qiniu/rfc3339.c
new file mode 100644
index 00000000..bc329655
--- /dev/null
+++ b/qiniu/rfc3339.c
@@ -0,0 +1,247 @@
+#include "rfc3339.h"
+
+static void _Strip_Spaces(char *source)
+{
+ char *i = source;
+
+ while (*source != 0)
+ {
+ *i = *source++;
+ if (*i != ' ')
+ i++;
+ }
+
+ *i = 0;
+}
+
+static void _Qiniu_Parse_Date(char *date_string, Qiniu_Date *d)
+{
+ char *const tokens = strdup(date_string);
+ _Strip_Spaces(tokens);
+ d->ok = 0;
+
+ if (strlen(tokens) < 10)
+ {
+ return;
+ }
+
+ int status = sscanf(
+ tokens, "%04d-%02d-%02d", &(d->year), &(d->month), &(d->day));
+ free((char *)tokens);
+
+ if (status != 3)
+ {
+ return;
+ }
+ if (d->year < 1 || d->year > 9999)
+ {
+ return;
+ }
+ if (d->month < 1 || d->month > 12)
+ {
+ return;
+ }
+ if (d->day < 1 || d->day > 31)
+ {
+ return;
+ }
+
+ const int days_in_month[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+
+ if (d->month == 2)
+ {
+ int leap = (d->year % 4 == 0 && d->year % 100 != 0) || (d->year % 400 == 0);
+ if (d->day > (days_in_month[2 - 1] + leap))
+ {
+ return;
+ }
+ }
+ else if (d->day > days_in_month[d->month - 1])
+ {
+ return;
+ }
+
+ d->ok = 1;
+}
+
+static void _Qiniu_Parse_Time(char *time_string, Qiniu_Time *t)
+{
+ char *tokens = strdup(time_string);
+ char *token_ptr = tokens;
+
+ _Strip_Spaces(tokens);
+
+ t->ok = 0;
+
+ if (strlen(tokens) < 8)
+ {
+ goto cleanup;
+ }
+
+ if ((strlen(tokens) > 11) && ((*(tokens + 10) == 'T') || (*(tokens + 10) == 't')))
+ {
+ tokens += 11;
+ }
+
+ int status = sscanf(
+ tokens, "%02d:%02d:%02d", &(t->hour), &(t->minute), &(t->second));
+
+ if (status != 3)
+ {
+ goto cleanup;
+ }
+ if (t->hour < 0 || t->hour > 23)
+ {
+ goto cleanup;
+ }
+ if (t->minute < 0 || t->minute > 59)
+ {
+ goto cleanup;
+ }
+ if (t->second < 0 || t->second > 59)
+ {
+ goto cleanup;
+ }
+
+ if (strlen(tokens) == 8)
+ {
+ t->offset = 0;
+ t->ok = 1;
+ goto cleanup;
+ }
+ else
+ {
+ tokens += 8;
+ }
+
+ if (*tokens == '.')
+ {
+ tokens++;
+ char fractions[7] = {0};
+
+ for (unsigned int i = 0; i < 6; i++)
+ {
+ if ((*(tokens + i) >= 48) && (*(tokens + i) <= 57))
+ {
+ fractions[i] = *(tokens + i);
+ }
+ else
+ {
+ break;
+ }
+ }
+
+ status = sscanf(fractions, "%d", &(t->fraction));
+
+ if (strlen(fractions) < 6 && strlen(fractions) > 0)
+ {
+ t->fraction = t->fraction * pow(10, 6 - strlen(fractions)); // convert msec to usec
+ }
+ else if (strlen(fractions) == 6)
+ {
+ // all fine, already in usec
+ }
+ else
+ {
+ // Invalid fractions must be msec or usec
+ goto cleanup;
+ }
+
+ if (status != 1)
+ {
+ goto cleanup;
+ }
+ if (t->fraction < 0 || t->fraction > 999999)
+ {
+ goto cleanup;
+ }
+
+ tokens += strlen(fractions);
+
+ if (strlen(tokens) == 0)
+ {
+ t->offset = 0;
+ t->ok = 1;
+ goto cleanup;
+ }
+ }
+
+ if ((*tokens == 'Z') || (*tokens == 'z'))
+ {
+ t->offset = 0;
+
+ tokens++;
+ if (strlen(tokens) == 0)
+ {
+ t->ok = 1;
+ }
+ else
+ {
+ t->ok = 0;
+ }
+ goto cleanup;
+ }
+ else if ((*tokens == '+') || (*tokens == '-'))
+ {
+ unsigned int tz_hour, tz_minute;
+
+ status = sscanf(tokens + 1, "%02d:%02d", &tz_hour, &tz_minute);
+
+ if (status != 2)
+ {
+ goto cleanup;
+ }
+ if ((tz_hour < 0) || (tz_hour > 23))
+ {
+ goto cleanup;
+ }
+ if ((tz_minute < 0) || (tz_minute > 59))
+ {
+ goto cleanup;
+ }
+
+ int tz_offset = (tz_hour * HOUR_IN_MINS) + tz_minute;
+
+ if (*tokens == '-')
+ {
+ tz_offset = tz_offset * -1;
+ }
+
+ t->offset = tz_offset;
+
+ tokens = tokens + 6;
+ if (strlen(tokens) == 0)
+ {
+ t->ok = 1;
+ }
+ else
+ {
+ t->ok = 0;
+ }
+ }
+
+cleanup:
+ free(token_ptr);
+ tokens = NULL;
+ token_ptr = NULL;
+ return;
+}
+
+void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt)
+{
+ dt->ok = 0;
+
+ _Qiniu_Parse_Date(datetime_string, &(dt->date));
+ if (dt->date.ok == 0)
+ {
+ return;
+ }
+
+ _Qiniu_Parse_Time(datetime_string, &(dt->time));
+ if (dt->time.ok == 0)
+ {
+ return;
+ }
+
+ dt->ok = 1;
+}
diff --git a/qiniu/rfc3339.h b/qiniu/rfc3339.h
new file mode 100644
index 00000000..1add59c5
--- /dev/null
+++ b/qiniu/rfc3339.h
@@ -0,0 +1,56 @@
+#ifndef QINIU_RFC3339_H
+#define QINIU_RFC3339_H
+
+#include
+#include
+#include
+#include
+
+#define DAY_IN_SECS 86400
+#define HOUR_IN_SECS 3600
+#define MINUTE_IN_SECS 60
+#define HOUR_IN_MINS 60
+
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+ typedef struct _Qiniu_Date
+ {
+ unsigned int year;
+ unsigned int month;
+ unsigned int day;
+ char ok;
+ } Qiniu_Date;
+
+ typedef struct _Qiniu_Time
+ {
+ unsigned int hour;
+ unsigned int minute;
+ unsigned int second;
+ unsigned int fraction;
+ int offset; // UTC offset in minutes
+ char ok;
+ } Qiniu_Time;
+
+ typedef struct _Qiniu_DateTime
+ {
+ Qiniu_Date date;
+ Qiniu_Time time;
+ char ok;
+ } Qiniu_DateTime;
+
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // QINIU_RFC3339_H
diff --git a/qiniu/rs.c b/qiniu/rs.c
index a6da08c9..1b0c4e99 100644
--- a/qiniu/rs.c
+++ b/qiniu/rs.c
@@ -108,6 +108,10 @@ char *Qiniu_RS_PutPolicy_Token(Qiniu_RS_PutPolicy *auth, Qiniu_Mac *mac)
{
cJSON_AddNumberToObject(root, "fileType", auth->fileType);
}
+ if (auth->persistentType)
+ {
+ cJSON_AddNumberToObject(root, "persistentType", auth->persistentType);
+ }
if (auth->expires)
{
diff --git a/qiniu/rs.h b/qiniu/rs.h
index 474144af..ee0da67d 100644
--- a/qiniu/rs.h
+++ b/qiniu/rs.h
@@ -12,7 +12,9 @@
#include "http.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -55,6 +57,7 @@ typedef struct _Qiniu_RS_PutPolicy {
Qiniu_Uint32 expires;
Qiniu_Uint32 deleteAfterDays;
Qiniu_Uint32 fileType;
+ Qiniu_Uint32 persistentType;
} Qiniu_RS_PutPolicy;
/* @endgist */
@@ -245,7 +248,9 @@ QINIU_DLLAPI extern Qiniu_Error Qiniu_RS_BatchRestoreArchive(Qiniu_Client *self,
Qiniu_RS_EntryRestoreArchive *entries,
Qiniu_ItemCount entryCount);
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/rsf.h b/qiniu/rsf.h
index 66acbb86..f0723f15 100644
--- a/qiniu/rsf.h
+++ b/qiniu/rsf.h
@@ -7,7 +7,9 @@
#include "http.h"
+#if defined(_WIN32)
#pragma pack(1)
+#endif
#ifdef __cplusplus
extern "C"
@@ -37,7 +39,9 @@ extern "C"
const char *prefix, const char *delimiter, const char *marker,
int limit);
QINIU_DLLAPI extern void Qiniu_RSF_ListRet_Cleanup(Qiniu_RSF_ListRet *self);
+#if defined(_WIN32)
#pragma pack()
+#endif
#ifdef __cplusplus
}
diff --git a/qiniu/tm.h b/qiniu/tm.h
index 6f0db071..7d1fbe11 100644
--- a/qiniu/tm.h
+++ b/qiniu/tm.h
@@ -12,6 +12,10 @@
#include "base.h"
+#if defined(_WIN32)
+#pragma pack(1)
+#endif
+
#ifdef __cplusplus
extern "C"
{
@@ -22,6 +26,10 @@ extern "C"
QINIU_DLLAPI extern const char *Qiniu_MD5_HexStr(const char *src);
QINIU_DLLAPI extern const char *Qiniu_MD5_HexStr_From_Reader(Qiniu_Reader r);
+#if defined(_WIN32)
+#pragma pack()
+#endif
+
#ifdef __cplusplus
}
#endif
diff --git a/vcpkg.json b/vcpkg.json
index 845d5be2..e70e89d3 100644
--- a/vcpkg.json
+++ b/vcpkg.json
@@ -1,6 +1,6 @@
{
"name": "qiniu-c-sdk",
- "version-string": "7.7.0",
+ "version-string": "7.8.0",
"dependencies": [
"curl",
"openssl"