From 66642cbe2037b62166fe2c66d08a854f94667ad6 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Tue, 24 Sep 2024 15:14:04 +0800 Subject: [PATCH] v7.8.0 (#213) --- .github/workflows/ci.yml | 4 +- CHANGELOG.md | 9 + CMakeLists.txt | 9 +- docs/README.md | 40 +- examples/fop_video_avthumb.c | 47 +- examples/up_form_upload.c | 7 +- examples/up_multipart_sample.c | 1 + examples/up_resumeable_upload.c | 1 + gtests/region_test.cc | 93 +- gtests/rfc3339_test.cc | 53 + hashmap/hashmap.c | 1154 +++++++++++++++++++++ hashmap/hashmap.h | 62 ++ qiniu/base.c | 5 +- qiniu/base.h | 4 + qiniu/base_io.c | 2 +- qiniu/cdn.c | 4 +- qiniu/cdn.h | 8 + qiniu/code.c | 47 + qiniu/conf.c | 5 +- qiniu/conf.h | 8 + qiniu/emu_posix.h | 8 + qiniu/fop.c | 138 ++- qiniu/fop.h | 62 +- qiniu/http.c | 63 +- qiniu/http.h | 33 +- qiniu/io.c | 165 ++- qiniu/io.h | 14 +- qiniu/multipart_upload.c | 239 +++-- qiniu/multipart_upload.h | 10 +- qiniu/private/code.h | 21 + qiniu/private/region.h | 40 +- qiniu/qetag.c | 8 +- qiniu/qetag.h | 34 +- qiniu/reader.h | 6 +- qiniu/recorder.c | 1 - qiniu/recorder.h | 4 + qiniu/recorder_key.h | 4 + qiniu/recorder_utils.h | 4 + qiniu/region.c | 1721 +++++++++++++++++++------------ qiniu/region.h | 16 +- qiniu/resumable_io.c | 343 ++++-- qiniu/resumable_io.h | 11 + qiniu/rfc3339.c | 247 +++++ qiniu/rfc3339.h | 56 + qiniu/rs.c | 4 + qiniu/rs.h | 5 + qiniu/rsf.h | 4 + qiniu/tm.h | 8 + vcpkg.json | 2 +- 49 files changed, 3828 insertions(+), 1006 deletions(-) create mode 100644 gtests/rfc3339_test.cc create mode 100644 hashmap/hashmap.c create mode 100644 hashmap/hashmap.h create mode 100644 qiniu/code.c create mode 100644 qiniu/private/code.h create mode 100644 qiniu/rfc3339.c create mode 100644 qiniu/rfc3339.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b85969f1..287b1268 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: run: | set -e cmake -S . -B build - cmake --build build + cmake --build build -j $(nproc) - name: Unit test working-directory: '${{ github.workspace }}/build/gtests' run: | @@ -38,7 +38,7 @@ jobs: run: | set -e cmake -DOPENSSL_ROOT_DIR="$(brew --prefix)/opt/openssl/" -S . -B build - cmake --build build + cmake --build build -j $(nproc) - name: Unit test working-directory: '${{ github.workspace }}/build/gtests' run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e01b9d3..dec93d77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ ## CHANGE LOG +## v7.8.0 (2024-08-27) + +- 支持闲时任务和 prefop 接口 +- 调整查询区域主备域名 +- 支持缓存多个 `v4/query` 查询结果 +- 上传重试支持切换上传域名 +- 支持上传加速域名 +- 增加上传进度回调支持 + ## v7.7.0 (2023-12-25) - 支持归档直读存储类型 diff --git a/CMakeLists.txt b/CMakeLists.txt index 18b8ac3d..eeab57fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ CMAKE_POLICY(SET CMP0074 NEW) SET(PROJECT_NAME qiniu) # 建立项目 -PROJECT(${PROJECT_NAME} VERSION 7.7.0 LANGUAGES C) +PROJECT(${PROJECT_NAME} VERSION 7.8.0 LANGUAGES C) set(CMAKE_C_STANDARD 99) @@ -19,16 +19,17 @@ ENDIF() AUX_SOURCE_DIRECTORY(b64 DIR_SRCS_B64) AUX_SOURCE_DIRECTORY(cJSON DIR_SRCS_CJSON) +AUX_SOURCE_DIRECTORY(hashmap DIR_SRCS_HASHMAP) AUX_SOURCE_DIRECTORY(qiniu DIR_SRCS_QINIU) -MESSAGE(STATUS "Src file: ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU}") +MESSAGE(STATUS "Src file: ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU}") # 编译可执行程序 # ADD_EXECUTABLE(${PROJECT_NAME} ${DIR_SRCS}) # 如果要生成静态链接库 -ADD_LIBRARY(${PROJECT_NAME}_static STATIC ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU}) +ADD_LIBRARY(${PROJECT_NAME}_static STATIC ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU}) # 如果要生成动态链接库 -ADD_LIBRARY(${PROJECT_NAME} SHARED ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_QINIU}) +ADD_LIBRARY(${PROJECT_NAME} SHARED ${DIR_SRCS_B64} ${DIR_SRCS_CJSON} ${DIR_SRCS_HASHMAP} ${DIR_SRCS_QINIU}) TARGET_INCLUDE_DIRECTORIES(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) LIST(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake") diff --git a/docs/README.md b/docs/README.md index acd00ac0..b3fe2f4e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -68,6 +68,24 @@ C-SDK 使用 [cURL](http://curl.haxx.se/) 进行网络相关操作。无论是 如果在项目构建过程中出现环境相关的编译错误和链接错误，请确认这些选项是否都已经正确配置，以及所依赖的库是否都已经正确的安装。 +#### 通过 CMake 编译 + +如果想在 CMake 项目里使用 C-SDK，可以直接在 CMake 里导入 C-SDK： + +```cmake +INCLUDE(FetchContent) +FetchContent_Declare( + qiniu + GIT_REPOSITORY https://github.com/qiniu/c-sdk.git + GIT_TAG v7.8.0 +) +FetchContent_MakeAvailable(qiniu) +FIND_PACKAGE(CURL REQUIRED) +FIND_PACKAGE(OpenSSL REQUIRED) +TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE qiniu m) +TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE ${CURL_LIBRARIES}) +TARGET_LINK_LIBRARIES(${PROJECT_NAME} PRIVATE ${OPENSSL_LIBRARIES}) +``` @@ -169,7 +187,7 @@ typedef struct _Qiniu_Error { typedef struct _Qiniu_RS_StatRet { const char* hash; const char* mimeType; - Qiniu_Int64 fsize; + Qiniu_Int64 fsize; Qiniu_Int64 putTime; } Qiniu_RS_StatRet; ``` @@ -427,7 +445,7 @@ void stat(Qiniu_Client* client, const char* bucket, const char* key) typedef struct _Qiniu_RS_StatRet { const char* hash; const char* mimeType; - Qiniu_Int64 fsize; + Qiniu_Int64 fsize; Qiniu_Int64 putTime; } Qiniu_RS_StatRet; ``` @@ -456,8 +474,8 @@ void delete(Qiniu_Client* client, const char* bucket, const char* key) 复制和移动操作，需要指定源路径和目标路径。 ```{c} -void copy(Qiniu_Client* client, - const char* bucketSrc, const char* keySrc, +void copy(Qiniu_Client* client, + const char* bucketSrc, const char* keySrc, const char* bucketDest, const char* keyDest) { Qiniu_Error err = Qiniu_RS_Copy(client, bucketSrc, keySrc, bucketDest, keyDest); @@ -470,8 +488,8 @@ void copy(Qiniu_Client* client, ``` ```{c} -void move(Qiniu_Client* client, - const char* bucketSrc, const char* keySrc, +void move(Qiniu_Client* client, + const char* bucketSrc, const char* keySrc, const char* bucketDest, const char* keyDest) { Qiniu_Error err = Qiniu_RS_Move(client, bucketSrc, keySrc, bucketDest, keyDest); @@ -494,7 +512,7 @@ void move(Qiniu_Client* client, 调用`Qiniu_RS_BatchStat`可以批量查看多个文件的属性信息。 ```{c} -void batchStat(Qiniu_Client* client, +void batchStat(Qiniu_Client* client, Qiniu_RS_EntryPath* entries, Qiniu_ItemCount entryCount) { Qiniu_RS_BatchStatRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchStatRet)); @@ -551,7 +569,7 @@ typedef struct _Qiniu_RS_BatchStatRet { typedef struct _Qiniu_RS_StatRet { const char* hash; const char* mimeType; - Qiniu_Int64 fsize; + Qiniu_Int64 fsize; Qiniu_Int64 putTime; } Qiniu_RS_StatRet; ``` @@ -563,7 +581,7 @@ typedef struct _Qiniu_RS_StatRet { 调用`Qiniu_RS_BatchDelete`可以批量删除多个文件。 ```{c} -void batchDelete(Qiniu_Client* client, +void batchDelete(Qiniu_Client* client, Qiniu_RS_EntryPath* entries, Qiniu_ItemCount entryCount) { Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet)); @@ -604,7 +622,7 @@ typedef struct _Qiniu_RS_BatchItemRet { 调用`Qiniu_RS_BatchCopy`可以批量复制多个文件。 ```{c} -void batchCopy(Qiniu_Client* client, +void batchCopy(Qiniu_Client* client, Qiniu_RS_EntryPathPair* entryPairs, Qiniu_ItemCount entryCount) { Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet)); @@ -644,7 +662,7 @@ typedef struct _Qiniu_RS_EntryPathPair { 批量移动和批量复制很类似，唯一的区别就是调用`Qiniu_RS_BatchMove`。 ```{c} -void batchMove(Qiniu_Client* client, +void batchMove(Qiniu_Client* client, Qiniu_RS_EntryPathPair* entryPairs, Qiniu_ItemCount entryCount) { Qiniu_RS_BatchItemRet* rets = calloc(entryCount, sizeof(Qiniu_RS_BatchItemRet)); diff --git a/examples/fop_video_avthumb.c b/examples/fop_video_avthumb.c index 36857c14..ed3da2bd 100644 --- a/examples/fop_video_avthumb.c +++ b/examples/fop_video_avthumb.c @@ -43,7 +43,16 @@ int main(int argc, char **argv) { char *fops[] = {avthumbMp4Fop, vframeJpgFop}; - Qiniu_Error error = Qiniu_FOP_Pfop(&client, &pfopRet, bucket, key, fops, 2, pipeline, notifyURL, force); + Qiniu_FOP_PfopParams params = { + .bucket = bucket, + .key = key, + .pipeline = pipeline, + .notifyURL = notifyURL, + .fops = fops, + .fopCount = 2, + .force = force, + }; + Qiniu_Error error = Qiniu_FOP_Pfop_v2(&client, &pfopRet, ¶ms); if (error.code != 200) { printf("video file pfop %s:%s error.\n", bucket, key); debug_log(&client, error); @@ -52,7 +61,41 @@ int main(int argc, char **argv) { printf("video file pfop %s:%s success, persistentId: %s .\n\n", bucket, key, pfopRet.persistentId); } + Qiniu_FOP_PrefopRet prefopRet; + Qiniu_FOP_PrefopItemRet prefopItemRet[2]; + Qiniu_ItemCount itemsCount; + error = Qiniu_FOP_Prefop(&client, &prefopRet, (Qiniu_FOP_PrefopItemRet *)&prefopItemRet, &itemsCount, pfopRet.persistentId, 2); + if (error.code != 200) + { + debug_log(&client, error); + } + else + { + printf("ID: %s\n", prefopRet.id); + printf("Code: %d\n", prefopRet.code); + printf("Desc: %s\n", prefopRet.desc); + printf("InputBucket: %s\n", prefopRet.inputBucket); + printf("InputKey: %s\n", prefopRet.inputKey); + printf("Type: %d\n", prefopRet.type); + printf("CreationDate: %d-%d-%d %d:%d:%d +%d\n", prefopRet.creationDate.date.year, prefopRet.creationDate.date.month, + prefopRet.creationDate.date.day, prefopRet.creationDate.time.hour, + prefopRet.creationDate.time.minute, prefopRet.creationDate.time.second, + prefopRet.creationDate.time.offset); + + for (Qiniu_ItemCount i = 0; i < itemsCount; i++) + { + printf("\tIndex: %d\n", i); + printf("\tCmd: %s\n", prefopItemRet[i].cmd); + printf("\tCode: %d\n", prefopItemRet[i].code); + printf("\tDesc: %s\n", prefopItemRet[i].desc); + printf("\tError: %s\n", prefopItemRet[i].error); + printf("\tHash: %s\n", prefopItemRet[i].hash); + printf("\tKey: %s\n", prefopItemRet[i].key); + printf("\tReturnOld: %d\n", prefopItemRet[i].returnOld); + } + } + Qiniu_Free(avthumbMp4Fop); Qiniu_Free(vframeJpgFop); Qiniu_Client_Cleanup(&client); -} \ No newline at end of file +} diff --git a/examples/up_form_upload.c b/examples/up_form_upload.c index a593b308..ed874059 100644 --- a/examples/up_form_upload.c +++ b/examples/up_form_upload.c @@ -31,12 +31,6 @@ int main(int argc, char **argv) { putPolicy.scope = bucket; char *uptoken = Qiniu_RS_PutPolicy_Token(&putPolicy, &mac); - //设置机房域名 - //Qiniu_Use_Zone_Beimei(Qiniu_False); - //Qiniu_Use_Zone_Huabei(Qiniu_True); - Qiniu_Use_Zone_Huadong(Qiniu_False); - //Qiniu_Use_Zone_Huanan(Qiniu_True); - //put extra //putExtra.upHost="http://nbxs-gate-up.qiniu.com"; @@ -60,6 +54,7 @@ int main(int argc, char **argv) { //init Qiniu_Client_InitMacAuth(&client, 1024, &mac); + Qiniu_Client_EnableAutoQuery(&client, Qiniu_True); Qiniu_Error error = Qiniu_Io_PutFile(&client, &putRet, uptoken, key, localFile, &putExtra); if (error.code != 200) { printf("upload file %s:%s error.\n", bucket, key); diff --git a/examples/up_multipart_sample.c b/examples/up_multipart_sample.c index 9bd84b6a..ba4d593d 100644 --- a/examples/up_multipart_sample.c +++ b/examples/up_multipart_sample.c @@ -62,6 +62,7 @@ int main(int argc, char **argv) char *uptoken = Qiniu_RS_PutPolicy_Token(&putPolicy, &mac); Qiniu_Client_InitMacAuth(&client, 1024, &mac); + Qiniu_Client_EnableAutoQuery(&client, Qiniu_True); Qiniu_RS_Delete(&client, bucket, key); //to avoid "file exist" err error = Qiniu_Multipart_PutFile(&client, uptoken, key, localFile, &putExtra, &putRet); diff --git a/examples/up_resumeable_upload.c b/examples/up_resumeable_upload.c index a08a3faf..f7e2b956 100644 --- a/examples/up_resumeable_upload.c +++ b/examples/up_resumeable_upload.c @@ -23,6 +23,7 @@ static void resumableUploadWithKey(Qiniu_Mac *mac, const char *bucket, const cha Qiniu_Client client; Qiniu_Global_Init(-1); Qiniu_Client_InitNoAuth(&client, 1024); + Qiniu_Client_EnableAutoQuery(&client, Qiniu_True); Qiniu_Error error; Qiniu_Rio_PutRet putRet; diff --git a/gtests/region_test.cc b/gtests/region_test.cc index 599d9ac9..102a7975 100644 --- a/gtests/region_test.cc +++ b/gtests/region_test.cc @@ -97,24 +97,21 @@ TEST(UnitTest, TestUseRegion) EXPECT_STREQ(hosts[0], "https://rs-z0.qiniuapi.com"); hosts = Qiniu_Region_Get_Rs_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "https://rs-z0.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "https://rsf-z0.qiniuapi.com"); hosts = Qiniu_Region_Get_Rsf_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "https://rsf-z0.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Api_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "https://api-z0.qiniuapi.com"); hosts = Qiniu_Region_Get_Api_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "https://api-z0.qbox.me"); + EXPECT_EQ(count, 0); Qiniu_Region_Free(region); @@ -126,40 +123,35 @@ TEST(UnitTest, TestUseRegion) EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com"); hosts = Qiniu_Region_Get_Up_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "http://up-z1.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Io_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "http://iovip-z1.qiniuio.com"); hosts = Qiniu_Region_Get_Io_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "http://iovip-z1.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "http://rs-z1.qiniuapi.com"); hosts = Qiniu_Region_Get_Rs_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "http://rs-z1.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "http://rsf-z1.qiniuapi.com"); hosts = Qiniu_Region_Get_Rsf_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "http://rsf-z1.qbox.me"); + EXPECT_EQ(count, 0); hosts = Qiniu_Region_Get_Api_Preferred_Hosts(region, &count); EXPECT_EQ(count, 1); EXPECT_STREQ(hosts[0], "http://api-z1.qiniuapi.com"); hosts = Qiniu_Region_Get_Api_Alternative_Hosts(region, &count); - EXPECT_EQ(count, 1); - EXPECT_STREQ(hosts[0], "http://api-z1.qbox.me"); + EXPECT_EQ(count, 0); Qiniu_Region_Free(region); } @@ -242,11 +234,17 @@ TEST(IntegrationTest, TestRegionAutoQuery) Qiniu_Client_EnableAutoQuery(&client, Qiniu_True); const char *host; + const char *const *hosts; + size_t count; Qiniu_Error err; - err = _Qiniu_Region_Get_Up_Host(&client, NULL, Test_bucket, &host); + err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count); EXPECT_EQ(err.code, 200); - EXPECT_STREQ(host, "https://upload.qiniup.com"); + EXPECT_EQ(count, 4); + EXPECT_STREQ(hosts[0], "https://upload.qiniup.com"); + EXPECT_STREQ(hosts[1], "https://up.qiniup.com"); + EXPECT_STREQ(hosts[2], "https://upload.qbox.me"); + EXPECT_STREQ(hosts[3], "https://up.qbox.me"); err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host); EXPECT_EQ(err.code, 200); @@ -276,11 +274,15 @@ TEST(IntegrationTest, TestRegionSpecify) Qiniu_Client_SpecifyRegion(&client, region); const char *host; + const char *const *hosts; + size_t count; Qiniu_Error err; - err = _Qiniu_Region_Get_Up_Host(&client, NULL, Test_bucket, &host); + err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count); EXPECT_EQ(err.code, 200); - EXPECT_STREQ(host, "http://upload-z1.qiniup.com"); + EXPECT_EQ(count, 2); + EXPECT_STREQ(hosts[0], "http://upload-z1.qiniup.com"); + EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com"); err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host); EXPECT_EQ(err.code, 200); @@ -300,3 +302,54 @@ TEST(IntegrationTest, TestRegionSpecify) Qiniu_Region_Free(region); } + +TEST(IntegrationTest, TestRegionSpecifyAcceleration) +{ + Qiniu_Client client; + Qiniu_Client_InitMacAuth(&client, 1024, NULL); + Qiniu_Client_SetTimeout(&client, 5000); + Qiniu_Client_SetConnectTimeout(&client, 3000); + + Qiniu_Region *region = Qiniu_Use_Region("z1", Qiniu_False); + const char *setHosts[1] = {"http://acc.upload-z1.qiniup.com"}; + Qiniu_Region_Set_Up_Accelerated_Hosts(region, (const char *const *)&setHosts, 1, Qiniu_False); + Qiniu_Client_SpecifyRegion(&client, region); + Qiniu_Client_EnableUploadingAcceleration(&client); + + const char *host; + const char *const *hosts; + size_t count; + Qiniu_Error err; + + err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count); + EXPECT_EQ(err.code, 200); + EXPECT_EQ(count, 3); + EXPECT_STREQ(hosts[0], "http://acc.upload-z1.qiniup.com"); + EXPECT_STREQ(hosts[1], "http://upload-z1.qiniup.com"); + EXPECT_STREQ(hosts[2], "http://up-z1.qiniup.com"); + + err = _Qiniu_Region_Get_Io_Host(&client, NULL, Test_bucket, &host); + EXPECT_EQ(err.code, 200); + EXPECT_STREQ(host, "http://iovip-z1.qiniuio.com"); + + err = _Qiniu_Region_Get_Rs_Host(&client, NULL, Test_bucket, &host); + EXPECT_EQ(err.code, 200); + EXPECT_STREQ(host, "http://rs-z1.qiniuapi.com"); + + err = _Qiniu_Region_Get_Rsf_Host(&client, NULL, Test_bucket, &host); + EXPECT_EQ(err.code, 200); + EXPECT_STREQ(host, "http://rsf-z1.qiniuapi.com"); + + err = _Qiniu_Region_Get_Api_Host(&client, NULL, Test_bucket, &host); + EXPECT_EQ(err.code, 200); + EXPECT_STREQ(host, "http://api-z1.qiniuapi.com"); + + Qiniu_Client_DisableUploadingAcceleration(&client); + err = _Qiniu_Region_Get_Up_Hosts(&client, NULL, Test_bucket, &hosts, &count); + EXPECT_EQ(err.code, 200); + EXPECT_EQ(count, 2); + EXPECT_STREQ(hosts[0], "http://upload-z1.qiniup.com"); + EXPECT_STREQ(hosts[1], "http://up-z1.qiniup.com"); + + Qiniu_Region_Free(region); +} diff --git a/gtests/rfc3339_test.cc b/gtests/rfc3339_test.cc new file mode 100644 index 00000000..fa4a4897 --- /dev/null +++ b/gtests/rfc3339_test.cc @@ -0,0 +1,53 @@ +#include +#include "test.h" +#include "qiniu/rfc3339.h" + +#ifdef _WIN32 +#include +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt); +#ifdef __cplusplus +} +#endif + +TEST(UnitTest, TestQiniuParseDateTime) +{ + Qiniu_DateTime dt; + Qiniu_Zero(dt); + _Qiniu_Parse_Date_Time((char *)"1937-02-02T12:00:27.87+00:20", &dt); + EXPECT_TRUE(dt.ok); + EXPECT_TRUE(dt.date.ok); + EXPECT_EQ(dt.date.year, 1937); + EXPECT_EQ(dt.date.month, 2); + EXPECT_EQ(dt.date.day, 2); + EXPECT_TRUE(dt.time.ok); + EXPECT_EQ(dt.time.hour, 12); + EXPECT_EQ(dt.time.minute, 0); + EXPECT_EQ(dt.time.second, 27); + EXPECT_EQ(dt.time.fraction, 870000); + EXPECT_EQ(dt.time.offset, 20); + + _Qiniu_Parse_Date_Time((char *)"1937-02-29T12:00:27.87+00:20", &dt); + EXPECT_FALSE(dt.ok); + + _Qiniu_Parse_Date_Time((char *)"1936-02-29T12:00:27.87Z", &dt); + EXPECT_TRUE(dt.ok); + EXPECT_TRUE(dt.date.ok); + EXPECT_EQ(dt.date.year, 1936); + EXPECT_EQ(dt.date.month, 2); + EXPECT_EQ(dt.date.day, 29); + EXPECT_TRUE(dt.time.ok); + EXPECT_EQ(dt.time.hour, 12); + EXPECT_EQ(dt.time.minute, 0); + EXPECT_EQ(dt.time.second, 27); + EXPECT_EQ(dt.time.fraction, 870000); + EXPECT_EQ(dt.time.offset, 0); + + _Qiniu_Parse_Date_Time((char *)"1900-02-29T12:00:27.87+00:20", &dt); + EXPECT_FALSE(dt.ok); +} diff --git a/hashmap/hashmap.c b/hashmap/hashmap.c new file mode 100644 index 00000000..d70b1627 --- /dev/null +++ b/hashmap/hashmap.c @@ -0,0 +1,1154 @@ +// Copyright 2020 Joshua J Baker. All rights reserved. +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +#include +#include +#include +#include +#include +#include "hashmap.h" + +#define GROW_AT 0.60 /* 60% */ +#define SHRINK_AT 0.10 /* 10% */ + +#ifndef HASHMAP_LOAD_FACTOR +#define HASHMAP_LOAD_FACTOR GROW_AT +#endif + +static void *(*__malloc)(size_t) = NULL; +static void *(*__realloc)(void *, size_t) = NULL; +static void (*__free)(void *) = NULL; + +// hashmap_set_allocator allows for configuring a custom allocator for +// all hashmap library operations. This function, if needed, should be called +// only once at startup and a prior to calling hashmap_new(). +void hashmap_set_allocator(void *(*malloc)(size_t), void (*free)(void*)) { + __malloc = malloc; + __free = free; +} + +struct bucket { + uint64_t hash:48; + uint64_t dib:16; +}; + +// hashmap is an open addressed hash map using robinhood hashing. +struct hashmap { + void *(*malloc)(size_t); + void *(*realloc)(void *, size_t); + void (*free)(void *); + size_t elsize; + size_t cap; + uint64_t seed0; + uint64_t seed1; + uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1); + int (*compare)(const void *a, const void *b, void *udata); + void (*elfree)(void *item); + void *udata; + size_t bucketsz; + size_t nbuckets; + size_t count; + size_t mask; + size_t growat; + size_t shrinkat; + uint8_t loadfactor; + uint8_t growpower; + bool oom; + void *buckets; + void *spare; + void *edata; +}; + +void hashmap_set_grow_by_power(struct hashmap *map, size_t power) { + map->growpower = power < 1 ? 1 : power > 16 ? 16 : power; +} + +static double clamp_load_factor(double factor, double default_factor) { + // Check for NaN and clamp between 50% and 90% + return factor != factor ? default_factor : + factor < 0.50 ? 0.50 : + factor > 0.95 ? 0.95 : + factor; +} + +void hashmap_set_load_factor(struct hashmap *map, double factor) { + factor = clamp_load_factor(factor, map->loadfactor / 100.0); + map->loadfactor = factor * 100; + map->growat = map->nbuckets * (map->loadfactor / 100.0); +} + +static struct bucket *bucket_at0(void *buckets, size_t bucketsz, size_t i) { + return (struct bucket*)(((char*)buckets)+(bucketsz*i)); +} + +static struct bucket *bucket_at(struct hashmap *map, size_t index) { + return bucket_at0(map->buckets, map->bucketsz, index); +} + +static void *bucket_item(struct bucket *entry) { + return ((char*)entry)+sizeof(struct bucket); +} + +static uint64_t clip_hash(uint64_t hash) { + return hash & 0xFFFFFFFFFFFF; +} + +static uint64_t get_hash(struct hashmap *map, const void *key) { + return clip_hash(map->hash(key, map->seed0, map->seed1)); +} + + +// hashmap_new_with_allocator returns a new hash map using a custom allocator. +// See hashmap_new for more information information +struct hashmap *hashmap_new_with_allocator(void *(*_malloc)(size_t), + void *(*_realloc)(void*, size_t), void (*_free)(void*), + size_t elsize, size_t cap, uint64_t seed0, uint64_t seed1, + uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1), + int (*compare)(const void *a, const void *b, void *udata), + void (*elfree)(void *item), + void *udata) +{ + _malloc = _malloc ? _malloc : __malloc ? __malloc : malloc; + _realloc = _realloc ? _realloc : __realloc ? __realloc : realloc; + _free = _free ? _free : __free ? __free : free; + size_t ncap = 16; + if (cap < ncap) { + cap = ncap; + } else { + while (ncap < cap) { + ncap *= 2; + } + cap = ncap; + } + size_t bucketsz = sizeof(struct bucket) + elsize; + while (bucketsz & (sizeof(uintptr_t)-1)) { + bucketsz++; + } + // hashmap + spare + edata + size_t size = sizeof(struct hashmap)+bucketsz*2; + struct hashmap *map = _malloc(size); + if (!map) { + return NULL; + } + memset(map, 0, sizeof(struct hashmap)); + map->elsize = elsize; + map->bucketsz = bucketsz; + map->seed0 = seed0; + map->seed1 = seed1; + map->hash = hash; + map->compare = compare; + map->elfree = elfree; + map->udata = udata; + map->spare = ((char*)map)+sizeof(struct hashmap); + map->edata = (char*)map->spare+bucketsz; + map->cap = cap; + map->nbuckets = cap; + map->mask = map->nbuckets-1; + map->buckets = _malloc(map->bucketsz*map->nbuckets); + if (!map->buckets) { + _free(map); + return NULL; + } + memset(map->buckets, 0, map->bucketsz*map->nbuckets); + map->growpower = 1; + map->loadfactor = clamp_load_factor(HASHMAP_LOAD_FACTOR, GROW_AT) * 100; + map->growat = map->nbuckets * (map->loadfactor / 100.0); + map->shrinkat = map->nbuckets * SHRINK_AT; + map->malloc = _malloc; + map->realloc = _realloc; + map->free = _free; + return map; +} + +// hashmap_new returns a new hash map. +// Param `elsize` is the size of each element in the tree. Every element that +// is inserted, deleted, or retrieved will be this size. +// Param `cap` is the default lower capacity of the hashmap. Setting this to +// zero will default to 16. +// Params `seed0` and `seed1` are optional seed values that are passed to the +// following `hash` function. These can be any value you wish but it's often +// best to use randomly generated values. +// Param `hash` is a function that generates a hash value for an item. It's +// important that you provide a good hash function, otherwise it will perform +// poorly or be vulnerable to Denial-of-service attacks. This implementation +// comes with two helper functions `hashmap_sip()` and `hashmap_murmur()`. +// Param `compare` is a function that compares items in the tree. See the +// qsort stdlib function for an example of how this function works. +// The hashmap must be freed with hashmap_free(). +// Param `elfree` is a function that frees a specific item. This should be NULL +// unless you're storing some kind of reference data in the hash. +struct hashmap *hashmap_new(size_t elsize, size_t cap, uint64_t seed0, + uint64_t seed1, + uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1), + int (*compare)(const void *a, const void *b, void *udata), + void (*elfree)(void *item), + void *udata) +{ + return hashmap_new_with_allocator(NULL, NULL, NULL, elsize, cap, seed0, + seed1, hash, compare, elfree, udata); +} + +static void free_elements(struct hashmap *map) { + if (map->elfree) { + for (size_t i = 0; i < map->nbuckets; i++) { + struct bucket *bucket = bucket_at(map, i); + if (bucket->dib) map->elfree(bucket_item(bucket)); + } + } +} + +// hashmap_clear quickly clears the map. +// Every item is called with the element-freeing function given in hashmap_new, +// if present, to free any data referenced in the elements of the hashmap. +// When the update_cap is provided, the map's capacity will be updated to match +// the currently number of allocated buckets. This is an optimization to ensure +// that this operation does not perform any allocations. +void hashmap_clear(struct hashmap *map, bool update_cap) { + map->count = 0; + free_elements(map); + if (update_cap) { + map->cap = map->nbuckets; + } else if (map->nbuckets != map->cap) { + void *new_buckets = map->malloc(map->bucketsz*map->cap); + if (new_buckets) { + map->free(map->buckets); + map->buckets = new_buckets; + } + map->nbuckets = map->cap; + } + memset(map->buckets, 0, map->bucketsz*map->nbuckets); + map->mask = map->nbuckets-1; + map->growat = map->nbuckets * (map->loadfactor / 100.0) ; + map->shrinkat = map->nbuckets * SHRINK_AT; +} + +static bool resize0(struct hashmap *map, size_t new_cap) { + struct hashmap *map2 = hashmap_new_with_allocator(map->malloc, map->realloc, + map->free, map->elsize, new_cap, map->seed0, map->seed1, map->hash, + map->compare, map->elfree, map->udata); + if (!map2) return false; + for (size_t i = 0; i < map->nbuckets; i++) { + struct bucket *entry = bucket_at(map, i); + if (!entry->dib) { + continue; + } + entry->dib = 1; + size_t j = entry->hash & map2->mask; + while(1) { + struct bucket *bucket = bucket_at(map2, j); + if (bucket->dib == 0) { + memcpy(bucket, entry, map->bucketsz); + break; + } + if (bucket->dib < entry->dib) { + memcpy(map2->spare, bucket, map->bucketsz); + memcpy(bucket, entry, map->bucketsz); + memcpy(entry, map2->spare, map->bucketsz); + } + j = (j + 1) & map2->mask; + entry->dib += 1; + } + } + map->free(map->buckets); + map->buckets = map2->buckets; + map->nbuckets = map2->nbuckets; + map->mask = map2->mask; + map->growat = map2->growat; + map->shrinkat = map2->shrinkat; + map->free(map2); + return true; +} + +static bool resize(struct hashmap *map, size_t new_cap) { + return resize0(map, new_cap); +} + +// hashmap_set_with_hash works like hashmap_set but you provide your +// own hash. The 'hash' callback provided to the hashmap_new function +// will not be called +const void *hashmap_set_with_hash(struct hashmap *map, const void *item, + uint64_t hash) +{ + hash = clip_hash(hash); + map->oom = false; + if (map->count >= map->growat) { + if (!resize(map, map->nbuckets*(1<growpower))) { + map->oom = true; + return NULL; + } + } + + struct bucket *entry = map->edata; + entry->hash = hash; + entry->dib = 1; + void *eitem = bucket_item(entry); + memcpy(eitem, item, map->elsize); + + void *bitem; + size_t i = entry->hash & map->mask; + while(1) { + struct bucket *bucket = bucket_at(map, i); + if (bucket->dib == 0) { + memcpy(bucket, entry, map->bucketsz); + map->count++; + return NULL; + } + bitem = bucket_item(bucket); + if (entry->hash == bucket->hash && (!map->compare || + map->compare(eitem, bitem, map->udata) == 0)) + { + memcpy(map->spare, bitem, map->elsize); + memcpy(bitem, eitem, map->elsize); + return map->spare; + } + if (bucket->dib < entry->dib) { + memcpy(map->spare, bucket, map->bucketsz); + memcpy(bucket, entry, map->bucketsz); + memcpy(entry, map->spare, map->bucketsz); + eitem = bucket_item(entry); + } + i = (i + 1) & map->mask; + entry->dib += 1; + } +} + +// hashmap_set inserts or replaces an item in the hash map. If an item is +// replaced then it is returned otherwise NULL is returned. This operation +// may allocate memory. If the system is unable to allocate additional +// memory then NULL is returned and hashmap_oom() returns true. +const void *hashmap_set(struct hashmap *map, const void *item) { + return hashmap_set_with_hash(map, item, get_hash(map, item)); +} + +// hashmap_get_with_hash works like hashmap_get but you provide your +// own hash. The 'hash' callback provided to the hashmap_new function +// will not be called +const void *hashmap_get_with_hash(struct hashmap *map, const void *key, + uint64_t hash) +{ + hash = clip_hash(hash); + size_t i = hash & map->mask; + while(1) { + struct bucket *bucket = bucket_at(map, i); + if (!bucket->dib) return NULL; + if (bucket->hash == hash) { + void *bitem = bucket_item(bucket); + if (!map->compare || map->compare(key, bitem, map->udata) == 0) { + return bitem; + } + } + i = (i + 1) & map->mask; + } +} + +// hashmap_get returns the item based on the provided key. If the item is not +// found then NULL is returned. +const void *hashmap_get(struct hashmap *map, const void *key) { + return hashmap_get_with_hash(map, key, get_hash(map, key)); +} + +// hashmap_probe returns the item in the bucket at position or NULL if an item +// is not set for that bucket. The position is 'moduloed' by the number of +// buckets in the hashmap. +const void *hashmap_probe(struct hashmap *map, uint64_t position) { + size_t i = position & map->mask; + struct bucket *bucket = bucket_at(map, i); + if (!bucket->dib) { + return NULL; + } + return bucket_item(bucket); +} + +// hashmap_delete_with_hash works like hashmap_delete but you provide your +// own hash. The 'hash' callback provided to the hashmap_new function +// will not be called +const void *hashmap_delete_with_hash(struct hashmap *map, const void *key, + uint64_t hash) +{ + hash = clip_hash(hash); + map->oom = false; + size_t i = hash & map->mask; + while(1) { + struct bucket *bucket = bucket_at(map, i); + if (!bucket->dib) { + return NULL; + } + void *bitem = bucket_item(bucket); + if (bucket->hash == hash && (!map->compare || + map->compare(key, bitem, map->udata) == 0)) + { + memcpy(map->spare, bitem, map->elsize); + bucket->dib = 0; + while(1) { + struct bucket *prev = bucket; + i = (i + 1) & map->mask; + bucket = bucket_at(map, i); + if (bucket->dib <= 1) { + prev->dib = 0; + break; + } + memcpy(prev, bucket, map->bucketsz); + prev->dib--; + } + map->count--; + if (map->nbuckets > map->cap && map->count <= map->shrinkat) { + // Ignore the return value. It's ok for the resize operation to + // fail to allocate enough memory because a shrink operation + // does not change the integrity of the data. + resize(map, map->nbuckets/2); + } + return map->spare; + } + i = (i + 1) & map->mask; + } +} + +// hashmap_delete removes an item from the hash map and returns it. If the +// item is not found then NULL is returned. +const void *hashmap_delete(struct hashmap *map, const void *key) { + return hashmap_delete_with_hash(map, key, get_hash(map, key)); +} + +// hashmap_count returns the number of items in the hash map. +size_t hashmap_count(struct hashmap *map) { + return map->count; +} + +// hashmap_free frees the hash map +// Every item is called with the element-freeing function given in hashmap_new, +// if present, to free any data referenced in the elements of the hashmap. +void hashmap_free(struct hashmap *map) { + if (!map) return; + free_elements(map); + map->free(map->buckets); + map->free(map); +} + +// hashmap_oom returns true if the last hashmap_set() call failed due to the +// system being out of memory. +bool hashmap_oom(struct hashmap *map) { + return map->oom; +} + +// hashmap_scan iterates over all items in the hash map +// Param `iter` can return false to stop iteration early. +// Returns false if the iteration has been stopped early. +bool hashmap_scan(struct hashmap *map, + bool (*iter)(const void *item, void *udata), void *udata) +{ + for (size_t i = 0; i < map->nbuckets; i++) { + struct bucket *bucket = bucket_at(map, i); + if (bucket->dib && !iter(bucket_item(bucket), udata)) { + return false; + } + } + return true; +} + +// hashmap_iter iterates one key at a time yielding a reference to an +// entry at each iteration. Useful to write simple loops and avoid writing +// dedicated callbacks and udata structures, as in hashmap_scan. +// +// map is a hash map handle. i is a pointer to a size_t cursor that +// should be initialized to 0 at the beginning of the loop. item is a void +// pointer pointer that is populated with the retrieved item. Note that this +// is NOT a copy of the item stored in the hash map and can be directly +// modified. +// +// Note that if hashmap_delete() is called on the hashmap being iterated, +// the buckets are rearranged and the iterator must be reset to 0, otherwise +// unexpected results may be returned after deletion. +// +// This function has not been tested for thread safety. +// +// The function returns true if an item was retrieved; false if the end of the +// iteration has been reached. +bool hashmap_iter(struct hashmap *map, size_t *i, void **item) { + struct bucket *bucket; + do { + if (*i >= map->nbuckets) return false; + bucket = bucket_at(map, *i); + (*i)++; + } while (!bucket->dib); + *item = bucket_item(bucket); + return true; +} + + +//----------------------------------------------------------------------------- +// SipHash reference C implementation +// +// Copyright (c) 2012-2016 Jean-Philippe Aumasson +// +// Copyright (c) 2012-2014 Daniel J. Bernstein +// +// To the extent possible under law, the author(s) have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// +// You should have received a copy of the CC0 Public Domain Dedication along +// with this software. If not, see +// . +// +// default: SipHash-2-4 +//----------------------------------------------------------------------------- +static uint64_t SIP64(const uint8_t *in, const size_t inlen, uint64_t seed0, + uint64_t seed1) +{ +#define U8TO64_LE(p) \ + { (((uint64_t)((p)[0])) | ((uint64_t)((p)[1]) << 8) | \ + ((uint64_t)((p)[2]) << 16) | ((uint64_t)((p)[3]) << 24) | \ + ((uint64_t)((p)[4]) << 32) | ((uint64_t)((p)[5]) << 40) | \ + ((uint64_t)((p)[6]) << 48) | ((uint64_t)((p)[7]) << 56)) } +#define U64TO8_LE(p, v) \ + { U32TO8_LE((p), (uint32_t)((v))); \ + U32TO8_LE((p) + 4, (uint32_t)((v) >> 32)); } +#define U32TO8_LE(p, v) \ + { (p)[0] = (uint8_t)((v)); \ + (p)[1] = (uint8_t)((v) >> 8); \ + (p)[2] = (uint8_t)((v) >> 16); \ + (p)[3] = (uint8_t)((v) >> 24); } +#define ROTL(x, b) (uint64_t)(((x) << (b)) | ((x) >> (64 - (b)))) +#define SIPROUND \ + { v0 += v1; v1 = ROTL(v1, 13); \ + v1 ^= v0; v0 = ROTL(v0, 32); \ + v2 += v3; v3 = ROTL(v3, 16); \ + v3 ^= v2; \ + v0 += v3; v3 = ROTL(v3, 21); \ + v3 ^= v0; \ + v2 += v1; v1 = ROTL(v1, 17); \ + v1 ^= v2; v2 = ROTL(v2, 32); } + uint64_t k0 = U8TO64_LE((uint8_t*)&seed0); + uint64_t k1 = U8TO64_LE((uint8_t*)&seed1); + uint64_t v3 = UINT64_C(0x7465646279746573) ^ k1; + uint64_t v2 = UINT64_C(0x6c7967656e657261) ^ k0; + uint64_t v1 = UINT64_C(0x646f72616e646f6d) ^ k1; + uint64_t v0 = UINT64_C(0x736f6d6570736575) ^ k0; + const uint8_t *end = in + inlen - (inlen % sizeof(uint64_t)); + for (; in != end; in += 8) { + uint64_t m = U8TO64_LE(in); + v3 ^= m; + SIPROUND; SIPROUND; + v0 ^= m; + } + const int left = inlen & 7; + uint64_t b = ((uint64_t)inlen) << 56; + switch (left) { + case 7: b |= ((uint64_t)in[6]) << 48; /* fall through */ + case 6: b |= ((uint64_t)in[5]) << 40; /* fall through */ + case 5: b |= ((uint64_t)in[4]) << 32; /* fall through */ + case 4: b |= ((uint64_t)in[3]) << 24; /* fall through */ + case 3: b |= ((uint64_t)in[2]) << 16; /* fall through */ + case 2: b |= ((uint64_t)in[1]) << 8; /* fall through */ + case 1: b |= ((uint64_t)in[0]); break; + case 0: break; + } + v3 ^= b; + SIPROUND; SIPROUND; + v0 ^= b; + v2 ^= 0xff; + SIPROUND; SIPROUND; SIPROUND; SIPROUND; + b = v0 ^ v1 ^ v2 ^ v3; + uint64_t out = 0; + U64TO8_LE((uint8_t*)&out, b); + return out; +} + +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. +// +// Murmur3_86_128 +//----------------------------------------------------------------------------- +static uint64_t MM86128(const void *key, const int len, uint32_t seed) { +#define ROTL32(x, r) ((x << r) | (x >> (32 - r))) +#define FMIX32(h) h^=h>>16; h*=0x85ebca6b; h^=h>>13; h*=0xc2b2ae35; h^=h>>16; + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + uint32_t c1 = 0x239b961b; + uint32_t c2 = 0xab0e9789; + uint32_t c3 = 0x38b34ae5; + uint32_t c4 = 0xa1e38b93; + const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); + for (int i = -nblocks; i; i++) { + uint32_t k1 = blocks[i*4+0]; + uint32_t k2 = blocks[i*4+1]; + uint32_t k3 = blocks[i*4+2]; + uint32_t k4 = blocks[i*4+3]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; + } + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + switch(len & 15) { + case 15: k4 ^= tail[14] << 16; /* fall through */ + case 14: k4 ^= tail[13] << 8; /* fall through */ + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + /* fall through */ + case 12: k3 ^= tail[11] << 24; /* fall through */ + case 11: k3 ^= tail[10] << 16; /* fall through */ + case 10: k3 ^= tail[ 9] << 8; /* fall through */ + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + /* fall through */ + case 8: k2 ^= tail[ 7] << 24; /* fall through */ + case 7: k2 ^= tail[ 6] << 16; /* fall through */ + case 6: k2 ^= tail[ 5] << 8; /* fall through */ + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + /* fall through */ + case 4: k1 ^= tail[ 3] << 24; /* fall through */ + case 3: k1 ^= tail[ 2] << 16; /* fall through */ + case 2: k1 ^= tail[ 1] << 8; /* fall through */ + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + /* fall through */ + }; + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + FMIX32(h1); FMIX32(h2); FMIX32(h3); FMIX32(h4); + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + return (((uint64_t)h2)<<32)|h1; +} + +//----------------------------------------------------------------------------- +// xxHash Library +// Copyright (c) 2012-2021 Yann Collet +// All rights reserved. +// +// BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) +// +// xxHash3 +//----------------------------------------------------------------------------- +#define XXH_PRIME_1 11400714785074694791ULL +#define XXH_PRIME_2 14029467366897019727ULL +#define XXH_PRIME_3 1609587929392839161ULL +#define XXH_PRIME_4 9650029242287828579ULL +#define XXH_PRIME_5 2870177450012600261ULL + +static uint64_t XXH_read64(const void* memptr) { + uint64_t val; + memcpy(&val, memptr, sizeof(val)); + return val; +} + +static uint32_t XXH_read32(const void* memptr) { + uint32_t val; + memcpy(&val, memptr, sizeof(val)); + return val; +} + +static uint64_t XXH_rotl64(uint64_t x, int r) { + return (x << r) | (x >> (64 - r)); +} + +static uint64_t xxh3(const void* data, size_t len, uint64_t seed) { + const uint8_t* p = (const uint8_t*)data; + const uint8_t* const end = p + len; + uint64_t h64; + + if (len >= 32) { + const uint8_t* const limit = end - 32; + uint64_t v1 = seed + XXH_PRIME_1 + XXH_PRIME_2; + uint64_t v2 = seed + XXH_PRIME_2; + uint64_t v3 = seed + 0; + uint64_t v4 = seed - XXH_PRIME_1; + + do { + v1 += XXH_read64(p) * XXH_PRIME_2; + v1 = XXH_rotl64(v1, 31); + v1 *= XXH_PRIME_1; + + v2 += XXH_read64(p + 8) * XXH_PRIME_2; + v2 = XXH_rotl64(v2, 31); + v2 *= XXH_PRIME_1; + + v3 += XXH_read64(p + 16) * XXH_PRIME_2; + v3 = XXH_rotl64(v3, 31); + v3 *= XXH_PRIME_1; + + v4 += XXH_read64(p + 24) * XXH_PRIME_2; + v4 = XXH_rotl64(v4, 31); + v4 *= XXH_PRIME_1; + + p += 32; + } while (p <= limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + + XXH_rotl64(v4, 18); + + v1 *= XXH_PRIME_2; + v1 = XXH_rotl64(v1, 31); + v1 *= XXH_PRIME_1; + h64 ^= v1; + h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4; + + v2 *= XXH_PRIME_2; + v2 = XXH_rotl64(v2, 31); + v2 *= XXH_PRIME_1; + h64 ^= v2; + h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4; + + v3 *= XXH_PRIME_2; + v3 = XXH_rotl64(v3, 31); + v3 *= XXH_PRIME_1; + h64 ^= v3; + h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4; + + v4 *= XXH_PRIME_2; + v4 = XXH_rotl64(v4, 31); + v4 *= XXH_PRIME_1; + h64 ^= v4; + h64 = h64 * XXH_PRIME_1 + XXH_PRIME_4; + } + else { + h64 = seed + XXH_PRIME_5; + } + + h64 += (uint64_t)len; + + while (p + 8 <= end) { + uint64_t k1 = XXH_read64(p); + k1 *= XXH_PRIME_2; + k1 = XXH_rotl64(k1, 31); + k1 *= XXH_PRIME_1; + h64 ^= k1; + h64 = XXH_rotl64(h64, 27) * XXH_PRIME_1 + XXH_PRIME_4; + p += 8; + } + + if (p + 4 <= end) { + h64 ^= (uint64_t)(XXH_read32(p)) * XXH_PRIME_1; + h64 = XXH_rotl64(h64, 23) * XXH_PRIME_2 + XXH_PRIME_3; + p += 4; + } + + while (p < end) { + h64 ^= (*p) * XXH_PRIME_5; + h64 = XXH_rotl64(h64, 11) * XXH_PRIME_1; + p++; + } + + h64 ^= h64 >> 33; + h64 *= XXH_PRIME_2; + h64 ^= h64 >> 29; + h64 *= XXH_PRIME_3; + h64 ^= h64 >> 32; + + return h64; +} + +// hashmap_sip returns a hash value for `data` using SipHash-2-4. +uint64_t hashmap_sip(const void *data, size_t len, uint64_t seed0, + uint64_t seed1) +{ + return SIP64((uint8_t*)data, len, seed0, seed1); +} + +// hashmap_murmur returns a hash value for `data` using Murmur3_86_128. +uint64_t hashmap_murmur(const void *data, size_t len, uint64_t seed0, + uint64_t seed1) +{ + (void)seed1; + return MM86128(data, len, seed0); +} + +uint64_t hashmap_xxhash3(const void *data, size_t len, uint64_t seed0, + uint64_t seed1) +{ + (void)seed1; + return xxh3(data, len ,seed0); +} + +//============================================================================== +// TESTS AND BENCHMARKS +// $ cc -DHASHMAP_TEST hashmap.c && ./a.out # run tests +// $ cc -DHASHMAP_TEST -O3 hashmap.c && BENCH=1 ./a.out # run benchmarks +//============================================================================== +#ifdef HASHMAP_TEST + +static size_t deepcount(struct hashmap *map) { + size_t count = 0; + for (size_t i = 0; i < map->nbuckets; i++) { + if (bucket_at(map, i)->dib) { + count++; + } + } + return count; +} + +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wpedantic" +#endif +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wunknown-warning-option" +#pragma GCC diagnostic ignored "-Wcompound-token-split-by-macro" +#pragma GCC diagnostic ignored "-Wgnu-statement-expression-from-macro-expansion" +#endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wunused-parameter" +#endif + +#include +#include +#include +#include +#include +#include "hashmap.h" + +static bool rand_alloc_fail = false; +static int rand_alloc_fail_odds = 3; // 1 in 3 chance malloc will fail. +static uintptr_t total_allocs = 0; +static uintptr_t total_mem = 0; + +static void *xmalloc(size_t size) { + if (rand_alloc_fail && rand()%rand_alloc_fail_odds == 0) { + return NULL; + } + void *mem = malloc(sizeof(uintptr_t)+size); + assert(mem); + *(uintptr_t*)mem = size; + total_allocs++; + total_mem += size; + return (char*)mem+sizeof(uintptr_t); +} + +static void xfree(void *ptr) { + if (ptr) { + total_mem -= *(uintptr_t*)((char*)ptr-sizeof(uintptr_t)); + free((char*)ptr-sizeof(uintptr_t)); + total_allocs--; + } +} + +static void shuffle(void *array, size_t numels, size_t elsize) { + char tmp[elsize]; + char *arr = array; + for (size_t i = 0; i < numels - 1; i++) { + int j = i + rand() / (RAND_MAX / (numels - i) + 1); + memcpy(tmp, arr + j * elsize, elsize); + memcpy(arr + j * elsize, arr + i * elsize, elsize); + memcpy(arr + i * elsize, tmp, elsize); + } +} + +static bool iter_ints(const void *item, void *udata) { + int *vals = *(int**)udata; + vals[*(int*)item] = 1; + return true; +} + +static int compare_ints_udata(const void *a, const void *b, void *udata) { + return *(int*)a - *(int*)b; +} + +static int compare_strs(const void *a, const void *b, void *udata) { + return strcmp(*(char**)a, *(char**)b); +} + +static uint64_t hash_int(const void *item, uint64_t seed0, uint64_t seed1) { + return hashmap_xxhash3(item, sizeof(int), seed0, seed1); + // return hashmap_sip(item, sizeof(int), seed0, seed1); + // return hashmap_murmur(item, sizeof(int), seed0, seed1); +} + +static uint64_t hash_str(const void *item, uint64_t seed0, uint64_t seed1) { + return hashmap_xxhash3(*(char**)item, strlen(*(char**)item), seed0, seed1); + // return hashmap_sip(*(char**)item, strlen(*(char**)item), seed0, seed1); + // return hashmap_murmur(*(char**)item, strlen(*(char**)item), seed0, seed1); +} + +static void free_str(void *item) { + xfree(*(char**)item); +} + +static void all(void) { + int seed = getenv("SEED")?atoi(getenv("SEED")):time(NULL); + int N = getenv("N")?atoi(getenv("N")):2000; + printf("seed=%d, count=%d, item_size=%zu\n", seed, N, sizeof(int)); + srand(seed); + + rand_alloc_fail = true; + + // test sip and murmur hashes + assert(hashmap_sip("hello", 5, 1, 2) == 2957200328589801622); + assert(hashmap_murmur("hello", 5, 1, 2) == 1682575153221130884); + assert(hashmap_xxhash3("hello", 5, 1, 2) == 2584346877953614258); + + int *vals; + while (!(vals = xmalloc(N * sizeof(int)))) {} + for (int i = 0; i < N; i++) { + vals[i] = i; + } + + struct hashmap *map; + + while (!(map = hashmap_new(sizeof(int), 0, seed, seed, + hash_int, compare_ints_udata, NULL, NULL))) {} + shuffle(vals, N, sizeof(int)); + for (int i = 0; i < N; i++) { + // // printf("== %d ==\n", vals[i]); + assert(map->count == (size_t)i); + assert(map->count == hashmap_count(map)); + assert(map->count == deepcount(map)); + const int *v; + assert(!hashmap_get(map, &vals[i])); + assert(!hashmap_delete(map, &vals[i])); + while (true) { + assert(!hashmap_set(map, &vals[i])); + if (!hashmap_oom(map)) { + break; + } + } + + for (int j = 0; j < i; j++) { + v = hashmap_get(map, &vals[j]); + assert(v && *v == vals[j]); + } + while (true) { + v = hashmap_set(map, &vals[i]); + if (!v) { + assert(hashmap_oom(map)); + continue; + } else { + assert(!hashmap_oom(map)); + assert(v && *v == vals[i]); + break; + } + } + v = hashmap_get(map, &vals[i]); + assert(v && *v == vals[i]); + v = hashmap_delete(map, &vals[i]); + assert(v && *v == vals[i]); + assert(!hashmap_get(map, &vals[i])); + assert(!hashmap_delete(map, &vals[i])); + assert(!hashmap_set(map, &vals[i])); + assert(map->count == (size_t)(i+1)); + assert(map->count == hashmap_count(map)); + assert(map->count == deepcount(map)); + } + + int *vals2; + while (!(vals2 = xmalloc(N * sizeof(int)))) {} + memset(vals2, 0, N * sizeof(int)); + assert(hashmap_scan(map, iter_ints, &vals2)); + + // Test hashmap_iter. This does the same as hashmap_scan above. + size_t iter = 0; + void *iter_val; + while (hashmap_iter (map, &iter, &iter_val)) { + assert (iter_ints(iter_val, &vals2)); + } + for (int i = 0; i < N; i++) { + assert(vals2[i] == 1); + } + xfree(vals2); + + shuffle(vals, N, sizeof(int)); + for (int i = 0; i < N; i++) { + const int *v; + v = hashmap_delete(map, &vals[i]); + assert(v && *v == vals[i]); + assert(!hashmap_get(map, &vals[i])); + assert(map->count == (size_t)(N-i-1)); + assert(map->count == hashmap_count(map)); + assert(map->count == deepcount(map)); + for (int j = N-1; j > i; j--) { + v = hashmap_get(map, &vals[j]); + assert(v && *v == vals[j]); + } + } + + for (int i = 0; i < N; i++) { + while (true) { + assert(!hashmap_set(map, &vals[i])); + if (!hashmap_oom(map)) { + break; + } + } + } + + assert(map->count != 0); + size_t prev_cap = map->cap; + hashmap_clear(map, true); + assert(prev_cap < map->cap); + assert(map->count == 0); + + + for (int i = 0; i < N; i++) { + while (true) { + assert(!hashmap_set(map, &vals[i])); + if (!hashmap_oom(map)) { + break; + } + } + } + + prev_cap = map->cap; + hashmap_clear(map, false); + assert(prev_cap == map->cap); + + hashmap_free(map); + + xfree(vals); + + + while (!(map = hashmap_new(sizeof(char*), 0, seed, seed, + hash_str, compare_strs, free_str, NULL))); + + for (int i = 0; i < N; i++) { + char *str; + while (!(str = xmalloc(16))); + snprintf(str, 16, "s%i", i); + while(!hashmap_set(map, &str)); + } + + hashmap_clear(map, false); + assert(hashmap_count(map) == 0); + + for (int i = 0; i < N; i++) { + char *str; + while (!(str = xmalloc(16))); + snprintf(str, 16, "s%i", i); + while(!hashmap_set(map, &str)); + } + + hashmap_free(map); + + if (total_allocs != 0) { + fprintf(stderr, "total_allocs: expected 0, got %lu\n", total_allocs); + exit(1); + } +} + +#define bench(name, N, code) {{ \ + if (strlen(name) > 0) { \ + printf("%-14s ", name); \ + } \ + size_t tmem = total_mem; \ + size_t tallocs = total_allocs; \ + uint64_t bytes = 0; \ + clock_t begin = clock(); \ + for (int i = 0; i < N; i++) { \ + (code); \ + } \ + clock_t end = clock(); \ + double elapsed_secs = (double)(end - begin) / CLOCKS_PER_SEC; \ + double bytes_sec = (double)bytes/elapsed_secs; \ + printf("%d ops in %.3f secs, %.0f ns/op, %.0f op/sec", \ + N, elapsed_secs, \ + elapsed_secs/(double)N*1e9, \ + (double)N/elapsed_secs \ + ); \ + if (bytes > 0) { \ + printf(", %.1f GB/sec", bytes_sec/1024/1024/1024); \ + } \ + if (total_mem > tmem) { \ + size_t used_mem = total_mem-tmem; \ + printf(", %.2f bytes/op", (double)used_mem/N); \ + } \ + if (total_allocs > tallocs) { \ + size_t used_allocs = total_allocs-tallocs; \ + printf(", %.2f allocs/op", (double)used_allocs/N); \ + } \ + printf("\n"); \ +}} + +static void benchmarks(void) { + int seed = getenv("SEED")?atoi(getenv("SEED")):time(NULL); + int N = getenv("N")?atoi(getenv("N")):5000000; + printf("seed=%d, count=%d, item_size=%zu\n", seed, N, sizeof(int)); + srand(seed); + + + int *vals = xmalloc(N * sizeof(int)); + for (int i = 0; i < N; i++) { + vals[i] = i; + } + + shuffle(vals, N, sizeof(int)); + + struct hashmap *map; + shuffle(vals, N, sizeof(int)); + + map = hashmap_new(sizeof(int), 0, seed, seed, hash_int, compare_ints_udata, + NULL, NULL); + bench("set", N, { + const int *v = hashmap_set(map, &vals[i]); + assert(!v); + }) + shuffle(vals, N, sizeof(int)); + bench("get", N, { + const int *v = hashmap_get(map, &vals[i]); + assert(v && *v == vals[i]); + }) + shuffle(vals, N, sizeof(int)); + bench("delete", N, { + const int *v = hashmap_delete(map, &vals[i]); + assert(v && *v == vals[i]); + }) + hashmap_free(map); + + map = hashmap_new(sizeof(int), N, seed, seed, hash_int, compare_ints_udata, + NULL, NULL); + bench("set (cap)", N, { + const int *v = hashmap_set(map, &vals[i]); + assert(!v); + }) + shuffle(vals, N, sizeof(int)); + bench("get (cap)", N, { + const int *v = hashmap_get(map, &vals[i]); + assert(v && *v == vals[i]); + }) + shuffle(vals, N, sizeof(int)); + bench("delete (cap)" , N, { + const int *v = hashmap_delete(map, &vals[i]); + assert(v && *v == vals[i]); + }) + + hashmap_free(map); + + + xfree(vals); + + if (total_allocs != 0) { + fprintf(stderr, "total_allocs: expected 0, got %lu\n", total_allocs); + exit(1); + } +} + +int main(void) { + hashmap_set_allocator(xmalloc, xfree); + + if (getenv("BENCH")) { + printf("Running hashmap.c benchmarks...\n"); + benchmarks(); + } else { + printf("Running hashmap.c tests...\n"); + all(); + printf("PASSED\n"); + } +} + + +#endif + + + diff --git a/hashmap/hashmap.h b/hashmap/hashmap.h new file mode 100644 index 00000000..e22990e0 --- /dev/null +++ b/hashmap/hashmap.h @@ -0,0 +1,62 @@ +// Copyright 2020 Joshua J Baker. All rights reserved. +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +#ifndef HASHMAP_H +#define HASHMAP_H + +#include +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif // __cplusplus + +struct hashmap; + +struct hashmap *hashmap_new(size_t elsize, size_t cap, uint64_t seed0, + uint64_t seed1, + uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1), + int (*compare)(const void *a, const void *b, void *udata), + void (*elfree)(void *item), + void *udata); + +struct hashmap *hashmap_new_with_allocator(void *(*malloc)(size_t), + void *(*realloc)(void *, size_t), void (*free)(void*), size_t elsize, + size_t cap, uint64_t seed0, uint64_t seed1, + uint64_t (*hash)(const void *item, uint64_t seed0, uint64_t seed1), + int (*compare)(const void *a, const void *b, void *udata), + void (*elfree)(void *item), + void *udata); + +void hashmap_free(struct hashmap *map); +void hashmap_clear(struct hashmap *map, bool update_cap); +size_t hashmap_count(struct hashmap *map); +bool hashmap_oom(struct hashmap *map); +const void *hashmap_get(struct hashmap *map, const void *item); +const void *hashmap_set(struct hashmap *map, const void *item); +const void *hashmap_delete(struct hashmap *map, const void *item); +const void *hashmap_probe(struct hashmap *map, uint64_t position); +bool hashmap_scan(struct hashmap *map, bool (*iter)(const void *item, void *udata), void *udata); +bool hashmap_iter(struct hashmap *map, size_t *i, void **item); + +uint64_t hashmap_sip(const void *data, size_t len, uint64_t seed0, uint64_t seed1); +uint64_t hashmap_murmur(const void *data, size_t len, uint64_t seed0, uint64_t seed1); +uint64_t hashmap_xxhash3(const void *data, size_t len, uint64_t seed0, uint64_t seed1); + +const void *hashmap_get_with_hash(struct hashmap *map, const void *key, uint64_t hash); +const void *hashmap_delete_with_hash(struct hashmap *map, const void *key, uint64_t hash); +const void *hashmap_set_with_hash(struct hashmap *map, const void *item, uint64_t hash); +void hashmap_set_grow_by_power(struct hashmap *map, size_t power); +void hashmap_set_load_factor(struct hashmap *map, double load_factor); + + +// DEPRECATED: use `hashmap_new_with_allocator` +void hashmap_set_allocator(void *(*malloc)(size_t), void (*free)(void*)); + +#if defined(__cplusplus) +} +#endif // __cplusplus + +#endif // HASHMAP_H diff --git a/qiniu/base.c b/qiniu/base.c index 76b1f313..00adda91 100644 --- a/qiniu/base.c +++ b/qiniu/base.c @@ -130,7 +130,7 @@ static char *Qiniu_escape(const char *s, escapeMode mode, Qiniu_Bool *fesc) { int spaceCount = 0; int hexCount = 0; - int i, j, len = strlen(s); + size_t i, j, len = strlen(s); int c; char *t; @@ -674,8 +674,7 @@ Qiniu_Writer Qiniu_FILE_Writer(FILE *fp) /*============================================================================*/ /* func Qiniu_Copy */ -Qiniu_Error Qiniu_OK = { - 200, "OK"}; +Qiniu_Error Qiniu_OK = {200, "OK"}; Qiniu_Error Qiniu_Copy(Qiniu_Writer w, Qiniu_Reader r, void *buf, size_t n, Qiniu_Int64 *ret) { diff --git a/qiniu/base.h b/qiniu/base.h index e57cac61..d3d85875 100644 --- a/qiniu/base.h +++ b/qiniu/base.h @@ -43,7 +43,9 @@ typedef off_t Qiniu_Off_T; #endif +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -357,7 +359,9 @@ typedef struct stat Qiniu_FileInfo; /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/base_io.c b/qiniu/base_io.c index 42ea25e3..572fef29 100644 --- a/qiniu/base_io.c +++ b/qiniu/base_io.c @@ -135,7 +135,7 @@ static ssize_t Qiniu_ReadBuf_ReadAt(Qiniu_ReadBuf *self, void *buf, size_t n, Qi n = max & (~(size_t)0L); } memcpy(buf, self->buf + off, n); - return n; + return (ssize_t)n; } Qiniu_Reader Qiniu_BufReader(Qiniu_ReadBuf *self, const char *buf, size_t bytes) diff --git a/qiniu/cdn.c b/qiniu/cdn.c index fa74dce9..13d932a8 100644 --- a/qiniu/cdn.c +++ b/qiniu/cdn.c @@ -13,7 +13,7 @@ char *Qiniu_CDN_CreateTimestampAntiLeechURL(const char *host, const char *fileNa char *queryStrEscaped = NULL; char expireHex[20]; - sprintf(expireHex, "%0llx", deadline); + snprintf(expireHex, 20, "%0llx", deadline); if (queryStr != NULL && strcmp("", queryStr) != 0) { queryStrEscaped = Qiniu_PathEscape(queryStr, &queryStrEscapeOk); @@ -611,4 +611,4 @@ void Qiniu_Free_CDNLogListRet(Qiniu_CDN_LogListRet *ret) { } Qiniu_Free(ret->data); } -} \ No newline at end of file +} diff --git a/qiniu/cdn.h b/qiniu/cdn.h index efd17924..1a883333 100644 --- a/qiniu/cdn.h +++ b/qiniu/cdn.h @@ -14,6 +14,10 @@ #include "macro.h" #include "http.h" +#if defined(_WIN32) +#pragma pack(1) +#endif + #ifdef __cplusplus extern "C" { @@ -164,6 +168,10 @@ QINIU_DLLAPI extern void Qiniu_Free_CDNLogListRet(Qiniu_CDN_LogListRet *ret); //===================================================================== +#if defined(_WIN32) +#pragma pack() +#endif + #ifdef __cplusplus } #endif diff --git a/qiniu/code.c b/qiniu/code.c new file mode 100644 index 00000000..c8d00333 --- /dev/null +++ b/qiniu/code.c @@ -0,0 +1,47 @@ +#include +#include "private/code.h" + +#define CURLE_WEIRD_SERVER_REPLY 8 +#define CURLE_HTTP2_STREAM 92 +#define CURLE_HTTP3 95 +#define CURLE_QUIC_CONNECT_ERROR 96 + +Qiniu_Retry_Decision _Qiniu_Should_Retry(int code) +{ + if (code / 100 == 4 && code != 406 && code != 429) + { + return QINIU_DONT_RETRY; + } + switch (code) + { + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_WEIRD_SERVER_REPLY: + case CURLE_PARTIAL_FILE: + case CURLE_UPLOAD_FAILED: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_SSL_CONNECT_ERROR: + case CURLE_SEND_ERROR: + case CURLE_RECV_ERROR: + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CIPHER: + case CURLE_PEER_FAILED_VERIFICATION: + case CURLE_HTTP2_STREAM: + case CURLE_HTTP3: + case CURLE_QUIC_CONNECT_ERROR: + case 406: + case 429: + case 500: + case 502: + case 503: + case 504: + case 509: + case 571: + case 573: + case 599: + return QINIU_TRY_NEXT_DOMAIN; + + default: + return QINIU_DONT_RETRY; + } +} diff --git a/qiniu/conf.c b/qiniu/conf.c index 5090a9dc..e356220b 100644 --- a/qiniu/conf.c +++ b/qiniu/conf.c @@ -22,8 +22,9 @@ const char *QINIU_RS_HOST = "http://rs.qbox.me"; const char *QINIU_RSF_HOST = "http://rsf.qbox.me"; const char *QINIU_API_HOST = "http://api.qiniu.com"; const char *QINIU_FUSION_HOST = "http://fusion.qiniuapi.com"; -const char *QINIU_UC_HOST = "http://kodo-config.qiniuapi.com"; -const char *QINIU_UC_HOST_BACKUP = "http://uc.qbox.me"; +const char *QINIU_UC_HOST = "http://uc.qiniuapi.com"; +const char *QINIU_UC_HOST_BACKUP = "http://kodo-config.qiniuapi.com"; +const char *QINIU_UC_HOST_BACKUP_2 = "http://uc.qbox.me"; // 默认华东机房 // diff --git a/qiniu/conf.h b/qiniu/conf.h index b8aaba60..f662d530 100644 --- a/qiniu/conf.h +++ b/qiniu/conf.h @@ -13,6 +13,10 @@ #include "macro.h" #include "base.h" +#if defined(_WIN32) +#pragma pack(1) +#endif + #ifdef __cplusplus extern "C" { @@ -64,6 +68,10 @@ extern "C" // 不再推荐使用，建议使用 Qiniu_Use_Region("cn-east-2") 方法替代 QINIU_DLLAPI extern void Qiniu_Use_Zone_Cn_East_2(Qiniu_Bool useHttps); +#if defined(_WIN32) +#pragma pack() +#endif + #ifdef __cplusplus } #endif diff --git a/qiniu/emu_posix.h b/qiniu/emu_posix.h index 5580865a..fee1acff 100644 --- a/qiniu/emu_posix.h +++ b/qiniu/emu_posix.h @@ -14,6 +14,10 @@ #include "macro.h" +#if defined(_WIN32) +#pragma pack(1) +#endif + #ifdef __cplusplus extern "C" { @@ -51,6 +55,10 @@ QINIU_DLLAPI extern char* Qiniu_Posix_strndup(const char *, size_t n); /*============================================================================*/ +#if defined(_WIN32) +#pragma pack() +#endif + #ifdef __cplusplus } #endif diff --git a/qiniu/fop.c b/qiniu/fop.c index 3fdefe90..dc387448 100644 --- a/qiniu/fop.c +++ b/qiniu/fop.c @@ -13,8 +13,18 @@ #include "private/region.h" #include "../cJSON/cJSON.h" -Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, const char *key, - char *fops[], int fopCount, const char *pipeline, const char *notifyURL, int force) +#ifdef __cplusplus +extern "C" +{ +#endif + + void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt); + +#ifdef __cplusplus +} +#endif + +Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_FOP_PfopParams *params) { Qiniu_Error err; cJSON *root; @@ -25,6 +35,7 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha char *encodedNotifyURL = NULL; char *encodedPipeline = NULL; char *forceStr = NULL; + char *typeStr = NULL; char *url = NULL; char *body = NULL; Qiniu_Bool escapeBucketOk; @@ -34,29 +45,29 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha Qiniu_Bool escapeNotifyURLOk; // Add encoded bucket - encodedBucket = Qiniu_QueryEscape(bucket, &escapeBucketOk); - encodedKey = Qiniu_QueryEscape(key, &escapeKeyOk); - fopsStr = Qiniu_String_Join(";", fops, fopCount); + encodedBucket = Qiniu_QueryEscape(params->bucket, &escapeBucketOk); + encodedKey = Qiniu_QueryEscape(params->key, &escapeKeyOk); + fopsStr = Qiniu_String_Join(";", params->fops, params->fopCount); encodedFops = Qiniu_QueryEscape(fopsStr, &escapeFopsOk); Qiniu_Free(fopsStr); - if (pipeline) + if (params->pipeline) { - encodedPipeline = Qiniu_QueryEscape(pipeline, &escapePipelineOk); + encodedPipeline = Qiniu_QueryEscape(params->pipeline, &escapePipelineOk); } else { encodedPipeline = ""; } - if (notifyURL) + if (params->notifyURL) { - encodedNotifyURL = Qiniu_QueryEscape(notifyURL, &escapeNotifyURLOk); + encodedNotifyURL = Qiniu_QueryEscape(params->notifyURL, &escapeNotifyURLOk); } else { encodedNotifyURL = ""; } - if (force == 1) + if (params->force == 1) { forceStr = "1"; } @@ -64,9 +75,18 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha { forceStr = "0"; } + if (params->type == 1) + { + typeStr = "1"; + } + else + { + typeStr = "0"; + } body = Qiniu_String_Concat("bucket=", encodedBucket, "&key=", encodedKey, "&fops=", encodedFops, - "&pipeline=", encodedPipeline, "¬ifyURL=", encodedNotifyURL, "&force=", forceStr, NULL); + "&pipeline=", encodedPipeline, "¬ifyURL=", encodedNotifyURL, "&force=", forceStr, + "&type=", typeStr, NULL); if (escapeBucketOk) { Qiniu_Free(encodedBucket); @@ -82,23 +102,22 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha Qiniu_Free(encodedFops); } - if (pipeline && escapePipelineOk) + if (params->pipeline && escapePipelineOk) { Qiniu_Free(encodedPipeline); } - if (notifyURL && escapeNotifyURLOk) + if (params->notifyURL && escapeNotifyURLOk) { Qiniu_Free(encodedNotifyURL); } const char *apiHost; - err = _Qiniu_Region_Get_Api_Host(self, NULL, bucket, &apiHost); + err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost); if (err.code != 200) { - goto error; + return err; } - url = Qiniu_String_Concat2(apiHost, "/pfop/"); err = Qiniu_Client_CallWithBuffer( self, @@ -107,13 +126,94 @@ Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const cha body, strlen(body), "application/x-www-form-urlencoded"); + Qiniu_Free((void *)body); + Qiniu_Free((void *)url); if (err.code == 200) { ret->persistentId = Qiniu_Json_GetString(root, "persistentId", 0); } -error: - Qiniu_Free(body); - Qiniu_Free(url); return err; +} + +Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, const char *key, + char *fops[], int fopCount, const char *pipeline, const char *notifyURL, int force) +{ + Qiniu_FOP_PfopParams params; + Qiniu_Zero(params); + params.bucket = bucket; + params.key = key; + params.fops = (char **)fops; + params.fopCount = fopCount; + params.pipeline = pipeline; + params.notifyURL = notifyURL; + params.force = force; + return Qiniu_FOP_Pfop_v2(self, ret, ¶ms); } // Qiniu_FOP_Pfop + +Qiniu_Error Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu_FOP_PrefopItemRet *itemsRet, Qiniu_ItemCount *itemsCount, const char *persistentId, Qiniu_ItemCount maxItemsCount) +{ + Qiniu_Error err; + cJSON *root, *items, *item; + Qiniu_ItemCount curIndex; + char *encodedPersistentId = NULL, *url = NULL, *creationDateStr = NULL; + Qiniu_ReadBuf emptyBodyBuf = {NULL, 0, 0}; + Qiniu_Reader emptyBody = Qiniu_BufReader(&emptyBodyBuf, 0, 0); + Qiniu_Bool escapePersistentIdOk; + + encodedPersistentId = Qiniu_QueryEscape(persistentId, &escapePersistentIdOk); + + const char *apiHost; + err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost); + if (err.code != 200) + { + return err; + } + url = Qiniu_String_Concat(apiHost, "/status/get/prefop?id=", encodedPersistentId, NULL); + if (escapePersistentIdOk) + { + Qiniu_Free(encodedPersistentId); + } + + err = Qiniu_Client_CallWithMethod( + self, + &root, + url, + emptyBody, + 0, + "application/x-www-form-urlencoded", + "GET", + NULL); + Qiniu_Free(url); + if (err.code != 200) + { + return err; + } + + ret->id = Qiniu_Json_GetString(root, "id", NULL); + ret->code = Qiniu_Json_GetInt(root, "code", 0); + ret->desc = Qiniu_Json_GetString(root, "desc", NULL); + ret->inputBucket = Qiniu_Json_GetString(root, "inputBucket", NULL); + ret->inputKey = Qiniu_Json_GetString(root, "inputBucket", NULL); + ret->type = Qiniu_Json_GetInt(root, "type", 0); + creationDateStr = (char *)Qiniu_Json_GetString(root, "creationDate", NULL); + if (creationDateStr != NULL) + { + _Qiniu_Parse_Date_Time(creationDateStr, &ret->creationDate); + } + + *itemsCount = Qiniu_Json_GetArraySize(root, "items", 0); + items = Qiniu_Json_GetObjectItem(root, "items", 0); + for (curIndex = 0; curIndex < *itemsCount && curIndex < maxItemsCount; curIndex++) + { + item = cJSON_GetArrayItem(items, curIndex); + itemsRet[curIndex].cmd = Qiniu_Json_GetString(item, "cmd", NULL); + itemsRet[curIndex].code = Qiniu_Json_GetInt(item, "code", 0); + itemsRet[curIndex].desc = Qiniu_Json_GetString(item, "desc", NULL); + itemsRet[curIndex].error = Qiniu_Json_GetString(item, "error", NULL); + itemsRet[curIndex].hash = Qiniu_Json_GetString(item, "hash", NULL); + itemsRet[curIndex].key = Qiniu_Json_GetString(item, "key", NULL); + itemsRet[curIndex].returnOld = Qiniu_Json_GetInt(item, "returnOld", 0); + } + return err; +} diff --git a/qiniu/fop.h b/qiniu/fop.h index 9300671f..27d4554d 100644 --- a/qiniu/fop.h +++ b/qiniu/fop.h @@ -3,16 +3,19 @@ Name : fop.h Author : Qiniu.com Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd. - Description : + Description : ============================================================================ */ #ifndef QINIU_FOP_H #define QINIU_FOP_H -#include "http.h" +#include "rs.h" +#include "rfc3339.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -22,6 +25,22 @@ extern "C" /*============================================================================*/ /* func Qiniu_FOP_Pfop */ +/* @gist pfop */ + +typedef struct _Qiniu_FOP_PfopParams +{ + const char *bucket; + const char *key; + const char *pipeline; + const char *notifyURL; + char **fops; + int fopCount; + int force; + int type; +} Qiniu_FOP_PfopParams; + +/* @endgist */ + /* @gist pfopret */ typedef struct _Qiniu_FOP_PfopRet { @@ -30,13 +49,46 @@ typedef struct _Qiniu_FOP_PfopRet { /* @endgist */ -QINIU_DLLAPI extern Qiniu_Error Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, - const char *key, char *fops[], int fopCount, const char *pipeline, - const char *notifyURL, int force); +/* @gist prefopret */ + +typedef struct _Qiniu_FOP_PrefopRet +{ + const char *id; + const char *desc; + const char *inputBucket; + const char *inputKey; + int code; + int type; + Qiniu_DateTime creationDate; +} Qiniu_FOP_PrefopRet; + +typedef struct _Qiniu_FOP_PrefopItemRet +{ + const char *cmd; + const char *desc; + const char *error; + const char *hash; + const char *key; + int code; + int returnOld; +} Qiniu_FOP_PrefopItemRet; + +/* @endgist */ + +QINIU_DLLAPI extern Qiniu_Error +Qiniu_FOP_Pfop(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, const char *bucket, + const char *key, char *fops[], int fopCount, const char *pipeline, + const char *notifyURL, int force); +QINIU_DLLAPI extern Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_FOP_PfopParams *params); +QINIU_DLLAPI extern Qiniu_Error +Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu_FOP_PrefopItemRet *itemsRet, + Qiniu_ItemCount *itemsCount, const char *persistentId, Qiniu_ItemCount maxItemsCount); /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/http.c b/qiniu/http.c index 2068ace5..ce5c31e0 100644 --- a/qiniu/http.c +++ b/qiniu/http.c @@ -9,6 +9,7 @@ #include "http.h" #include "../cJSON/cJSON.h" +#include "../hashmap/hashmap.h" #include Qiniu_Error Qiniu_Client_config(Qiniu_Client *self); @@ -347,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize) Qiniu_Zero_Ptr(self); self->curl = curl_easy_init(); self->auth = auth; + self->hostsRetriesMax = 3; Qiniu_Buffer_Init(&self->b, bufSize); Qiniu_Buffer_Init(&self->respHeader, bufSize); @@ -377,8 +379,12 @@ void Qiniu_Client_Cleanup(Qiniu_Client *self) Qiniu_Buffer_Cleanup(&self->b); Qiniu_Buffer_Cleanup(&self->respHeader); - Qiniu_FreeV2((void **)&self->cachedRegion); - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); + if (self->cachedRegions != NULL) + { + hashmap_free(self->cachedRegions); + self->cachedRegions = NULL; + } + self->enableUploadingAcceleration = Qiniu_False; } void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic) @@ -392,6 +398,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long self->lowSpeedTime = lowSpeedTime; } // Qiniu_Client_SetLowSpeedLimit +void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t hostsRetriesMax) +{ + self->hostsRetriesMax = hostsRetriesMax; +} // Qiniu_Client_SetMaximumHostsRetries + void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs) { self->timeoutMs = timeoutMs; @@ -408,6 +419,16 @@ void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps) self->autoQueryHttpsRegion = useHttps; } +void Qiniu_Client_EnableUploadingAcceleration(Qiniu_Client *self) +{ + self->enableUploadingAcceleration = Qiniu_True; +} + +void Qiniu_Client_DisableUploadingAcceleration(Qiniu_Client *self) +{ + self->enableUploadingAcceleration = Qiniu_False; +} + void Qiniu_Client_SpecifyRegion(Qiniu_Client *self, Qiniu_Region *region) { self->specifiedRegion = region; @@ -471,7 +492,7 @@ static Qiniu_Error Qiniu_Client_callWithBody( Qiniu_Error err; const char *ctxType; char ctxLength[64], userAgent[64]; - Qiniu_Header *headers = NULL; + Qiniu_Header *headers; CURL *curl = (CURL *)self->curl; err = Qiniu_Client_config(self); if (err.code != 200) @@ -532,12 +553,30 @@ static Qiniu_Error Qiniu_Client_callWithBody( Qiniu_Error Qiniu_Client_CallWithMethod( Qiniu_Client *self, Qiniu_Json **ret, const char *url, Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5) +{ + return Qiniu_Client_CallWithMethodAndProgressCallback(self, ret, url, body, bodyLen, mimeType, httpMethod, md5, NULL, NULL); +} + +Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback( + Qiniu_Client *self, Qiniu_Json **ret, const char *url, + Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5, + int (*callback)(void *, double, double, double, double), void *callbackData) { CURL *curl = Qiniu_Client_initcall_withMethod(self, url, httpMethod); curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen); curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read); curl_easy_setopt(curl, CURLOPT_READDATA, body.self); + if (callback != NULL) + { + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + } + else + { + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); + } return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, md5); } @@ -545,12 +584,30 @@ Qiniu_Error Qiniu_Client_CallWithMethod( Qiniu_Error Qiniu_Client_CallWithBinary( Qiniu_Client *self, Qiniu_Json **ret, const char *url, Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType) +{ + return Qiniu_Client_CallWithBinaryAndProgressCallback(self, ret, url, body, bodyLen, mimeType, NULL, NULL); +} + +Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback( + Qiniu_Client *self, Qiniu_Json **ret, const char *url, + Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, + int (*callback)(void *, double, double, double, double), void *callbackData) { CURL *curl = Qiniu_Client_initcall(self, url); curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, bodyLen); curl_easy_setopt(curl, CURLOPT_READFUNCTION, body.Read); curl_easy_setopt(curl, CURLOPT_READDATA, body.self); + if (callback != NULL) + { + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, callbackData); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); + } + else + { + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); + } return Qiniu_Client_callWithBody(self, ret, url, NULL, bodyLen, mimeType, NULL); } diff --git a/qiniu/http.h b/qiniu/http.h index 240082b1..c64f1600 100644 --- a/qiniu/http.h +++ b/qiniu/http.h @@ -79,7 +79,9 @@ extern "C" /*============================================================================*/ /* type Qiniu_Auth */ +#if defined(_WIN32) #pragma pack(1) +#endif typedef struct curl_slist Qiniu_Header; @@ -106,7 +108,9 @@ extern "C" typedef struct _Qiniu_Region Qiniu_Region; QINIU_DLLAPI extern void Qiniu_Region_Free(Qiniu_Region *region); - struct _Qiniu_Client + struct hashmap; + + typedef struct _Qiniu_Client { void *curl; Qiniu_Auth auth; @@ -116,8 +120,7 @@ extern "C" Qiniu_Bool autoQueryRegion; Qiniu_Bool autoQueryHttpsRegion; - const char *cachedRegionBucketName; - Qiniu_Region *cachedRegion; + struct hashmap *cachedRegions; Qiniu_Region *specifiedRegion; // Use the following field to specify which NIC to use for sending packets. @@ -138,16 +141,24 @@ extern "C" // Millisecond timeout for the connection phase. long connectTimeoutMs; - }; - typedef struct _Qiniu_Client Qiniu_Client; + + // Max retries count. + size_t hostsRetriesMax; + + // Is uploading acceleration enabled. + Qiniu_Bool enableUploadingAcceleration; + } Qiniu_Client; QINIU_DLLAPI extern void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize); QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self); QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic); QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime); + QINIU_DLLAPI extern void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t retries); QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs); QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs); QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps); + QINIU_DLLAPI extern void Qiniu_Client_EnableUploadingAcceleration(Qiniu_Client *self); + QINIU_DLLAPI extern void Qiniu_Client_DisableUploadingAcceleration(Qiniu_Client *self); QINIU_DLLAPI extern void Qiniu_Client_SpecifyRegion(Qiniu_Client *self, Qiniu_Region *region); QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_Call(Qiniu_Client *self, Qiniu_Json **ret, const char *url); @@ -166,6 +177,16 @@ extern "C" QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethod( Qiniu_Client *self, Qiniu_Json **ret, const char *url, Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5); + + QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithBinaryAndProgressCallback( + Qiniu_Client *self, Qiniu_Json **ret, const char *url, + Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, + int (*callback)(void *, double, double, double, double), void *callbackData); + + QINIU_DLLAPI extern Qiniu_Error Qiniu_Client_CallWithMethodAndProgressCallback( + Qiniu_Client *self, Qiniu_Json **ret, const char *url, + Qiniu_Reader body, Qiniu_Int64 bodyLen, const char *mimeType, const char *httpMethod, const char *md5, + int (*callback)(void *, double, double, double, double), void *callbackData); /*============================================================================*/ /* func Qiniu_Client_InitNoAuth/InitMacAuth */ @@ -185,7 +206,9 @@ extern "C" /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/io.c b/qiniu/io.c index 31ea14e2..5ad35b51 100644 --- a/qiniu/io.c +++ b/qiniu/io.c @@ -11,6 +11,7 @@ #include "reader.h" #include "recorder_utils.h" #include "private/region.h" +#include "private/code.h" #include /*============================================================================*/ @@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self); Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph); -static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost) +static Qiniu_Error _Qiniu_Get_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount) { if (extra && extra->ipCount != 0) { Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex); - *upHost = extra->upIps[abs(oldIndex % extra->ipCount)]; + *upHosts = &extra->upIps[labs(oldIndex % extra->ipCount)]; + *upHostsCount = 1; } else if (extra && extra->upHost != NULL) { - *upHost = extra->upHost; + *upHosts = &extra->upHost; + *upHostsCount = 1; + } + else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0) + { + *upHosts = extra->upHosts; + *upHostsCount = extra->upHostsCount; } else { - Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost); + Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount); if (err.code != 200) { return err; @@ -144,54 +152,85 @@ Qiniu_Error Qiniu_Client_config(Qiniu_Client *self) return err; } +static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) +{ + void (*uploadingProgress)(size_t, size_t) = (void (*)(size_t, size_t))clientp; + uploadingProgress((size_t)ultotal, (size_t)ulnow); + return 0; +} + static Qiniu_Error Qiniu_Io_call( Qiniu_Client *self, const char *accessKey, const char *bucketName, Qiniu_Io_PutRet *ret, struct curl_httppost *formpost, Qiniu_Io_PutExtra *extra) { int retCode = 0; Qiniu_Error err; - struct curl_slist *headers = NULL; - const char *upHost = NULL; + struct curl_slist *headers; + const char *const *upHosts; + const char *defaultUpHosts[] = {QINIU_UP_HOST}; + size_t upHostsCount; - err = Qiniu_Client_config(self); + headers = curl_slist_append(NULL, "Expect:"); + err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount); if (err.code != 200) { return err; } - err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost); - if (err.code != 200) + if (upHostsCount == 0) { - return err; + upHosts = defaultUpHosts; + upHostsCount = 1; } - headers = curl_slist_append(NULL, "Expect:"); - - CURL *curl = Qiniu_Client_reset(self); - curl_easy_setopt(curl, CURLOPT_URL, upHost); - curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - - //// For aborting uploading file. - if (extra->upAbortCallback) + for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++) { - curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback); - } // if + CURL *curl = Qiniu_Client_reset(self); + err = Qiniu_Client_config(self); + if (err.code != 200) + { + return err; + } - err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); - if (err.code == 200 && ret != NULL) - { - if (extra->callbackRetParser != NULL) + curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + if (extra->uploadingProgress != NULL) { - err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); } else { - ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); - ret->key = Qiniu_Json_GetString(self->root, "key", NULL); - ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); } - } + //// For aborting uploading file. + if (extra->upAbortCallback) + { + curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback); + } // if + + err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); + if (err.code == 200) + { + if (extra->callbackRetParser != NULL) + { + err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + } + else if (ret != NULL) + { + ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); + ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL); + } + break; + } + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + break; + } + } curl_formfree(formpost); curl_slist_free_all(headers); return err; @@ -313,7 +352,6 @@ Qiniu_Error Qiniu_Io_PutBuffer( CURLFORM_BUFFER, key, CURLFORM_BUFFERPTR, buf, CURLFORM_BUFFERLENGTH, fsize, CURLFORM_END); err = Qiniu_Io_call(self, accessKey, bucketName, ret, form.formpost, extra); -error: Qiniu_Free((void *)accessKey); Qiniu_Free((void *)bucketName); return err; @@ -327,42 +365,67 @@ static Qiniu_Error Qiniu_Io_call_with_callback( { int retCode = 0; Qiniu_Error err; - struct curl_slist *headers = NULL; - const char *upHost; + struct curl_slist *headers; + const char *const *upHosts; + const char *defaultUpHosts[] = {QINIU_UP_HOST}; + size_t upHostsCount; - CURL *curl = Qiniu_Client_reset(self); - err = Qiniu_Client_config(self); + headers = curl_slist_append(NULL, "Expect:"); + err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount); if (err.code != 200) { return err; } - err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost); - if (err.code != 200) + if (upHostsCount == 0) { - return err; + upHosts = defaultUpHosts; + upHostsCount = 1; } - headers = curl_slist_append(NULL, "Expect:"); + for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++) + { + CURL *curl = Qiniu_Client_reset(self); + err = Qiniu_Client_config(self); + if (err.code != 200) + { + return err; + } - curl_easy_setopt(curl, CURLOPT_URL, upHost); - curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); - curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr); + curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]); + curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr); - err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); - if (err.code == 200 && ret != NULL) - { - if (extra->callbackRetParser != NULL) + if (extra->uploadingProgress != NULL) { - err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, _Qiniu_Progress_Callback); + curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, extra->uploadingProgress); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); } else { - ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); - ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); } - } + err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader); + if (err.code == 200) + { + if (extra->callbackRetParser != NULL) + { + err = (*extra->callbackRetParser)(extra->callbackRet, self->root); + } + else if (ret != NULL) + { + ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL); + ret->key = Qiniu_Json_GetString(self->root, "key", NULL); + } + break; + } + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + break; + } + } curl_formfree(formpost); curl_slist_free_all(headers); return err; diff --git a/qiniu/io.h b/qiniu/io.h index 2329bcaa..e5eb036b 100644 --- a/qiniu/io.h +++ b/qiniu/io.h @@ -3,7 +3,7 @@ Name : io.h Author : Qiniu.com Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd. - Description : + Description : ============================================================================ */ @@ -13,7 +13,9 @@ #include "http.h" #include "reader.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -50,10 +52,18 @@ extern "C" void *upAbortUserData; Qiniu_Rd_FnAbort upAbortCallback; + // Deprecated fields, prefer upHosts. const char *upHost; const char **upIps; Qiniu_Count ipCount; Qiniu_Count ipIndex; + + // Specify multiple upHosts + const char *const *upHosts; + size_t upHostsCount; + + // Uploading file progress + void (*uploadingProgress)(size_t ultotal, size_t ulnow); } Qiniu_Io_PutExtra; /*============================================================================*/ @@ -98,7 +108,9 @@ extern "C" void *self, Qiniu_Header **header, const char *url, const char *addition, size_t addlen); /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/multipart_upload.c b/qiniu/multipart_upload.c index cea290bd..d217b08b 100644 --- a/qiniu/multipart_upload.c +++ b/qiniu/multipart_upload.c @@ -16,6 +16,7 @@ #include "../cJSON/cJSON.h" #include "tm.h" #include "private/region.h" +#include "private/code.h" /*============================================================================*/ #if defined(_WIN32) @@ -49,6 +50,12 @@ typedef struct int totalPartNum; } Qiniu_UploadParts_Ret; +struct _Qiniu_Progress_Callback_Data +{ + size_t base, totalSize, previousUlNow; + void (*callback)(size_t, size_t); +}; + static Qiniu_Error readMedium(struct Qiniu_Record_Medium *medium, char **uploadId, Qiniu_Uint64 *expireAt, Qiniu_UploadPartResp *ret); static Qiniu_Error writeMedium(struct Qiniu_Record_Medium *medium, const Qiniu_InitPart_Ret *, const Qiniu_UploadPartResp *); static Qiniu_Error initializeRecorder(Qiniu_Multipart_PutExtra *param, const char *uptoken, const char *key, const char *fileName, Qiniu_FileInfo *fi, Qiniu_Record_Medium *medium, Qiniu_Multipart_Recorder *recorder); @@ -124,67 +131,116 @@ getTotalPartNum(Qiniu_Int64 fileSize, Qiniu_Int64 partSize, int *totalPartNum) static Qiniu_Error init_upload(Qiniu_Client *client, const char *bucket, const char *encodedKey, Qiniu_Multipart_PutExtra *extraParam, Qiniu_InitPart_Ret *initPartRet) { Qiniu_Error err = Qiniu_OK; - const char *uphost = extraParam->upHost; - char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL); + const char *const *upHosts; + const char *upHost, *reqUrl; + size_t upHostsCount; + Qiniu_Json *callRet; - for (int i = 0; i < extraParam->tryTimes; i++) + if (extraParam->upHost != NULL) { - Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } + + for (int i = 0; i < extraParam->tryTimes && i <= client->hostsRetriesMax; i++) + { + upHost = upHosts[i % upHostsCount]; + reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads", NULL); + + callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. err = Qiniu_Client_Call(client, &callRet, reqUrl); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx , net error will retry + Qiniu_Free((void *)reqUrl); + if (err.code == 200) { - Qiniu_Log_Error("init_upload retry: %E", err); - continue; + if (initPartRet != NULL) + { + initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL)); + initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0); + } + break; } - else if (err.code != 200) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("init_upload: %E", err); break; } - - initPartRet->uploadId = Qiniu_String_Dup(Qiniu_Json_GetString(callRet, "uploadId", NULL)); - initPartRet->expireAt = Qiniu_Json_GetUInt64(callRet, "expireAt", 0); - break; } - Qiniu_Free(reqUrl); return err; } -Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *reqUrl, int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str, Qiniu_UploadPartResp *ret) +static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) +{ + struct _Qiniu_Progress_Callback_Data *data = (struct _Qiniu_Progress_Callback_Data *)clientp; + if (data->previousUlNow != (size_t)ulnow) + { + data->callback((size_t)data->totalSize, data->base + (size_t)ulnow); + data->previousUlNow = (size_t)ulnow; + } + return 0; +} + +Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extraParam, const char *path, + int partNum, Qiniu_ReaderAt reader, Qiniu_Int64 partOffset, Qiniu_Int64 partSize, const char *md5str, + Qiniu_UploadPartResp *ret, struct _Qiniu_Progress_Callback_Data *progressCallback) { Qiniu_Error err = Qiniu_OK; - for (int try = 0; try < extraParam->tryTimes; try++) + const char *const *upHosts; + size_t upHostsCount; + int (*callback)(void *, double, double, double, double) = NULL; + void *callbackData = NULL; + if (progressCallback != NULL && progressCallback->callback != NULL) + { + callback = _Qiniu_Progress_Callback; + callbackData = (void *)progressCallback; + } + + if (extraParam->upHost != NULL) + { + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } + for (int tries = 0; tries < extraParam->tryTimes && tries <= client->hostsRetriesMax; tries++) { Qiniu_Section section; Qiniu_Zero(section); Qiniu_Reader thisPartBody = Qiniu_SectionReader(§ion, reader, (Qiniu_Off_T)partOffset, partSize); Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. - err = Qiniu_Client_CallWithMethod(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry + const char *upHost = upHosts[tries % upHostsCount]; + const char *reqUrl = Qiniu_String_Concat(upHost, path, NULL); + err = Qiniu_Client_CallWithMethodAndProgressCallback(client, &callRet, reqUrl, thisPartBody, partSize, NULL, "PUT", md5str, callback, callbackData); + Qiniu_Free((void *)reqUrl); + if (err.code == 200) { - Qiniu_Log_Error("upload_part retry: partNum:%d, %E", partNum, err); - continue; + if (ret != NULL) + { + const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL); + const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL); + ret->etag = Qiniu_String_Dup(etag); + ret->md5 = Qiniu_String_Dup(md5); + ret->partNum = partNum; + } + break; } - else if (err.code != 200) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("upload_part: partNum:%d, %E", partNum, err); break; } - - const char *md5 = Qiniu_Json_GetString(callRet, "md5", NULL); - const char *etag = Qiniu_Json_GetString(callRet, "etag", NULL); - ret->etag = Qiniu_String_Dup(etag); - ret->md5 = Qiniu_String_Dup(md5); - ret->partNum = partNum; - // Qiniu_Log_Debug("partNum:%d,remote md5:%s ", partNum, md5); - break; } // notify callback - if ((err.code == 200) && extraParam->notify) + if (err.code == 200 && extraParam->notify) { extraParam->notify(ret); } - if ((err.code != 200) && extraParam->notifyErr) + else if (err.code != 200 && extraParam->notifyErr) { extraParam->notifyErr(partNum, err); } @@ -195,22 +251,19 @@ Qiniu_Error upload_one_part(Qiniu_Client *client, Qiniu_Multipart_PutExtra *extr static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const char *encodedKey, const Qiniu_InitPart_Ret *initParts, Qiniu_ReaderAt *reader, Qiniu_Int64 fsize, int totalPartNum, Qiniu_Multipart_PutExtra *extraParam, Qiniu_Multipart_Recorder *recorder, Qiniu_UploadParts_Ret *uploadPartsRet) { Qiniu_Error err = Qiniu_OK; - const char *uphost = extraParam->upHost; - Qiniu_Int64 partSize = extraParam->partSize; - int lastPart = totalPartNum - 1; - for (int partNum = 0; partNum < totalPartNum; partNum++) - { - if ((uploadPartsRet->PartsRet + partNum)->etag != NULL) - { - continue; - } + const int lastPart = totalPartNum - 1; + struct _Qiniu_Progress_Callback_Data progressCallbackData; + Qiniu_Zero(progressCallbackData); - int partNumInReq = partNum + 1; // partNum start from 1 - char partNumStr[10]; // valid partNum ={"1"~"10000"} - snprintf(partNumStr, 10, "%d", partNumInReq); - char *reqUrl = Qiniu_String_Concat(uphost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL); + if (extraParam->uploadingProgress != NULL) + { + progressCallbackData.callback = extraParam->uploadingProgress; + progressCallbackData.totalSize = (size_t)fsize; + } + for (int partNum = 0; partNum < totalPartNum; partNum++) + { Qiniu_Int64 thisPartOffset = partNum * partSize; Qiniu_Int64 thisPartSize = partSize; if (partNum == lastPart) @@ -218,29 +271,36 @@ static Qiniu_Error upload_parts(Qiniu_Client *client, const char *bucket, const thisPartSize = fsize - (totalPartNum - 1) * partSize; } - const char *md5str = NULL; - if (extraParam->enableContentMd5) + if ((uploadPartsRet->PartsRet + partNum)->etag == NULL) { - md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize); - // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str); - } - err = upload_one_part(client, extraParam, reqUrl, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum]); - - Qiniu_Multi_Free(2, (void *)reqUrl, (void *)md5str); + const int partNumInReq = partNum + 1; // partNum start from 1 + char partNumStr[10]; // valid partNum ={"1"~"10000"} + snprintf(partNumStr, 10, "%d", partNumInReq); + const char *path = Qiniu_String_Concat("/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initParts->uploadId, "/", partNumStr, NULL); - if (err.code != 200) - { - return err; - } + const char *md5str = NULL; + if (extraParam->enableContentMd5) + { + md5str = caculatePartMd5(*reader, thisPartOffset, thisPartSize); + // Qiniu_Log_Debug("partNum:%d, local Md5:%s ", partNumInReq, md5str); + } + err = upload_one_part(client, extraParam, path, partNumInReq, *reader, thisPartOffset, thisPartSize, md5str, &uploadPartsRet->PartsRet[partNum], &progressCallbackData); + Qiniu_Multi_Free(2, (void *)path, (void *)md5str); - if (recorder != NULL && recorder->recorderMedium != NULL) - { - err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]); if (err.code != 200) { return err; } + if (recorder != NULL && recorder->recorderMedium != NULL) + { + err = writeMedium(recorder->recorderMedium, initParts, &uploadPartsRet->PartsRet[partNum]); + if (err.code != 200) + { + return err; + } + } } + progressCallbackData.base += thisPartSize; } return err; } @@ -283,35 +343,54 @@ static Qiniu_Error complete_upload( // step2:send req Qiniu_Error err = Qiniu_OK; - char *reqUrl = Qiniu_String_Concat(extraParam->upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL); + const char *const *upHosts; + size_t upHostsCount; - for (int try = 0; try < extraParam->tryTimes; try++) + if (extraParam->upHost != NULL) { + upHosts = &extraParam->upHost; + upHostsCount = 1; + } + else + { + upHosts = extraParam->upHosts; + upHostsCount = extraParam->upHostsCount; + } + + for (int tries = 0; tries < extraParam->tryTimes && tries <= client->hostsRetriesMax; tries++) + { + const char *upHost = upHosts[tries % upHostsCount]; + const char *reqUrl = Qiniu_String_Concat(upHost, "/buckets/", bucket, "/objects/", encodedKey, "/uploads/", initPartsRet->uploadId, NULL); Qiniu_Json *callRet = NULL; // don't cJSON_Delete(callRet), it will be automatically freed on next http request by Qiniu_Client_Call. err = Qiniu_Client_CallWithBuffer(client, &callRet, reqUrl, body, strlen(body), "application/json"); - if ((err.code / 100 == 5) || (callRet == NULL)) // 5xx,net error will retry - { - Qiniu_Log_Error("cupload: retry uploadId:%s, %E", initPartsRet->uploadId, err); - continue; - } - if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) + Qiniu_Free((void *)reqUrl); + + if (err.code == 200) { - extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + if (extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) + { + extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + } + if (callRet != NULL) + { + const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen. + const char *key = Qiniu_Json_GetString(callRet, "key", NULL); + completeRet->hash = Qiniu_String_Dup(hash); + completeRet->key = Qiniu_String_Dup(key); + } + break; } - if (err.code != 200) + else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) { - Qiniu_Log_Error("cupload: uploadId:%s, %E", initPartsRet->uploadId, err); + if (err.code / 100 != 4 && extraParam->recorder != NULL && recorder != NULL && recorder->recorderKey != NULL) + { + extraParam->recorder->remove(extraParam->recorder, recorder->recorderKey); + } break; } - - const char *hash = Qiniu_Json_GetString(callRet, "hash", NULL); // don't free(hash) since no malloc happen. - const char *key = Qiniu_Json_GetString(callRet, "key", NULL); - completeRet->hash = Qiniu_String_Dup(hash); - completeRet->key = Qiniu_String_Dup(key); - break; } // Qiniu_Log_Debug("Upload result: uploadid:%s, hash:%s, key:%s ", uploadId, completeRet->hash, completeRet->key); - Qiniu_Multi_Free(2, (void *)reqUrl, (void *)body); + Qiniu_Free((void *)body); return err; } @@ -506,9 +585,9 @@ Qiniu_Error verifyParam(Qiniu_Client *client, const char *accessKey, const char { param->tryTimes = 3; } - if (param->upHost == NULL) + if (param->upHost == NULL && (param->upHosts == NULL || param->upHostsCount == 0)) { - err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, ¶m->upHost); + err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, ¶m->upHosts, ¶m->upHostsCount); } return err; diff --git a/qiniu/multipart_upload.h b/qiniu/multipart_upload.h index 3d265ca7..4c536d7e 100644 --- a/qiniu/multipart_upload.h +++ b/qiniu/multipart_upload.h @@ -17,7 +17,9 @@ #include "stdio.h" #include "recorder.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -37,7 +39,7 @@ extern "C" typedef struct _Qiniu_Multipart_PutExtra { - const char *upHost; //if not set explicitly ,will use global QINIU_UP_HOST; + const char *upHost; // if not set explicitly, will use global upHosts; Qiniu_Int64 partSize; //size for each part const char *mimeType; int tryTimes; //at least 1, default=3 @@ -58,6 +60,12 @@ extern "C" Qiniu_Recorder *recorder; + // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST + const char *const *upHosts; + size_t upHostsCount; + + // Uploading file progress + void (*uploadingProgress)(size_t ultotal, size_t ulnow); } Qiniu_Multipart_PutExtra; typedef struct diff --git a/qiniu/private/code.h b/qiniu/private/code.h new file mode 100644 index 00000000..492bc735 --- /dev/null +++ b/qiniu/private/code.h @@ -0,0 +1,21 @@ +#ifndef QINIU_CODE_H +#define QINIU_CODE_H + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef enum + { + QINIU_DONT_RETRY, + QINIU_TRY_NEXT_DOMAIN + } Qiniu_Retry_Decision; + + Qiniu_Retry_Decision _Qiniu_Should_Retry(int code); + +#ifdef __cplusplus +} +#endif + +#endif // QINIU_CODE_H diff --git a/qiniu/private/region.h b/qiniu/private/region.h index 8a5cc465..1dc9f0ea 100644 --- a/qiniu/private/region.h +++ b/qiniu/private/region.h @@ -1,26 +1,17 @@ #ifndef QINIU_PRIVATE_REGION_H #define QINIU_PRIVATE_REGION_H #include -#include "../base.h" +#include "../http.h" #ifdef __cplusplus extern "C" { #endif - struct _Qiniu_Client; - typedef struct _Qiniu_Client Qiniu_Client; - - struct _Qiniu_Region_Hosts - { - const char *const *hosts; - size_t hostsCount; - }; - struct _Qiniu_Region_Service { - struct _Qiniu_Region_Hosts *preferredHosts; - struct _Qiniu_Region_Hosts *alternativeHosts; + const char *const *hosts; + size_t acceleratedHostsCount, preferredHostsCount, alternativeHostsCount; }; struct _Qiniu_Region @@ -37,20 +28,31 @@ extern "C" struct _Qiniu_Region_Service *apiService; }; - static struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New(const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); - static struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New_without_dup(const char *const *hosts, size_t hostsCount); - static void _Qiniu_Region_Hosts_Free(struct _Qiniu_Region_Hosts *hosts); + struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(const char *const *acceleratedHosts, size_t acceleratedHostsCount, + const char *const *preferredHosts, size_t preferredHostsCount, + const char *const *alternativeHosts, size_t alternativeHostsCount, + Qiniu_Bool useHttps); + void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service); - static struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(struct _Qiniu_Region_Hosts *preferredHosts, struct _Qiniu_Region_Hosts *alternativeHosts); - static void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service); - - Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host); + + Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); + Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); + Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); + Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); + Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count); Qiniu_Error _Qiniu_Region_Query(Qiniu_Client *self, struct _Qiniu_Region **region, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps); + const char *const *_Qiniu_Region_Get_All_Up_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Io_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Io_Src_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Bucket_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Rs_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Rsf_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); + const char *const *_Qiniu_Region_Get_All_Api_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled); #ifdef __cplusplus } #endif diff --git a/qiniu/qetag.c b/qiniu/qetag.c index 9e9bcaa5..5a87823e 100644 --- a/qiniu/qetag.c +++ b/qiniu/qetag.c @@ -10,7 +10,9 @@ extern "C" { #endif +#if defined(_WIN32) #pragma pack(1) +#endif #define BLOCK_ELEMENT_MAX_COUNT 16 #define BLOCK_MAX_SIZE (1 << 22) @@ -60,7 +62,7 @@ static Qiniu_Error Qiniu_Qetag_mergeBlocks(struct _Qiniu_Qetag_Context * ctx) { } blk->done = NO; - + ctx->blkBegin += 1; if (ctx->blkBegin >= ctx->blkElementCount) { ctx->blkBegin = 0; @@ -266,7 +268,7 @@ Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context * ctx, char ** digest) Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block ** blk, size_t * remainderSize) { Qiniu_Error err; - + if (ctx->blk) { *blk = ctx->blk; ctx->blk = NULL; @@ -394,7 +396,9 @@ Qiniu_Error Qiniu_Qetag_DigestBuffer(const char * buf, size_t bufSize, char ** d return err; } // Qiniu_Qetag_DigestBuffer +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/qetag.h b/qiniu/qetag.h index 165b8821..ef12ed99 100644 --- a/qiniu/qetag.h +++ b/qiniu/qetag.h @@ -3,7 +3,7 @@ Name : qetag.h Author : Qiniu.com Copyright : 2012(c) Shanghai Qiniu Information Technologies Co., Ltd. - Description : + Description : ============================================================================ */ @@ -18,27 +18,31 @@ extern "C" { #endif +#if defined(_WIN32) #pragma pack(1) +#endif -struct _Qiniu_Qetag_Context; -struct _Qiniu_Qetag_Block; + struct _Qiniu_Qetag_Context; + struct _Qiniu_Qetag_Block; -// 底层函数 -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_New(struct _Qiniu_Qetag_Context ** ctx, unsigned int concurrency); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Reset(struct _Qiniu_Qetag_Context * ctx); -QINIU_DLLAPI extern void Qiniu_Qetag_Destroy(struct _Qiniu_Qetag_Context * ctx); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Update(struct _Qiniu_Qetag_Context * ctx, const char * buf, size_t bufSize); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context * ctx, char ** digest); + // 底层函数 + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_New(struct _Qiniu_Qetag_Context **ctx, unsigned int concurrency); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Reset(struct _Qiniu_Qetag_Context *ctx); + QINIU_DLLAPI extern void Qiniu_Qetag_Destroy(struct _Qiniu_Qetag_Context *ctx); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Update(struct _Qiniu_Qetag_Context *ctx, const char *buf, size_t bufSize); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_Final(struct _Qiniu_Qetag_Context *ctx, char **digest); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block ** blk, size_t * blkCapacity); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_UpdateBlock(struct _Qiniu_Qetag_Block * blk, const char * buf, size_t bufSize, size_t * blkCapacity); -QINIU_DLLAPI extern void Qiniu_Qetag_CommitBlock(struct _Qiniu_Qetag_Context * ctx, struct _Qiniu_Qetag_Block * blk); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_AllocateBlock(struct _Qiniu_Qetag_Context *ctx, struct _Qiniu_Qetag_Block **blk, size_t *blkCapacity); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_UpdateBlock(struct _Qiniu_Qetag_Block *blk, const char *buf, size_t bufSize, size_t *blkCapacity); + QINIU_DLLAPI extern void Qiniu_Qetag_CommitBlock(struct _Qiniu_Qetag_Context *ctx, struct _Qiniu_Qetag_Block *blk); -// 单线程计算 QETAG -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestFile(const char * localFile, char ** digest); -QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestBuffer(const char * buf, size_t fsize, char ** digest); + // 单线程计算 QETAG + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestFile(const char *localFile, char **digest); + QINIU_DLLAPI extern Qiniu_Error Qiniu_Qetag_DigestBuffer(const char *buf, size_t fsize, char **digest); +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/reader.h b/qiniu/reader.h index 8515475f..2a07f5ca 100644 --- a/qiniu/reader.h +++ b/qiniu/reader.h @@ -3,7 +3,7 @@ Name : reader.h Author : Qiniu.com Copyright : 2012-2016(c) Shanghai Qiniu Information Technologies Co., Ltd. - Description : + Description : ============================================================================ */ @@ -12,7 +12,9 @@ #include "base.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -47,7 +49,9 @@ QINIU_DLLAPI extern void Qiniu_Rd_Reader_Close(Qiniu_Rd_Reader * rdr); QINIU_DLLAPI extern size_t Qiniu_Rd_Reader_Callback(char * buffer, size_t size, size_t nitems, void * rdr); +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/recorder.c b/qiniu/recorder.c index 3afc31dc..d786c32d 100644 --- a/qiniu/recorder.c +++ b/qiniu/recorder.c @@ -157,7 +157,6 @@ Qiniu_Error _Qiniu_FileSystem_RecordMedium_Read_Entry(const struct Qiniu_Record_ Qiniu_Error _Qiniu_FileSystem_RecordMedium_Has_Next_Entry(const struct Qiniu_Record_Medium *medium, Qiniu_Bool *has) { struct _FileSystem_Recorder_Data *data = (struct _FileSystem_Recorder_Data *)medium->data; - size_t i; Qiniu_Bool hasEntry = Qiniu_True; if (data->buf == NULL || data->bufOffset >= data->bufSize) diff --git a/qiniu/recorder.h b/qiniu/recorder.h index 08ae4fe0..49698b32 100644 --- a/qiniu/recorder.h +++ b/qiniu/recorder.h @@ -14,7 +14,9 @@ #define Qiniu_Recorder_Read_Error 9801 #define Qiniu_Recorder_Write_Error 9802 +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -45,7 +47,9 @@ extern "C" QINIU_DLLAPI extern Qiniu_Error Qiniu_FileSystem_Recorder_New(const char *rootPath, struct Qiniu_Recorder *recorder); /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/recorder_key.h b/qiniu/recorder_key.h index a7f56602..c5d10e55 100644 --- a/qiniu/recorder_key.h +++ b/qiniu/recorder_key.h @@ -13,7 +13,9 @@ #include "base.h" #include "private/crypto.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -36,7 +38,9 @@ QINIU_DLLAPI extern void Qiniu_Recorder_Key_Generator_Free(Qiniu_Recorder_Key_Ge /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/recorder_utils.h b/qiniu/recorder_utils.h index a8623805..4b717f46 100644 --- a/qiniu/recorder_utils.h +++ b/qiniu/recorder_utils.h @@ -13,7 +13,9 @@ #include "base.h" #include "recorder.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -29,7 +31,9 @@ extern "C" /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/region.c b/qiniu/region.c index 821a69b5..7d3d7871 100644 --- a/qiniu/region.c +++ b/qiniu/region.c @@ -11,6 +11,7 @@ #include "region.h" #include "tm.h" #include "../cJSON/cJSON.h" +#include "../hashmap/hashmap.h" /*============================================================================*/ @@ -18,8 +19,7 @@ #include #endif -static const char * -duplicate_host(const char *host, Qiniu_Bool useHttps) +static const char *_Qiniu_Duplicate_Host(const char *host, Qiniu_Bool useHttps) { if (host == NULL) { @@ -39,83 +39,82 @@ duplicate_host(const char *host, Qiniu_Bool useHttps) } } -struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New(const char *const *const hosts, size_t hostsCount, Qiniu_Bool useHttps) +static Qiniu_Bool _Qiniu_Region_Service_Copy_Hosts(const char **targetHosts, size_t copyOffset, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) { - const char **newHosts = malloc(sizeof(const char *) * hostsCount); - if (newHosts == NULL) - { - return NULL; - } - - struct _Qiniu_Region_Hosts *newRegionHosts = malloc(sizeof(struct _Qiniu_Region_Hosts)); - if (newRegionHosts == NULL) - { - free((void *)newHosts); - return NULL; - } - newRegionHosts->hosts = (const char *const *)newHosts; - - for (size_t i = 0; i < hostsCount; i++) + size_t i, j; + for (i = 0; i < hostsCount; i++) { if (hosts[i] == NULL) { break; } - newHosts[i] = duplicate_host(hosts[i], useHttps); - if (newHosts[i] == NULL) + targetHosts[copyOffset + i] = _Qiniu_Duplicate_Host(hosts[i], useHttps); + if (targetHosts[copyOffset + i] == NULL) { - for (size_t j = 0; j < i; j++) - { - free((void *)newHosts[j]); - } - free((void *)newHosts); - free((void *)newRegionHosts); - return NULL; + goto handleErr; } - newRegionHosts->hostsCount = i + 1; } - return newRegionHosts; -} - -struct _Qiniu_Region_Hosts *_Qiniu_Region_Hosts_New_without_dup(const char *const *hosts, size_t hostsCount) -{ - struct _Qiniu_Region_Hosts *newRegionHosts = malloc(sizeof(struct _Qiniu_Region_Hosts)); - if (newRegionHosts == NULL) + return Qiniu_True; +handleErr: + for (j = 0; j < i; j++) { - return NULL; + free((void *)targetHosts[copyOffset + j]); } - newRegionHosts->hosts = hosts; - newRegionHosts->hostsCount = hostsCount; - return newRegionHosts; + return Qiniu_False; } -void _Qiniu_Region_Hosts_Free(struct _Qiniu_Region_Hosts *hosts) +struct _Qiniu_Region_Service *_Qiniu_Region_Service_New( + const char *const *acceleratedHosts, size_t acceleratedHostsCount, + const char *const *preferredHosts, size_t preferredHostsCount, + const char *const *alternativeHosts, size_t alternativeHostsCount, + Qiniu_Bool useHttps) { - if (hosts == NULL) + struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service)); + if (newService == NULL) { - return; + goto handleErr; } - for (int i = 0; i < hosts->hostsCount; i++) + size_t allHostsCount = acceleratedHostsCount + preferredHostsCount + alternativeHostsCount; + const char **newHosts = malloc(sizeof(const char *) * allHostsCount); + if (newHosts == NULL) { - free((void *)hosts->hosts[i]); + goto handleErr; } - free((void *)hosts->hosts); - hosts->hosts = NULL; - hosts->hostsCount = 0; - free((void *)hosts); -} - -struct _Qiniu_Region_Service *_Qiniu_Region_Service_New(struct _Qiniu_Region_Hosts *preferredHosts, struct _Qiniu_Region_Hosts *alternativeHosts) -{ - struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service)); - if (newService == NULL) + size_t offset = 0; + if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, acceleratedHosts, acceleratedHostsCount, useHttps) == Qiniu_False) { - return NULL; + goto handleErr; } - newService->preferredHosts = preferredHosts; - newService->alternativeHosts = alternativeHosts; - + offset += acceleratedHostsCount; + if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, preferredHosts, preferredHostsCount, useHttps) == Qiniu_False) + { + goto handleErr; + } + offset += preferredHostsCount; + if (_Qiniu_Region_Service_Copy_Hosts(newHosts, offset, alternativeHosts, alternativeHostsCount, useHttps) == Qiniu_False) + { + goto handleErr; + } + offset += alternativeHostsCount; + newService->hosts = newHosts; + newService->acceleratedHostsCount = acceleratedHostsCount; + newService->preferredHostsCount = preferredHostsCount; + newService->alternativeHostsCount = alternativeHostsCount; return newService; +handleErr: + if (newHosts != NULL) + { + for (size_t j = 0; j < offset; j++) + { + free((void *)newHosts[offset + j]); + } + free((void *)newHosts); + } + if (newService != NULL) + { + free((void *)newService); + } + return NULL; } void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service) @@ -124,11 +123,13 @@ void _Qiniu_Region_Service_Free(struct _Qiniu_Region_Service *service) { return; } - _Qiniu_Region_Hosts_Free(service->preferredHosts); - service->preferredHosts = NULL; - _Qiniu_Region_Hosts_Free(service->alternativeHosts); - service->alternativeHosts = NULL; - free(service); + size_t allHostsCount = service->acceleratedHostsCount + service->preferredHostsCount + service->alternativeHostsCount; + for (size_t i = 0; i < allHostsCount; i++) + { + free((void *)service->hosts[i]); + } + free((void *)service->hosts); + free((void *)service); } Qiniu_Region *Qiniu_Region_New(const char *regionId, Qiniu_Uint64 ttl) @@ -182,22 +183,48 @@ void Qiniu_Region_Free(Qiniu_Region *region) free((void *)region); } +void Qiniu_Region_Set_Up_Accelerated_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) +{ + if (region == NULL) + { + return; + } + if (region->upService != NULL) + { + struct _Qiniu_Region_Service *oldUpService = region->upService; + region->upService = _Qiniu_Region_Service_New( + hosts, hostsCount, + oldUpService->hosts + oldUpService->acceleratedHostsCount, oldUpService->preferredHostsCount, + oldUpService->hosts + oldUpService->acceleratedHostsCount + oldUpService->preferredHostsCount, oldUpService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldUpService); + } + else + { + region->upService = _Qiniu_Region_Service_New(hosts, hostsCount, NULL, 0, NULL, 0, useHttps); + } +} + void Qiniu_Region_Set_Up_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) { if (region == NULL) { return; } - if (region->upService == NULL) + if (region->upService != NULL) { - region->upService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldUpService = region->upService; + region->upService = _Qiniu_Region_Service_New( + oldUpService->hosts, oldUpService->acceleratedHostsCount, + hosts, hostsCount, + oldUpService->hosts + oldUpService->acceleratedHostsCount + oldUpService->preferredHostsCount, oldUpService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldUpService); } - if (region->upService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->upService->preferredHosts); - region->upService->preferredHosts = NULL; + region->upService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->upService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -206,16 +233,20 @@ void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *con { return; } - if (region->upService == NULL) + if (region->upService != NULL) { - region->upService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldUpService = region->upService; + region->upService = _Qiniu_Region_Service_New( + oldUpService->hosts, oldUpService->acceleratedHostsCount, + oldUpService->hosts + oldUpService->acceleratedHostsCount, oldUpService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldUpService); } - if (region->upService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->upService->alternativeHosts); - region->upService->alternativeHosts = NULL; + region->upService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->upService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -224,16 +255,20 @@ void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const { return; } - if (region->ioService == NULL) + if (region->ioService != NULL) { - region->ioService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldIoService = region->ioService; + region->ioService = _Qiniu_Region_Service_New( + oldIoService->hosts, oldIoService->acceleratedHostsCount, + hosts, hostsCount, + oldIoService->hosts + oldIoService->acceleratedHostsCount + oldIoService->preferredHostsCount, oldIoService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldIoService); } - if (region->ioService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ioService->preferredHosts); - region->ioService->preferredHosts = NULL; + region->ioService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->ioService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -242,16 +277,20 @@ void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *con { return; } - if (region->ioService == NULL) + if (region->ioService != NULL) { - region->ioService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldIoService = region->ioService; + region->ioService = _Qiniu_Region_Service_New( + oldIoService->hosts, oldIoService->acceleratedHostsCount, + oldIoService->hosts + oldIoService->acceleratedHostsCount, oldIoService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldIoService); } - if (region->ioService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ioService->alternativeHosts); - region->ioService->alternativeHosts = NULL; + region->ioService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->ioService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -260,16 +299,20 @@ void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *c { return; } - if (region->ioSrcService == NULL) + if (region->ioSrcService != NULL) { - region->ioSrcService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldIoSrcService = region->ioSrcService; + region->ioSrcService = _Qiniu_Region_Service_New( + oldIoSrcService->hosts, oldIoSrcService->acceleratedHostsCount, + hosts, hostsCount, + oldIoSrcService->hosts + oldIoSrcService->acceleratedHostsCount + oldIoSrcService->preferredHostsCount, oldIoSrcService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldIoSrcService); } - if (region->ioSrcService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ioSrcService->preferredHosts); - region->ioSrcService->preferredHosts = NULL; + region->ioSrcService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->ioSrcService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Io_Src_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -278,16 +321,20 @@ void Qiniu_Region_Set_Io_Src_Alternative_Hosts(Qiniu_Region *region, const char { return; } - if (region->ioSrcService == NULL) + if (region->ioSrcService != NULL) { - region->ioSrcService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldIoSrcService = region->ioSrcService; + region->ioSrcService = _Qiniu_Region_Service_New( + oldIoSrcService->hosts, oldIoSrcService->acceleratedHostsCount, + oldIoSrcService->hosts + oldIoSrcService->acceleratedHostsCount, oldIoSrcService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldIoSrcService); } - if (region->ioSrcService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ioSrcService->alternativeHosts); - region->ioSrcService->alternativeHosts = NULL; + region->ioSrcService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->ioSrcService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Bucket_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -296,16 +343,20 @@ void Qiniu_Region_Set_Bucket_Preferred_Hosts(Qiniu_Region *region, const char *c { return; } - if (region->ucService == NULL) + if (region->ucService != NULL) { - region->ucService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldUcService = region->ucService; + region->ucService = _Qiniu_Region_Service_New( + oldUcService->hosts, oldUcService->acceleratedHostsCount, + hosts, hostsCount, + oldUcService->hosts + oldUcService->acceleratedHostsCount + oldUcService->preferredHostsCount, oldUcService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldUcService); } - if (region->ucService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ucService->preferredHosts); - region->ucService->preferredHosts = NULL; + region->ucService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->ucService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Bucket_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -314,16 +365,20 @@ void Qiniu_Region_Set_Bucket_Alternative_Hosts(Qiniu_Region *region, const char { return; } - if (region->ucService == NULL) + if (region->ucService != NULL) { - region->ucService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldUcService = region->ucService; + region->ucService = _Qiniu_Region_Service_New( + oldUcService->hosts, oldUcService->acceleratedHostsCount, + oldUcService->hosts + oldUcService->acceleratedHostsCount, oldUcService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldUcService); } - if (region->ucService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->ucService->alternativeHosts); - region->ucService->alternativeHosts = NULL; + region->ucService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->ucService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Rs_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -332,16 +387,20 @@ void Qiniu_Region_Set_Rs_Preferred_Hosts(Qiniu_Region *region, const char *const { return; } - if (region->rsService == NULL) + if (region->rsService != NULL) { - region->rsService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldRsService = region->rsService; + region->rsService = _Qiniu_Region_Service_New( + oldRsService->hosts, oldRsService->acceleratedHostsCount, + hosts, hostsCount, + oldRsService->hosts + oldRsService->acceleratedHostsCount + oldRsService->preferredHostsCount, oldRsService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldRsService); } - if (region->rsService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->rsService->preferredHosts); - region->rsService->preferredHosts = NULL; + region->rsService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->rsService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Rs_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -350,16 +409,20 @@ void Qiniu_Region_Set_Rs_Alternative_Hosts(Qiniu_Region *region, const char *con { return; } - if (region->rsService == NULL) + if (region->rsService != NULL) { - region->rsService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldRsService = region->rsService; + region->rsService = _Qiniu_Region_Service_New( + oldRsService->hosts, oldRsService->acceleratedHostsCount, + oldRsService->hosts + oldRsService->acceleratedHostsCount, oldRsService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldRsService); } - if (region->rsService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->rsService->alternativeHosts); - region->rsService->alternativeHosts = NULL; + region->rsService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->rsService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Rsf_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -368,16 +431,20 @@ void Qiniu_Region_Set_Rsf_Preferred_Hosts(Qiniu_Region *region, const char *cons { return; } - if (region->rsfService == NULL) + if (region->rsfService != NULL) { - region->rsfService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldRsfService = region->rsfService; + region->rsfService = _Qiniu_Region_Service_New( + oldRsfService->hosts, oldRsfService->acceleratedHostsCount, + hosts, hostsCount, + oldRsfService->hosts + oldRsfService->acceleratedHostsCount + oldRsfService->preferredHostsCount, oldRsfService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldRsfService); } - if (region->rsfService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->rsfService->preferredHosts); - region->rsfService->preferredHosts = NULL; + region->rsfService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->rsfService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Rsf_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -386,16 +453,20 @@ void Qiniu_Region_Set_Rsf_Alternative_Hosts(Qiniu_Region *region, const char *co { return; } - if (region->rsfService == NULL) + if (region->rsfService != NULL) { - region->rsfService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldRsfService = region->rsfService; + region->rsfService = _Qiniu_Region_Service_New( + oldRsfService->hosts, oldRsfService->acceleratedHostsCount, + oldRsfService->hosts + oldRsfService->acceleratedHostsCount, oldRsfService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldRsfService); } - if (region->rsfService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->rsfService->alternativeHosts); - region->rsfService->alternativeHosts = NULL; + region->rsfService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->rsfService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Api_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -404,16 +475,20 @@ void Qiniu_Region_Set_Api_Preferred_Hosts(Qiniu_Region *region, const char *cons { return; } - if (region->apiService == NULL) + if (region->apiService != NULL) { - region->apiService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldApiService = region->apiService; + region->apiService = _Qiniu_Region_Service_New( + oldApiService->hosts, oldApiService->acceleratedHostsCount, + hosts, hostsCount, + oldApiService->hosts + oldApiService->acceleratedHostsCount + oldApiService->preferredHostsCount, oldApiService->alternativeHostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldApiService); } - if (region->apiService->preferredHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->apiService->preferredHosts); - region->apiService->preferredHosts = NULL; + region->apiService = _Qiniu_Region_Service_New(NULL, 0, hosts, hostsCount, NULL, 0, useHttps); } - region->apiService->preferredHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } void Qiniu_Region_Set_Api_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps) @@ -422,19 +497,23 @@ void Qiniu_Region_Set_Api_Alternative_Hosts(Qiniu_Region *region, const char *co { return; } - if (region->apiService == NULL) + if (region->apiService != NULL) { - region->apiService = _Qiniu_Region_Service_New(NULL, NULL); + struct _Qiniu_Region_Service *oldApiService = region->apiService; + region->apiService = _Qiniu_Region_Service_New( + oldApiService->hosts, oldApiService->acceleratedHostsCount, + oldApiService->hosts + oldApiService->acceleratedHostsCount, oldApiService->preferredHostsCount, + hosts, hostsCount, + useHttps); + _Qiniu_Region_Service_Free(oldApiService); } - if (region->apiService->alternativeHosts != NULL) + else { - _Qiniu_Region_Hosts_Free(region->apiService->alternativeHosts); - region->apiService->alternativeHosts = NULL; + region->apiService = _Qiniu_Region_Service_New(NULL, 0, NULL, 0, hosts, hostsCount, useHttps); } - region->apiService->alternativeHosts = _Qiniu_Region_Hosts_New(hosts, hostsCount, useHttps); } -const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Up_Accelerated_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -448,18 +527,18 @@ const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, siz { return NULL; } - if (region->upService->preferredHosts == NULL) + if (region->upService->hosts == NULL || region->upService->acceleratedHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->upService->preferredHosts->hostsCount; + *count = region->upService->acceleratedHostsCount; } - return region->upService->preferredHosts->hosts; + return region->upService->hosts; } -const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -473,18 +552,18 @@ const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, s { return NULL; } - if (region->upService->alternativeHosts == NULL) + if (region->upService->hosts == NULL || region->upService->preferredHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->upService->alternativeHosts->hostsCount; + *count = region->upService->preferredHostsCount; } - return region->upService->alternativeHosts->hosts; + return region->upService->hosts + region->upService->acceleratedHostsCount; } -const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -494,22 +573,22 @@ const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, siz { return NULL; } - if (region->ioService == NULL) + if (region->upService == NULL) { return NULL; } - if (region->ioService->preferredHosts == NULL) + if (region->upService->hosts == NULL || region->upService->alternativeHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ioService->preferredHosts->hostsCount; + *count = region->upService->alternativeHostsCount; } - return region->ioService->preferredHosts->hosts; + return region->upService->hosts + region->upService->acceleratedHostsCount + region->upService->preferredHostsCount; } -const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *_Qiniu_Region_Get_All_Up_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) { if (count != NULL) { @@ -519,22 +598,34 @@ const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, s { return NULL; } - if (region->ioService == NULL) + if (region->upService == NULL) { return NULL; } - if (region->ioService->alternativeHosts == NULL) + size_t allHostsCount = region->upService->preferredHostsCount + region->upService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->upService->acceleratedHostsCount; + } + if (region->upService->hosts == NULL || allHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ioService->alternativeHosts->hostsCount; + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->upService->hosts; + } + else + { + return region->upService->hosts + region->upService->acceleratedHostsCount; } - return region->ioService->alternativeHosts->hosts; } -const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -544,22 +635,22 @@ const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, { return NULL; } - if (region->ioSrcService == NULL) + if (region->ioService == NULL) { return NULL; } - if (region->ioSrcService->preferredHosts == NULL) + if (region->ioService->hosts == NULL || region->ioService->preferredHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ioSrcService->preferredHosts->hostsCount; + *count = region->ioService->preferredHostsCount; } - return region->ioSrcService->preferredHosts->hosts; + return region->ioService->hosts + region->ioService->acceleratedHostsCount; } -const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -569,22 +660,22 @@ const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *regio { return NULL; } - if (region->ioSrcService == NULL) + if (region->ioService == NULL) { return NULL; } - if (region->ioSrcService->alternativeHosts == NULL) + if (region->ioService->hosts == NULL || region->ioService->alternativeHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ioSrcService->alternativeHosts->hostsCount; + *count = region->ioService->alternativeHostsCount; } - return region->ioSrcService->alternativeHosts->hosts; + return region->ioService->hosts + region->ioService->acceleratedHostsCount + region->ioService->preferredHostsCount; } -const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *_Qiniu_Region_Get_All_Io_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) { if (count != NULL) { @@ -594,22 +685,34 @@ const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region, { return NULL; } - if (region->ucService == NULL) + if (region->ioService == NULL) { return NULL; } - if (region->ucService->preferredHosts == NULL) + size_t allHostsCount = region->ioService->preferredHostsCount + region->ioService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->ioService->acceleratedHostsCount; + } + if (region->ioService->hosts == NULL || allHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ucService->preferredHosts->hostsCount; + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->ioService->hosts; + } + else + { + return region->ioService->hosts + region->ioService->acceleratedHostsCount; } - return region->ucService->preferredHosts->hosts; } -const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -619,22 +722,22 @@ const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *regio { return NULL; } - if (region->ucService == NULL) + if (region->ioSrcService == NULL) { return NULL; } - if (region->ucService->alternativeHosts == NULL) + if (region->ioSrcService->hosts == NULL || region->ioSrcService->preferredHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->ucService->alternativeHosts->hostsCount; + *count = region->ioSrcService->preferredHostsCount; } - return region->ucService->alternativeHosts->hosts; + return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount; } -const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Io_Src_Alternative_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -644,22 +747,22 @@ const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, siz { return NULL; } - if (region->rsService == NULL) + if (region->ioSrcService == NULL) { return NULL; } - if (region->rsService->preferredHosts == NULL) + if (region->ioSrcService->hosts == NULL || region->ioSrcService->alternativeHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->rsService->preferredHosts->hostsCount; + *count = region->ioSrcService->alternativeHostsCount; } - return region->rsService->preferredHosts->hosts; + return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount + region->ioSrcService->preferredHostsCount; } -const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *_Qiniu_Region_Get_All_Io_Src_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) { if (count != NULL) { @@ -669,22 +772,34 @@ const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, s { return NULL; } - if (region->rsService == NULL) + if (region->ioSrcService == NULL) { return NULL; } - if (region->rsService->alternativeHosts == NULL) + size_t allHostsCount = region->ioSrcService->preferredHostsCount + region->ioSrcService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->ioSrcService->acceleratedHostsCount; + } + if (region->ioSrcService->hosts == NULL || allHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->rsService->alternativeHosts->hostsCount; + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->ioSrcService->hosts; + } + else + { + return region->ioSrcService->hosts + region->ioSrcService->acceleratedHostsCount; } - return region->rsService->alternativeHosts->hosts; } -const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Bucket_Preferred_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -694,22 +809,22 @@ const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, si { return NULL; } - if (region->rsfService == NULL) + if (region->ucService == NULL) { return NULL; } - if (region->rsfService->preferredHosts == NULL) + if (region->ucService->hosts == NULL || region->ucService->preferredHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->rsfService->preferredHosts->hostsCount; + *count = region->ucService->preferredHostsCount; } - return region->rsfService->preferredHosts->hosts; + return region->ucService->hosts + region->ucService->acceleratedHostsCount; } -const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Bucket_Alternative_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -719,22 +834,22 @@ const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region, { return NULL; } - if (region->rsfService == NULL) + if (region->ucService == NULL) { return NULL; } - if (region->rsfService->alternativeHosts == NULL) + if (region->ucService->hosts == NULL || region->ucService->alternativeHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->rsfService->alternativeHosts->hostsCount; + *count = region->ucService->alternativeHostsCount; } - return region->rsfService->alternativeHosts->hosts; + return region->ucService->hosts + region->ucService->acceleratedHostsCount + region->ucService->preferredHostsCount; } -const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, size_t *count) +const char *const *_Qiniu_Region_Get_All_Bucket_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) { if (count != NULL) { @@ -744,22 +859,34 @@ const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, si { return NULL; } - if (region->apiService == NULL) + if (region->ucService == NULL) { return NULL; } - if (region->apiService->preferredHosts == NULL) + size_t allHostsCount = region->ucService->preferredHostsCount + region->ucService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->ucService->acceleratedHostsCount; + } + if (region->ucService->hosts == NULL || allHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->apiService->preferredHosts->hostsCount; + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->ucService->hosts; + } + else + { + return region->ucService->hosts + region->ucService->acceleratedHostsCount; } - return region->apiService->preferredHosts->hosts; } -const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region, size_t *count) +const char *const *Qiniu_Region_Get_Rs_Preferred_Hosts(Qiniu_Region *region, size_t *count) { if (count != NULL) { @@ -769,153 +896,485 @@ const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region, { return NULL; } - if (region->apiService == NULL) + if (region->rsService == NULL) { return NULL; } - if (region->apiService->alternativeHosts == NULL) + if (region->rsService->hosts == NULL || region->rsService->preferredHostsCount == 0) { return NULL; } if (count != NULL) { - *count = region->apiService->alternativeHosts->hostsCount; + *count = region->rsService->preferredHostsCount; } - return region->apiService->alternativeHosts->hosts; + return region->rsService->hosts + region->rsService->acceleratedHostsCount; } -Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps) +const char *const *Qiniu_Region_Get_Rs_Alternative_Hosts(Qiniu_Region *region, size_t *count) { - Qiniu_Region *region = Qiniu_Region_New(regionId, 0); + if (count != NULL) + { + *count = 0; + } if (region == NULL) { return NULL; } - - char buf[32], buf2[32]; - Qiniu_Zero(buf); - Qiniu_Zero(buf2); - - if (strcmp(regionId, "z0") == 0) + if (region->rsService == NULL) { - Qiniu_Region_Set_Up_Preferred_Hosts(region, (const char *const[]){"upload.qiniup.com", "up.qiniup.com"}, 2, useHttps); - Qiniu_Region_Set_Up_Alternative_Hosts(region, (const char *const[]){"up.qbox.me"}, 1, useHttps); - Qiniu_Region_Set_Io_Preferred_Hosts(region, (const char *const[]){"iovip.qiniuio.com"}, 1, useHttps); - Qiniu_Region_Set_Io_Alternative_Hosts(region, (const char *const[]){"iovip.qbox.me"}, 1, useHttps); + return NULL; } - else + if (region->rsService->hosts == NULL || region->rsService->alternativeHostsCount == 0) { - Qiniu_snprintf(buf, sizeof(buf), "upload-%s.qiniup.com", regionId); - Qiniu_snprintf(buf2, sizeof(buf2), "up-%s.qiniup.com", regionId); - Qiniu_Region_Set_Up_Preferred_Hosts(region, (const char *const[]){buf, buf2}, 2, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "up-%s.qbox.me", regionId); - Qiniu_Region_Set_Up_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qiniuio.com", regionId); - Qiniu_Region_Set_Io_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qbox.me", regionId); - Qiniu_Region_Set_Io_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps); + return NULL; } - Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qiniuapi.com", regionId); - Qiniu_Region_Set_Rs_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qbox.me", regionId); - Qiniu_Region_Set_Rs_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qiniuapi.com", regionId); - Qiniu_Region_Set_Rsf_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qbox.me", regionId); - Qiniu_Region_Set_Rsf_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "api-%s.qiniuapi.com", regionId); - Qiniu_Region_Set_Api_Preferred_Hosts(region, (const char *const[]){buf}, 1, useHttps); - Qiniu_snprintf(buf, sizeof(buf), "api-%s.qbox.me", regionId); - Qiniu_Region_Set_Api_Alternative_Hosts(region, (const char *const[]){buf}, 1, useHttps); - return region; -} - -static Qiniu_Bool isRegionQueryRetryable(Qiniu_Error err) -{ - switch (err.code) + if (count != NULL) { - case 501: - case 509: - case 573: - case 579: - case 608: - case 612: - case 614: - case 616: - case 618: - case 630: - case 631: - case 632: - case 640: - case 701: - return Qiniu_False; - default: - return err.code >= 500 || err.code < 100; + *count = region->rsService->alternativeHostsCount; } + return region->rsService->hosts + region->rsService->acceleratedHostsCount + region->rsService->preferredHostsCount; } -static Qiniu_Error _Qiniu_Region_Query_call(Qiniu_Client *self, const char *accessKey, const char *const bucketName, cJSON **ret) +const char *const *_Qiniu_Region_Get_All_Rs_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) { - char *url; - Qiniu_Error err; - Qiniu_Zero(err); - const char *hosts[3] = {NULL, NULL, NULL}; - int hostsLen = 0; - - if (QINIU_UC_HOST != NULL) + if (count != NULL) { - hosts[hostsLen++] = QINIU_UC_HOST; + *count = 0; } - if (QINIU_UC_HOST_BACKUP != NULL) + if (region == NULL) { - hosts[hostsLen++] = QINIU_UC_HOST_BACKUP; + return NULL; } - if (QINIU_API_HOST != NULL) + if (region->rsService == NULL) { - hosts[hostsLen++] = QINIU_API_HOST; + return NULL; } - - for (int i = 0; i < hostsLen; i++) + size_t allHostsCount = region->rsService->preferredHostsCount + region->rsService->alternativeHostsCount; + if (accelerationEnabled) { - url = Qiniu_String_Concat(hosts[i], "/v4/query?ak=", accessKey, "&bucket=", bucketName, NULL); - err = Qiniu_Client_Call(self, ret, url); - Qiniu_Free(url); - if (err.code == 200) - { - cJSON *hostsJson = Qiniu_Json_GetObjectItem(*ret, "hosts", NULL); - if (hostsJson == NULL) - { - continue; - } - cJSON *firstHostJson = Qiniu_Json_GetArrayItem(hostsJson, 0, NULL); - if (firstHostJson == NULL) - { - continue; - } - return err; - } - else if (!isRegionQueryRetryable(err)) - { - return err; - } + allHostsCount += region->rsService->acceleratedHostsCount; } - if (err.code == 0) + if (region->rsService->hosts == NULL || allHostsCount == 0) { - err.code = 599; - err.message = "no uc hosts available"; + return NULL; + } + if (count != NULL) + { + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->rsService->hosts; + } + else + { + return region->rsService->hosts + region->rsService->acceleratedHostsCount; } - return err; } -Qiniu_Error -_Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps) +const char *const *Qiniu_Region_Get_Rsf_Preferred_Hosts(Qiniu_Region *region, size_t *count) { - Qiniu_Error err; - cJSON *root; - - if (accessKey == NULL) + if (count != NULL) { - accessKey = self->auth.itbl->GetAccessKey(self->auth.self); + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->rsfService == NULL) + { + return NULL; + } + if (region->rsfService->hosts == NULL || region->rsfService->preferredHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = region->rsfService->preferredHostsCount; + } + return region->rsfService->hosts + region->rsfService->acceleratedHostsCount; +} + +const char *const *Qiniu_Region_Get_Rsf_Alternative_Hosts(Qiniu_Region *region, size_t *count) +{ + if (count != NULL) + { + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->rsfService == NULL) + { + return NULL; + } + if (region->rsfService->hosts == NULL || region->rsfService->alternativeHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = region->rsfService->alternativeHostsCount; + } + return region->rsfService->hosts + region->rsfService->acceleratedHostsCount + region->rsfService->preferredHostsCount; +} + +const char *const *_Qiniu_Region_Get_All_Rsf_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) +{ + if (count != NULL) + { + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->rsfService == NULL) + { + return NULL; + } + size_t allHostsCount = region->rsfService->preferredHostsCount + region->rsfService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->rsfService->acceleratedHostsCount; + } + if (region->rsfService->hosts == NULL || allHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->rsfService->hosts; + } + else + { + return region->rsfService->hosts + region->rsfService->acceleratedHostsCount; + } +} + +const char *const *Qiniu_Region_Get_Api_Preferred_Hosts(Qiniu_Region *region, size_t *count) +{ + if (count != NULL) + { + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->apiService == NULL) + { + return NULL; + } + if (region->apiService->hosts == NULL || region->apiService->preferredHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = region->apiService->preferredHostsCount; + } + return region->apiService->hosts + region->apiService->acceleratedHostsCount; +} + +const char *const *Qiniu_Region_Get_Api_Alternative_Hosts(Qiniu_Region *region, size_t *count) +{ + if (count != NULL) + { + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->apiService == NULL) + { + return NULL; + } + if (region->apiService->hosts == NULL || region->apiService->alternativeHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = region->apiService->alternativeHostsCount; + } + return region->apiService->hosts + region->apiService->acceleratedHostsCount + region->apiService->preferredHostsCount; +} + +const char *const *_Qiniu_Region_Get_All_Api_Hosts(Qiniu_Region *region, size_t *count, Qiniu_Bool accelerationEnabled) +{ + if (count != NULL) + { + *count = 0; + } + if (region == NULL) + { + return NULL; + } + if (region->apiService == NULL) + { + return NULL; + } + size_t allHostsCount = region->apiService->preferredHostsCount + region->apiService->alternativeHostsCount; + if (accelerationEnabled) + { + allHostsCount += region->apiService->acceleratedHostsCount; + } + if (region->apiService->hosts == NULL || allHostsCount == 0) + { + return NULL; + } + if (count != NULL) + { + *count = allHostsCount; + } + if (accelerationEnabled) + { + return region->apiService->hosts; + } + else + { + return region->apiService->hosts + region->apiService->acceleratedHostsCount; + } +} + +Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps) +{ + Qiniu_Region *region = Qiniu_Region_New(regionId, 0); + if (region == NULL) + { + return NULL; + } + + char buf[32], buf2[32]; + Qiniu_Zero(buf); + Qiniu_Zero(buf2); + + if (strcmp(regionId, "z0") == 0) + { + region->upService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){"upload.qiniup.com", "up.qiniup.com"}, 2, (const char *const[]){"up.qbox.me"}, 1, useHttps); + region->ioService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){"iovip.qiniuio.com"}, 1, (const char *const[]){"iovip.qbox.me"}, 1, useHttps); + } + else + { + Qiniu_snprintf(buf, sizeof(buf), "upload-%s.qiniup.com", regionId); + Qiniu_snprintf(buf2, sizeof(buf2), "up-%s.qiniup.com", regionId); + region->upService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf, buf2}, 2, NULL, 0, useHttps); + Qiniu_snprintf(buf, sizeof(buf), "iovip-%s.qiniuio.com", regionId); + region->ioService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps); + } + Qiniu_snprintf(buf, sizeof(buf), "rs-%s.qiniuapi.com", regionId); + region->rsService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps); + Qiniu_snprintf(buf, sizeof(buf), "rsf-%s.qiniuapi.com", regionId); + region->rsfService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps); + Qiniu_snprintf(buf, sizeof(buf), "api-%s.qiniuapi.com", regionId); + region->apiService = _Qiniu_Region_Service_New(NULL, 0, (const char *const[]){buf}, 1, NULL, 0, useHttps); + return region; +} + +static Qiniu_Bool isRegionQueryRetryable(Qiniu_Error err) +{ + switch (err.code) + { + case 501: + case 509: + case 573: + case 579: + case 608: + case 612: + case 614: + case 616: + case 618: + case 630: + case 631: + case 632: + case 640: + case 701: + return Qiniu_False; + default: + return err.code >= 500 || err.code < 100; + } +} + +QINIU_DLLAPI extern const char *QINIU_UC_HOST_BACKUP_2; + +static const char **_Qiniu_Get_Bucket_Hosts(Qiniu_Client *self, size_t *count, Qiniu_Bool *needToFree) +{ + const char **hosts = NULL; + *count = 0; + *needToFree = Qiniu_False; + + if (self->specifiedRegion == NULL) + { + hosts = (const char **)malloc(sizeof(const char *) * 3); + *needToFree = Qiniu_True; + if (QINIU_UC_HOST != NULL) + { + hosts[(*count)++] = QINIU_UC_HOST; + } + if (QINIU_UC_HOST_BACKUP != NULL) + { + hosts[(*count)++] = QINIU_UC_HOST_BACKUP; + } + if (QINIU_UC_HOST_BACKUP_2 != NULL) + { + hosts[(*count)++] = QINIU_UC_HOST_BACKUP_2; + } + } + else + { + hosts = (const char **)_Qiniu_Region_Get_All_Bucket_Hosts(self->specifiedRegion, count, Qiniu_False); + } + return hosts; +} + +static Qiniu_Error _Qiniu_Region_Query_call(Qiniu_Client *self, const char *accessKey, const char *const bucketName, cJSON **ret) +{ + char *url; + Qiniu_Error err; + Qiniu_Zero(err); + const char *const *hosts; + size_t hostsLen; + Qiniu_Bool needToFreeHosts; + + hosts = _Qiniu_Get_Bucket_Hosts(self, &hostsLen, &needToFreeHosts); + + for (size_t i = 0; i < hostsLen; i++) + { + url = Qiniu_String_Concat(hosts[i], "/v4/query?ak=", accessKey, "&bucket=", bucketName, NULL); + err = Qiniu_Client_Call(self, ret, url); + Qiniu_Free(url); + if (err.code == 200) + { + cJSON *hostsJson = Qiniu_Json_GetObjectItem(*ret, "hosts", NULL); + if (hostsJson == NULL) + { + continue; + } + cJSON *firstHostJson = Qiniu_Json_GetArrayItem(hostsJson, 0, NULL); + if (firstHostJson == NULL) + { + continue; + } + goto handleErr; + } + else if (!isRegionQueryRetryable(err)) + { + goto handleErr; + } + } + if (err.code == 0) + { + err.code = 599; + err.message = "no uc hosts available"; + } +handleErr: + if (needToFreeHosts) + { + Qiniu_Free((void *)hosts); + } + return err; +} + +static struct _Qiniu_Region_Service *_Qiniu_Region_New_From_JSON(cJSON *json, Qiniu_Bool useHttps) +{ + cJSON *accDomainsJson = Qiniu_Json_GetObjectItem(json, "acc_domains", NULL); + cJSON *domainsJson = Qiniu_Json_GetObjectItem(json, "domains", NULL); + cJSON *oldJson = Qiniu_Json_GetObjectItem(json, "old", NULL); + size_t accDomainsCount = 0, domainsCount = 0, oldCount = 0; + if (accDomainsJson != NULL) + { + accDomainsCount = cJSON_GetArraySize(accDomainsJson); + } + if (domainsJson != NULL) + { + domainsCount = cJSON_GetArraySize(domainsJson); + } + if (oldJson != NULL) + { + oldCount = cJSON_GetArraySize(oldJson); + } + + struct _Qiniu_Region_Service *newService = malloc(sizeof(struct _Qiniu_Region_Service)); + if (newService == NULL) + { + goto handleErr; + } + const char **newHosts = malloc(sizeof(const char *) * (accDomainsCount + domainsCount + oldCount)); + if (newHosts == NULL) + { + goto handleErr; + } + size_t offset = 0, i; + const char *h; + for (i = 0; i < accDomainsCount; i++) + { + h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(accDomainsJson, i, NULL), useHttps); + if (h == NULL) + { + goto handleErr; + } + newHosts[offset + i] = h; + } + offset += accDomainsCount; + for (i = 0; i < domainsCount; i++) + { + h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(domainsJson, i, NULL), useHttps); + if (h == NULL) + { + goto handleErr; + } + newHosts[offset + i] = h; + } + offset += domainsCount; + for (i = 0; i < oldCount; i++) + { + h = _Qiniu_Duplicate_Host(Qiniu_Json_GetStringAt(oldJson, i, NULL), useHttps); + if (h == NULL) + { + goto handleErr; + } + newHosts[offset + i] = h; + } + newService->hosts = newHosts; + newService->acceleratedHostsCount = accDomainsCount; + newService->preferredHostsCount = domainsCount; + newService->alternativeHostsCount = oldCount; + return newService; +handleErr: + if (newHosts != NULL) + { + for (size_t j = 0; j < offset + i; j++) + { + free((void *)newHosts[j]); + } + free((void *)newHosts); + } + if (newService != NULL) + { + free((void *)newService); + } + return NULL; +} + +Qiniu_Error +_Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *accessKey, const char *const bucketName, Qiniu_Bool useHttps) +{ + Qiniu_Error err; + cJSON *root; + + if (accessKey == NULL) + { + accessKey = self->auth.itbl->GetAccessKey(self->auth.self); } err = _Qiniu_Region_Query_call(self, accessKey, bucketName, &root); if (err.code == 200) @@ -943,253 +1402,43 @@ _Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const char *acce cJSON *upJson = Qiniu_Json_GetObjectItem(firstHostJson, "up", NULL); if (upJson != NULL) { - struct _Qiniu_Region_Hosts *upPreferredRegionHosts = NULL, *upAlternativeRegionHosts = NULL; - cJSON *upDomainsJson = Qiniu_Json_GetObjectItem(upJson, "domains", NULL); - if (upDomainsJson != NULL) - { - int upPreferredHostsCount = cJSON_GetArraySize(upDomainsJson); - if (upPreferredHostsCount > 0) - { - const char **upPreferredHosts = (const char **)malloc(sizeof(const char *) * upPreferredHostsCount); - for (int i = 0; i < upPreferredHostsCount; i++) - { - upPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(upDomainsJson, i, NULL), useHttps); - } - upPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(upPreferredHosts, upPreferredHostsCount); - } - } - cJSON *upOldJson = Qiniu_Json_GetObjectItem(upJson, "old", NULL); - if (upOldJson != NULL) - { - int upAlternativeHostsCount = cJSON_GetArraySize(upOldJson); - if (upAlternativeHostsCount > 0) - { - const char **upAlternativeHosts = (const char **)malloc(sizeof(const char *) * upAlternativeHostsCount); - for (int i = 0; i < upAlternativeHostsCount; i++) - { - upAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(upOldJson, i, NULL), useHttps); - } - upAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(upAlternativeHosts, upAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *upService = _Qiniu_Region_Service_New(upPreferredRegionHosts, upAlternativeRegionHosts); - region->upService = upService; + region->upService = _Qiniu_Region_New_From_JSON(upJson, useHttps); } cJSON *ioJson = Qiniu_Json_GetObjectItem(firstHostJson, "io", NULL); if (ioJson != NULL) { - struct _Qiniu_Region_Hosts *ioPreferredRegionHosts = NULL, *ioAlternativeRegionHosts = NULL; - cJSON *ioDomainsJson = Qiniu_Json_GetObjectItem(ioJson, "domains", NULL); - if (ioDomainsJson != NULL) - { - int ioPreferredHostsCount = cJSON_GetArraySize(ioDomainsJson); - if (ioPreferredHostsCount > 0) - { - const char **ioPreferredHosts = (const char **)malloc(sizeof(const char *) * ioPreferredHostsCount); - for (int i = 0; i < ioPreferredHostsCount; i++) - { - ioPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioDomainsJson, i, NULL), useHttps); - } - ioPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioPreferredHosts, ioPreferredHostsCount); - } - } - cJSON *ioOldJson = Qiniu_Json_GetObjectItem(ioJson, "old", NULL); - if (ioOldJson != NULL) - { - int ioAlternativeHostsCount = cJSON_GetArraySize(ioOldJson); - if (ioAlternativeHostsCount > 0) - { - const char **ioAlternativeHosts = (const char **)malloc(sizeof(const char *) * ioAlternativeHostsCount); - for (int i = 0; i < ioAlternativeHostsCount; i++) - { - ioAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioOldJson, i, NULL), useHttps); - } - ioAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioAlternativeHosts, ioAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *ioService = _Qiniu_Region_Service_New(ioPreferredRegionHosts, ioAlternativeRegionHosts); - region->ioService = ioService; + region->ioService = _Qiniu_Region_New_From_JSON(ioJson, useHttps); } cJSON *ioSrcJson = Qiniu_Json_GetObjectItem(firstHostJson, "io_src", NULL); if (ioSrcJson != NULL) { - struct _Qiniu_Region_Hosts *ioSrcPreferredRegionHosts = NULL, *ioSrcAlternativeRegionHosts = NULL; - cJSON *ioSrcDomainsJson = Qiniu_Json_GetObjectItem(ioSrcJson, "domains", NULL); - if (ioSrcDomainsJson != NULL) - { - int ioSrcPreferredHostsCount = cJSON_GetArraySize(ioSrcDomainsJson); - if (ioSrcPreferredHostsCount > 0) - { - const char **ioSrcPreferredHosts = (const char **)malloc(sizeof(const char *) * ioSrcPreferredHostsCount); - for (int i = 0; i < ioSrcPreferredHostsCount; i++) - { - ioSrcPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioSrcDomainsJson, i, NULL), useHttps); - } - ioSrcPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioSrcPreferredHosts, ioSrcPreferredHostsCount); - } - } - cJSON *ioSrcOldJson = Qiniu_Json_GetObjectItem(ioSrcJson, "old", NULL); - if (ioSrcOldJson != NULL) - { - int ioSrcAlternativeHostsCount = cJSON_GetArraySize(ioSrcOldJson); - if (ioSrcAlternativeHostsCount > 0) - { - const char **ioSrcAlternativeHosts = (const char **)malloc(sizeof(const char *) * ioSrcAlternativeHostsCount); - for (int i = 0; i < ioSrcAlternativeHostsCount; i++) - { - ioSrcAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ioSrcOldJson, i, NULL), useHttps); - } - ioSrcAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ioSrcAlternativeHosts, ioSrcAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *ioSrcService = _Qiniu_Region_Service_New(ioSrcPreferredRegionHosts, ioSrcAlternativeRegionHosts); - region->ioSrcService = ioSrcService; + region->ioSrcService = _Qiniu_Region_New_From_JSON(ioSrcJson, useHttps); } cJSON *ucJson = Qiniu_Json_GetObjectItem(firstHostJson, "uc", NULL); if (ucJson != NULL) { - struct _Qiniu_Region_Hosts *ucPreferredRegionHosts = NULL, *ucAlternativeRegionHosts = NULL; - cJSON *ucDomainsJson = Qiniu_Json_GetObjectItem(ucJson, "domains", NULL); - if (ucDomainsJson != NULL) - { - int ucPreferredHostsCount = cJSON_GetArraySize(ucDomainsJson); - if (ucPreferredHostsCount > 0) - { - const char **ucPreferredHosts = (const char **)malloc(sizeof(const char *) * ucPreferredHostsCount); - for (int i = 0; i < ucPreferredHostsCount; i++) - { - ucPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ucDomainsJson, i, NULL), useHttps); - } - ucPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ucPreferredHosts, ucPreferredHostsCount); - } - } - cJSON *ucOldJson = Qiniu_Json_GetObjectItem(ucJson, "old", NULL); - if (ucOldJson != NULL) - { - int ucAlternativeHostsCount = cJSON_GetArraySize(ucOldJson); - if (ucAlternativeHostsCount > 0) - { - const char **ucAlternativeHosts = (const char **)malloc(sizeof(const char *) * ucAlternativeHostsCount); - for (int i = 0; i < ucAlternativeHostsCount; i++) - { - ucAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(ucOldJson, i, NULL), useHttps); - } - ucAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(ucAlternativeHosts, ucAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *ucService = _Qiniu_Region_Service_New(ucPreferredRegionHosts, ucAlternativeRegionHosts); - region->ucService = ucService; + region->ucService = _Qiniu_Region_New_From_JSON(ucJson, useHttps); } cJSON *rsJson = Qiniu_Json_GetObjectItem(firstHostJson, "rs", NULL); if (rsJson != NULL) { - struct _Qiniu_Region_Hosts *rsPreferredRegionHosts = NULL, *rsAlternativeRegionHosts = NULL; - cJSON *rsDomainsJson = Qiniu_Json_GetObjectItem(rsJson, "domains", NULL); - if (rsDomainsJson != NULL) - { - int rsPreferredHostsCount = cJSON_GetArraySize(rsDomainsJson); - if (rsPreferredHostsCount > 0) - { - const char **rsPreferredHosts = (const char **)malloc(sizeof(const char *) * rsPreferredHostsCount); - for (int i = 0; i < rsPreferredHostsCount; i++) - { - rsPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsDomainsJson, i, NULL), useHttps); - } - rsPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsPreferredHosts, rsPreferredHostsCount); - } - } - cJSON *rsOldJson = Qiniu_Json_GetObjectItem(rsJson, "old", NULL); - if (rsOldJson != NULL) - { - int rsAlternativeHostsCount = cJSON_GetArraySize(rsOldJson); - if (rsAlternativeHostsCount > 0) - { - const char **rsAlternativeHosts = (const char **)malloc(sizeof(const char *) * rsAlternativeHostsCount); - for (int i = 0; i < rsAlternativeHostsCount; i++) - { - rsAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsOldJson, i, NULL), useHttps); - } - rsAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsAlternativeHosts, rsAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *rsService = _Qiniu_Region_Service_New(rsPreferredRegionHosts, rsAlternativeRegionHosts); - region->rsService = rsService; + region->rsService = _Qiniu_Region_New_From_JSON(rsJson, useHttps); } cJSON *rsfJson = Qiniu_Json_GetObjectItem(firstHostJson, "rsf", NULL); if (rsfJson != NULL) { - struct _Qiniu_Region_Hosts *rsfPreferredRegionHosts = NULL, *rsfAlternativeRegionHosts = NULL; - cJSON *rsfDomainsJson = Qiniu_Json_GetObjectItem(rsfJson, "domains", NULL); - if (rsfDomainsJson != NULL) - { - int rsfPreferredHostsCount = cJSON_GetArraySize(rsfDomainsJson); - if (rsfPreferredHostsCount > 0) - { - const char **rsfPreferredHosts = (const char **)malloc(sizeof(const char *) * rsfPreferredHostsCount); - for (int i = 0; i < rsfPreferredHostsCount; i++) - { - rsfPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsfDomainsJson, i, NULL), useHttps); - } - rsfPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsfPreferredHosts, rsfPreferredHostsCount); - } - } - cJSON *rsfOldJson = Qiniu_Json_GetObjectItem(rsfJson, "old", NULL); - if (rsfOldJson != NULL) - { - int rsfAlternativeHostsCount = cJSON_GetArraySize(rsfOldJson); - if (rsfAlternativeHostsCount > 0) - { - const char **rsfAlternativeHosts = (const char **)malloc(sizeof(const char *) * rsfAlternativeHostsCount); - for (int i = 0; i < rsfAlternativeHostsCount; i++) - { - rsfAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(rsfOldJson, i, NULL), useHttps); - } - rsfAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(rsfAlternativeHosts, rsfAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *rsfService = _Qiniu_Region_Service_New(rsfPreferredRegionHosts, rsfAlternativeRegionHosts); - region->rsfService = rsfService; + region->rsfService = _Qiniu_Region_New_From_JSON(rsfJson, useHttps); } cJSON *apiJson = Qiniu_Json_GetObjectItem(firstHostJson, "api", NULL); if (apiJson != NULL) { - struct _Qiniu_Region_Hosts *apiPreferredRegionHosts = NULL, *apiAlternativeRegionHosts = NULL; - cJSON *apiDomainsJson = Qiniu_Json_GetObjectItem(apiJson, "domains", NULL); - if (apiDomainsJson != NULL) - { - int apiPreferredHostsCount = cJSON_GetArraySize(apiDomainsJson); - if (apiPreferredHostsCount > 0) - { - const char **apiPreferredHosts = (const char **)malloc(sizeof(const char *) * apiPreferredHostsCount); - for (int i = 0; i < apiPreferredHostsCount; i++) - { - apiPreferredHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(apiDomainsJson, i, NULL), useHttps); - } - apiPreferredRegionHosts = _Qiniu_Region_Hosts_New_without_dup(apiPreferredHosts, apiPreferredHostsCount); - } - } - cJSON *apiOldJson = Qiniu_Json_GetObjectItem(apiJson, "old", NULL); - if (apiOldJson != NULL) - { - int apiAlternativeHostsCount = cJSON_GetArraySize(apiOldJson); - if (apiAlternativeHostsCount > 0) - { - const char **apiAlternativeHosts = (const char **)malloc(sizeof(const char *) * apiAlternativeHostsCount); - for (int i = 0; i < apiAlternativeHostsCount; i++) - { - apiAlternativeHosts[i] = duplicate_host(Qiniu_Json_GetStringAt(apiOldJson, i, NULL), useHttps); - } - apiAlternativeRegionHosts = _Qiniu_Region_Hosts_New_without_dup(apiAlternativeHosts, apiAlternativeHostsCount); - } - } - struct _Qiniu_Region_Service *apiService = _Qiniu_Region_Service_New(apiPreferredRegionHosts, apiAlternativeRegionHosts); - region->apiService = apiService; + region->apiService = _Qiniu_Region_New_From_JSON(apiJson, useHttps); } *pRegion = region; @@ -1204,99 +1453,237 @@ Qiniu_Error Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **pRegion, const return _Qiniu_Region_Query(self, pRegion, NULL, bucketName, useHttps); } -Qiniu_Error _Qiniu_Region_Get_Up_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +struct _Qiniu_Region_Cache { - const char *const *hosts; - size_t count; - Qiniu_Error err = Qiniu_OK; - Qiniu_Region **foundRegion = &self->cachedRegion; + Qiniu_Region *region; + const char *accessKey, *bucketName, *const *bucketHosts; + size_t bucketHostsLen; + Qiniu_Bool needToFreeBucketHosts, uploadingAccelerationEnabled; +}; - if (self->specifiedRegion) +static int _Qiniu_Compare_Str(const char *a, const char *b) +{ + if (a == NULL && b == NULL) { - foundRegion = &self->specifiedRegion; - goto foundCache; + return 0; + } + else if (a == NULL) + { + return -1; + } + else if (b == NULL) + { + return 1; + } + else + { + return strcmp(a, b); + } +} + +static int _Qiniu_Region_Cache_Compare(const void *a, const void *b, void *user_data) +{ + int result; + struct _Qiniu_Region_Cache *cacheA = (struct _Qiniu_Region_Cache *)a; + struct _Qiniu_Region_Cache *cacheB = (struct _Qiniu_Region_Cache *)b; + result = _Qiniu_Compare_Str(cacheA->accessKey, cacheB->accessKey); + if (!result) + { + return result; } - else if (self->autoQueryRegion) + result = _Qiniu_Compare_Str(cacheA->bucketName, cacheB->bucketName); + if (!result) { - if (self->cachedRegion != NULL) + return result; + } + if (cacheA->bucketHostsLen == cacheB->bucketHostsLen) + { + for (size_t i = 0; i < cacheA->bucketHostsLen; i++) { - if (strcmp(self->cachedRegionBucketName, bucketName) == 0) - { - goto foundCache; - } - else + result = _Qiniu_Compare_Str(cacheA->bucketHosts[i], cacheB->bucketHosts[i]); + if (!result) { - Qiniu_Region_Free(self->cachedRegion); - self->cachedRegion = NULL; - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); + return result; } } - err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion); - if (err.code != 200) - { - goto useDefault; - } - self->cachedRegionBucketName = Qiniu_String_Dup(bucketName); - goto foundCache; + return result; } -useDefault: - *host = QINIU_UP_HOST; - return err; + else + { + return cacheA->bucketHostsLen - cacheB->bucketHostsLen; + } +} -foundCache: - hosts = Qiniu_Region_Get_Up_Preferred_Hosts(*foundRegion, &count); - if (count == 0) +static void _Qiniu_Region_Cache_Free(void *r) +{ + struct _Qiniu_Region_Cache *cache = (struct _Qiniu_Region_Cache *)r; + Qiniu_Free((void *)cache->accessKey); + Qiniu_Free((void *)cache->bucketName); + if (cache->needToFreeBucketHosts) { - *host = QINIU_UP_HOST; + Qiniu_Free((void *)cache->bucketHosts); } - else + Qiniu_Region_Free(cache->region); + Qiniu_Zero_Ptr(cache); +} + +static uint64_t _Qiniu_Region_Cache_Hash(const void *r, uint64_t seed0, uint64_t seed1) +{ + struct _Qiniu_Region_Cache *cache = (struct _Qiniu_Region_Cache *)r; + uint64_t hash = 0; + if (cache->accessKey != NULL) { - *host = hosts[0]; + hash ^= hashmap_sip(cache->accessKey, strlen(cache->accessKey), seed0, seed1); } - return err; + if (cache->bucketName != NULL) + { + hash ^= hashmap_sip(cache->bucketName, strlen(cache->bucketName), seed0, seed1); + } + hash ^= hashmap_sip(&cache->bucketHostsLen, sizeof(cache->bucketHostsLen), seed0, seed1); + for (size_t i = 0; i < cache->bucketHostsLen; i++) + { + hash ^= hashmap_sip(cache->bucketHosts[i], strlen(cache->bucketHosts[i]), seed0, seed1); + } + hash ^= hashmap_sip(&cache->uploadingAccelerationEnabled, sizeof(cache->uploadingAccelerationEnabled), seed0, seed1); + return hash; } -Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +static Qiniu_Error _Qiniu_Region_Auto_Query_With_Cache(Qiniu_Client *self, const char *accessKey, const char *bucketName, Qiniu_Region **foundRegion) { const char *const *hosts; - size_t count; + size_t hostsLen; + Qiniu_Bool needToFreeHosts; Qiniu_Error err = Qiniu_OK; - Qiniu_Region **foundRegion = &self->cachedRegion; - if (self->specifiedRegion) + if (accessKey == NULL) { - foundRegion = &self->specifiedRegion; - goto foundCache; + accessKey = QINIU_ACCESS_KEY; } - else if (self->autoQueryRegion) + + hosts = _Qiniu_Get_Bucket_Hosts(self, &hostsLen, &needToFreeHosts); + const struct _Qiniu_Region_Cache cacheKey = { + .accessKey = accessKey, + .bucketName = bucketName, + .bucketHosts = hosts, + .bucketHostsLen = hostsLen, + .needToFreeBucketHosts = needToFreeHosts, + .uploadingAccelerationEnabled = self->enableUploadingAcceleration, + }; + struct _Qiniu_Region_Cache *cache = NULL; + if (self->cachedRegions != NULL) { - if (self->cachedRegion != NULL) + cache = (struct _Qiniu_Region_Cache *)hashmap_get(self->cachedRegions, &cacheKey); + if (cache != NULL && !Qiniu_Region_Is_Expired(cache->region)) { - if (strcmp(self->cachedRegionBucketName, bucketName) == 0 && !Qiniu_Region_Is_Expired(self->cachedRegion)) - { - goto foundCache; - } - else + *foundRegion = cache->region; + err = Qiniu_OK; + goto handleErr; + } + } + err = _Qiniu_Region_Query(self, foundRegion, accessKey, bucketName, self->autoQueryHttpsRegion); + if (err.code != 200) + { + if (cache != NULL) + { // 有已经过期的缓存区域可以使用 + *foundRegion = cache->region; + err = Qiniu_OK; + } + goto handleErr; + } + if (self->cachedRegions == NULL) + { + self->cachedRegions = hashmap_new(sizeof(struct _Qiniu_Region_Cache), 0, rand(), rand(), _Qiniu_Region_Cache_Hash, _Qiniu_Region_Cache_Compare, _Qiniu_Region_Cache_Free, NULL); + } + if (self->cachedRegions != NULL) + { + if (cache != NULL) + { // 复用前面已经过期的缓存的内存 + Qiniu_Region_Free(cache->region); + cache->region = *foundRegion; + } + else + { + struct _Qiniu_Region_Cache *newCache = malloc(sizeof(struct _Qiniu_Region_Cache)); + if (newCache != NULL) { - Qiniu_Region_Free(self->cachedRegion); - self->cachedRegion = NULL; - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); + *newCache = (struct _Qiniu_Region_Cache){ + .region = *foundRegion, + .accessKey = Qiniu_String_Dup(accessKey), + .bucketName = Qiniu_String_Dup(bucketName), + .bucketHosts = hosts, + .bucketHostsLen = hostsLen, + .needToFreeBucketHosts = needToFreeHosts, + }; + hashmap_set(self->cachedRegions, newCache); + needToFreeHosts = Qiniu_False; } } - err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion); - if (err.code != 200) + } +handleErr: + if (needToFreeHosts) + { + Qiniu_Free((void *)hosts); + } + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Up_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) +{ + Qiniu_Error err = Qiniu_OK; + Qiniu_Region *foundRegion = NULL; + + if (self->specifiedRegion) + { + foundRegion = self->specifiedRegion; + goto foundCache; + } + else if (self->autoQueryRegion && bucketName != NULL) + { + err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion); + if (err.code == 200) { - goto useDefault; + goto foundCache; } - self->cachedRegionBucketName = Qiniu_String_Dup(bucketName); - goto foundCache; + return err; } -useDefault: - *host = QINIU_IOVIP_HOST; +foundCache: + *hosts = _Qiniu_Region_Get_All_Up_Hosts(foundRegion, count, self->enableUploadingAcceleration); return err; +} +Qiniu_Error _Qiniu_Region_Get_Io_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) +{ + Qiniu_Error err = Qiniu_OK; + Qiniu_Region *foundRegion = NULL; + + if (self->specifiedRegion) + { + foundRegion = self->specifiedRegion; + goto foundCache; + } + else if (self->autoQueryRegion && bucketName != NULL) + { + err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion); + if (err.code == 200) + { + goto foundCache; + } + return err; + } foundCache: - hosts = Qiniu_Region_Get_Io_Preferred_Hosts(*foundRegion, &count); + *hosts = _Qiniu_Region_Get_All_Io_Hosts(foundRegion, count, Qiniu_False); + return err; +} + +Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Io_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != 200) + { + return err; + } if (count == 0) { *host = QINIU_IOVIP_HOST; @@ -1308,47 +1695,39 @@ Qiniu_Error _Qiniu_Region_Get_Io_Host(Qiniu_Client *self, const char *accessKey, return err; } -Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Rs_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; - Qiniu_Region **foundRegion = &self->cachedRegion; + Qiniu_Region *foundRegion = NULL; if (self->specifiedRegion) { - foundRegion = &self->specifiedRegion; + foundRegion = self->specifiedRegion; goto foundCache; } - else if (self->autoQueryRegion) + else if (self->autoQueryRegion && bucketName != NULL) { - if (self->cachedRegion != NULL) - { - if (strcmp(self->cachedRegionBucketName, bucketName) == 0) - { - goto foundCache; - } - else - { - Qiniu_Region_Free(self->cachedRegion); - self->cachedRegion = NULL; - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); - } - } - err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion); - if (err.code != 200) + err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion); + if (err.code == 200) { - goto useDefault; + goto foundCache; } - self->cachedRegionBucketName = Qiniu_String_Dup(bucketName); - goto foundCache; + return err; } -useDefault: - *host = QINIU_RS_HOST; +foundCache: + *hosts = _Qiniu_Region_Get_All_Rs_Hosts(foundRegion, count, Qiniu_False); return err; +} -foundCache: - hosts = Qiniu_Region_Get_Rs_Preferred_Hosts(*foundRegion, &count); +Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Rs_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != 200) + { + return err; + } if (count == 0) { *host = QINIU_RS_HOST; @@ -1360,47 +1739,39 @@ Qiniu_Error _Qiniu_Region_Get_Rs_Host(Qiniu_Client *self, const char *accessKey, return err; } -Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Rsf_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; - Qiniu_Region **foundRegion = &self->cachedRegion; + Qiniu_Region *foundRegion = NULL; if (self->specifiedRegion) { - foundRegion = &self->specifiedRegion; + foundRegion = self->specifiedRegion; goto foundCache; } - else if (self->autoQueryRegion) + else if (self->autoQueryRegion && bucketName != NULL) { - if (self->cachedRegion != NULL) - { - if (strcmp(self->cachedRegionBucketName, bucketName) == 0) - { - goto foundCache; - } - else - { - Qiniu_Region_Free(self->cachedRegion); - self->cachedRegion = NULL; - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); - } - } - err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion); - if (err.code != 200) + err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion); + if (err.code == 200) { - goto useDefault; + goto foundCache; } - self->cachedRegionBucketName = Qiniu_String_Dup(bucketName); - goto foundCache; + return err; } -useDefault: - *host = QINIU_RSF_HOST; +foundCache: + *hosts = _Qiniu_Region_Get_All_Rsf_Hosts(foundRegion, count, Qiniu_False); return err; +} -foundCache: - hosts = Qiniu_Region_Get_Rsf_Preferred_Hosts(*foundRegion, &count); +Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Rsf_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != 200) + { + return err; + } if (count == 0) { *host = QINIU_RSF_HOST; @@ -1412,47 +1783,39 @@ Qiniu_Error _Qiniu_Region_Get_Rsf_Host(Qiniu_Client *self, const char *accessKey return err; } -Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +Qiniu_Error _Qiniu_Region_Get_Api_Hosts(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char *const **hosts, size_t *count) { - const char *const *hosts; - size_t count; Qiniu_Error err = Qiniu_OK; - Qiniu_Region **foundRegion = &self->cachedRegion; + Qiniu_Region *foundRegion = NULL; if (self->specifiedRegion) { - foundRegion = &self->specifiedRegion; + foundRegion = self->specifiedRegion; goto foundCache; } - else if (self->autoQueryRegion) + else if (self->autoQueryRegion && bucketName != NULL) { - if (self->cachedRegion != NULL) - { - if (strcmp(self->cachedRegionBucketName, bucketName) == 0) - { - goto foundCache; - } - else - { - Qiniu_Region_Free(self->cachedRegion); - self->cachedRegion = NULL; - Qiniu_FreeV2((void **)&self->cachedRegionBucketName); - } - } - err = _Qiniu_Region_Query(self, &self->cachedRegion, accessKey, bucketName, self->autoQueryHttpsRegion); - if (err.code != 200) + err = _Qiniu_Region_Auto_Query_With_Cache(self, accessKey, bucketName, &foundRegion); + if (err.code == 200) { - goto useDefault; + goto foundCache; } - self->cachedRegionBucketName = Qiniu_String_Dup(bucketName); - goto foundCache; + return err; } -useDefault: - *host = QINIU_API_HOST; +foundCache: + *hosts = _Qiniu_Region_Get_All_Api_Hosts(foundRegion, count, Qiniu_False); return err; +} -foundCache: - hosts = Qiniu_Region_Get_Api_Preferred_Hosts(*foundRegion, &count); +Qiniu_Error _Qiniu_Region_Get_Api_Host(Qiniu_Client *self, const char *accessKey, const char *bucketName, const char **host) +{ + const char *const *hosts; + size_t count; + Qiniu_Error err = _Qiniu_Region_Get_Api_Hosts(self, accessKey, bucketName, &hosts, &count); + if (err.code != 200) + { + return err; + } if (count == 0) { *host = QINIU_API_HOST; diff --git a/qiniu/region.h b/qiniu/region.h index 9d3d0194..b9d54b23 100644 --- a/qiniu/region.h +++ b/qiniu/region.h @@ -11,19 +11,18 @@ #define QINIU_REGION_H #include "http.h" +#if defined(_WIN32) +#pragma pack(1) +#endif + #ifdef __cplusplus extern "C" { #endif - struct _Qiniu_Client; - typedef struct _Qiniu_Client Qiniu_Client; - - struct _Qiniu_Region; - typedef struct _Qiniu_Region Qiniu_Region; - QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Preferred_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Alternative_Hosts(Qiniu_Region *region, size_t *count); + QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Up_Accelerated_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Preferred_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Alternative_Hosts(Qiniu_Region *region, size_t *count); QINIU_DLLAPI extern const char *const *Qiniu_Region_Get_Io_Src_Preferred_Hosts(Qiniu_Region *region, size_t *count); @@ -44,6 +43,7 @@ extern "C" QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); + QINIU_DLLAPI extern void Qiniu_Region_Set_Up_Accelerated_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Alternative_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); QINIU_DLLAPI extern void Qiniu_Region_Set_Io_Src_Preferred_Hosts(Qiniu_Region *region, const char *const *hosts, size_t hostsCount, Qiniu_Bool useHttps); @@ -60,6 +60,10 @@ extern "C" QINIU_DLLAPI extern Qiniu_Region *Qiniu_Use_Region(const char *const regionId, Qiniu_Bool useHttps); QINIU_DLLAPI extern Qiniu_Error Qiniu_Region_Query(Qiniu_Client *self, Qiniu_Region **region, const char *const bucketName, Qiniu_Bool useHttps); +#if defined(_WIN32) +#pragma pack() +#endif + #ifdef __cplusplus } #endif diff --git a/qiniu/resumable_io.c b/qiniu/resumable_io.c index e916c458..11315547 100644 --- a/qiniu/resumable_io.c +++ b/qiniu/resumable_io.c @@ -16,7 +16,9 @@ #include "recorder_key.h" #include "recorder_utils.h" #include "../cJSON/cJSON.h" +#include "../hashmap/hashmap.h" #include "private/region.h" +#include "private/code.h" #define blockBits 22 #define blockMask ((1 << blockBits) - 1) @@ -277,7 +279,7 @@ static Qiniu_Error Qiniu_Rio_PutExtra_Init( } else { - memset(self, 0, sizeof(Qiniu_Rio_PutExtra)); + Qiniu_Zero_Ptr(self); } cbprog = sizeof(Qiniu_Rio_BlkputRet) * blockCnt; @@ -349,19 +351,133 @@ static void Qiniu_Io_PutExtra_initFrom(Qiniu_Io_PutExtra *self, Qiniu_Rio_PutExt } else { - memset(self, 0, sizeof(*self)); + Qiniu_Zero_Ptr(self); } } /*============================================================================*/ -static Qiniu_Error Qiniu_Rio_bput( - Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength, const char *url) +struct _Qiniu_Uploading_Parts_Progress +{ + size_t uploaded; + struct hashmap *uploading; + Qiniu_Mutex mutex; +}; + +struct _Qiniu_Uploading_Parts_Progress_Pair +{ + int blkIdx; + size_t uploaded; +}; + +static uint64_t _Qiniu_Uploading_Parts_Progress_Hash(const void *item, uint64_t seed0, uint64_t seed1) +{ + const struct _Qiniu_Uploading_Parts_Progress_Pair *pair = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)item; + return hashmap_sip(&pair->blkIdx, sizeof(pair->blkIdx), seed0, seed1); +} + +static int _Qiniu_Uploading_Parts_Progress_Compare(const void *a, const void *b, void *udata) +{ + const struct _Qiniu_Uploading_Parts_Progress_Pair *left = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)a; + const struct _Qiniu_Uploading_Parts_Progress_Pair *right = (const struct _Qiniu_Uploading_Parts_Progress_Pair *)b; + return left->blkIdx - right->blkIdx; +} + +static struct _Qiniu_Uploading_Parts_Progress *_Qiniu_Uploading_Parts_Progress_New() +{ + struct _Qiniu_Uploading_Parts_Progress *data = (struct _Qiniu_Uploading_Parts_Progress *)malloc(sizeof(struct _Qiniu_Uploading_Parts_Progress)); + Qiniu_Zero_Ptr(data); + Qiniu_Mutex_Init(&data->mutex); + data->uploading = hashmap_new( + sizeof(struct _Qiniu_Uploading_Parts_Progress_Pair), 0, rand(), rand(), + _Qiniu_Uploading_Parts_Progress_Hash, _Qiniu_Uploading_Parts_Progress_Compare, + free, NULL); + return data; +} + +static void _Qiniu_Uploading_Parts_Progress_Free(struct _Qiniu_Uploading_Parts_Progress *data) +{ + hashmap_free(data->uploading); + Qiniu_Mutex_Cleanup(&data->mutex); + free((void *)data); +} + +static void _Qiniu_Uploading_Parts_Progress_Set_Progress(struct _Qiniu_Uploading_Parts_Progress *progress, int blkIdx, size_t uploaded) +{ + Qiniu_Mutex_Lock(&progress->mutex); + const struct _Qiniu_Uploading_Parts_Progress_Pair pair = {.blkIdx = blkIdx}; + struct _Qiniu_Uploading_Parts_Progress_Pair *pPair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)hashmap_get(progress->uploading, &pair); + if (pPair == NULL) + { + pPair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)malloc(sizeof(struct _Qiniu_Uploading_Parts_Progress_Pair)); + pPair->blkIdx = blkIdx; + pPair->uploaded = uploaded; + hashmap_set(progress->uploading, pPair); + } + else + { + pPair->uploaded = uploaded; + } + + Qiniu_Mutex_Unlock(&progress->mutex); +} + +static void _Qiniu_Uploading_Parts_Progress_Part_Uploaded(struct _Qiniu_Uploading_Parts_Progress *progress, int blkIdx, size_t partSize) +{ + Qiniu_Mutex_Lock(&progress->mutex); + const struct _Qiniu_Uploading_Parts_Progress_Pair pair = {.blkIdx = blkIdx}; + hashmap_delete(progress->uploading, (const void *)&pair); + progress->uploaded += partSize; + Qiniu_Mutex_Unlock(&progress->mutex); +} + +static size_t _Qiniu_Uploading_Parts_Progress_Get_Total_Size(struct _Qiniu_Uploading_Parts_Progress *progress) +{ + Qiniu_Mutex_Lock(&progress->mutex); + size_t total = progress->uploaded, iter = 0; + void *iter_val; + while (hashmap_iter(progress->uploading, &iter, &iter_val)) + { + struct _Qiniu_Uploading_Parts_Progress_Pair *pair = (struct _Qiniu_Uploading_Parts_Progress_Pair *)iter_val; + total += pair->uploaded; + } + Qiniu_Mutex_Unlock(&progress->mutex); + return total; +} + +struct _Qiniu_Progress_Callback_Data +{ + size_t totalSize, previousUlNow; + int blkIdx; + struct _Qiniu_Uploading_Parts_Progress *progress; + void (*callback)(size_t, size_t); +}; + +static int _Qiniu_Progress_Callback(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow) +{ + struct _Qiniu_Progress_Callback_Data *data = (struct _Qiniu_Progress_Callback_Data *)clientp; + if (data->previousUlNow != (size_t)ulnow) + { + _Qiniu_Uploading_Parts_Progress_Set_Progress(data->progress, data->blkIdx, (size_t)ulnow); + data->callback((size_t)data->totalSize, _Qiniu_Uploading_Parts_Progress_Get_Total_Size(data->progress)); + data->previousUlNow = (size_t)ulnow; + } + return 0; +} + +static Qiniu_Error Qiniu_Rio_bput(Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength, const char *url, struct _Qiniu_Progress_Callback_Data *progressCallback) { Qiniu_Rio_BlkputRet retFromResp; Qiniu_Json *root; + int (*callback)(void *, double, double, double, double) = NULL; + void *callbackData = NULL; + if (progressCallback != NULL && progressCallback->callback != NULL) + { + callback = _Qiniu_Progress_Callback; + callbackData = (void *)progressCallback; + } - Qiniu_Error err = Qiniu_Client_CallWithBinary(self, &root, url, body, bodyLength, NULL); + Qiniu_Error err = Qiniu_Client_CallWithBinaryAndProgressCallback(self, &root, url, body, bodyLength, NULL, callback, callbackData); if (err.code == 200) { retFromResp.ctx = Qiniu_Json_GetString(root, "ctx", NULL); @@ -385,22 +501,21 @@ static Qiniu_Error Qiniu_Rio_bput( } static Qiniu_Error Qiniu_Rio_Mkblock( - Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, Qiniu_Reader body, int bodyLength, - Qiniu_Rio_PutExtra *extra) + Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, int blkSize, const char *upHost, Qiniu_Reader body, int bodyLength, struct _Qiniu_Progress_Callback_Data *progressCallback) { - char *url = Qiniu_String_Format(128, "%s/mkblk/%d", extra->upHost, blkSize); - Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url); - Qiniu_Free(url); + const char *url = Qiniu_String_Format(128, "%s/mkblk/%d", upHost, blkSize); + Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url, progressCallback); + Qiniu_Free((void *)url); return err; } static Qiniu_Error Qiniu_Rio_Blockput( - Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, Qiniu_Reader body, int bodyLength) + Qiniu_Client *self, Qiniu_Rio_BlkputRet *ret, const char *upHost, Qiniu_Reader body, int bodyLength, struct _Qiniu_Progress_Callback_Data *progressCallback) { - char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", ret->host, ret->ctx, (int)ret->offset); - Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url); - Qiniu_Free(url); + const char *url = Qiniu_String_Format(1024, "%s/bput/%s/%d", upHost, ret->ctx, (int)ret->offset); + Qiniu_Error err = Qiniu_Rio_bput(self, ret, body, bodyLength, url, progressCallback); + Qiniu_Free((void *)url); return err; } @@ -409,19 +524,18 @@ static Qiniu_Error Qiniu_Rio_Blockput( static Qiniu_Error ErrUnmatchedChecksum = { Qiniu_Rio_UnmatchedChecksum, "unmatched checksum"}; -static int Qiniu_TemporaryError(int code) -{ - return code / 100 != 4; -} - -static Qiniu_Error Qiniu_Rio_ResumableBlockput( - Qiniu_Client *c, Qiniu_Rio_BlkputRet *ret, Qiniu_ReaderAt f, int blkIdx, int blkSize, - Qiniu_Rio_PutExtra *extra, size_t *chunksUploaded) +static Qiniu_Error +Qiniu_Rio_ResumableBlockput( + Qiniu_Client *c, Qiniu_Rio_BlkputRet *ret, Qiniu_ReaderAt f, int blkIdx, int blkSize, int fsize, + struct _Qiniu_Uploading_Parts_Progress *uploadingPartsProgress, Qiniu_Rio_PutExtra *extra, size_t *pChunksUploaded) { Qiniu_Error err = Qiniu_OK; Qiniu_Tee tee; Qiniu_Section section; Qiniu_Reader body, body1; + const char *const *upHosts; + const char *upHost; + size_t upHostsCount; Qiniu_Crc32 crc32; Qiniu_Writer h = Qiniu_Crc32Writer(&crc32, 0); @@ -431,11 +545,32 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( int bodyLength; int tryTimes; int notifyRet = 0; - size_t httpCalled = 0; + size_t chunksUploaded = 0; - if (ret->ctx == NULL) + struct _Qiniu_Progress_Callback_Data progressCallbackData; + Qiniu_Zero(progressCallbackData); + + if (extra->uploadingProgress != NULL) { + progressCallbackData.callback = extra->uploadingProgress; + progressCallbackData.totalSize = (size_t)fsize; + progressCallbackData.progress = uploadingPartsProgress; + progressCallbackData.blkIdx = blkIdx; + } + if (extra->upHost != NULL) + { + upHosts = &extra->upHost; + upHostsCount = 1; + } + else + { + upHosts = extra->upHosts; + upHostsCount = extra->upHostsCount; + } + + if (ret->ctx == NULL) + { if (chunkSize < blkSize) { bodyLength = chunkSize; @@ -445,19 +580,42 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( bodyLength = blkSize; } - body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength); - body = Qiniu_TeeReader(&tee, body1, h); + for (int tries = 0; tries < extra->tryTimes && tries <= c->hostsRetriesMax; tries++) + { + body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase, bodyLength); + body = Qiniu_TeeReader(&tee, body1, h); - err = Qiniu_Rio_Mkblock(c, ret, blkSize, body, bodyLength, extra); + upHost = upHosts[tries % upHostsCount]; + err = Qiniu_Rio_Mkblock(c, ret, blkSize, upHost, body, bodyLength, &progressCallbackData); + if (err.code == 200) + { + if (extra->uploadingProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, bodyLength); + } + break; + } + else + { + if (extra->uploadingProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0); + } + if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + goto handleErr; + } + } + } if (err.code != 200) { - return err; + goto handleErr; } - - httpCalled++; + chunksUploaded++; if (ret->crc32 != crc32.val || (int)(ret->offset) != bodyLength) { - return ErrUnmatchedChecksum; + err = ErrUnmatchedChecksum; + goto handleErr; } notifyRet = extra->notify(extra->notifyRecvr, blkIdx, blkSize, ret); if (notifyRet == QINIU_RIO_NOTIFY_EXIT) @@ -465,9 +623,13 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( // Terminate the upload process if the caller requests err.code = Qiniu_Rio_PutInterrupted; err.message = "Interrupted by the caller"; - return err; + goto handleErr; } } + else if (extra->uploadingProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, ret->offset); + } while ((int)(ret->offset) < blkSize) { @@ -482,16 +644,21 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( } tryTimes = extra->tryTimes; + if (tryTimes > c->hostsRetriesMax + 1) + { + tryTimes = c->hostsRetriesMax + 1; + } lzRetry: crc32.val = 0; body1 = Qiniu_SectionReader(§ion, f, (Qiniu_Off_T)offbase + (ret->offset), bodyLength); body = Qiniu_TeeReader(&tee, body1, h); + upHost = upHosts[tryTimes % upHostsCount]; - err = Qiniu_Rio_Blockput(c, ret, body, bodyLength); + err = Qiniu_Rio_Blockput(c, ret, upHost, body, bodyLength, &progressCallbackData); if (err.code == 200) { - httpCalled++; + chunksUploaded++; if (ret->crc32 == crc32.val) { notifyRet = extra->notify(extra->notifyRecvr, blkIdx, blkSize, ret); @@ -500,25 +667,39 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( // Terminate the upload process if the caller requests err.code = Qiniu_Rio_PutInterrupted; err.message = "Interrupted by the caller"; - return err; + goto handleErr; + } + if (extra->uploadingProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Part_Uploaded(uploadingPartsProgress, blkIdx, bodyLength); } - continue; } - Qiniu_Log_Warn("ResumableBlockput: invalid checksum, retry"); - err = ErrUnmatchedChecksum; + else + { + Qiniu_Log_Warn("ResumableBlockput: invalid checksum, retry"); + err = ErrUnmatchedChecksum; + if (extra->uploadingProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0); + } + } + } + else if (err.code == Qiniu_Rio_InvalidCtx) + { + Qiniu_Rio_BlkputRet_Cleanup(ret); // reset + Qiniu_Log_Warn("ResumableBlockput: invalid ctx, please retry"); + goto handleErr; } else { - if (err.code == Qiniu_Rio_InvalidCtx) + Qiniu_Log_Warn("ResumableBlockput %d off:%d failed - %E", blkIdx, (int)ret->offset, err); + if (extra->uploadingProgress != NULL) { - Qiniu_Rio_BlkputRet_Cleanup(ret); // reset - Qiniu_Log_Warn("ResumableBlockput: invalid ctx, please retry"); - return err; + _Qiniu_Uploading_Parts_Progress_Set_Progress(uploadingPartsProgress, blkIdx, 0); } - Qiniu_Log_Warn("ResumableBlockput %d off:%d failed - %E", blkIdx, (int)ret->offset, err); } - if (tryTimes > 1 && Qiniu_TemporaryError(err.code)) + if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY) { tryTimes--; Qiniu_Log_Info("ResumableBlockput %E, retrying ...", err); @@ -527,9 +708,10 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( break; } - if (chunksUploaded != NULL) +handleErr: + if (pChunksUploaded != NULL) { - *chunksUploaded = httpCalled; + *pChunksUploaded = chunksUploaded; } return err; @@ -538,7 +720,7 @@ static Qiniu_Error Qiniu_Rio_ResumableBlockput( /*============================================================================*/ static Qiniu_Error Qiniu_Rio_Mkfile( - Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra) + Qiniu_Client *c, Qiniu_Rio_PutRet *ret, const char *upHost, const char *key, Qiniu_Int64 fsize, Qiniu_Rio_PutExtra *extra) { size_t i, blkCount = extra->blockCnt; Qiniu_Json *root; @@ -547,7 +729,7 @@ static Qiniu_Error Qiniu_Rio_Mkfile( int j = 0; Qiniu_Buffer_Init(&url, 2048); - Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", extra->upHost, fsize); + Qiniu_Buffer_AppendFormat(&url, "%s/mkfile/%D", upHost, fsize); if (key != NULL) { @@ -674,6 +856,8 @@ typedef struct _Qiniu_Rio_task Qiniu_Count *ninterrupts; int blkIdx; int blkSize1; + int fsize; + struct _Qiniu_Uploading_Parts_Progress *progress; } Qiniu_Rio_task; static void Qiniu_Rio_doTask(void *params) @@ -701,7 +885,7 @@ static void Qiniu_Rio_doTask(void *params) lzRetry: Qiniu_Rio_BlkputRet_Assign(&ret, &extra->progresses[blkIdx]); - Qiniu_Error err = Qiniu_Rio_ResumableBlockput(c, &ret, task->f, blkIdx, task->blkSize1, extra, &chunksUploaded); + Qiniu_Error err = Qiniu_Rio_ResumableBlockput(c, &ret, task->f, blkIdx, task->blkSize1, task->fsize, task->progress, extra, &chunksUploaded); if (err.code != 200) { if (err.code == Qiniu_Rio_PutInterrupted) @@ -714,7 +898,7 @@ static void Qiniu_Rio_doTask(void *params) return; } - if (tryTimes > 1 && Qiniu_TemporaryError(err.code)) + if (tryTimes > 1 && _Qiniu_Should_Retry(err.code) != QINIU_DONT_RETRY) { tryTimes--; Qiniu_Log_Info("resumable.Put %E, retrying ...", err); @@ -740,10 +924,8 @@ static void Qiniu_Rio_doTask(void *params) /*============================================================================*/ /* func Qiniu_Rio_PutXXX */ -static Qiniu_Error ErrPutFailed = { - Qiniu_Rio_PutFailed, "resumable put failed"}; -static Qiniu_Error ErrPutInterrupted = { - Qiniu_Rio_PutInterrupted, "resumable put interrupted"}; +static Qiniu_Error ErrPutFailed; +static Qiniu_Error ErrPutInterrupted; static Qiniu_Error Qiniu_Rio_loadProgresses(Qiniu_Rio_PutRet *ret, Qiniu_Rio_PutExtra *extra, Qiniu_Rio_Recorder *recorder) { @@ -807,12 +989,13 @@ static Qiniu_Error _Qiniu_Rio_Put( int nfails; int retCode; Qiniu_Count ninterrupts; + struct _Qiniu_Uploading_Parts_Progress *uploadingPartProgress = NULL; Qiniu_Error err = Qiniu_Rio_PutExtra_Init(&extra, fsize, extra1); if (err.code != 200) { return err; } - if (extra.upHost == NULL) + if (extra.upHost == NULL && (extra.upHosts == NULL || extra.upHostsCount == 0)) { if (!Qiniu_Utils_Extract_Bucket(uptoken, &accessKey, &bucketName)) { @@ -820,7 +1003,7 @@ static Qiniu_Error _Qiniu_Rio_Put( err.message = "parse uptoken failed"; return err; } - err = _Qiniu_Region_Get_Up_Host(self, accessKey, bucketName, &extra.upHost); + err = _Qiniu_Region_Get_Up_Hosts(self, accessKey, bucketName, &extra.upHosts, &extra.upHostsCount); if (err.code != 200) { Qiniu_Free((void *)accessKey); @@ -828,6 +1011,10 @@ static Qiniu_Error _Qiniu_Rio_Put( return err; } } + if (extra.uploadingProgress != NULL) + { + uploadingPartProgress = _Qiniu_Uploading_Parts_Progress_New(); + } tm = extra.threadModel; wg = tm.itbl->WaitGroup(tm.self); @@ -853,6 +1040,8 @@ static Qiniu_Error _Qiniu_Rio_Put( task->ninterrupts = &ninterrupts; task->blkIdx = i; task->blkSize1 = blkSize; + task->fsize = fsize; + task->progress = uploadingPartProgress; if (i == last) { offbase = (Qiniu_Int64)(i) << blockBits; @@ -865,7 +1054,7 @@ static Qiniu_Error _Qiniu_Rio_Put( { wg.itbl->Done(wg.self); Qiniu_Count_Inc(&ninterrupts); - free(task); + Qiniu_Free((void *)task); } if (ninterrupts > 0) @@ -885,19 +1074,45 @@ static Qiniu_Error _Qiniu_Rio_Put( } else { - err = Qiniu_Rio_Mkfile(self, ret, key, fsize, &extra); - if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL) + const char *const *upHosts; + size_t upHostsCount; + + if (extra.upHost != NULL) { - Qiniu_Rio_PutExtra_Clear(&extra); - reinitializeRecorder(&extra, fi, recorder); - goto reinit; + upHosts = &extra.upHost; + upHostsCount = 1; } - if (err.code / 100 == 2 || err.code / 100 == 4 || err.code == Qiniu_Rio_InvalidCtx) + else { - Qiniu_Rio_clearProgresses(&extra, recorder); + upHosts = extra.upHosts; + upHostsCount = extra.upHostsCount; + } + + for (int tries = 0; tries < extra.tryTimes && tries <= self->hostsRetriesMax; tries++) + { + const char *upHost = upHosts[tries % upHostsCount]; + err = Qiniu_Rio_Mkfile(self, ret, upHost, key, fsize, &extra); + if (err.code == Qiniu_Rio_InvalidCtx && recorder != NULL && recorder->toLoadProgresses == Qiniu_True && fi != NULL) + { + Qiniu_Rio_PutExtra_Clear(&extra); + reinitializeRecorder(&extra, fi, recorder); + goto reinit; + } + else if (err.code == 200 || _Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY) + { + if (err.code / 100 != 4 || err.code == Qiniu_Rio_InvalidCtx) + { + Qiniu_Rio_clearProgresses(&extra, recorder); + } + break; + } } } + if (uploadingPartProgress != NULL) + { + _Qiniu_Uploading_Parts_Progress_Free(uploadingPartProgress); + } Qiniu_Rio_PutExtra_Cleanup(&extra); Qiniu_Free((void *)accessKey); Qiniu_Free((void *)bucketName); diff --git a/qiniu/resumable_io.h b/qiniu/resumable_io.h index a1efacf7..091da120 100644 --- a/qiniu/resumable_io.h +++ b/qiniu/resumable_io.h @@ -14,7 +14,9 @@ #include "io.h" #include "recorder.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -137,6 +139,13 @@ extern "C" // For those who want to send request to specific host. const char *upHost; + + // Specify multiple upHosts, if not set explicitly, will global QINIU_UP_HOST + const char *const *upHosts; + size_t upHostsCount; + + // Uploading file progress + void (*uploadingProgress)(size_t ultotal, size_t ulnow); } Qiniu_Rio_PutExtra; /*============================================================================*/ @@ -166,7 +175,9 @@ extern "C" /*============================================================================*/ +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/rfc3339.c b/qiniu/rfc3339.c new file mode 100644 index 00000000..bc329655 --- /dev/null +++ b/qiniu/rfc3339.c @@ -0,0 +1,247 @@ +#include "rfc3339.h" + +static void _Strip_Spaces(char *source) +{ + char *i = source; + + while (*source != 0) + { + *i = *source++; + if (*i != ' ') + i++; + } + + *i = 0; +} + +static void _Qiniu_Parse_Date(char *date_string, Qiniu_Date *d) +{ + char *const tokens = strdup(date_string); + _Strip_Spaces(tokens); + d->ok = 0; + + if (strlen(tokens) < 10) + { + return; + } + + int status = sscanf( + tokens, "%04d-%02d-%02d", &(d->year), &(d->month), &(d->day)); + free((char *)tokens); + + if (status != 3) + { + return; + } + if (d->year < 1 || d->year > 9999) + { + return; + } + if (d->month < 1 || d->month > 12) + { + return; + } + if (d->day < 1 || d->day > 31) + { + return; + } + + const int days_in_month[] = {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + + if (d->month == 2) + { + int leap = (d->year % 4 == 0 && d->year % 100 != 0) || (d->year % 400 == 0); + if (d->day > (days_in_month[2 - 1] + leap)) + { + return; + } + } + else if (d->day > days_in_month[d->month - 1]) + { + return; + } + + d->ok = 1; +} + +static void _Qiniu_Parse_Time(char *time_string, Qiniu_Time *t) +{ + char *tokens = strdup(time_string); + char *token_ptr = tokens; + + _Strip_Spaces(tokens); + + t->ok = 0; + + if (strlen(tokens) < 8) + { + goto cleanup; + } + + if ((strlen(tokens) > 11) && ((*(tokens + 10) == 'T') || (*(tokens + 10) == 't'))) + { + tokens += 11; + } + + int status = sscanf( + tokens, "%02d:%02d:%02d", &(t->hour), &(t->minute), &(t->second)); + + if (status != 3) + { + goto cleanup; + } + if (t->hour < 0 || t->hour > 23) + { + goto cleanup; + } + if (t->minute < 0 || t->minute > 59) + { + goto cleanup; + } + if (t->second < 0 || t->second > 59) + { + goto cleanup; + } + + if (strlen(tokens) == 8) + { + t->offset = 0; + t->ok = 1; + goto cleanup; + } + else + { + tokens += 8; + } + + if (*tokens == '.') + { + tokens++; + char fractions[7] = {0}; + + for (unsigned int i = 0; i < 6; i++) + { + if ((*(tokens + i) >= 48) && (*(tokens + i) <= 57)) + { + fractions[i] = *(tokens + i); + } + else + { + break; + } + } + + status = sscanf(fractions, "%d", &(t->fraction)); + + if (strlen(fractions) < 6 && strlen(fractions) > 0) + { + t->fraction = t->fraction * pow(10, 6 - strlen(fractions)); // convert msec to usec + } + else if (strlen(fractions) == 6) + { + // all fine, already in usec + } + else + { + // Invalid fractions must be msec or usec + goto cleanup; + } + + if (status != 1) + { + goto cleanup; + } + if (t->fraction < 0 || t->fraction > 999999) + { + goto cleanup; + } + + tokens += strlen(fractions); + + if (strlen(tokens) == 0) + { + t->offset = 0; + t->ok = 1; + goto cleanup; + } + } + + if ((*tokens == 'Z') || (*tokens == 'z')) + { + t->offset = 0; + + tokens++; + if (strlen(tokens) == 0) + { + t->ok = 1; + } + else + { + t->ok = 0; + } + goto cleanup; + } + else if ((*tokens == '+') || (*tokens == '-')) + { + unsigned int tz_hour, tz_minute; + + status = sscanf(tokens + 1, "%02d:%02d", &tz_hour, &tz_minute); + + if (status != 2) + { + goto cleanup; + } + if ((tz_hour < 0) || (tz_hour > 23)) + { + goto cleanup; + } + if ((tz_minute < 0) || (tz_minute > 59)) + { + goto cleanup; + } + + int tz_offset = (tz_hour * HOUR_IN_MINS) + tz_minute; + + if (*tokens == '-') + { + tz_offset = tz_offset * -1; + } + + t->offset = tz_offset; + + tokens = tokens + 6; + if (strlen(tokens) == 0) + { + t->ok = 1; + } + else + { + t->ok = 0; + } + } + +cleanup: + free(token_ptr); + tokens = NULL; + token_ptr = NULL; + return; +} + +void _Qiniu_Parse_Date_Time(char *datetime_string, Qiniu_DateTime *dt) +{ + dt->ok = 0; + + _Qiniu_Parse_Date(datetime_string, &(dt->date)); + if (dt->date.ok == 0) + { + return; + } + + _Qiniu_Parse_Time(datetime_string, &(dt->time)); + if (dt->time.ok == 0) + { + return; + } + + dt->ok = 1; +} diff --git a/qiniu/rfc3339.h b/qiniu/rfc3339.h new file mode 100644 index 00000000..1add59c5 --- /dev/null +++ b/qiniu/rfc3339.h @@ -0,0 +1,56 @@ +#ifndef QINIU_RFC3339_H +#define QINIU_RFC3339_H + +#include +#include +#include +#include + +#define DAY_IN_SECS 86400 +#define HOUR_IN_SECS 3600 +#define MINUTE_IN_SECS 60 +#define HOUR_IN_MINS 60 + +#if defined(_WIN32) +#pragma pack(1) +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + + typedef struct _Qiniu_Date + { + unsigned int year; + unsigned int month; + unsigned int day; + char ok; + } Qiniu_Date; + + typedef struct _Qiniu_Time + { + unsigned int hour; + unsigned int minute; + unsigned int second; + unsigned int fraction; + int offset; // UTC offset in minutes + char ok; + } Qiniu_Time; + + typedef struct _Qiniu_DateTime + { + Qiniu_Date date; + Qiniu_Time time; + char ok; + } Qiniu_DateTime; + +#if defined(_WIN32) +#pragma pack() +#endif + +#ifdef __cplusplus +} +#endif + +#endif // QINIU_RFC3339_H diff --git a/qiniu/rs.c b/qiniu/rs.c index a6da08c9..1b0c4e99 100644 --- a/qiniu/rs.c +++ b/qiniu/rs.c @@ -108,6 +108,10 @@ char *Qiniu_RS_PutPolicy_Token(Qiniu_RS_PutPolicy *auth, Qiniu_Mac *mac) { cJSON_AddNumberToObject(root, "fileType", auth->fileType); } + if (auth->persistentType) + { + cJSON_AddNumberToObject(root, "persistentType", auth->persistentType); + } if (auth->expires) { diff --git a/qiniu/rs.h b/qiniu/rs.h index 474144af..ee0da67d 100644 --- a/qiniu/rs.h +++ b/qiniu/rs.h @@ -12,7 +12,9 @@ #include "http.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -55,6 +57,7 @@ typedef struct _Qiniu_RS_PutPolicy { Qiniu_Uint32 expires; Qiniu_Uint32 deleteAfterDays; Qiniu_Uint32 fileType; + Qiniu_Uint32 persistentType; } Qiniu_RS_PutPolicy; /* @endgist */ @@ -245,7 +248,9 @@ QINIU_DLLAPI extern Qiniu_Error Qiniu_RS_BatchRestoreArchive(Qiniu_Client *self, Qiniu_RS_EntryRestoreArchive *entries, Qiniu_ItemCount entryCount); +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/rsf.h b/qiniu/rsf.h index 66acbb86..f0723f15 100644 --- a/qiniu/rsf.h +++ b/qiniu/rsf.h @@ -7,7 +7,9 @@ #include "http.h" +#if defined(_WIN32) #pragma pack(1) +#endif #ifdef __cplusplus extern "C" @@ -37,7 +39,9 @@ extern "C" const char *prefix, const char *delimiter, const char *marker, int limit); QINIU_DLLAPI extern void Qiniu_RSF_ListRet_Cleanup(Qiniu_RSF_ListRet *self); +#if defined(_WIN32) #pragma pack() +#endif #ifdef __cplusplus } diff --git a/qiniu/tm.h b/qiniu/tm.h index 6f0db071..7d1fbe11 100644 --- a/qiniu/tm.h +++ b/qiniu/tm.h @@ -12,6 +12,10 @@ #include "base.h" +#if defined(_WIN32) +#pragma pack(1) +#endif + #ifdef __cplusplus extern "C" { @@ -22,6 +26,10 @@ extern "C" QINIU_DLLAPI extern const char *Qiniu_MD5_HexStr(const char *src); QINIU_DLLAPI extern const char *Qiniu_MD5_HexStr_From_Reader(Qiniu_Reader r); +#if defined(_WIN32) +#pragma pack() +#endif + #ifdef __cplusplus } #endif diff --git a/vcpkg.json b/vcpkg.json index 845d5be2..e70e89d3 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,6 +1,6 @@ { "name": "qiniu-c-sdk", - "version-string": "7.7.0", + "version-string": "7.8.0", "dependencies": [ "curl", "openssl"