Skip to content

Commit

Permalink
now upload APIs could switch upHosts
Browse files Browse the repository at this point in the history
  • Loading branch information
@bachue
bachue committed Aug 23, 2024
1 parent 61a8b1c commit 685df3a
Show file tree
Hide file tree
Showing 15 changed files with 473 additions and 209 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
- 支持闲时任务和 prefop 接口
- 调整查询区域主备域名
- 支持缓存多个 `v4/query` 查询结果
- 上传重试支持切换上传域名

## v7.7.0 (2023-12-25)

Expand Down
42 changes: 42 additions & 0 deletions qiniu/code.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#include <curl/curl.h>
#include "private/code.h"

Qiniu_Retry_Decision _Qiniu_Should_Retry(int code)
{
if (code / 100 == 4 && code != 406 && code != 429)
{
return QINIU_DONT_RETRY;
}
switch (code)
{
case CURLE_COULDNT_RESOLVE_HOST:
case CURLE_COULDNT_CONNECT:
case CURLE_WEIRD_SERVER_REPLY:
case CURLE_PARTIAL_FILE:
case CURLE_UPLOAD_FAILED:
case CURLE_OPERATION_TIMEDOUT:
case CURLE_SSL_CONNECT_ERROR:
case CURLE_SEND_ERROR:
case CURLE_RECV_ERROR:
case CURLE_SSL_CERTPROBLEM:
case CURLE_SSL_CIPHER:
case CURLE_PEER_FAILED_VERIFICATION:
case CURLE_HTTP2_STREAM:
case CURLE_HTTP3:
case CURLE_QUIC_CONNECT_ERROR:
case 406:
case 429:
case 500:
case 502:
case 503:
case 504:
case 509:
case 571:
case 573:
case 599:
return QINIU_TRY_NEXT_DOMAIN;

default:
return QINIU_DONT_RETRY;
}
}
15 changes: 6 additions & 9 deletions qiniu/fop.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_
err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost);
if (err.code != 200)
{
goto error;
return err;
}
url = Qiniu_String_Concat2(apiHost, "/pfop/");
err = Qiniu_Client_CallWithBuffer(
Expand All @@ -117,14 +117,13 @@ Qiniu_Error Qiniu_FOP_Pfop_v2(Qiniu_Client *self, Qiniu_FOP_PfopRet *ret, Qiniu_
body,
strlen(body),
"application/x-www-form-urlencoded");
Qiniu_Free((void *)body);
Qiniu_Free((void *)url);
if (err.code == 200)
{
ret->persistentId = Qiniu_Json_GetString(root, "persistentId", 0);
}

error:
Qiniu_Free(body);
Qiniu_Free(url);
return err;
}

Expand Down Expand Up @@ -159,7 +158,7 @@ Qiniu_Error Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu
err = _Qiniu_Region_Get_Api_Host(self, NULL, NULL, &apiHost);
if (err.code != 200)
{
goto error;
return err;
}
url = Qiniu_String_Concat(apiHost, "/status/get/prefop?id=", encodedPersistentId, NULL);
if (escapePersistentIdOk)
Expand All @@ -176,9 +175,10 @@ Qiniu_Error Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu
"application/x-www-form-urlencoded",
"GET",
NULL);
Qiniu_Free(url);
if (err.code != 200)
{
goto error;
return err;
}

ret->id = Qiniu_Json_GetString(root, "id", NULL);
Expand Down Expand Up @@ -206,8 +206,5 @@ Qiniu_Error Qiniu_FOP_Prefop(Qiniu_Client *self, Qiniu_FOP_PrefopRet *ret, Qiniu
itemsRet[curIndex].key = Qiniu_Json_GetString(item, "key", NULL);
itemsRet[curIndex].returnOld = Qiniu_Json_GetInt(item, "returnOld", 0);
}

error:
Qiniu_Free(url);
return err;
}
8 changes: 7 additions & 1 deletion qiniu/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize)
Qiniu_Zero_Ptr(self);
self->curl = curl_easy_init();
self->auth = auth;
self->hostsRetriesMax = 3;

Qiniu_Buffer_Init(&self->b, bufSize);
Qiniu_Buffer_Init(&self->respHeader, bufSize);
Expand Down Expand Up @@ -396,6 +397,11 @@ void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long
self->lowSpeedTime = lowSpeedTime;
} // Qiniu_Client_SetLowSpeedLimit

void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t hostsRetriesMax)
{
self->hostsRetriesMax = hostsRetriesMax;
} // Qiniu_Client_SetMaximumHostsRetries

void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs)
{
self->timeoutMs = timeoutMs;
Expand Down Expand Up @@ -475,7 +481,7 @@ static Qiniu_Error Qiniu_Client_callWithBody(
Qiniu_Error err;
const char *ctxType;
char ctxLength[64], userAgent[64];
Qiniu_Header *headers = NULL;
Qiniu_Header *headers;
CURL *curl = (CURL *)self->curl;
err = Qiniu_Client_config(self);
if (err.code != 200)
Expand Down
4 changes: 4 additions & 0 deletions qiniu/http.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,12 +141,16 @@ extern "C"

// Millisecond timeout for the connection phase.
long connectTimeoutMs;

// Max retries count.
size_t hostsRetriesMax;
} Qiniu_Client;

QINIU_DLLAPI extern void Qiniu_Client_InitEx(Qiniu_Client *self, Qiniu_Auth auth, size_t bufSize);
QINIU_DLLAPI extern void Qiniu_Client_Cleanup(Qiniu_Client *self);
QINIU_DLLAPI extern void Qiniu_Client_BindNic(Qiniu_Client *self, const char *nic);
QINIU_DLLAPI extern void Qiniu_Client_SetLowSpeedLimit(Qiniu_Client *self, long lowSpeedLimit, long lowSpeedTime);
QINIU_DLLAPI extern void Qiniu_Client_SetMaximumHostsRetries(Qiniu_Client *self, size_t retries);
QINIU_DLLAPI extern void Qiniu_Client_SetTimeout(Qiniu_Client *self, long timeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_SetConnectTimeout(Qiniu_Client *self, long connectTimeoutMs);
QINIU_DLLAPI extern void Qiniu_Client_EnableAutoQuery(Qiniu_Client *self, Qiniu_Bool useHttps);
Expand Down
139 changes: 87 additions & 52 deletions qiniu/io.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#include "reader.h"
#include "recorder_utils.h"
#include "private/region.h"
#include "private/code.h"
#include <curl/curl.h>

/*============================================================================*/
Expand Down Expand Up @@ -58,20 +59,27 @@ CURL *Qiniu_Client_reset(Qiniu_Client *self);

Qiniu_Error Qiniu_callex(CURL *curl, Qiniu_Buffer *resp, Qiniu_Json **ret, Qiniu_Bool simpleError, Qiniu_Buffer *resph);

static Qiniu_Error Get_Qiniu_UpHost(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char **upHost)
static Qiniu_Error _Qiniu_Get_UpHosts(Qiniu_Client *client, const char *accessKey, const char *bucketName, Qiniu_Io_PutExtra *extra, const char *const **upHosts, size_t *upHostsCount)
{
if (extra && extra->ipCount != 0)
{
Qiniu_Count oldIndex = Qiniu_Count_Inc(&extra->ipIndex);
*upHost = extra->upIps[labs(oldIndex % extra->ipCount)];
*upHosts = &extra->upIps[labs(oldIndex % extra->ipCount)];
*upHostsCount = 1;
}
else if (extra && extra->upHost != NULL)
{
*upHost = extra->upHost;
*upHosts = &extra->upHost;
*upHostsCount = 1;
}
else if (extra && extra->upHosts != NULL && extra->upHostsCount != 0)
{
*upHosts = extra->upHosts;
*upHostsCount = extra->upHostsCount;
}
else
{
Qiniu_Error err = _Qiniu_Region_Get_Up_Host(client, accessKey, bucketName, upHost);
Qiniu_Error err = _Qiniu_Region_Get_Up_Hosts(client, accessKey, bucketName, upHosts, upHostsCount);
if (err.code != 200)
{
return err;
Expand Down Expand Up @@ -150,48 +158,62 @@ static Qiniu_Error Qiniu_Io_call(
{
int retCode = 0;
Qiniu_Error err;
struct curl_slist *headers = NULL;
const char *upHost = NULL;
struct curl_slist *headers;
const char *const *upHosts;
const char *defaultUpHosts[] = {QINIU_UP_HOST};
size_t upHostsCount;

err = Qiniu_Client_config(self);
headers = curl_slist_append(NULL, "Expect:");
err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != 200)
{
return err;
}
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
if (err.code != 200)
if (upHostsCount == 0)
{
return err;
upHosts = defaultUpHosts;
upHostsCount = 1;
}

headers = curl_slist_append(NULL, "Expect:");
for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++)
{
CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
if (err.code != 200)
{
return err;
}

CURL *curl = Qiniu_Client_reset(self);
curl_easy_setopt(curl, CURLOPT_URL, upHost);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

//// For aborting uploading file.
if (extra->upAbortCallback)
{
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
} // if
//// For aborting uploading file.
if (extra->upAbortCallback)
{
curl_easy_setopt(curl, CURLOPT_READFUNCTION, Qiniu_Rd_Reader_Callback);
} // if

err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == 200 && ret != NULL)
{
if (extra->callbackRetParser != NULL)
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == 200)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
if (extra->callbackRetParser != NULL)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
}
else if (ret != NULL)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
}
break;
}
else
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
ret->persistentId = Qiniu_Json_GetString(self->root, "persistentId", NULL);
break;
}
}

curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
Expand Down Expand Up @@ -313,7 +335,6 @@ Qiniu_Error Qiniu_Io_PutBuffer(
CURLFORM_BUFFER, key, CURLFORM_BUFFERPTR, buf, CURLFORM_BUFFERLENGTH, fsize, CURLFORM_END);
err = Qiniu_Io_call(self, accessKey, bucketName, ret, form.formpost, extra);

error:
Qiniu_Free((void *)accessKey);
Qiniu_Free((void *)bucketName);
return err;
Expand All @@ -327,42 +348,56 @@ static Qiniu_Error Qiniu_Io_call_with_callback(
{
int retCode = 0;
Qiniu_Error err;
struct curl_slist *headers = NULL;
const char *upHost;
struct curl_slist *headers;
const char *const *upHosts;
const char *defaultUpHosts[] = {QINIU_UP_HOST};
size_t upHostsCount;

CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
headers = curl_slist_append(NULL, "Expect:");
err = _Qiniu_Get_UpHosts(self, accessKey, bucketName, extra, &upHosts, &upHostsCount);
if (err.code != 200)
{
return err;
}
err = Get_Qiniu_UpHost(self, accessKey, bucketName, extra, &upHost);
if (err.code != 200)
if (upHostsCount == 0)
{
return err;
upHosts = defaultUpHosts;
upHostsCount = 1;
}

headers = curl_slist_append(NULL, "Expect:");
for (size_t retries = 0; retries <= self->hostsRetriesMax; retries++)
{
CURL *curl = Qiniu_Client_reset(self);
err = Qiniu_Client_config(self);
if (err.code != 200)
{
return err;
}

curl_easy_setopt(curl, CURLOPT_URL, upHost);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);
curl_easy_setopt(curl, CURLOPT_URL, upHosts[retries % upHostsCount]);
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
curl_easy_setopt(curl, CURLOPT_READFUNCTION, rdr);

err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == 200 && ret != NULL)
{
if (extra->callbackRetParser != NULL)
err = Qiniu_callex(curl, &self->b, &self->root, Qiniu_False, &self->respHeader);
if (err.code == 200)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
if (extra->callbackRetParser != NULL)
{
err = (*extra->callbackRetParser)(extra->callbackRet, self->root);
}
else if (ret != NULL)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
}
break;
}
else
else if (_Qiniu_Should_Retry(err.code) == QINIU_DONT_RETRY)
{
ret->hash = Qiniu_Json_GetString(self->root, "hash", NULL);
ret->key = Qiniu_Json_GetString(self->root, "key", NULL);
break;
}
}

curl_formfree(formpost);
curl_slist_free_all(headers);
return err;
Expand Down
5 changes: 5 additions & 0 deletions qiniu/io.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,15 @@ extern "C"
void *upAbortUserData;
Qiniu_Rd_FnAbort upAbortCallback;

// Deprecated fields, prefer upHosts.
const char *upHost;
const char **upIps;
Qiniu_Count ipCount;
Qiniu_Count ipIndex;

// Specify multiple upHosts
const char *const *upHosts;
size_t upHostsCount;
} Qiniu_Io_PutExtra;

/*============================================================================*/
Expand Down
Loading

0 comments on commit 685df3a

Please sign in to comment.