From 84a2fdfe426fdc7f45438084e8a770ed67a25f63 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Wed, 23 Aug 2017 06:45:38 -0400 Subject: [PATCH 01/10] refine speed --- pcs/pcs_http.c | 29 ++++++++++++++++++++--------- shell.c | 12 +++++------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pcs/pcs_http.c b/pcs/pcs_http.c index 8212a2bd..875aceae 100644 --- a/pcs/pcs_http.c +++ b/pcs/pcs_http.c @@ -89,9 +89,11 @@ enum HttpMethod HTTP_METHOD_POST }; -static inline void pcs_http_prepare(struct pcs_http *http, enum HttpMethod method, const char *url, PcsBool follow_location, +static inline void pcs_http_prepare(struct pcs_http *http, enum HttpMethod method, + const char *url, PcsBool follow_location, PcsHttpWriteFunction write_func, void *state, curl_off_t max_recv_speed, curl_off_t max_send_speed, + long min_recv_speed, long min_recv_speed_time, curl_off_t resume_from, curl_off_t max_length) { pcs_http_reset_response(http); @@ -113,6 +115,15 @@ static inline void pcs_http_prepare(struct pcs_http *http, enum HttpMethod metho curl_easy_setopt(http->curl, CURLOPT_WRITEFUNCTION, write_func); curl_easy_setopt(http->curl, CURLOPT_WRITEDATA, state); + if (min_recv_speed > 0 && min_recv_speed_time > 0) { + curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_LIMIT, min_recv_speed); + curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_TIME, min_recv_speed_time); + } + else { + curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); + curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_TIME, 30L); + } + // 设置文件续传的位置给libcurl if (resume_from && max_length) { char *range = pcs_utils_sprintf("%" PRId64 "-%" PRId64, resume_from, resume_from + max_length - 1); @@ -455,8 +466,6 @@ PCS_API PcsHttp pcs_http_create(const char *cookie_file) } curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L); - curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_LIMIT, 1024L); - curl_easy_setopt(http->curl, CURLOPT_LOW_SPEED_TIME, 60L); curl_easy_setopt(http->curl, CURLOPT_SSL_VERIFYHOST, 0L); curl_easy_setopt(http->curl, CURLOPT_USERAGENT, USAGE); curl_easy_setopt(http->curl, CURLOPT_FOLLOWLOCATION, 1L); @@ -807,7 +816,7 @@ PCS_API int pcs_http_get_response_size(PcsHttp handle) PCS_API char *pcs_http_get(PcsHttp handle, const char *url, PcsBool follow_location) { struct pcs_http *http = (struct pcs_http *)handle; - pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0, 0); return pcs_http_perform(http, url); } @@ -815,7 +824,7 @@ PCS_API char *pcs_http_get_raw(PcsHttp handle, const char *url, PcsBool follow_l { struct pcs_http *http = (struct pcs_http *)handle; char *data; - pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0); http->res_type = PCS_HTTP_RES_TYPE_RAW; data = pcs_http_perform(http, url); if (sz) *sz = http->res_body_size; @@ -825,7 +834,7 @@ PCS_API char *pcs_http_get_raw(PcsHttp handle, const char *url, PcsBool follow_l PCS_API char *pcs_http_post(PcsHttp handle, const char *url, char *post_data, PcsBool follow_location) { struct pcs_http *http = (struct pcs_http *)handle; - pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0, 0); if (post_data) curl_easy_setopt(http->curl, CURLOPT_POSTFIELDS, post_data); else @@ -837,7 +846,9 @@ PCS_API PcsBool pcs_http_get_download(PcsHttp handle, const char *url, PcsBool f { struct pcs_http *http = (struct pcs_http *)handle; pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, - max_speed, 0, resume_from, max_length); + max_speed, 0, + 5 * 1024L, 10L, /* 10秒内的平均下载速度低于 5KB/s 的话，则取消下载 */ + resume_from, max_length); http->res_type = PCS_HTTP_RES_TYPE_DOWNLOAD; pcs_http_perform(http, url); return http->strerror == NULL ? PcsTrue : PcsFalse; @@ -853,7 +864,7 @@ PCS_API int64_t pcs_http_get_download_filesize(PcsHttp handle, const char *url, CURLcode res; double downloadFileLenth = 0; struct pcs_http *http = (struct pcs_http *)handle; - pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, pcs_http_null_write, NULL, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, pcs_http_null_write, NULL, 0, 0, 0, 0, 0, 0); curl_easy_setopt(http->curl, CURLOPT_NOBODY, 1L); //不需要body res = curl_easy_perform(http->curl); if (res == CURLE_OK) { @@ -967,7 +978,7 @@ PCS_API char *pcs_post_httpform(PcsHttp handle, const char *url, PcsHttpForm dat struct pcs_http *http = (struct pcs_http *)handle; struct http_post *formpost = (struct http_post *)data; char *rc; - pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, max_speed, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, max_speed, 0, 0, 0, 0); if (data){ curl_easy_setopt(http->curl, CURLOPT_HTTPPOST, formpost->formpost); if (formpost->read_func) { diff --git a/shell.c b/shell.c index 2d3f500a..c42fc492 100644 --- a/shell.c +++ b/shell.c @@ -47,7 +47,7 @@ #define USAGE "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36" #define TIMEOUT 60 -#define CONNECTTIMEOUT 10 +#define CONNECTTIMEOUT 1 #define MAX_THREAD_NUM 100 #define MIN_SLICE_SIZE (512 * 1024) /*最小分片大小*/ #define MAX_SLICE_SIZE (10 * 1024 * 1024) /*最大分片大小*/ @@ -3365,6 +3365,7 @@ static void *download_thread(void *params) ShellContext *context = ds->context; struct DownloadThreadState *ts = pop_download_threadstate(ds); Pcs *pcs; + srand((unsigned int)time(NULL)); if (ts == NULL) { lock_for_download(ds); ds->num_of_running_thread--; @@ -3417,12 +3418,9 @@ static void *download_thread(void *params) } //ds->status = DOWNLOAD_STATUS_FAIL; unlock_for_download(ds); - delay = (rand() % 10); -#ifdef _WIN32 - printf("Download slice failed, retry delay %d second, tid: %x. message: %s\n", delay, GetCurrentThreadId(), pcs_strerror(pcs)); -#else - printf("Download slice failed, retry delay %d second, tid: %p. message: %s\n", delay, pthread_self(), pcs_strerror(pcs)); -#endif + delay = rand(); + delay %= 10; + printf("Low speed, retry delay %d second...\n", delay); sleep(delay); /*10秒后重试*/ continue; } From e41d8f913143b25adf385c5acd4cd6fd5d0cbd88 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Wed, 23 Aug 2017 06:50:01 -0400 Subject: [PATCH 02/10] typo --- shell.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.c b/shell.c index c42fc492..be004064 100644 --- a/shell.c +++ b/shell.c @@ -3420,7 +3420,7 @@ static void *download_thread(void *params) unlock_for_download(ds); delay = rand(); delay %= 10; - printf("Low speed, retry delay %d second...\n", delay); + printf("Low speed, retry after %d seconds.\n", delay); sleep(delay); /*10秒后重试*/ continue; } From cabf074951631ef1bd7ac5702afa2781a58b7cd7 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Wed, 23 Aug 2017 06:56:14 -0400 Subject: [PATCH 03/10] tiny fix --- pcs/pcs_http.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcs/pcs_http.c b/pcs/pcs_http.c index 875aceae..6f264912 100644 --- a/pcs/pcs_http.c +++ b/pcs/pcs_http.c @@ -816,7 +816,7 @@ PCS_API int pcs_http_get_response_size(PcsHttp handle) PCS_API char *pcs_http_get(PcsHttp handle, const char *url, PcsBool follow_location) { struct pcs_http *http = (struct pcs_http *)handle; - pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_GET, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0); return pcs_http_perform(http, url); } @@ -834,7 +834,7 @@ PCS_API char *pcs_http_get_raw(PcsHttp handle, const char *url, PcsBool follow_l PCS_API char *pcs_http_post(PcsHttp handle, const char *url, char *post_data, PcsBool follow_location) { struct pcs_http *http = (struct pcs_http *)handle; - pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0, 0); + pcs_http_prepare(http, HTTP_METHOD_POST, url, follow_location, &pcs_http_write, http, 0, 0, 0, 0, 0, 0); if (post_data) curl_easy_setopt(http->curl, CURLOPT_POSTFIELDS, post_data); else From 4d5a6541b56b56d83b4d17d13752812711362a29 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Wed, 23 Aug 2017 07:06:50 -0400 Subject: [PATCH 04/10] typo --- shell.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.c b/shell.c index be004064..053f5d21 100644 --- a/shell.c +++ b/shell.c @@ -3420,7 +3420,7 @@ static void *download_thread(void *params) unlock_for_download(ds); delay = rand(); delay %= 10; - printf("Low speed, retry after %d seconds.\n", delay); + printf("Low speed, retry after %d seconds. \n", delay); sleep(delay); /*10秒后重试*/ continue; } From ec4beb1863ac51ec06b663300e821ad2111c8845 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Wed, 23 Aug 2017 22:50:40 -0400 Subject: [PATCH 05/10] remove low speed message --- shell.c | 1 - 1 file changed, 1 deletion(-) diff --git a/shell.c b/shell.c index 053f5d21..7fda27c5 100644 --- a/shell.c +++ b/shell.c @@ -3420,7 +3420,6 @@ static void *download_thread(void *params) unlock_for_download(ds); delay = rand(); delay %= 10; - printf("Low speed, retry after %d seconds. \n", delay); sleep(delay); /*10秒后重试*/ continue; } From 56b3ffbb89c36f869ee8a2662baea132aaed7233 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Thu, 24 Aug 2017 01:42:39 -0400 Subject: [PATCH 06/10] disk cache --- Makefile.am | 4 +- Makefile.old | 5 +- README.md | 22 ++++- cache.c | 115 ++++++++++++++++++++++++++ cache.h | 40 +++++++++ pcs_shell.vcxproj | 2 + pcs_shell.vcxproj.filters | 6 ++ shell.c | 166 ++++++++++++++++++++++++++++---------- shell.h | 2 +- 9 files changed, 311 insertions(+), 51 deletions(-) create mode 100644 cache.c create mode 100644 cache.h diff --git a/Makefile.am b/Makefile.am index 384af98c..1dad386c 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,7 +31,8 @@ shell_src = shell.c \ rb_tree/red_black_tree.c \ rb_tree/stack.c \ utils.c \ - hashtable.c + hashtable.c \ + cache.c baidupcs_SOURCES = \ $(shell_src) \ @@ -68,6 +69,7 @@ noinst_HEADERS = pcs/BaiduPCS.h \ arg.h \ dir.h \ hashtable.h \ + cache.h \ rb_tree/misc.h \ rb_tree/red_black_tree.h \ rb_tree/stack.h diff --git a/Makefile.old b/Makefile.old index 97f4cc3e..764d3ffb 100644 --- a/Makefile.old +++ b/Makefile.old @@ -24,7 +24,8 @@ SHELL_OBJS = bin/shell_arg.o \ bin/rb_tree_stack.o \ bin/red_black_tree.o \ bin/shell_utils.o \ - bin/hashtable.o + bin/hashtable.o \ + bin/cache.o #CCFLAGS = -DHAVE_ASPRINTF -DHAVE_ICONV ifeq ($(LC_OS_NAME), cygwin) @@ -75,6 +76,8 @@ bin/rb_tree_stack.o: rb_tree/stack.c rb_tree/stack.h $(CC) -o $@ -c $(PCS_CCFLAGS) rb_tree/stack.c bin/red_black_tree.o: rb_tree/red_black_tree.c rb_tree/red_black_tree.h $(CC) -o $@ -c $(PCS_CCFLAGS) rb_tree/red_black_tree.c +bin/cache.o: cache.c cache.h + $(CC) -o $@ -c $(PCS_CCFLAGS) cache.c bin/cJSON.o: pcs/cJSON.c pcs/cJSON.h $(CC) -o $@ -c $(PCS_CCFLAGS) pcs/cJSON.c diff --git a/README.md b/README.md index 7f8dad9e..5a911831 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,7 @@ C/C++写的一个百度网盘工具，可以在linux终端中使用。 cd .. sudo apt install ./baidupcs_*.deb - -编译 (Debian)： +编译 (Debian) (新方法)： =================================== 程序依赖于 libcurl。 @@ -35,12 +34,27 @@ C/C++写的一个百度网盘工具，可以在linux终端中使用。 git clone https://github.com/GangZhuo/BaiduPCS.git ### 3. 编译源代码 cd BaiduPCS - make clean - make + ./configure && make make install #将安装到/usr/local/bin下 ### 4. 手动安装到其他目录，例如 /usr/bin 下 cp ./baidupcs /usr/bin/ +编译 (Debian)： +=================================== +程序依赖于 libcurl。 + +### 1. 安装依赖 + apt-get install build-essential libcurl4-openssl-dev libssl-dev +### 2. 获取源代码 + git clone https://github.com/GangZhuo/BaiduPCS.git +### 3. 编译源代码 + cd BaiduPCS + make clean -f Makefile.old + make -f Makefile.old + make install -f Makefile.old #将安装到/usr/local/bin下 +### 4. 手动安装到其他目录，例如 /usr/bin 下 + cp ./bin/pcs /usr/bin/ + 编译 (Windows)： =================================== ### 1. 获取源代码 diff --git a/cache.c b/cache.c new file mode 100644 index 00000000..253ff85d --- /dev/null +++ b/cache.c @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include +#include +#ifdef WIN32 +# include +# include +#else +# include +#endif + +#include "cache.h" +#include "pcs/pcs_mem.h" + +#ifdef WIN32 +#ifndef __MINGW32__ +# define lseek _lseek +# define fileno _fileno +# define fseeko _fseeki64 +# define ftello _ftelli64 +#endif +#endif + +int cache_init(cathe_t *cache) +{ + memset(cache, 0, sizeof(cathe_t)); + cache->blocks.prev = &cache->blocks; + cache->blocks.next = &cache->blocks; + return 0; +} + +void cache_uninit(cathe_t *cache) +{ + if (cache) { + block_t *block = cache->blocks.next, *tmp; + while (block != &cache->blocks) { + tmp = block; + block = block->next; + pcs_free(tmp->data); + pcs_free(tmp); + } + } +} + +int cache_reset(cathe_t *cache) +{ + cache_uninit(cache); + cache->blocks.prev = &cache->blocks; + cache->blocks.next = &cache->blocks; + cache->total_size = 0; + return 0; +} + +block_t *cache_newblock(curl_off_t start, char *data, size_t size) +{ + block_t *block; + block = pcs_malloc(sizeof(block_t)); + if (!block) + return NULL; + memset(block, 0, sizeof(block_t)); + block->start = start; + block->size = size; + block->data = pcs_malloc(size); + if (!block->data) { + free(block); + return NULL; + } + memcpy(block->data, data, size); + return block; +} + +int cache_addblock(cathe_t *cache, block_t *block) +{ + block_t *pos = cache->blocks.next; + cache->total_size += block->size; + /* find position */ + while (pos != &cache->blocks) { + if (pos->start > block->start) + break; + pos = pos->next; + } + block->next = pos; + block->prev = pos->prev; + pos->prev->next = block; + pos->prev = block; + return 0; +} + +int cache_add(cathe_t *cache, curl_off_t start, char *data, size_t size) +{ + block_t *block, *pos = cache->blocks.next; + block = cache_newblock(start, data, size); + if (!block) + return -1; + return cache_addblock(cache, block); +} + +int cache_flush(cathe_t *cache) +{ + block_t *block = cache->blocks.next; + int r; + while (block != &cache->blocks) { + r = fseeko((cache->fp), block->start, SEEK_SET); + if (r) + return -1; + r = fwrite(block->data, 1, block->size, cache->fp); + if (r != block->size) + return -1; + block = block->next; + } + return 0; +} + diff --git a/cache.h b/cache.h new file mode 100644 index 00000000..670fdb17 --- /dev/null +++ b/cache.h @@ -0,0 +1,40 @@ +#ifndef _PCS_SHELL_CAHCE_H +#define _PCS_SHELL_CAHCE_H + +#include "pcs/pcs.h" + +#define MAX_CACHE_SIZE (1 * 1024) /* 1MiB */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct block { + curl_off_t start; + size_t size; + char *data; + struct block *prev; + struct block *next; +} block_t; + +typedef struct cache { + block_t blocks; + FILE *fp; + size_t total_size; +} cathe_t; + +int cache_init(cathe_t *cache); + +void cache_uninit(cathe_t *cache); + +int cache_add(cathe_t *cache, curl_off_t start, char *data, size_t size); + +int cache_flush(cathe_t *cache); + +int cache_reset(cathe_t *cache); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/pcs_shell.vcxproj b/pcs_shell.vcxproj index a8cff59d..774cd8b1 100644 --- a/pcs_shell.vcxproj +++ b/pcs_shell.vcxproj @@ -102,6 +102,7 @@ + @@ -124,6 +125,7 @@ + diff --git a/pcs_shell.vcxproj.filters b/pcs_shell.vcxproj.filters index e2c69c89..82cb42dc 100644 --- a/pcs_shell.vcxproj.filters +++ b/pcs_shell.vcxproj.filters @@ -87,6 +87,9 @@ Source Files\pcs + + Source Files + @@ -161,6 +164,9 @@ Header Files\pcs + + Header Files + diff --git a/shell.c b/shell.c index 7fda27c5..d3fac3ef 100644 --- a/shell.c +++ b/shell.c @@ -34,6 +34,7 @@ #include "dir.h" #include "utils.h" #include "arg.h" +#include "cache.h" #ifdef WIN32 # include "pcs/utf8.h" #ifndef __MINGW32__ @@ -58,6 +59,8 @@ #define convert_to_real_speed(speed) ((speed) * 1024) +#define convert_to_real_cache_size(size) ((size) * 1024) + #define PCS_CONTEXT_ENV "PCS_CONTEXT" #define PCS_COOKIE_ENV "PCS_COOKIE" #define PCS_CAPTCHA_ENV "PCS_CAPTCHA" @@ -181,7 +184,6 @@ struct DownloadState FILE *pf; int64_t downloaded_size; /*已经下载的字节数*/ curl_off_t resume_from; /*断点续传时，从这个位置开始续传*/ - size_t noflush_size; /*未执行fflush()的字节大小*/ time_t time; /*最后一次在屏幕打印信息的时间*/ size_t speed; /*用于统计下载速度*/ int64_t file_size; /*完整的文件的字节大小*/ @@ -193,6 +195,7 @@ struct DownloadState int status; const char *remote_file; struct DownloadThreadState *threads; + cathe_t cache; }; struct DownloadThreadState @@ -1031,6 +1034,7 @@ static const char *captchafile() static void init_download_state(struct DownloadState *ds) { memset(ds, 0, sizeof(struct DownloadState)); + cache_init(&ds->cache); #ifdef _WIN32 ds->mutex = CreateMutex(NULL, FALSE, NULL); #else @@ -1053,6 +1057,7 @@ static void uninit_download_state(struct DownloadState *ds) ts = ts->next; pcs_free(ps); } + cache_uninit(&ds->cache); } static void lock_for_download(struct DownloadState *ds) @@ -1252,22 +1257,21 @@ static int download_write(char *ptr, size_t size, size_t contentlength, void *us { struct DownloadState *ds = (struct DownloadState *)userdata; FILE *pf = ds->pf; - size_t i; time_t tm; char tmp[64]; + int rc; tmp[63] = '\0'; - i = fwrite(ptr, 1, size, pf); - if (i != size) { - unlock_for_download(ds); + rc = cache_add(&ds->cache, ds->downloaded_size, ptr, size); + if (rc) return 0; + ds->downloaded_size += size; + ds->speed += size; + if (ds->cache.total_size >= convert_to_real_cache_size(ds->context->cache_size)) { + rc = cache_flush(&ds->cache); + if (rc) + return 0; } - ds->downloaded_size += i; - ds->speed += i; - ds->noflush_size += i; - if (ds->noflush_size > MAX_FFLUSH_SIZE) { - fflush(pf); - ds->noflush_size = 0; - } + tm = time(&tm); if (tm != ds->time) { int64_t left_size = ds->file_size - ds->downloaded_size; @@ -1281,7 +1285,7 @@ static int download_write(char *ptr, size_t size, size_t contentlength, void *us fflush(stdout); ds->speed = 0; } - return i; + return size; } static int download_write_for_multy_thread(char *ptr, size_t size, size_t contentlength, void *userdata) @@ -1295,27 +1299,16 @@ static int download_write_for_multy_thread(char *ptr, size_t size, size_t conten int rc; tmp[63] = '\0'; lock_for_download(ds); - //lseek(fileno(pf), ts->start, SEEK_SET); - rc = fseeko((pf), ts->start, SEEK_SET); - if (rc) { - if (ds->pErrMsg) { - if (*(ds->pErrMsg)) pcs_free(*(ds->pErrMsg)); - (*(ds->pErrMsg)) = pcs_utils_sprintf("fseeko() error."); - } - ds->status = DOWNLOAD_STATUS_WRITE_FILE_FAIL; - unlock_for_download(ds); - return 0; - } if (ts->start + size > ts->end) { size = (size_t)(ts->end - ts->start); ts->status = DOWNLOAD_STATUS_OK; } if (size > 0) { - rc = fwrite(ptr, 1, size, pf); - if (rc != size) { + rc = cache_add(&ds->cache, ts->start, ptr, size); + if (rc) { if (ds->pErrMsg) { if (*(ds->pErrMsg)) pcs_free(*(ds->pErrMsg)); - (*(ds->pErrMsg)) = pcs_utils_sprintf("fwrite() error."); + (*(ds->pErrMsg)) = pcs_utils_sprintf("cache_add() error."); } ds->status = DOWNLOAD_STATUS_WRITE_FILE_FAIL; unlock_for_download(ds); @@ -1325,26 +1318,35 @@ static int download_write_for_multy_thread(char *ptr, size_t size, size_t conten ds->downloaded_size += size; ts->start += size; ds->speed += size; - ds->noflush_size += size; if (ts->start == ts->end) { ts->status = DOWNLOAD_STATUS_OK; size = 0; } - if (save_thread_states_to_file(pf, ds->file_size, ds->threads)) { - if (ds->pErrMsg) { - if (*(ds->pErrMsg)) pcs_free(*(ds->pErrMsg)); - (*(ds->pErrMsg)) = pcs_utils_sprintf("save slices error."); + if (ds->cache.total_size >= convert_to_real_cache_size(context->cache_size)) { + rc = cache_flush(&ds->cache); + if (rc) { + if (ds->pErrMsg) { + if (*(ds->pErrMsg)) pcs_free(*(ds->pErrMsg)); + (*(ds->pErrMsg)) = pcs_utils_sprintf("cache_flush() error."); + } + ds->status = DOWNLOAD_STATUS_WRITE_FILE_FAIL; + unlock_for_download(ds); + return 0; } - ds->status = DOWNLOAD_STATUS_WRITE_FILE_FAIL; - unlock_for_download(ds); - return 0; + rc = save_thread_states_to_file(pf, ds->file_size, ds->threads); + if (rc) { + if (ds->pErrMsg) { + if (*(ds->pErrMsg)) pcs_free(*(ds->pErrMsg)); + (*(ds->pErrMsg)) = pcs_utils_sprintf("save slices error."); + } + ds->status = DOWNLOAD_STATUS_WRITE_FILE_FAIL; + unlock_for_download(ds); + return 0; + } + cache_reset(&ds->cache); } - //if (ds->noflush_size > MAX_FFLUSH_SIZE) { - // fflush(pf); - // ds->noflush_size = 0; - //} tm = time(&tm); if (tm != ds->time) { int64_t left_size = ds->file_size - ds->downloaded_size; @@ -1359,7 +1361,6 @@ static int download_write_for_multy_thread(char *ptr, size_t size, size_t conten fflush(stdout); ds->speed = 0; } - unlock_for_download(ds); return size; } @@ -1796,6 +1797,10 @@ static char *context2str(ShellContext *context) assert(item); cJSON_AddItemToObject(root, "user_agent", item); + item = cJSON_CreateNumber(context->cache_size); + assert(item); + cJSON_AddItemToObject(root, "cache_size", item); + json = cJSON_Print(root); assert(json); @@ -1994,6 +1999,16 @@ static int restore_context(ShellContext *context, const char *filename) context->user_agent = pcs_utils_strdup(item->valuestring); } + item = cJSON_GetObjectItem(root, "cache_size"); + if (item) { + if (((int)item->valueint) < 0) { + printf("warning: Invalid context.cache_size, the value should be >= 0, use default value: %d.\n", context->cache_size); + } + else { + context->cache_size = (int)item->valueint; + } + } + cJSON_Delete(root); pcs_free(filecontent); return 0; @@ -2019,6 +2034,7 @@ static void init_context(ShellContext *context, struct args *arg) context->max_thread = 1; context->max_speed_per_thread = 0; context->max_upload_speed_per_thread = 0; + context->cache_size = MAX_CACHE_SIZE; context->user_agent = pcs_utils_strdup(USAGE); } @@ -2438,6 +2454,7 @@ static void usage_set() printf(" max_speed_per_thread Int >= 0. The max speed in KiB per thread.\n"); printf(" max_upload_speed_per_thread Int >= 0. The max speed in KiB per thread.\n"); printf(" user_agent String set user agent.\n"); + printf(" cache_size Int >= 0. The max cache size in KiB.\n"); printf("\nSamples:\n"); printf(" %s set -h\n", app_name); printf(" %s set --cookie_file=\"/tmp/pcs.cookie\"\n", app_name); @@ -2780,6 +2797,23 @@ static int set_max_upload_speed_per_thread(ShellContext *context, const char *va return 0; } +/*设置上下文中的cache_size值*/ +static int set_cache_size(ShellContext *context, const char *val) +{ + const char *p = val; + int v; + if (!val || !val[0]) return -1; + while (*p) { + if (*p < '0' || *p > '9') + return -1; + p++; + } + v = atoi(val); + if (v < 0) return -1; + context->cache_size = v; + return 0; +} + /*设置上下文中的 user_agent 值*/ static int set_user_agent(ShellContext *context, const char *val) { @@ -3691,11 +3725,28 @@ static inline int do_download(ShellContext *context, uninit_download_state(&ds); return -1; } + ds.cache.fp = ds.pf; pcs_setopts(context->pcs, PCS_OPTION_DOWNLOAD_WRITE_FUNCTION, &download_write, PCS_OPTION_DOWNLOAD_WRITE_FUNCTION_DATA, &ds, PCS_OPTION_END); res = pcs_download(context->pcs, remote_path, convert_to_real_speed(context->max_speed_per_thread), 0, 0); + if (ds.cache.total_size > 0) { + int rc = cache_flush(&ds.cache); + if (rc) { + if (pErrMsg) { + if (*pErrMsg) pcs_free(*pErrMsg); + (*pErrMsg) = pcs_utils_sprintf("%s\n", "cache_flush() error."); + } + if (op_st) (*op_st) = OP_ST_FAIL; + fclose(ds.pf); + pcs_free(tmp_local_path); + pcs_free(local_path); + pcs_free(remote_path); + uninit_download_state(&ds); + return -1; + } + } fclose(ds.pf); if (res != PCS_OK) { if (pErrMsg) { @@ -3790,13 +3841,14 @@ static inline int do_download(ShellContext *context, uninit_download_state(&ds); return -1; } + ds.cache.fp = ds.pf; //保存分片数据 - printf("Saving slices...\r"); + printf("Create/Load temporary file...\r"); fflush(stdout); if (save_thread_states_to_file(ds.pf, ds.file_size, ds.threads)) { if (pErrMsg) { if (*pErrMsg) pcs_free(*pErrMsg); - (*pErrMsg) = pcs_utils_sprintf("Can't save slices into temp file: %s \n", tmp_local_path); + (*pErrMsg) = pcs_utils_sprintf("Can't save slices into temporary file: %s \n", tmp_local_path); } if (op_st) (*op_st) = OP_ST_FAIL; DeleteFileRecursive(tmp_local_path); @@ -3817,6 +3869,7 @@ static inline int do_download(ShellContext *context, thread_count = context->max_thread; ds.num_of_running_thread = thread_count; //printf("\nthread: %d, slice: %d\n", thread_count, ds.num_of_slice); + srand((unsigned int)time(NULL)); #ifdef _WIN32 handles = (HANDLE *)pcs_malloc(sizeof(HANDLE) * thread_count); memset(handles, 0, sizeof(HANDLE) * thread_count); @@ -3841,6 +3894,24 @@ static inline int do_download(ShellContext *context, sleep(1); } + if (ds.cache.total_size > 0) { + int rc = cache_flush(&ds.cache); + if (rc) { + if (pErrMsg) { + if (!(*pErrMsg)) + (*pErrMsg) = pcs_utils_sprintf("%s\n", "cache_flush() error."); + } + if (op_st) (*op_st) = OP_ST_FAIL; + fclose(ds.pf); + DeleteFileRecursive(tmp_local_path); + pcs_free(tmp_local_path); + pcs_free(local_path); + pcs_free(remote_path); + uninit_download_state(&ds); + return -1; + } + } + fclose(ds.pf); #ifdef _WIN32 @@ -6240,7 +6311,7 @@ static int cmd_set(ShellContext *context, struct args *arg) "cookie_file", "captcha_file", "list_page_size", "list_sort_name", "list_sort_direction", "secure_method", "secure_key", "secure_enable", "timeout_retry", "max_thread", "max_speed_per_thread", - "user-agent", + "user-agent", "cache_size", "h", "help", NULL) && arg->optc == 0) { usage_set(); return -1; @@ -6354,6 +6425,14 @@ static int cmd_set(ShellContext *context, struct args *arg) } } + if (has_optEx(arg, "cache_size", &val)) { + count++; + if (set_cache_size(context, val)) { + usage_set(); + return -1; + } + } + if (!count) { usage_set(); return -1; @@ -6930,7 +7009,6 @@ int main(int argc, char *argv[]) printf("Can't create pcs context.\n"); goto exit_main; } - srand((unsigned int)time(NULL)); rc = exec_cmd(&context, &arg); destroy_pcs(context.pcs); save_context(&context); diff --git a/shell.h b/shell.h index fcb6aaff..112c8057 100644 --- a/shell.h +++ b/shell.h @@ -29,7 +29,7 @@ typedef struct ShellContext int max_thread; /*指定最大线程数量*/ int max_speed_per_thread; /*指定单个线程的最多下载速度*/ int max_upload_speed_per_thread; /*指定单个线程的最大上传速度*/ - + int cache_size; /* 磁盘缓存的大小 */ char *user_agent; /**/ } ShellContext; From efcadf7cd7a17e972e44d20937fe43c395a7d620 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Thu, 24 Aug 2017 01:56:28 -0400 Subject: [PATCH 07/10] tiny fix --- pcs/pcs.c | 11 +++-------- pcs/pcs_utils.c | 4 +--- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/pcs/pcs.c b/pcs/pcs.c index 13a42d18..750dc970 100644 --- a/pcs/pcs.c +++ b/pcs/pcs.c @@ -34,12 +34,6 @@ #define PCS_SKIP_SPACE(p) while((*p) && (*p == ' ' || *p == '\f' || *p == '\n' || *p == '\r' || *p == '\t' || *p == '\v')) p++ -//#define PCS_IS_TOKEN_CHAR(ch) (((ch) >= '0' && (ch) <= '9')\ -// || ((ch) >= 'a' && (ch) <= 'z')\ -// || ((ch) >= 'A' && (ch) <= 'Z')\ -// || (ch) == '_'\ -// || (ch) == '-') - #define PCS_IS_TOKEN_CHAR(ch) ((ch) != '&' && (ch) != '?' && (ch) != '#' && (ch) > 32 && (ch) < 127) #define URL_HOME "http://www.baidu.com" @@ -128,9 +122,7 @@ PCS_API void pcs_set_errmsg(Pcs handle, const char *fmt, ...) PCS_API void pcs_cat_errmsg(Pcs handle, const char *fmt, ...) { - struct pcs *pcs = (struct pcs *)handle; char *errmsg; - size_t sz = 0; va_list args; va_start(args, fmt); errmsg = pcs_utils_vsprintf(fmt, args); @@ -323,6 +315,9 @@ static char *pcs_get_yunData(const char *html, const char *key) p++; } return val; +#undef st_push +#undef st_pop +#undef st_top } /*从URL地址的QueryString中解析出类似于 &error=123&a= 的值。此例中，key传入"&error"，将返回123*/ diff --git a/pcs/pcs_utils.c b/pcs/pcs_utils.c index bf4f78fc..3856715d 100644 --- a/pcs/pcs_utils.c +++ b/pcs/pcs_utils.c @@ -132,8 +132,6 @@ PCS_API char* pcs_utils_readable_size(double size/*in bytes*/, char *buf, int bu /* Human-readable left time */ PCS_API char* pcs_utils_readable_left_time(int64_t second, char *buf, int buf_size, char *sp) { - int i = 0; - const char* units[] = { "B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB" }; int day = (int)(second / (24 * 60 * 60)); int hour = (int)((second % (24 * 60 * 60)) / (60 * 60)); int minute = (int)(((second % (24 * 60 * 60)) % (60 * 60)) / 60); @@ -310,7 +308,7 @@ PCS_API const char *pcs_md5_file_s(const char *file_name) printf("%s can't be openedn", file_name); return 0; } - while (length = fread(buffer, 1, 1024, file)) + while ((length = fread(buffer, 1, 1024, file))) MD5_Update(&md5, buffer, length); MD5_Final(md, &md5); fclose(file); From 339371d4a0b29f7da46e0a3f7ac76d12aadccba3 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Thu, 24 Aug 2017 02:00:32 -0400 Subject: [PATCH 08/10] changable 'MAX_CACHE_SIZE' when build --- cache.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cache.h b/cache.h index 670fdb17..664bf610 100644 --- a/cache.h +++ b/cache.h @@ -3,7 +3,9 @@ #include "pcs/pcs.h" +#ifndef MAX_CACHE_SIZE #define MAX_CACHE_SIZE (1 * 1024) /* 1MiB */ +#endif #ifdef __cplusplus extern "C" { From 4e67014d45aa33b96633a4a978a6a853bf454739 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Thu, 24 Aug 2017 02:03:52 -0400 Subject: [PATCH 09/10] bump 0.3 --- pcs/pcs.h | 2 +- version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pcs/pcs.h b/pcs/pcs.h index 7686ac02..468e7277 100644 --- a/pcs/pcs.h +++ b/pcs/pcs.h @@ -9,7 +9,7 @@ #include "pcs_slist.h" #include "pcs_utils.h" -#define PCS_API_VERSION "v1.1.4" +#define PCS_API_VERSION "v1.1.5" #define PCS_RAPIDUPLOAD_THRESHOLD (256 * 1024) diff --git a/version.h b/version.h index 599e772f..6b3c48f7 100644 --- a/version.h +++ b/version.h @@ -4,7 +4,7 @@ #ifdef PRG_VER # define program_version PRG_VER #else -# define program_version "v0.2.6" +# define program_version "v0.3" #endif #define program_name "pcs" From bcd77d4a1a40fa8ec94110ea7bccadc0eae2f1b5 Mon Sep 17 00:00:00 2001 From: Gang Zhuo Date: Thu, 24 Aug 2017 02:05:49 -0400 Subject: [PATCH 10/10] bump 0.3 (openwrt) --- openwrt/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/Makefile b/openwrt/Makefile index c9bb1021..8da216ad 100644 --- a/openwrt/Makefile +++ b/openwrt/Makefile @@ -1,7 +1,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=BaiduPCS -PKG_VERSION:=0.2.6 +PKG_VERSION:=0.3 PKG_RELEASE:=$(PKG_SOURCE_VERSION) PKG_BUILD_DIR := $(BUILD_DIR)/$(PKG_NAME)-$(PKG_VERSION)