mirror/curl
2
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2025-02-17 14:59:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

example/crawler: make it use a few more options

Browse Source 
For show, but reasonable
This commit is contained in:
Daniel Stenberg 2023-06-19 09:06:41 +02:00
parent 7ed832a245
commit aef74ae416
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
docs/examples/crawler.c
View File

@ -99,7 +99,13 @@ CURL *make_handle(char *url)
curl_easy_setopt(handle, CURLOPT_REDIR_PROTOCOLS_STR, "http,https");
curl_easy_setopt(handle, CURLOPT_AUTOREFERER, 1L);
curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);
curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);
/* each transfer needs to be done within 20 seconds! */
curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 20000L);
/* connect fast or fail */
curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 2000L);
/* skip files larger than a gigabyte */
curl_easy_setopt(handle, CURLOPT_MAXFILESIZE_LARGE,
(curl_off_t)1024*1024*1024);
curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");
curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);
curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");