Skip to content

Commit b889408

Browse files
bagderbagder
committed
curl: support parallel transfers
This is done by making sure each individual transfer is first added to a linked list as then they can be performed serially, or at will, in parallel. Closes #3804
1 parent 14a385b commit b889408

34 files changed

Lines changed: 1501 additions & 784 deletions

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,7 @@ check_symbol_exists(strlcat "${CURL_INCLUDES}" HAVE_STRLCAT)
860860
check_symbol_exists(getpwuid "${CURL_INCLUDES}" HAVE_GETPWUID)
861861
check_symbol_exists(getpwuid_r "${CURL_INCLUDES}" HAVE_GETPWUID_R)
862862
check_symbol_exists(geteuid "${CURL_INCLUDES}" HAVE_GETEUID)
863+
check_symbol_exists(usleep "${CURL_INCLUDES}" HAVE_USLEEP)
863864
check_symbol_exists(utime "${CURL_INCLUDES}" HAVE_UTIME)
864865
check_symbol_exists(gmtime_r "${CURL_INCLUDES}" HAVE_GMTIME_R)
865866
check_symbol_exists(localtime_r "${CURL_INCLUDES}" HAVE_LOCALTIME_R)

‎configure.ac‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3671,6 +3671,7 @@ AC_CHECK_FUNCS([fnmatch \
36713671
setlocale \
36723672
setmode \
36733673
setrlimit \
3674+
usleep \
36743675
utime \
36753676
utimes
36763677
],[

‎docs/cmdline-opts/Makefile.inc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,10 @@ DPAGES = \
100100
noproxy.d \
101101
ntlm.d ntlm-wb.d \
102102
oauth2-bearer.d \
103-
output.d pass.d \
103+
output.d \
104+
pass.d \
105+
parallel.d \
106+
parallel-max.d \
104107
path-as-is.d \
105108
pinnedpubkey.d \
106109
post301.d \

‎docs/cmdline-opts/parallel-max.d‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Long: parallel-max
2+
Help: Maximum concurrency for parallel transfers
3+
Added: 7.66.0
4+
See-also: parallel
5+
---
6+
When asked to do parallel transfers, using --parallel, this option controls
7+
the maximum amount of transfers to do simultaneously.
8+
9+
The default is 50.

‎docs/cmdline-opts/parallel.d‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Short: Z
2+
Long: parallel
3+
Help: Perform transfers in parallel
4+
Added: 7.66.0
5+
---
6+
Makes curl perform its transfers in parallel as compared to the regular serial
7+
manner.

‎src/Makefile.inc‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ CURL_CFILES = \
5454
tool_panykey.c \
5555
tool_paramhlp.c \
5656
tool_parsecfg.c \
57+
tool_progress.c \
5758
tool_strdup.c \
5859
tool_setopt.c \
5960
tool_sleep.c \
@@ -95,6 +96,7 @@ CURL_HFILES = \
9596
tool_panykey.h \
9697
tool_paramhlp.h \
9798
tool_parsecfg.h \
99+
tool_progress.h \
98100
tool_sdecls.h \
99101
tool_setopt.h \
100102
tool_setup.h \

‎src/tool_cb_hdr.c‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* | (__| |_| | _ <| |___
66
* \___|\___/|_| \_\_____|
77
*
8-
* Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
8+
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
99
*
1010
* This software is licensed as described in the file COPYING, which
1111
* you should have received as part of this distribution. The terms
@@ -32,6 +32,7 @@
3232
#include "tool_msgs.h"
3333
#include "tool_cb_hdr.h"
3434
#include "tool_cb_wrt.h"
35+
#include "tool_operate.h"
3536

3637
#include "memdebug.h" /* keep this as LAST include */
3738

@@ -54,9 +55,10 @@ static char *parse_filename(const char *ptr, size_t len);
5455

5556
size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
5657
{
57-
struct HdrCbData *hdrcbdata = userdata;
58-
struct OutStruct *outs = hdrcbdata->outs;
59-
struct OutStruct *heads = hdrcbdata->heads;
58+
struct per_transfer *per = userdata;
59+
struct HdrCbData *hdrcbdata = &per->hdrcbdata;
60+
struct OutStruct *outs = &per->outs;
61+
struct OutStruct *heads = &per->heads;
6062
const char *str = ptr;
6163
const size_t cb = size * nmemb;
6264
const char *end = (char *)ptr + cb;
@@ -100,7 +102,7 @@ size_t tool_header_cb(char *ptr, size_t size, size_t nmemb, void *userdata)
100102
* Content-Disposition header specifying a filename property.
101103
*/
102104

103-
curl_easy_getinfo(outs->config->easy, CURLINFO_PROTOCOL, &protocol);
105+
curl_easy_getinfo(per->curl, CURLINFO_PROTOCOL, &protocol);
104106
if(hdrcbdata->honor_cd_filename &&
105107
(cb > 20) && checkprefix("Content-disposition:", str) &&
106108
(protocol & (CURLPROTO_HTTPS|CURLPROTO_HTTP))) {

‎src/tool_cb_wrt.c‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* | (__| |_| | _ <| |___
66
* \___|\___/|_| \_\_____|
77
*
8-
* Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
8+
* Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
99
*
1010
* This software is licensed as described in the file COPYING, which
1111
* you should have received as part of this distribution. The terms
@@ -28,6 +28,7 @@
2828
#include "tool_cfgable.h"
2929
#include "tool_msgs.h"
3030
#include "tool_cb_wrt.h"
31+
#include "tool_operate.h"
3132

3233
#include "memdebug.h" /* keep this as LAST include */
3334

@@ -75,7 +76,8 @@ bool tool_create_output_file(struct OutStruct *outs)
7576
size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
7677
{
7778
size_t rc;
78-
struct OutStruct *outs = userdata;
79+
struct per_transfer *per = userdata;
80+
struct OutStruct *outs = &per->outs;
7981
struct OperationConfig *config = outs->config;
8082
size_t bytes = sz * nmemb;
8183
bool is_tty = config->global->isatty;
@@ -202,7 +204,7 @@ size_t tool_write_cb(char *buffer, size_t sz, size_t nmemb, void *userdata)
202204

203205
if(config->readbusy) {
204206
config->readbusy = FALSE;
205-
curl_easy_pause(config->easy, CURLPAUSE_CONT);
207+
curl_easy_pause(per->curl, CURLPAUSE_CONT);
206208
}
207209

208210
if(config->nobuffer) {

‎src/tool_cfgable.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ typedef enum {
3838
struct GlobalConfig;
3939

4040
struct OperationConfig {
41-
CURL *easy; /* A copy of the handle from GlobalConfig */
4241
bool remote_time;
4342
char *random_file;
4443
char *egd_file;
@@ -242,9 +241,6 @@ struct OperationConfig {
242241
bool use_metalink; /* process given URLs as metalink XML file */
243242
metalinkfile *metalinkfile_list; /* point to the first node */
244243
metalinkfile *metalinkfile_last; /* point to the last/current node */
245-
#ifdef CURLDEBUG
246-
bool test_event_based;
247-
#endif
248244
char *oauth_bearer; /* OAuth 2.0 bearer token */
249245
bool nonpn; /* enable/disable TLS NPN extension */
250246
bool noalpn; /* enable/disable TLS ALPN extension */
@@ -268,7 +264,6 @@ struct OperationConfig {
268264
};
269265

270266
struct GlobalConfig {
271-
CURL *easy; /* Once we have one, we keep it here */
272267
int showerror; /* -1 == unset, default => show errors
273268
0 => -s is used to NOT show errors
274269
1 => -S has been used to show errors */
@@ -286,6 +281,11 @@ struct GlobalConfig {
286281
char *libcurl; /* Output libcurl code to this file name */
287282
bool fail_early; /* exit on first transfer error */
288283
bool styled_output; /* enable fancy output style detection */
284+
#ifdef CURLDEBUG
285+
bool test_event_based;
286+
#endif
287+
bool parallel;
288+
long parallel_max;
289289
struct OperationConfig *first;
290290
struct OperationConfig *current;
291291
struct OperationConfig *last; /* Always last in the struct */

‎src/tool_getparam.c‎

Lines changed: 36 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "tool_msgs.h"
4141
#include "tool_paramhlp.h"
4242
#include "tool_parsecfg.h"
43+
#include "tool_main.h"
4344

4445
#include "memdebug.h" /* keep this as LAST include */
4546

@@ -316,6 +317,8 @@ static const struct LongShort aliases[]= {
316317
{"Y", "speed-limit", ARG_STRING},
317318
{"y", "speed-time", ARG_STRING},
318319
{"z", "time-cond", ARG_STRING},
320+
{"Z", "parallel", ARG_BOOL},
321+
{"Zb", "parallel-max", ARG_STRING},
319322
{"#", "progress-bar", ARG_BOOL},
320323
{":", "next", ARG_NONE},
321324
};
@@ -1104,7 +1107,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
11041107
break;
11051108
case 'L': /* --test-event */
11061109
#ifdef CURLDEBUG
1107-
config->test_event_based = toggle;
1110+
global->test_event_based = toggle;
11081111
#else
11091112
warnf(global, "--test-event is ignored unless a debug build!\n");
11101113
#endif
@@ -1356,7 +1359,7 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
13561359
size = 0;
13571360
}
13581361
else {
1359-
char *enc = curl_easy_escape(config->easy, postdata, (int)size);
1362+
char *enc = curl_easy_escape(NULL, postdata, (int)size);
13601363
Curl_safefree(postdata); /* no matter if it worked or not */
13611364
if(enc) {
13621365
/* now make a string with the name from above and append the
@@ -2127,6 +2130,21 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
21272130
if(!config->low_speed_time)
21282131
config->low_speed_time = 30;
21292132
break;
2133+
case 'Z':
2134+
switch(subletter) {
2135+
case '\0': /* --parallel */
2136+
global->parallel = toggle;
2137+
break;
2138+
case 'b': /* --parallel-max */
2139+
err = str2unum(&global->parallel_max, nextarg);
2140+
if(err)
2141+
return err;
2142+
if((global->parallel_max > MAX_PARALLEL) ||
2143+
(global->parallel_max < 1))
2144+
global->parallel_max = PARALLEL_DEFAULT;
2145+
break;
2146+
}
2147+
break;
21302148
case 'z': /* time condition coming up */
21312149
switch(*nextarg) {
21322150
case '+':
@@ -2176,14 +2194,14 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
21762194
return PARAM_OK;
21772195
}
21782196

2179-
ParameterError parse_args(struct GlobalConfig *config, int argc,
2197+
ParameterError parse_args(struct GlobalConfig *global, int argc,
21802198
argv_item_t argv[])
21812199
{
21822200
int i;
21832201
bool stillflags;
21842202
char *orig_opt = NULL;
21852203
ParameterError result = PARAM_OK;
2186-
struct OperationConfig *operation = config->first;
2204+
struct OperationConfig *config = global->first;
21872205

21882206
for(i = 1, stillflags = TRUE; i < argc && !result; i++) {
21892207
orig_opt = argv[i];
@@ -2199,31 +2217,28 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
21992217
else {
22002218
char *nextarg = (i < (argc - 1)) ? argv[i + 1] : NULL;
22012219

2202-
result = getparameter(flag, nextarg, &passarg, config, operation);
2220+
result = getparameter(flag, nextarg, &passarg, global, config);
22032221
if(result == PARAM_NEXT_OPERATION) {
22042222
/* Reset result as PARAM_NEXT_OPERATION is only used here and not
22052223
returned from this function */
22062224
result = PARAM_OK;
22072225

2208-
if(operation->url_list && operation->url_list->url) {
2226+
if(config->url_list && config->url_list->url) {
22092227
/* Allocate the next config */
2210-
operation->next = malloc(sizeof(struct OperationConfig));
2211-
if(operation->next) {
2228+
config->next = malloc(sizeof(struct OperationConfig));
2229+
if(config->next) {
22122230
/* Initialise the newly created config */
2213-
config_init(operation->next);
2214-
2215-
/* Copy the easy handle */
2216-
operation->next->easy = config->easy;
2231+
config_init(config->next);
22172232

22182233
/* Set the global config pointer */
2219-
operation->next->global = config;
2234+
config->next->global = global;
22202235

2221-
/* Update the last operation pointer */
2222-
config->last = operation->next;
2236+
/* Update the last config pointer */
2237+
global->last = config->next;
22232238

22242239
/* Move onto the new config */
2225-
operation->next->prev = operation;
2226-
operation = operation->next;
2240+
config->next->prev = config;
2241+
config = config->next;
22272242
}
22282243
else
22292244
result = PARAM_NO_MEM;
@@ -2237,8 +2252,8 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
22372252
bool used;
22382253

22392254
/* Just add the URL please */
2240-
result = getparameter((char *)"--url", argv[i], &used, config,
2241-
operation);
2255+
result = getparameter((char *)"--url", argv[i], &used, global,
2256+
config);
22422257
}
22432258
}
22442259

@@ -2249,9 +2264,9 @@ ParameterError parse_args(struct GlobalConfig *config, int argc,
22492264
const char *reason = param2text(result);
22502265

22512266
if(orig_opt && strcmp(":", orig_opt))
2252-
helpf(config->errors, "option %s: %s\n", orig_opt, reason);
2267+
helpf(global->errors, "option %s: %s\n", orig_opt, reason);
22532268
else
2254-
helpf(config->errors, "%s\n", reason);
2269+
helpf(global->errors, "%s\n", reason);
22552270
}
22562271

22572272
return result;

0 commit comments

Comments
 (0)