Skip to content

Commit 4830868

Browse files
stefanbellergitster
stefanbeller
authored and
gitster
committed
git submodule update: have a dedicated helper for cloning
This introduces a new helper function in git submodule--helper which takes care of cloning all submodules, which we want to parallelize eventually. Some tests (such as empty URL, update_mode=none) are required in the helper to make the decision for cloning. These checks have been moved into the C function as well (no need to repeat them in the shell script). Reviewed-by: Jonathan Nieder <jrnieder@gmail.com> Signed-off-by: Stefan Beller <sbeller@google.com> Signed-off-by: Junio C Hamano <gitster@pobox.com>
1 parent aa71049 commit 4830868

File tree

2 files changed

+259
-34
lines changed

2 files changed

+259
-34
lines changed

‎builtin/submodule--helper.c‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,251 @@ static int module_clone(int argc, const char **argv, const char *prefix)
255255
return 0;
256256
}
257257

258+
struct submodule_update_clone {
259+
/* index into 'list', the list of submodules to look into for cloning */
260+
int current;
261+
struct module_list list;
262+
unsigned warn_if_uninitialized : 1;
263+
264+
/* update parameter passed via commandline */
265+
struct submodule_update_strategy update;
266+
267+
/* configuration parameters which are passed on to the children */
268+
int quiet;
269+
const char *reference;
270+
const char *depth;
271+
const char *recursive_prefix;
272+
const char *prefix;
273+
274+
/* Machine-readable status lines to be consumed by git-submodule.sh */
275+
struct string_list projectlines;
276+
277+
/* If we want to stop as fast as possible and return an error */
278+
unsigned quickstop : 1;
279+
};
280+
#define SUBMODULE_UPDATE_CLONE_INIT {0, MODULE_LIST_INIT, 0, \
281+
SUBMODULE_UPDATE_STRATEGY_INIT, 0, NULL, NULL, NULL, NULL, \
282+
STRING_LIST_INIT_DUP, 0}
283+
284+
/**
285+
* Determine whether 'ce' needs to be cloned. If so, prepare the 'child' to
286+
* run the clone. Returns 1 if 'ce' needs to be cloned, 0 otherwise.
287+
*/
288+
static int prepare_to_clone_next_submodule(const struct cache_entry *ce,
289+
struct child_process *child,
290+
struct submodule_update_clone *suc,
291+
struct strbuf *out)
292+
{
293+
const struct submodule *sub = NULL;
294+
struct strbuf displaypath_sb = STRBUF_INIT;
295+
struct strbuf sb = STRBUF_INIT;
296+
const char *displaypath = NULL;
297+
char *url = NULL;
298+
int needs_cloning = 0;
299+
300+
if (ce_stage(ce)) {
301+
if (suc->recursive_prefix)
302+
strbuf_addf(&sb, "%s/%s", suc->recursive_prefix, ce->name);
303+
else
304+
strbuf_addf(&sb, "%s", ce->name);
305+
strbuf_addf(out, _("Skipping unmerged submodule %s"), sb.buf);
306+
strbuf_addch(out, '\n');
307+
goto cleanup;
308+
}
309+
310+
sub = submodule_from_path(null_sha1, ce->name);
311+
312+
if (suc->recursive_prefix)
313+
displaypath = relative_path(suc->recursive_prefix,
314+
ce->name, &displaypath_sb);
315+
else
316+
displaypath = ce->name;
317+
318+
if (suc->update.type == SM_UPDATE_NONE
319+
|| (suc->update.type == SM_UPDATE_UNSPECIFIED
320+
&& sub->update_strategy.type == SM_UPDATE_NONE)) {
321+
strbuf_addf(out, _("Skipping submodule '%s'"), displaypath);
322+
strbuf_addch(out, '\n');
323+
goto cleanup;
324+
}
325+
326+
/*
327+
* Looking up the url in .git/config.
328+
* We must not fall back to .gitmodules as we only want
329+
* to process configured submodules.
330+
*/
331+
strbuf_reset(&sb);
332+
strbuf_addf(&sb, "submodule.%s.url", sub->name);
333+
git_config_get_string(sb.buf, &url);
334+
if (!url) {
335+
/*
336+
* Only mention uninitialized submodules when their
337+
* path have been specified
338+
*/
339+
if (suc->warn_if_uninitialized) {
340+
strbuf_addf(out,
341+
_("Submodule path '%s' not initialized"),
342+
displaypath);
343+
strbuf_addch(out, '\n');
344+
strbuf_addstr(out,
345+
_("Maybe you want to use 'update --init'?"));
346+
strbuf_addch(out, '\n');
347+
}
348+
goto cleanup;
349+
}
350+
351+
strbuf_reset(&sb);
352+
strbuf_addf(&sb, "%s/.git", ce->name);
353+
needs_cloning = !file_exists(sb.buf);
354+
355+
strbuf_reset(&sb);
356+
strbuf_addf(&sb, "%06o %s %d %d\t%s\n", ce->ce_mode,
357+
sha1_to_hex(ce->sha1), ce_stage(ce),
358+
needs_cloning, ce->name);
359+
string_list_append(&suc->projectlines, sb.buf);
360+
361+
if (!needs_cloning)
362+
goto cleanup;
363+
364+
child->git_cmd = 1;
365+
child->no_stdin = 1;
366+
child->stdout_to_stderr = 1;
367+
child->err = -1;
368+
argv_array_push(&child->args, "submodule--helper");
369+
argv_array_push(&child->args, "clone");
370+
if (suc->quiet)
371+
argv_array_push(&child->args, "--quiet");
372+
if (suc->prefix)
373+
argv_array_pushl(&child->args, "--prefix", suc->prefix, NULL);
374+
argv_array_pushl(&child->args, "--path", sub->path, NULL);
375+
argv_array_pushl(&child->args, "--name", sub->name, NULL);
376+
argv_array_pushl(&child->args, "--url", url, NULL);
377+
if (suc->reference)
378+
argv_array_push(&child->args, suc->reference);
379+
if (suc->depth)
380+
argv_array_push(&child->args, suc->depth);
381+
382+
cleanup:
383+
free(url);
384+
strbuf_reset(&displaypath_sb);
385+
strbuf_reset(&sb);
386+
387+
return needs_cloning;
388+
}
389+
390+
static int update_clone_get_next_task(struct child_process *child,
391+
struct strbuf *err,
392+
void *suc_cb,
393+
void **void_task_cb)
394+
{
395+
struct submodule_update_clone *suc = suc_cb;
396+
397+
for (; suc->current < suc->list.nr; suc->current++) {
398+
const struct cache_entry *ce = suc->list.entries[suc->current];
399+
if (prepare_to_clone_next_submodule(ce, child, suc, err)) {
400+
suc->current++;
401+
return 1;
402+
}
403+
}
404+
return 0;
405+
}
406+
407+
static int update_clone_start_failure(struct strbuf *err,
408+
void *suc_cb,
409+
void *void_task_cb)
410+
{
411+
struct submodule_update_clone *suc = suc_cb;
412+
suc->quickstop = 1;
413+
return 1;
414+
}
415+
416+
static int update_clone_task_finished(int result,
417+
struct strbuf *err,
418+
void *suc_cb,
419+
void *void_task_cb)
420+
{
421+
struct submodule_update_clone *suc = suc_cb;
422+
423+
if (!result)
424+
return 0;
425+
426+
suc->quickstop = 1;
427+
return 1;
428+
}
429+
430+
static int update_clone(int argc, const char **argv, const char *prefix)
431+
{
432+
const char *update = NULL;
433+
struct string_list_item *item;
434+
struct pathspec pathspec;
435+
struct submodule_update_clone suc = SUBMODULE_UPDATE_CLONE_INIT;
436+
437+
struct option module_update_clone_options[] = {
438+
OPT_STRING(0, "prefix", &prefix,
439+
N_("path"),
440+
N_("path into the working tree")),
441+
OPT_STRING(0, "recursive-prefix", &suc.recursive_prefix,
442+
N_("path"),
443+
N_("path into the working tree, across nested "
444+
"submodule boundaries")),
445+
OPT_STRING(0, "update", &update,
446+
N_("string"),
447+
N_("rebase, merge, checkout or none")),
448+
OPT_STRING(0, "reference", &suc.reference, N_("repo"),
449+
N_("reference repository")),
450+
OPT_STRING(0, "depth", &suc.depth, "<depth>",
451+
N_("Create a shallow clone truncated to the "
452+
"specified number of revisions")),
453+
OPT__QUIET(&suc.quiet, N_("don't print cloning progress")),
454+
OPT_END()
455+
};
456+
457+
const char *const git_submodule_helper_usage[] = {
458+
N_("git submodule--helper update_clone [--prefix=<path>] [<path>...]"),
459+
NULL
460+
};
461+
suc.prefix = prefix;
462+
463+
argc = parse_options(argc, argv, prefix, module_update_clone_options,
464+
git_submodule_helper_usage, 0);
465+
466+
if (update)
467+
if (parse_submodule_update_strategy(update, &suc.update) < 0)
468+
die(_("bad value for update parameter"));
469+
470+
if (module_list_compute(argc, argv, prefix, &pathspec, &suc.list) < 0)
471+
return 1;
472+
473+
if (pathspec.nr)
474+
suc.warn_if_uninitialized = 1;
475+
476+
/* Overlay the parsed .gitmodules file with .git/config */
477+
gitmodules_config();
478+
git_config(submodule_config, NULL);
479+
480+
run_processes_parallel(1,
481+
update_clone_get_next_task,
482+
update_clone_start_failure,
483+
update_clone_task_finished,
484+
&suc);
485+
486+
/*
487+
* We saved the output and put it out all at once now.
488+
* That means:
489+
* - the listener does not have to interleave their (checkout)
490+
* work with our fetching. The writes involved in a
491+
* checkout involve more straightforward sequential I/O.
492+
* - the listener can avoid doing any work if fetching failed.
493+
*/
494+
if (suc.quickstop)
495+
return 1;
496+
497+
for_each_string_list_item(item, &suc.projectlines)
498+
utf8_fprintf(stdout, "%s", item->string);
499+
500+
return 0;
501+
}
502+
258503
struct cmd_struct {
259504
const char *cmd;
260505
int (*fn)(int, const char **, const char *);
@@ -264,6 +509,7 @@ static struct cmd_struct commands[] = {
264509
{"list", module_list},
265510
{"name", module_name},
266511
{"clone", module_clone},
512+
{"update-clone", update_clone}
267513
};
268514

269515
int cmd_submodule__helper(int argc, const char **argv, const char *prefix)

‎git-submodule.sh‎

Lines changed: 13 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -664,17 +664,20 @@ cmd_update()
664664
cmd_init "--" "$@" || return
665665
fi
666666

667-
cloned_modules=
668-
git submodule--helper list --prefix "$wt_prefix" "$@" | {
667+
{
668+
git submodule--helper update-clone ${GIT_QUIET:+--quiet} \
669+
${wt_prefix:+--prefix "$wt_prefix"} \
670+
${prefix:+--recursive-prefix "$prefix"} \
671+
${update:+--update "$update"} \
672+
${reference:+--reference "$reference"} \
673+
${depth:+--depth "$depth"} \
674+
"$@" || echo "#unmatched"
675+
} | {
669676
err=
670-
while read mode sha1 stage sm_path
677+
while read mode sha1 stage just_cloned sm_path
671678
do
672679
die_if_unmatched "$mode"
673-
if test "$stage" = U
674-
then
675-
echo >&2 "Skipping unmerged submodule $prefix$sm_path"
676-
continue
677-
fi
680+
678681
name=$(git submodule--helper name "$sm_path") || exit
679682
url=$(git config submodule."$name".url)
680683
branch=$(get_submodule_config "$name" branch master)
@@ -691,27 +694,10 @@ cmd_update()
691694

692695
displaypath=$(relative_path "$prefix$sm_path")
693696

694-
if test "$update_module" = "none"
695-
then
696-
echo >&2 "Skipping submodule '$displaypath'"
697-
continue
698-
fi
699-
700-
if test -z "$url"
701-
then
702-
# Only mention uninitialized submodules when its
703-
# path have been specified
704-
test "$#" != "0" &&
705-
say >&2 "$(eval_gettext "Submodule path '\$displaypath' not initialized
706-
Maybe you want to use 'update --init'?")"
707-
continue
708-
fi
709-
710-
if ! test -d "$sm_path"/.git && ! test -f "$sm_path"/.git
697+
if test $just_cloned -eq 1
711698
then
712-
git submodule--helper clone ${GIT_QUIET:+--quiet} --prefix "$prefix" --path "$sm_path" --name "$name" --url "$url" "$reference" "$depth" || exit
713-
cloned_modules="$cloned_modules;$name"
714699
subsha1=
700+
update_module=checkout
715701
else
716702
subsha1=$(clear_local_git_env; cd "$sm_path" &&
717703
git rev-parse --verify HEAD) ||
@@ -751,13 +737,6 @@ Maybe you want to use 'update --init'?")"
751737
die "$(eval_gettext "Unable to fetch in submodule path '\$displaypath'")"
752738
fi
753739

754-
# Is this something we just cloned?
755-
case ";$cloned_modules;" in
756-
*";$name;"*)
757-
# then there is no local change to integrate
758-
update_module=checkout ;;
759-
esac
760-
761740
must_die_on_failure=
762741
case "$update_module" in
763742
checkout)

0 commit comments

Comments
 (0)