Skip to content

Commit eb39b33

Browse files
gitstergitster
committed
Merge branch 'hn/reftable' into seen
The "reftable" backend for the refs API. * hn/reftable: (28 commits) t1404: annotate test cases with REFFILES t1401,t2011: parameterize HEAD.lock for REFTABLE t1301: document what needs to be done for REFTABLE Add "test-tool dump-reftable" command. git-prompt: prepare for reftable refs backend Reftable support for git-core reftable: add dump utility reftable: implement stack, a mutable database of reftable files. reftable: implement refname validation reftable: add merged table view reftable: add a heap-based priority queue for reftable records reftable: reftable file level tests reftable: read reftable files reftable: generic interface to tables reftable: write reftable files reftable: a generic binary tree implementation reftable: reading/writing blocks Provide zlib's uncompress2 from compat/zlib-compat.c reftable: (de)serialization for the polymorphic record type. reftable: add blocksource, an abstraction for random access reads ...
2 parents 3c0a9d4 + 53f6ba3 commit eb39b33

95 files changed

Lines changed: 13051 additions & 72 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Documentation/config/extensions.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,12 @@ extensions.objectFormat::
66
Note that this setting should only be set by linkgit:git-init[1] or
77
linkgit:git-clone[1]. Trying to change it after initialization will not
88
work and will produce hard-to-diagnose issues.
9+
+
10+
extensions.refStorage::
11+
Specify the ref storage mechanism to use. The acceptable values are `files` and
12+
`reftable`. If not specified, `files` is assumed. It is an error to specify
13+
this key unless `core.repositoryFormatVersion` is 1.
14+
+
15+
Note that this setting should only be set by linkgit:git-init[1] or
16+
linkgit:git-clone[1]. Trying to change it after initialization will not
17+
work and will produce hard-to-diagnose issues.

‎Documentation/technical/repository-version.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,3 +100,10 @@ If set, by default "git config" reads from both "config" and
100100
multiple working directory mode, "config" file is shared while
101101
"config.worktree" is per-working directory (i.e., it's in
102102
GIT_COMMON_DIR/worktrees/<id>/config.worktree)
103+
104+
==== `refStorage`
105+
106+
Specifies the file format for the ref database. Values are `files`
107+
(for the traditional packed + loose ref format) and `reftable` for the
108+
binary reftable format. See https://github.com/google/reftable for
109+
more information.

‎Makefile‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,8 @@ all::
256256
#
257257
# Define NO_DEFLATE_BOUND if your zlib does not have deflateBound.
258258
#
259+
# Define NO_UNCOMPRESS2 if your zlib does not have uncompress2.
260+
#
259261
# Define NO_NORETURN if using buggy versions of gcc 4.6+ and profile feedback,
260262
# as the compiler can crash (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=49299)
261263
#
@@ -738,6 +740,7 @@ TEST_BUILTINS_OBJS += test-read-cache.o
738740
TEST_BUILTINS_OBJS += test-read-graph.o
739741
TEST_BUILTINS_OBJS += test-read-midx.o
740742
TEST_BUILTINS_OBJS += test-ref-store.o
743+
TEST_BUILTINS_OBJS += test-reftable.o
741744
TEST_BUILTINS_OBJS += test-regex.o
742745
TEST_BUILTINS_OBJS += test-repository.o
743746
TEST_BUILTINS_OBJS += test-revision-walking.o
@@ -816,6 +819,8 @@ TEST_SHELL_PATH = $(SHELL_PATH)
816819

817820
LIB_FILE = libgit.a
818821
XDIFF_LIB = xdiff/lib.a
822+
REFTABLE_LIB = reftable/libreftable.a
823+
REFTABLE_TEST_LIB = reftable/libreftable_test.a
819824

820825
GENERATED_H += command-list.h
821826
GENERATED_H += config-list.h
@@ -980,6 +985,7 @@ LIB_OBJS += reflog-walk.o
980985
LIB_OBJS += refs.o
981986
LIB_OBJS += refs/debug.o
982987
LIB_OBJS += refs/files-backend.o
988+
LIB_OBJS += refs/reftable-backend.o
983989
LIB_OBJS += refs/iterator.o
984990
LIB_OBJS += refs/packed-backend.o
985991
LIB_OBJS += refs/ref-cache.o
@@ -1194,7 +1200,7 @@ THIRD_PARTY_SOURCES += compat/regex/%
11941200
THIRD_PARTY_SOURCES += sha1collisiondetection/%
11951201
THIRD_PARTY_SOURCES += sha1dc/%
11961202

1197-
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB)
1203+
GITLIBS = common-main.o $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB)
11981204
EXTLIBS =
11991205

12001206
GIT_USER_AGENT = git/$(GIT_VERSION)
@@ -1716,6 +1722,11 @@ ifdef NO_DEFLATE_BOUND
17161722
BASIC_CFLAGS += -DNO_DEFLATE_BOUND
17171723
endif
17181724

1725+
ifdef NO_UNCOMPRESS2
1726+
BASIC_CFLAGS += -DNO_UNCOMPRESS2
1727+
LIB_OBJS += compat/zlib-uncompress2.o
1728+
endif
1729+
17191730
ifdef NO_POSIX_GOODIES
17201731
BASIC_CFLAGS += -DNO_POSIX_GOODIES
17211732
endif
@@ -2416,7 +2427,36 @@ XDIFF_OBJS += xdiff/xutils.o
24162427
.PHONY: xdiff-objs
24172428
xdiff-objs: $(XDIFF_OBJS)
24182429

2430+
REFTABLE_OBJS += reftable/basics.o
2431+
REFTABLE_OBJS += reftable/error.o
2432+
REFTABLE_OBJS += reftable/block.o
2433+
REFTABLE_OBJS += reftable/blocksource.o
2434+
REFTABLE_OBJS += reftable/iter.o
2435+
REFTABLE_OBJS += reftable/publicbasics.o
2436+
REFTABLE_OBJS += reftable/merged.o
2437+
REFTABLE_OBJS += reftable/pq.o
2438+
REFTABLE_OBJS += reftable/reader.o
2439+
REFTABLE_OBJS += reftable/record.o
2440+
REFTABLE_OBJS += reftable/refname.o
2441+
REFTABLE_OBJS += reftable/generic.o
2442+
REFTABLE_OBJS += reftable/stack.o
2443+
REFTABLE_OBJS += reftable/tree.o
2444+
REFTABLE_OBJS += reftable/writer.o
2445+
2446+
REFTABLE_TEST_OBJS += reftable/basics_test.o
2447+
REFTABLE_TEST_OBJS += reftable/block_test.o
2448+
REFTABLE_TEST_OBJS += reftable/dump.o
2449+
REFTABLE_TEST_OBJS += reftable/merged_test.o
2450+
REFTABLE_TEST_OBJS += reftable/pq_test.o
2451+
REFTABLE_TEST_OBJS += reftable/record_test.o
2452+
REFTABLE_TEST_OBJS += reftable/readwrite_test.o
2453+
REFTABLE_TEST_OBJS += reftable/refname_test.o
2454+
REFTABLE_TEST_OBJS += reftable/stack_test.o
2455+
REFTABLE_TEST_OBJS += reftable/test_framework.o
2456+
REFTABLE_TEST_OBJS += reftable/tree_test.o
2457+
24192458
TEST_OBJS := $(patsubst %$X,%.o,$(TEST_PROGRAMS)) $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
2459+
24202460
.PHONY: test-objs
24212461
test-objs: $(TEST_OBJS)
24222462

@@ -2432,6 +2472,8 @@ OBJECTS += $(PROGRAM_OBJS)
24322472
OBJECTS += $(TEST_OBJS)
24332473
OBJECTS += $(XDIFF_OBJS)
24342474
OBJECTS += $(FUZZ_OBJS)
2475+
OBJECTS += $(REFTABLE_OBJS) $(REFTABLE_TEST_OBJS)
2476+
24352477
ifndef NO_CURL
24362478
OBJECTS += http.o http-walker.o remote-curl.o
24372479
endif
@@ -2583,6 +2625,12 @@ $(LIB_FILE): $(LIB_OBJS)
25832625
$(XDIFF_LIB): $(XDIFF_OBJS)
25842626
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
25852627

2628+
$(REFTABLE_LIB): $(REFTABLE_OBJS)
2629+
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
2630+
2631+
$(REFTABLE_TEST_LIB): $(REFTABLE_TEST_OBJS)
2632+
$(QUIET_AR)$(RM) $@ && $(AR) $(ARFLAGS) $@ $^
2633+
25862634
export DEFAULT_EDITOR DEFAULT_PAGER
25872635

25882636
Documentation/GIT-EXCLUDED-PROGRAMS: FORCE
@@ -2866,7 +2914,7 @@ perf: all
28662914

28672915
t/helper/test-tool$X: $(patsubst %,t/helper/%,$(TEST_BUILTINS_OBJS))
28682916

2869-
t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS)
2917+
t/helper/test-%$X: t/helper/test-%.o GIT-LDFLAGS $(GITLIBS) $(REFTABLE_TEST_LIB)
28702918
$(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(filter %.a,$^) $(LIBS)
28712919

28722920
check-sha1:: t/helper/test-tool$X
@@ -3196,7 +3244,7 @@ cocciclean:
31963244
clean: profile-clean coverage-clean cocciclean
31973245
$(RM) *.res
31983246
$(RM) $(OBJECTS)
3199-
$(RM) $(LIB_FILE) $(XDIFF_LIB)
3247+
$(RM) $(LIB_FILE) $(XDIFF_LIB) $(REFTABLE_LIB) $(REFTABLE_TEST_LIB)
32003248
$(RM) $(ALL_PROGRAMS) $(SCRIPT_LIB) $(BUILT_INS) git$X
32013249
$(RM) $(TEST_PROGRAMS)
32023250
$(RM) $(FUZZ_PROGRAMS)

‎builtin/clone.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,7 +1148,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
11481148
}
11491149

11501150
init_db(git_dir, real_git_dir, option_template, GIT_HASH_UNKNOWN, NULL,
1151-
INIT_DB_QUIET);
1151+
default_ref_storage(), INIT_DB_QUIET);
11521152

11531153
if (real_git_dir)
11541154
git_dir = real_git_dir;
@@ -1299,7 +1299,8 @@ int cmd_clone(int argc, const char **argv, const char *prefix)
12991299
* Now that we know what algorithm the remote side is using,
13001300
* let's set ours to the same thing.
13011301
*/
1302-
initialize_repository_version(hash_algo, 1);
1302+
initialize_repository_version(hash_algo, 1,
1303+
default_ref_storage());
13031304
repo_set_hash_algo(the_repository, hash_algo);
13041305

13051306
mapped_refs = wanted_peer_refs(refs, &remote->fetch);

‎builtin/init-db.c‎

Lines changed: 62 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -167,12 +167,14 @@ static int needs_work_tree_config(const char *git_dir, const char *work_tree)
167167
return 1;
168168
}
169169

170-
void initialize_repository_version(int hash_algo, int reinit)
170+
void initialize_repository_version(int hash_algo, int reinit,
171+
const char *ref_storage_format)
171172
{
172173
char repo_version_string[10];
173174
int repo_version = GIT_REPO_VERSION;
174175

175-
if (hash_algo != GIT_HASH_SHA1)
176+
if (hash_algo != GIT_HASH_SHA1 ||
177+
!strcmp(ref_storage_format, "reftable"))
176178
repo_version = GIT_REPO_VERSION_READ;
177179

178180
/* This forces creation of new config file */
@@ -225,6 +227,7 @@ static int create_default_files(const char *template_path,
225227
is_bare_repository_cfg = init_is_bare_repository || !work_tree;
226228
if (init_shared_repository != -1)
227229
set_shared_repository(init_shared_repository);
230+
the_repository->ref_storage_format = xstrdup(fmt->ref_storage);
228231

229232
/*
230233
* We would have created the above under user's umask -- under
@@ -234,6 +237,24 @@ static int create_default_files(const char *template_path,
234237
adjust_shared_perm(get_git_dir());
235238
}
236239

240+
/*
241+
* Check to see if .git/HEAD exists; this must happen before
242+
* initializing the ref db, because we want to see if there is an
243+
* existing HEAD.
244+
*/
245+
path = git_path_buf(&buf, "HEAD");
246+
reinit = (!access(path, R_OK) ||
247+
readlink(path, junk, sizeof(junk) - 1) != -1);
248+
249+
/*
250+
* refs/heads is a file when using reftable. We can't reinitialize with
251+
* a reftable because it will overwrite HEAD
252+
*/
253+
if (reinit && (!strcmp(fmt->ref_storage, "reftable")) ==
254+
is_directory(git_path_buf(&buf, "refs/heads"))) {
255+
die("cannot switch ref storage format.");
256+
}
257+
237258
/*
238259
* We need to create a "refs" dir in any case so that older
239260
* versions of git can tell that this is a repository.
@@ -248,9 +269,6 @@ static int create_default_files(const char *template_path,
248269
* Point the HEAD symref to the initial branch with if HEAD does
249270
* not yet exist.
250271
*/
251-
path = git_path_buf(&buf, "HEAD");
252-
reinit = (!access(path, R_OK)
253-
|| readlink(path, junk, sizeof(junk)-1) != -1);
254272
if (!reinit) {
255273
char *ref;
256274

@@ -267,7 +285,7 @@ static int create_default_files(const char *template_path,
267285
free(ref);
268286
}
269287

270-
initialize_repository_version(fmt->hash_algo, 0);
288+
initialize_repository_version(fmt->hash_algo, 0, fmt->ref_storage);
271289

272290
/* Check filemode trustability */
273291
path = git_path_buf(&buf, "config");
@@ -382,7 +400,7 @@ static void validate_hash_algorithm(struct repository_format *repo_fmt, int hash
382400

383401
int init_db(const char *git_dir, const char *real_git_dir,
384402
const char *template_dir, int hash, const char *initial_branch,
385-
unsigned int flags)
403+
const char *ref_storage_format, unsigned int flags)
386404
{
387405
int reinit;
388406
int exist_ok = flags & INIT_DB_EXIST_OK;
@@ -421,9 +439,31 @@ int init_db(const char *git_dir, const char *real_git_dir,
421439
* is an attempt to reinitialize new repository with an old tool.
422440
*/
423441
check_repository_format(&repo_fmt);
442+
repo_fmt.ref_storage = xstrdup(ref_storage_format);
424443

425444
validate_hash_algorithm(&repo_fmt, hash);
426445

446+
/*
447+
* At this point, the_repository we have in-core does not look
448+
* anything like one that we would see initialized in an already
449+
* working repository after calling setup_git_directory().
450+
*
451+
* Calling repository.c::initialize_the_repository() may have
452+
* prepared the .index .objects and .parsed_objects members, but
453+
* other members like .gitdir, .commondir, etc. have not been
454+
* initialized.
455+
*
456+
* Many API functions assume they are working with the_repository
457+
* that has sensibly been initialized, but because we haven't
458+
* really read from an existing repository, we need to hand-craft
459+
* the necessary members of the structure to get out of this
460+
* chicken-and-egg situation.
461+
*
462+
* For now, we update the hash algorithm member to what the
463+
* validate_hash_algorithm() call decided for us.
464+
*/
465+
repo_set_hash_algo(the_repository, repo_fmt.hash_algo);
466+
427467
reinit = create_default_files(template_dir, original_git_dir,
428468
initial_branch, &repo_fmt,
429469
flags & INIT_DB_QUIET);
@@ -454,6 +494,9 @@ int init_db(const char *git_dir, const char *real_git_dir,
454494
git_config_set("receive.denyNonFastforwards", "true");
455495
}
456496

497+
if (!strcmp(ref_storage_format, "reftable"))
498+
git_config_set("extensions.refStorage", ref_storage_format);
499+
457500
if (!(flags & INIT_DB_QUIET)) {
458501
int len = strlen(git_dir);
459502

@@ -527,6 +570,7 @@ static const char *const init_db_usage[] = {
527570
int cmd_init_db(int argc, const char **argv, const char *prefix)
528571
{
529572
const char *git_dir;
573+
const char *ref_storage_format = default_ref_storage();
530574
const char *real_git_dir = NULL;
531575
const char *work_tree;
532576
const char *template_dir = NULL;
@@ -535,15 +579,18 @@ int cmd_init_db(int argc, const char **argv, const char *prefix)
535579
const char *initial_branch = NULL;
536580
int hash_algo = GIT_HASH_UNKNOWN;
537581
const struct option init_db_options[] = {
538-
OPT_STRING(0, "template", &template_dir, N_("template-directory"),
539-
N_("directory from which templates will be used")),
582+
OPT_STRING(0, "template", &template_dir,
583+
N_("template-directory"),
584+
N_("directory from which templates will be used")),
540585
OPT_SET_INT(0, "bare", &is_bare_repository_cfg,
541-
N_("create a bare repository"), 1),
586+
N_("create a bare repository"), 1),
542587
{ OPTION_CALLBACK, 0, "shared", &init_shared_repository,
543-
N_("permissions"),
544-
N_("specify that the git repository is to be shared amongst several users"),
545-
PARSE_OPT_OPTARG | PARSE_OPT_NONEG, shared_callback, 0},
588+
N_("permissions"),
589+
N_("specify that the git repository is to be shared amongst several users"),
590+
PARSE_OPT_OPTARG | PARSE_OPT_NONEG, shared_callback, 0 },
546591
OPT_BIT('q', "quiet", &flags, N_("be quiet"), INIT_DB_QUIET),
592+
OPT_STRING(0, "ref-storage", &ref_storage_format, N_("backend"),
593+
N_("the ref storage format to use")),
547594
OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"),
548595
N_("separate git dir from working tree")),
549596
OPT_STRING('b', "initial-branch", &initial_branch, N_("name"),
@@ -686,10 +733,11 @@ int cmd_init_db(int argc, const char **argv, const char *prefix)
686733
}
687734

688735
UNLEAK(real_git_dir);
736+
UNLEAK(ref_storage_format);
689737
UNLEAK(git_dir);
690738
UNLEAK(work_tree);
691739

692740
flags |= INIT_DB_EXIST_OK;
693741
return init_db(git_dir, real_git_dir, template_dir, hash_algo,
694-
initial_branch, flags);
742+
initial_branch, ref_storage_format, flags);
695743
}

‎builtin/stash.c‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,16 @@ static int get_stash_info(struct stash_info *info, int argc, const char **argv)
207207
static int do_clear_stash(void)
208208
{
209209
struct object_id obj;
210+
int result;
210211
if (get_oid(ref_stash, &obj))
211212
return 0;
212213

213-
return delete_ref(NULL, ref_stash, &obj, 0);
214+
result = delete_ref(NULL, ref_stash, &obj, 0);
215+
216+
/* Ignore error; this is necessary for reftable, which keeps reflogs
217+
* even when refs are deleted. */
218+
delete_reflog(ref_stash);
219+
return result;
214220
}
215221

216222
static int clear_stash(int argc, const char **argv, const char *prefix)

‎builtin/worktree.c‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include "utf8.h"
1414
#include "worktree.h"
1515
#include "quote.h"
16+
#include "../refs/refs-internal.h"
1617

1718
static const char * const worktree_usage[] = {
1819
N_("git worktree add [<options>] <path> [<commit-ish>]"),
@@ -330,9 +331,29 @@ static int add_worktree(const char *path, const char *refname,
330331
* worktree.
331332
*/
332333
strbuf_reset(&sb);
333-
strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
334-
write_file(sb.buf, "%s", oid_to_hex(null_oid()));
335-
strbuf_reset(&sb);
334+
if (get_main_ref_store(the_repository)->be == &refs_be_reftable) {
335+
/* XXX this is cut & paste from reftable_init_db. */
336+
strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
337+
write_file(sb.buf, "%s", "ref: refs/heads/.invalid\n");
338+
strbuf_reset(&sb);
339+
340+
strbuf_addf(&sb, "%s/refs", sb_repo.buf);
341+
safe_create_dir(sb.buf, 1);
342+
strbuf_reset(&sb);
343+
344+
strbuf_addf(&sb, "%s/refs/heads", sb_repo.buf);
345+
write_file(sb.buf, "this repository uses the reftable format");
346+
strbuf_reset(&sb);
347+
348+
strbuf_addf(&sb, "%s/reftable", sb_repo.buf);
349+
safe_create_dir(sb.buf, 1);
350+
strbuf_reset(&sb);
351+
} else {
352+
strbuf_addf(&sb, "%s/HEAD", sb_repo.buf);
353+
write_file(sb.buf, "%s", oid_to_hex(null_oid()));
354+
strbuf_reset(&sb);
355+
}
356+
336357
strbuf_addf(&sb, "%s/commondir", sb_repo.buf);
337358
write_file(sb.buf, "../..");
338359

0 commit comments

Comments
 (0)