Skip to content

Commit 8af2691

Browse files
ByronByron
authored
Merge pull request #2581 from GitoxideLabs/improvements
basic SHA-256 support
2 parents a791ea3 + 011316e commit 8af2691

35 files changed

Lines changed: 731 additions & 284 deletions

File tree

‎gix-pack/src/bundle/write/mod.rs‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -103,17 +103,13 @@ impl crate::Bundle {
103103
thin_pack_lookup,
104104
);
105105
let pack_version = pack_entries_iter.inner.version();
106-
#[cfg(feature = "sha1")]
107-
let thin_pack_object_hash = gix_hash::Kind::Sha1; // Thin packs imply a pack being transported, and there we only ever know SHA1 at the moment.
108-
#[cfg(not(feature = "sha1"))]
109-
let thin_pack_object_hash = object_hash;
110106
let pack_entries_iter = data::input::EntriesToBytesIter::new(
111107
pack_entries_iter,
112108
LockWriter {
113109
writer: data_file.clone(),
114110
},
115111
pack_version,
116-
thin_pack_object_hash,
112+
object_hash,
117113
);
118114
(Box::new(pack_entries_iter), pack_version)
119115
}

‎gix-pack/src/data/input/entries_to_bytes.rs‎

Lines changed: 13 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -37,32 +37,20 @@ where
3737
///
3838
/// # Panics
3939
///
40-
/// Not all combinations of `object_hash` and `version` are supported currently triggering assertion errors.
40+
/// Only [Version::V2](crate::data::Version::V2) is allowed for `version.
4141
pub fn new(input: I, output: W, version: crate::data::Version, object_hash: gix_hash::Kind) -> Self {
42-
#[cfg(not(feature = "sha1"))]
43-
{
44-
let _ = (input, output, version, object_hash);
45-
unreachable!("currently only Sha1 is supported, right now we don't know how other hashes are encoded");
46-
}
47-
#[cfg(feature = "sha1")]
48-
{
49-
assert!(
50-
matches!(version, crate::data::Version::V2),
51-
"currently only pack version 2 can be written",
52-
);
53-
assert!(
54-
matches!(object_hash, gix_hash::Kind::Sha1),
55-
"currently only Sha1 is supported, right now we don't know how other hashes are encoded",
56-
);
57-
EntriesToBytesIter {
58-
input: input.peekable(),
59-
output,
60-
object_hash,
61-
num_entries: 0,
62-
trailer: None,
63-
data_version: version,
64-
is_done: false,
65-
}
42+
assert!(
43+
matches!(version, crate::data::Version::V2),
44+
"currently only pack version 2 can be written",
45+
);
46+
EntriesToBytesIter {
47+
input: input.peekable(),
48+
output,
49+
object_hash,
50+
num_entries: 0,
51+
trailer: None,
52+
data_version: version,
53+
is_done: false,
6654
}
6755
}
6856

‎gix-pack/src/data/mod.rs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,17 @@ pub type EntryRange = std::ops::Range<Offset>;
5151
/// Supported versions of a pack data file
5252
#[derive(Default, PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone, Copy)]
5353
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
54-
#[allow(missing_docs)]
5554
pub enum Version {
55+
/// The default pack data version.
56+
///
57+
/// This is the version generated by Git and by `gix-pack` writers.
5658
#[default]
5759
V2,
60+
/// A pack data version accepted by Git and recognized by `gix-pack` readers.
61+
///
62+
/// Git does not generate this version, and `gix-pack` writers currently reject it.
63+
/// Entries are decoded with the same layout as [`V2`](Version::V2); the difference
64+
/// visible to this crate is the version number stored in the pack header.
5865
V3,
5966
}
6067

‎gix-pack/src/index/encode.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ mod function {
7272
entries_sorted_by_oid: Vec<crate::cache::delta::Item<crate::index::write::TreeEntry>>,
7373
pack_hash: &gix_hash::ObjectId,
7474
kind: crate::index::Version,
75+
object_hash: gix_hash::Kind,
7576
progress: &mut dyn DynNestedProgress,
7677
) -> Result<gix_hash::ObjectId, gix_hash::io::Error> {
7778
use io::Write;
@@ -84,7 +85,7 @@ mod function {
8485
// Write header
8586
let mut out = Count::new(std::io::BufWriter::with_capacity(
8687
8 * 4096,
87-
gix_hash::io::Write::new(out, kind.hash()),
88+
gix_hash::io::Write::new(out, object_hash),
8889
));
8990
out.write_all(V2_SIGNATURE)?;
9091
out.write_all(&(kind as u32).to_be_bytes())?;
@@ -143,7 +144,7 @@ mod function {
143144
progress.inc();
144145
progress.show_throughput_with(
145146
start,
146-
(bytes_written_without_trailer + 20) as usize,
147+
(bytes_written_without_trailer + object_hash.len_in_bytes() as u64) as usize,
147148
progress::bytes().expect("unit always set"),
148149
progress::MessageLevel::Success,
149150
);

‎gix-pack/src/index/write/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ pub(super) mod function {
194194
entry,
195195
decompressed: bytes,
196196
..
197-
}| { modify_base(data, entry, bytes, version.hash()) },
197+
}| { modify_base(data, entry, bytes, object_hash) },
198198
traverse::Options {
199199
object_progress: Box::new(
200200
root_progress.add_child_with_id("Resolving".into(), ProgressId::ResolveObjects.into()),
@@ -235,6 +235,7 @@ pub(super) mod function {
235235
sorted_pack_offsets_by_oid,
236236
&pack_hash,
237237
version,
238+
object_hash,
238239
&mut root_progress.add_child_with_id("writing index file".into(), ProgressId::IndexBytesWritten.into()),
239240
)?;
240241
root_progress.show_throughput_with(

‎gix-pack/tests/Cargo.toml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,13 @@ name = "pack"
2020
path = "integrate.rs"
2121

2222
[dev-dependencies]
23-
gix-pack = { path = "..", features = ["generate", "streaming-input", "sha1" ] }
23+
gix-pack = { path = "..", features = ["generate", "streaming-input", "sha1", "sha256" ] }
2424
gix-features = { path = "../../gix-features" }
2525
gix-testtools = { path = "../../tests/tools" }
26-
gix-odb = { path = "../../gix-odb" }
26+
gix-odb = { path = "../../gix-odb", features = ["sha1", "sha256"] }
2727
bstr = { version = "1.12.0", default-features = false, features = ["std"] }
2828
maplit = "1.0.2"
2929
gix-object = { path = "../../gix-object" }
3030
gix-traverse = { path = "../../gix-traverse" }
31-
gix-hash = { path = "../../gix-hash", features = ["sha256"] }
31+
gix-hash = { path = "../../gix-hash", features = ["sha1", "sha256"] }
3232
memmap2 = "0.9.10"

‎gix-pack/tests/pack/data/output/count_and_entries.rs‎

Lines changed: 73 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ use gix_pack::data::{
1212

1313
use crate::pack::{
1414
data::output::{DbKind, db},
15-
hex_to_id,
15+
hex_to_id, hex_to_id_for_hash, object_hash,
1616
};
1717

1818
#[test]
@@ -64,12 +64,25 @@ fn traversals() -> crate::Result {
6464
}
6565
}
6666
}
67+
let object_hash = object_hash();
6768
let whole_pack = Count {
68-
trees: 21,
69+
trees: match object_hash {
70+
gix_hash::Kind::Sha1 => 21,
71+
gix_hash::Kind::Sha256 => 20,
72+
_ => unimplemented!(),
73+
},
6974
commits: 16,
70-
blobs: 288,
75+
blobs: match object_hash {
76+
gix_hash::Kind::Sha1 => 288,
77+
gix_hash::Kind::Sha256 => 323,
78+
_ => unimplemented!(),
79+
},
7180
tags: 1,
72-
delta_ref: 542,
81+
delta_ref: match object_hash {
82+
gix_hash::Kind::Sha1 => 542,
83+
gix_hash::Kind::Sha256 => 508,
84+
_ => unimplemented!(),
85+
},
7386
delta_oid: 0, // these are basically none-existing in non-legacy packs, but are used only in thin packs on the wire
7487
};
7588
let whole_pack_obj_count = ObjectCount {
@@ -82,7 +95,7 @@ fn traversals() -> crate::Result {
8295
DbKind::DeterministicGeneratedContent,
8396
DbKind::DeterministicGeneratedContentMultiIndex,
8497
] {
85-
let db = db(db_kind)?;
98+
let db = db(db_kind, object_hash)?;
8699
for (
87100
expansion_mode,
88101
expected_count,
@@ -132,10 +145,18 @@ fn traversals() -> crate::Result {
132145
Count {
133146
trees: 3,
134147
commits: 2, // todo: why more?
135-
blobs: 19,
148+
blobs: match object_hash {
149+
gix_hash::Kind::Sha1 => 19,
150+
gix_hash::Kind::Sha256 => 22,
151+
_ => unimplemented!(),
152+
},
136153
tags: 0,
137154
delta_ref: 5,
138-
delta_oid: 74,
155+
delta_oid: match object_hash {
156+
gix_hash::Kind::Sha1 => 74,
157+
gix_hash::Kind::Sha256 => 71,
158+
_ => unimplemented!(),
159+
},
139160
},
140161
ObjectCount {
141162
trees: 5,
@@ -153,7 +174,11 @@ fn traversals() -> crate::Result {
153174
decoded_and_recompressed_objects: 0,
154175
missing_objects: 0,
155176
objects_copied_from_pack: 103,
156-
ref_delta_objects: 74,
177+
ref_delta_objects: match object_hash {
178+
gix_hash::Kind::Sha1 => 74,
179+
gix_hash::Kind::Sha256 => 71,
180+
_ => unimplemented!(),
181+
},
157182
},
158183
hex_to_id("25114bd8820b393c402cd53ad8ec7f6a84bb0633"),
159184
Some(hex_to_id("29ab9797aff1ca826afb699680356695d19c5acb")),
@@ -183,9 +208,17 @@ fn traversals() -> crate::Result {
183208
total_objects: 103,
184209
},
185210
output::entry::iter_from_counts::Outcome {
186-
decoded_and_recompressed_objects: 74,
211+
decoded_and_recompressed_objects: match object_hash {
212+
gix_hash::Kind::Sha1 => 74,
213+
gix_hash::Kind::Sha256 => 71,
214+
_ => unimplemented!(),
215+
},
187216
missing_objects: 0,
188-
objects_copied_from_pack: 29,
217+
objects_copied_from_pack: match object_hash {
218+
gix_hash::Kind::Sha1 => 29,
219+
gix_hash::Kind::Sha256 => 32,
220+
_ => unimplemented!(),
221+
},
189222
ref_delta_objects: 0,
190223
},
191224
hex_to_id("d83d42128e40957c5174920189a0390b5a70f446"),
@@ -239,13 +272,17 @@ fn traversals() -> crate::Result {
239272
.iter()
240273
.copied()
241274
{
242-
let head = hex_to_id("dfcb5e39ac6eb30179808bbab721e8a28ce1b52e");
275+
let head = hex_to_id_for_hash(
276+
object_hash,
277+
"dfcb5e39ac6eb30179808bbab721e8a28ce1b52e",
278+
"ad454f92f046c2873aebac2686d30d5b100ee10fae1a28e2994df52a0c097cae",
279+
);
243280
let mut commits = gix_traverse::commit::Simple::new(Some(head), db.clone())
244281
.map(Result::unwrap)
245282
.map(|c| c.id)
246283
.collect::<Vec<_>>();
247284
if let Some(take) = take {
248-
commits.resize(take, gix_hash::Kind::Sha1.null());
285+
commits.resize(take, object_hash.null());
249286
}
250287

251288
let deterministic_count_needs_single_thread = Some(1);
@@ -254,11 +291,15 @@ fn traversals() -> crate::Result {
254291
Box::new(
255292
commits
256293
.into_iter()
257-
.chain(std::iter::once(hex_to_id(if take.is_some() {
258-
"0000000000000000000000000000000000000000"
294+
.chain(std::iter::once(if take.is_some() {
295+
object_hash.null()
259296
} else {
260-
"e3fb53cbb4c346d48732a24f09cf445e49bc63d6"
261-
})))
297+
hex_to_id_for_hash(
298+
object_hash,
299+
"e3fb53cbb4c346d48732a24f09cf445e49bc63d6",
300+
"859b1fd3fd3cc6a0a016edc9f439afd33f87e524dc1fd2befd1ac550953d3b3b",
301+
)
302+
}))
262303
.filter(|o| !o.is_null())
263304
.map(Ok),
264305
),
@@ -313,7 +354,13 @@ fn traversals() -> crate::Result {
313354
"two different ways of counting, still the same in the end"
314355
);
315356

316-
write_and_verify(db.clone(), entries, expected_pack_hash, expected_thin_pack_hash)?;
357+
write_and_verify(
358+
db.clone(),
359+
entries,
360+
object_hash,
361+
expected_pack_hash,
362+
expected_thin_pack_hash,
363+
)?;
317364
}
318365
}
319366

@@ -324,8 +371,9 @@ fn traversals() -> crate::Result {
324371
fn empty_pack_is_allowed() {
325372
assert_eq!(
326373
write_and_verify(
327-
db(DbKind::DeterministicGeneratedContent).unwrap(),
374+
db(DbKind::DeterministicGeneratedContent, object_hash()).unwrap(),
328375
vec![],
376+
object_hash(),
329377
hex_to_id("029d08823bd8a8eab510ad6ac75c823cfd3ed31e"),
330378
None,
331379
)
@@ -339,6 +387,7 @@ fn empty_pack_is_allowed() {
339387
fn write_and_verify(
340388
db: gix_odb::HandleArc,
341389
entries: Vec<output::Entry>,
390+
object_hash: gix_hash::Kind,
342391
_expected_pack_hash: gix_hash::ObjectId,
343392
_expected_thin_pack_hash: Option<gix_hash::ObjectId>,
344393
) -> crate::Result {
@@ -355,7 +404,7 @@ fn write_and_verify(
355404
&mut pack_file,
356405
num_entries as u32,
357406
pack::data::Version::V2,
358-
gix_hash::Kind::Sha1,
407+
object_hash,
359408
);
360409
let mut n = pack_writer.next().expect("one entries bundle was written")?;
361410
n += pack_writer.next().expect("the trailer was written")?;
@@ -374,7 +423,7 @@ fn write_and_verify(
374423
pack_file.metadata()?.len(),
375424
"it reports the correct amount of written bytes"
376425
);
377-
let pack = pack::data::File::at(&pack_file_path, gix_hash::Kind::Sha1)?;
426+
let pack = pack::data::File::at(&pack_file_path, object_hash)?;
378427
let should_interrupt = AtomicBool::new(false);
379428
let hash = pack.verify_checksum(&mut progress::Discard, &should_interrupt)?;
380429
assert_eq!(
@@ -386,15 +435,17 @@ fn write_and_verify(
386435
// assert_eq!(hash, expected_pack_hash, "pack hashes are stable if the input is");
387436

388437
// Re-generate the index from the pack for validation.
389-
let object_hash = gix_hash::Kind::Sha1; // TODO: parameterize this
390438
let bundle = pack::Bundle::at(
391439
pack::Bundle::write_to_directory(
392440
&mut std::io::BufReader::new(std::fs::File::open(pack_file_path)?),
393441
Some(tmp_dir.path()),
394442
&mut progress::Discard,
395443
&should_interrupt,
396444
Some(&db),
397-
pack::bundle::write::Options::default(),
445+
pack::bundle::write::Options {
446+
object_hash,
447+
..Default::default()
448+
},
398449
)?
399450
.data_path
400451
.ok_or("pack data directory should be set")?,

‎gix-pack/tests/pack/data/output/mod.rs‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,27 @@ enum DbKind {
3232
DeterministicGeneratedContentMultiIndex,
3333
}
3434

35-
fn db(kind: DbKind) -> crate::Result<gix_odb::HandleArc> {
35+
fn db(kind: DbKind, object_hash: gix_hash::Kind) -> crate::Result<gix_odb::HandleArc> {
3636
use DbKind::*;
3737
let name = match kind {
3838
DeterministicGeneratedContent => "make_pack_gen_repo.sh",
3939
DeterministicGeneratedContentMultiIndex => "make_pack_gen_repo_multi_index.sh",
4040
};
4141
let path: PathBuf = crate::scripted_fixture_read_only(name)?.join(".git").join("objects");
42-
gix_odb::Store::at_opts(path, &mut None.into_iter(), gix_odb::store::init::Options::default())
43-
.map_err(Into::into)
44-
.map(|store| {
45-
let mut cache = Arc::new(store).to_cache_arc();
46-
cache.prevent_pack_unload();
47-
cache
48-
})
42+
gix_odb::Store::at_opts(
43+
path,
44+
&mut None.into_iter(),
45+
gix_odb::store::init::Options {
46+
object_hash,
47+
..Default::default()
48+
},
49+
)
50+
.map_err(Into::into)
51+
.map(|store| {
52+
let mut cache = Arc::new(store).to_cache_arc();
53+
cache.prevent_pack_unload();
54+
cache
55+
})
4956
}
5057

5158
mod count_and_entries;

0 commit comments

Comments
 (0)