Skip to content

Commit 7dc9416

Browse files
edolstraedolstra
authored
Merge pull request #215 from DeterminateSystems/export-import
Add `nix nario` command
2 parents 1520150 + db4e80b commit 7dc9416

23 files changed

Lines changed: 490 additions & 121 deletions

File tree

‎src/libcmd/include/nix/cmd/command.hh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -352,6 +352,20 @@ struct MixEnvironment : virtual Args
352352
void setEnviron();
353353
};
354354

355+
struct MixNoCheckSigs : virtual Args
356+
{
357+
CheckSigsFlag checkSigs = CheckSigs;
358+
359+
MixNoCheckSigs()
360+
{
361+
addFlag({
362+
.longName = "no-check-sigs",
363+
.description = "Do not require that paths are signed by trusted keys.",
364+
.handler = {&checkSigs, NoCheckSigs},
365+
});
366+
}
367+
};
368+
355369
void completeFlakeInputAttrPath(
356370
AddCompletions & completions,
357371
ref<EvalState> evalState,

‎src/libmain/shared.cc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ int handleExceptions(const std::string & programName, std::function<void()> fun)
334334
return e.status;
335335
} catch (UsageError & e) {
336336
logError(e.info());
337-
printError("Try '%1% --help' for more information.", programName);
337+
printError("\nTry '%1% --help' for more information.", programName);
338338
return 1;
339339
} catch (BaseError & e) {
340340
logError(e.info());

‎src/libstore/common-protocol.cc‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ void CommonProto::Serialise<std::string>::write(
2626

2727
StorePath CommonProto::Serialise<StorePath>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
2828
{
29-
return store.parseStorePath(readString(conn.from));
29+
return conn.shortStorePaths ? StorePath(readString(conn.from)) : store.parseStorePath(readString(conn.from));
3030
}
3131

3232
void CommonProto::Serialise<StorePath>::write(
3333
const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath)
3434
{
35-
conn.to << store.printStorePath(storePath);
35+
conn.to << (conn.shortStorePaths ? storePath.to_string() : store.printStorePath(storePath));
3636
}
3737

3838
ContentAddress CommonProto::Serialise<ContentAddress>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
@@ -73,13 +73,15 @@ std::optional<StorePath>
7373
CommonProto::Serialise<std::optional<StorePath>>::read(const StoreDirConfig & store, CommonProto::ReadConn conn)
7474
{
7575
auto s = readString(conn.from);
76-
return s == "" ? std::optional<StorePath>{} : store.parseStorePath(s);
76+
return s == "" ? std::optional<StorePath>{} : conn.shortStorePaths ? StorePath(s) : store.parseStorePath(s);
7777
}
7878

7979
void CommonProto::Serialise<std::optional<StorePath>>::write(
8080
const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional<StorePath> & storePathOpt)
8181
{
82-
conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : "");
82+
conn.to
83+
<< (storePathOpt ? (conn.shortStorePaths ? storePathOpt->to_string() : store.printStorePath(*storePathOpt))
84+
: "");
8385
}
8486

8587
std::optional<ContentAddress>

‎src/libstore/export-import.cc‎

Lines changed: 132 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -4,92 +4,152 @@
44
#include "nix/util/archive.hh"
55
#include "nix/store/common-protocol.hh"
66
#include "nix/store/common-protocol-impl.hh"
7-
8-
#include <algorithm>
7+
#include "nix/store/worker-protocol.hh"
98

109
namespace nix {
1110

12-
static void exportPath(Store & store, const StorePath & path, Sink & sink)
13-
{
14-
auto info = store.queryPathInfo(path);
15-
16-
HashSink hashSink(HashAlgorithm::SHA256);
17-
TeeSink teeSink(sink, hashSink);
18-
19-
store.narFromPath(path, teeSink);
20-
21-
/* Refuse to export paths that have changed. This prevents
22-
filesystem corruption from spreading to other machines.
23-
Don't complain if the stored hash is zero (unknown). */
24-
Hash hash = hashSink.currentHash().hash;
25-
if (hash != info->narHash && info->narHash != Hash(info->narHash.algo))
26-
throw Error(
27-
"hash of path '%s' has changed from '%s' to '%s'!",
28-
store.printStorePath(path),
29-
info->narHash.to_string(HashFormat::Nix32, true),
30-
hash.to_string(HashFormat::Nix32, true));
31-
32-
teeSink << exportMagic << store.printStorePath(path);
33-
CommonProto::write(store, CommonProto::WriteConn{.to = teeSink}, info->references);
34-
teeSink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0;
35-
}
11+
static const uint32_t exportMagicV1 = 0x4558494e;
12+
static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2'
3613

37-
void exportPaths(Store & store, const StorePathSet & paths, Sink & sink)
14+
void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version)
3815
{
3916
auto sorted = store.topoSortPaths(paths);
4017
std::reverse(sorted.begin(), sorted.end());
4118

42-
for (auto & path : sorted) {
43-
sink << 1;
44-
exportPath(store, path, sink);
19+
auto dumpNar = [&](const ValidPathInfo & info) {
20+
HashSink hashSink(HashAlgorithm::SHA256);
21+
TeeSink teeSink(sink, hashSink);
22+
23+
store.narFromPath(info.path, teeSink);
24+
25+
/* Refuse to export paths that have changed. This prevents
26+
filesystem corruption from spreading to other machines.
27+
Don't complain if the stored hash is zero (unknown). */
28+
Hash hash = hashSink.currentHash().hash;
29+
if (hash != info.narHash && info.narHash != Hash(info.narHash.algo))
30+
throw Error(
31+
"hash of path '%s' has changed from '%s' to '%s'!",
32+
store.printStorePath(info.path),
33+
info.narHash.to_string(HashFormat::Nix32, true),
34+
hash.to_string(HashFormat::Nix32, true));
35+
};
36+
37+
switch (version) {
38+
39+
case 1:
40+
for (auto & path : sorted) {
41+
sink << 1;
42+
auto info = store.queryPathInfo(path);
43+
dumpNar(*info);
44+
sink << exportMagicV1 << store.printStorePath(path);
45+
CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references);
46+
sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0;
47+
}
48+
sink << 0;
49+
break;
50+
51+
case 2:
52+
sink << exportMagicV2;
53+
54+
for (auto & path : sorted) {
55+
Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path)));
56+
sink << 1;
57+
auto info = store.queryPathInfo(path);
58+
// FIXME: move to CommonProto?
59+
WorkerProto::Serialise<ValidPathInfo>::write(
60+
store, WorkerProto::WriteConn{.to = sink, .version = 16, .shortStorePaths = true}, *info);
61+
dumpNar(*info);
62+
}
63+
64+
sink << 0;
65+
break;
66+
67+
default:
68+
throw Error("unsupported nario version %d", version);
4569
}
46-
47-
sink << 0;
4870
}
4971

5072
StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs)
5173
{
5274
StorePaths res;
53-
while (true) {
54-
auto n = readNum<uint64_t>(source);
55-
if (n == 0)
56-
break;
57-
if (n != 1)
58-
throw Error("input doesn't look like something created by 'nix-store --export'");
59-
60-
/* Extract the NAR from the source. */
61-
StringSink saved;
62-
TeeSource tee{source, saved};
63-
NullFileSystemObjectSink ether;
64-
parseDump(ether, tee);
65-
66-
uint32_t magic = readInt(source);
67-
if (magic != exportMagic)
68-
throw Error("Nix archive cannot be imported; wrong format");
69-
70-
auto path = store.parseStorePath(readString(source));
71-
72-
// Activity act(*logger, lvlInfo, "importing path '%s'", info.path);
73-
74-
auto references = CommonProto::Serialise<StorePathSet>::read(store, CommonProto::ReadConn{.from = source});
75-
auto deriver = readString(source);
76-
auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
77-
78-
ValidPathInfo info{path, narHash};
79-
if (deriver != "")
80-
info.deriver = store.parseStorePath(deriver);
81-
info.references = references;
82-
info.narSize = saved.s.size();
83-
84-
// Ignore optional legacy signature.
85-
if (readInt(source) == 1)
86-
readString(source);
87-
88-
// Can't use underlying source, which would have been exhausted
89-
auto source = StringSource(saved.s);
90-
store.addToStore(info, source, NoRepair, checkSigs);
91-
92-
res.push_back(info.path);
75+
76+
auto version = readNum<uint64_t>(source);
77+
78+
/* Note: nario version 1 lacks an explicit header. The first
79+
integer denotes whether a store path follows or not. So look
80+
for 0 or 1. */
81+
switch (version) {
82+
83+
case 0:
84+
/* Empty version 1 nario, nothing to do. */
85+
break;
86+
87+
case 1:
88+
/* Non-empty version 1 nario. */
89+
while (true) {
90+
/* Extract the NAR from the source. */
91+
StringSink saved;
92+
TeeSource tee{source, saved};
93+
NullFileSystemObjectSink ether;
94+
parseDump(ether, tee);
95+
96+
uint32_t magic = readInt(source);
97+
if (magic != exportMagicV1)
98+
throw Error("nario cannot be imported; wrong format");
99+
100+
auto path = store.parseStorePath(readString(source));
101+
102+
auto references = CommonProto::Serialise<StorePathSet>::read(store, CommonProto::ReadConn{.from = source});
103+
auto deriver = readString(source);
104+
auto narHash = hashString(HashAlgorithm::SHA256, saved.s);
105+
106+
ValidPathInfo info{path, narHash};
107+
if (deriver != "")
108+
info.deriver = store.parseStorePath(deriver);
109+
info.references = references;
110+
info.narSize = saved.s.size();
111+
112+
// Ignore optional legacy signature.
113+
if (readInt(source) == 1)
114+
readString(source);
115+
116+
// Can't use underlying source, which would have been exhausted.
117+
auto source2 = StringSource(saved.s);
118+
store.addToStore(info, source2, NoRepair, checkSigs);
119+
120+
res.push_back(info.path);
121+
122+
auto n = readNum<uint64_t>(source);
123+
if (n == 0)
124+
break;
125+
if (n != 1)
126+
throw Error("input doesn't look like a nario");
127+
}
128+
break;
129+
130+
case exportMagicV2:
131+
while (true) {
132+
auto n = readNum<uint64_t>(source);
133+
if (n == 0)
134+
break;
135+
if (n != 1)
136+
throw Error("input doesn't look like a nario");
137+
138+
auto info = WorkerProto::Serialise<ValidPathInfo>::read(
139+
store, WorkerProto::ReadConn{.from = source, .version = 16, .shortStorePaths = true});
140+
141+
Activity act(
142+
*logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path)));
143+
144+
store.addToStore(info, source, NoRepair, checkSigs);
145+
146+
res.push_back(info.path);
147+
}
148+
149+
break;
150+
151+
default:
152+
throw Error("input doesn't look like a nario");
93153
}
94154

95155
return res;

‎src/libstore/include/nix/store/common-protocol.hh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ struct CommonProto
3030
struct ReadConn
3131
{
3232
Source & from;
33+
bool shortStorePaths = false;
3334
};
3435

3536
/**
@@ -39,6 +40,7 @@ struct CommonProto
3940
struct WriteConn
4041
{
4142
Sink & to;
43+
bool shortStorePaths = false;
4244
};
4345

4446
template<typename T>

‎src/libstore/include/nix/store/export-import.hh‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,11 @@
44

55
namespace nix {
66

7-
/**
8-
* Magic header of exportPath() output (obsolete).
9-
*/
10-
const uint32_t exportMagic = 0x4558494e;
11-
127
/**
138
* Export multiple paths in the format expected by `nix-store
149
* --import`. The paths will be sorted topologically.
1510
*/
16-
void exportPaths(Store & store, const StorePathSet & paths, Sink & sink);
11+
void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version);
1712

1813
/**
1914
* Import a sequence of NAR dumps created by `exportPaths()` into the

‎src/libstore/include/nix/store/worker-protocol-impl.hh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,12 +45,14 @@ struct WorkerProto::Serialise
4545
{
4646
static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn)
4747
{
48-
return CommonProto::Serialise<T>::read(store, CommonProto::ReadConn{.from = conn.from});
48+
return CommonProto::Serialise<T>::read(
49+
store, CommonProto::ReadConn{.from = conn.from, .shortStorePaths = conn.shortStorePaths});
4950
}
5051

5152
static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t)
5253
{
53-
CommonProto::Serialise<T>::write(store, CommonProto::WriteConn{.to = conn.to}, t);
54+
CommonProto::Serialise<T>::write(
55+
store, CommonProto::WriteConn{.to = conn.to, .shortStorePaths = conn.shortStorePaths}, t);
5456
}
5557
};
5658

‎src/libstore/include/nix/store/worker-protocol.hh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ struct WorkerProto
6666
{
6767
Source & from;
6868
Version version;
69+
bool shortStorePaths = false;
6970
};
7071

7172
/**
@@ -76,6 +77,7 @@ struct WorkerProto
7677
{
7778
Sink & to;
7879
Version version;
80+
bool shortStorePaths = false;
7981
};
8082

8183
/**

‎src/libstore/worker-protocol.cc‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -219,11 +219,10 @@ void WorkerProto::Serialise<ValidPathInfo>::write(
219219

220220
UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const StoreDirConfig & store, ReadConn conn)
221221
{
222-
auto deriver = readString(conn.from);
222+
auto deriver = WorkerProto::Serialise<std::optional<StorePath>>::read(store, conn);
223223
auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256);
224224
UnkeyedValidPathInfo info(narHash);
225-
if (deriver != "")
226-
info.deriver = store.parseStorePath(deriver);
225+
info.deriver = std::move(deriver);
227226
info.references = WorkerProto::Serialise<StorePathSet>::read(store, conn);
228227
conn.from >> info.registrationTime >> info.narSize;
229228
if (GET_PROTOCOL_MINOR(conn.version) >= 16) {
@@ -237,8 +236,8 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St
237236
void WorkerProto::Serialise<UnkeyedValidPathInfo>::write(
238237
const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo)
239238
{
240-
conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "")
241-
<< pathInfo.narHash.to_string(HashFormat::Base16, false);
239+
WorkerProto::write(store, conn, pathInfo.deriver);
240+
conn.to << pathInfo.narHash.to_string(HashFormat::Base16, false);
242241
WorkerProto::write(store, conn, pathInfo.references);
243242
conn.to << pathInfo.registrationTime << pathInfo.narSize;
244243
if (GET_PROTOCOL_MINOR(conn.version) >= 16) {

0 commit comments

Comments
 (0)