Skip to content

Commit 8d8d47a

Browse files
edolstraedolstra
committed
Only substitute wanted outputs of a derivation
If a derivation has multiple outputs, then we only want to download those outputs that are actuallty needed. So if we do "nix-build -A openssl.man", then only the "man" output should be downloaded. Likewise if another package depends on ${openssl.man}. The tricky part is that different derivations can depend on different outputs of a given derivation, so we may need to restart the corresponding derivation goal if that happens.
1 parent 46a369a commit 8d8d47a

4 files changed

Lines changed: 77 additions & 21 deletions

File tree

‎src/libstore/build.cc‎

Lines changed: 50 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,7 @@ class Worker
240240
~Worker();
241241

242242
/* Make a goal (with caching). */
243-
GoalPtr makeDerivationGoal(const Path & drvPath, bool repair = false);
243+
GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, bool repair = false);
244244
GoalPtr makeSubstitutionGoal(const Path & storePath, bool repair = false);
245245

246246
/* Remove a dead goal. */
@@ -756,6 +756,13 @@ class DerivationGoal : public Goal
756756
/* The path of the derivation. */
757757
Path drvPath;
758758

759+
/* The specific outputs that we need to build. Empty means all of
760+
them. */
761+
StringSet wantedOutputs;
762+
763+
/* Whether additional wanted outputs have been added. */
764+
bool needRestart;
765+
759766
/* The derivation stored at drvPath. */
760767
Derivation drv;
761768

@@ -831,7 +838,7 @@ class DerivationGoal : public Goal
831838
const static int childSetupFailed = 189;
832839

833840
public:
834-
DerivationGoal(const Path & drvPath, Worker & worker, bool repair = false);
841+
DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair = false);
835842
~DerivationGoal();
836843

837844
void cancel();
@@ -843,6 +850,9 @@ class DerivationGoal : public Goal
843850
return drvPath;
844851
}
845852

853+
/* Add wanted outputs to an already existing derivation goal. */
854+
void addWantedOutputs(const StringSet & outputs);
855+
846856
private:
847857
/* The states. */
848858
void init();
@@ -897,8 +907,10 @@ class DerivationGoal : public Goal
897907
};
898908

899909

900-
DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker, bool repair)
910+
DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, Worker & worker, bool repair)
901911
: Goal(worker)
912+
, wantedOutputs(wantedOutputs)
913+
, needRestart(false)
902914
, fLogFile(0)
903915
, bzLogFile(0)
904916
, useChroot(false)
@@ -967,6 +979,23 @@ void DerivationGoal::work()
967979
}
968980

969981

982+
void DerivationGoal::addWantedOutputs(const StringSet & outputs)
983+
{
984+
/* If we already want all outputs, there is nothing to do. */
985+
if (wantedOutputs.empty()) return;
986+
987+
if (outputs.empty()) {
988+
wantedOutputs.clear();
989+
needRestart = true;
990+
} else
991+
foreach (StringSet::const_iterator, i, outputs)
992+
if (wantedOutputs.find(*i) == wantedOutputs.end()) {
993+
wantedOutputs.insert(*i);
994+
needRestart = true;
995+
}
996+
}
997+
998+
970999
void DerivationGoal::init()
9711000
{
9721001
trace("init");
@@ -1043,6 +1072,12 @@ void DerivationGoal::outputsSubstituted()
10431072

10441073
nrFailed = nrNoSubstituters = 0;
10451074

1075+
if (needRestart) {
1076+
needRestart = false;
1077+
haveDerivation();
1078+
return;
1079+
}
1080+
10461081
if (checkPathValidity(false, repair).size() == 0) {
10471082
if (repair) repairClosure(); else amDone(ecSuccess);
10481083
return;
@@ -1051,9 +1086,13 @@ void DerivationGoal::outputsSubstituted()
10511086
/* Otherwise, at least one of the output paths could not be
10521087
produced using a substitute. So we have to build instead. */
10531088

1089+
/* Make sure checkPathValidity() from now on checks all
1090+
outputs. */
1091+
wantedOutputs = PathSet();
1092+
10541093
/* The inputs must be built before we can build this goal. */
10551094
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
1056-
addWaitee(worker.makeDerivationGoal(i->first, repair));
1095+
addWaitee(worker.makeDerivationGoal(i->first, i->second, repair));
10571096

10581097
foreach (PathSet::iterator, i, drv.inputSrcs)
10591098
addWaitee(worker.makeSubstitutionGoal(*i));
@@ -1103,7 +1142,7 @@ void DerivationGoal::repairClosure()
11031142
if (drvPath2 == "")
11041143
addWaitee(worker.makeSubstitutionGoal(*i, true));
11051144
else
1106-
addWaitee(worker.makeDerivationGoal(drvPath2, true));
1145+
addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), true));
11071146
}
11081147

11091148
if (waitees.empty()) {
@@ -2385,6 +2424,7 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
23852424
{
23862425
PathSet result;
23872426
foreach (DerivationOutputs::iterator, i, drv.outputs) {
2427+
if (!wantOutput(i->first, wantedOutputs)) continue;
23882428
bool good =
23892429
worker.store.isValidPath(i->second.path) &&
23902430
(!checkHash || worker.store.pathContentsGood(i->second.path));
@@ -2851,14 +2891,15 @@ Worker::~Worker()
28512891
}
28522892

28532893

2854-
GoalPtr Worker::makeDerivationGoal(const Path & path, bool repair)
2894+
GoalPtr Worker::makeDerivationGoal(const Path & path, const StringSet & wantedOutputs, bool repair)
28552895
{
28562896
GoalPtr goal = derivationGoals[path].lock();
28572897
if (!goal) {
2858-
goal = GoalPtr(new DerivationGoal(path, *this, repair));
2898+
goal = GoalPtr(new DerivationGoal(path, wantedOutputs, *this, repair));
28592899
derivationGoals[path] = goal;
28602900
wakeUp(goal);
2861-
}
2901+
} else
2902+
(dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs);
28622903
return goal;
28632904
}
28642905

@@ -3200,7 +3241,7 @@ void LocalStore::buildPaths(const PathSet & drvPaths, bool repair)
32003241
foreach (PathSet::const_iterator, i, drvPaths) {
32013242
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
32023243
if (isDerivation(i2.first))
3203-
goals.insert(worker.makeDerivationGoal(i2.first, repair));
3244+
goals.insert(worker.makeDerivationGoal(i2.first, i2.second, repair));
32043245
else
32053246
goals.insert(worker.makeSubstitutionGoal(*i, repair));
32063247
}

‎src/libstore/derivations.cc‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,12 +261,18 @@ DrvPathWithOutputs parseDrvPathWithOutputs(const string & s)
261261
}
262262

263263

264-
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs)
264+
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs)
265265
{
266266
return outputs.empty()
267267
? drvPath
268268
: drvPath + "!" + concatStringsSep(",", outputs);
269269
}
270270

271271

272+
bool wantOutput(const string & output, const std::set<string> & wanted)
273+
{
274+
return wanted.empty() || wanted.find(output) != wanted.end();
275+
}
276+
277+
272278
}

‎src/libstore/derivations.hh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,9 @@ extern DrvHashes drvHashes;
8585
typedef std::pair<string, std::set<string> > DrvPathWithOutputs;
8686
DrvPathWithOutputs parseDrvPathWithOutputs(const string & s);
8787

88-
Path makeDrvPathWithOutputs(const Path & drvPath, std::set<string> outputs);
88+
Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs);
89+
90+
bool wantOutput(const string & output, const std::set<string> & wanted);
8991

9092

9193
}

‎src/libstore/misc.cc‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -93,12 +93,13 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
9393
Derivation drv = derivationFromPath(store, i2.first);
9494

9595
PathSet invalid;
96-
// FIXME: only fetch the desired outputs
9796
foreach (DerivationOutputs::iterator, j, drv.outputs)
98-
if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
97+
if (wantOutput(j->first, i2.second)
98+
&& !store.isValidPath(j->second.path))
99+
invalid.insert(j->second.path);
99100
if (invalid.empty()) continue;
100101

101-
todoDrv.insert(i2.first);
102+
todoDrv.insert(*i);
102103
if (settings.useSubstitutes) query.insert(invalid.begin(), invalid.end());
103104
}
104105

@@ -115,26 +116,32 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
115116
store.querySubstitutablePathInfos(query, infos);
116117

117118
foreach (PathSet::iterator, i, todoDrv) {
119+
DrvPathWithOutputs i2 = parseDrvPathWithOutputs(*i);
120+
118121
// FIXME: cache this
119-
Derivation drv = derivationFromPath(store, *i);
122+
Derivation drv = derivationFromPath(store, i2.first);
120123

124+
PathSet outputs;
121125
bool mustBuild = false;
122126
if (settings.useSubstitutes) {
123-
foreach (DerivationOutputs::iterator, j, drv.outputs)
127+
foreach (DerivationOutputs::iterator, j, drv.outputs) {
128+
if (!wantOutput(j->first, i2.second)) continue;
124129
if (!store.isValidPath(j->second.path) &&
125130
infos.find(j->second.path) == infos.end())
126131
mustBuild = true;
132+
else
133+
outputs.insert(j->second.path);
134+
}
127135
} else
128136
mustBuild = true;
129137

130138
if (mustBuild) {
131-
willBuild.insert(*i);
139+
willBuild.insert(i2.first);
132140
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
133-
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
134-
todo.insert(i->first);
141+
foreach (DerivationInputs::iterator, j, drv.inputDrvs)
142+
todo.insert(makeDrvPathWithOutputs(j->first, j->second));
135143
} else
136-
foreach (DerivationOutputs::iterator, i, drv.outputs)
137-
todoNonDrv.insert(i->second.path);
144+
todoNonDrv.insert(outputs.begin(), outputs.end());
138145
}
139146

140147
foreach (PathSet::iterator, i, todoNonDrv) {

0 commit comments

Comments
 (0)