nix-store -r: do substitutions in parallel

I.e. when multiple non-derivation arguments are passed to ‘nix-store
-r’ to be substituted, do them in parallel.
This commit is contained in:
Eelco Dolstra 2012-06-27 16:58:15 -04:00
parent 42f5a2fc29
commit 1aba0bf0fa
11 changed files with 30 additions and 25 deletions

View File

@ -53,7 +53,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
try { try {
/* !!! If using a substitute, we only need to fetch /* !!! If using a substitute, we only need to fetch
the selected output of this derivation. */ the selected output of this derivation. */
store->buildDerivations(singleton<PathSet>(ctx)); store->buildPaths(singleton<PathSet>(ctx));
} catch (Error & e) { } catch (Error & e) {
throw ImportError(e.msg()); throw ImportError(e.msg());
} }

View File

@ -2275,6 +2275,8 @@ public:
/* Callback used by the worker to write to the log. */ /* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data); void handleChildOutput(int fd, const string & data);
void handleEOF(int fd); void handleEOF(int fd);
Path getStorePath() { return storePath; }
}; };
@ -2938,7 +2940,7 @@ unsigned int Worker::exitStatus()
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
void LocalStore::buildDerivations(const PathSet & drvPaths) void LocalStore::buildPaths(const PathSet & drvPaths)
{ {
startNest(nest, lvlDebug, startNest(nest, lvlDebug,
format("building %1%") % showPaths(drvPaths)); format("building %1%") % showPaths(drvPaths));
@ -2947,7 +2949,10 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
Goals goals; Goals goals;
foreach (PathSet::const_iterator, i, drvPaths) foreach (PathSet::const_iterator, i, drvPaths)
goals.insert(worker.makeDerivationGoal(*i)); if (isDerivation(*i))
goals.insert(worker.makeDerivationGoal(*i));
else
goals.insert(worker.makeSubstitutionGoal(*i));
worker.run(goals); worker.run(goals);
@ -2955,8 +2960,8 @@ void LocalStore::buildDerivations(const PathSet & drvPaths)
foreach (Goals::iterator, i, goals) foreach (Goals::iterator, i, goals)
if ((*i)->getExitCode() == Goal::ecFailed) { if ((*i)->getExitCode() == Goal::ecFailed) {
DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get()); DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i->get());
assert(i2); if (i2) failed.insert(i2->getDrvPath());
failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i->get())->getStorePath());
} }
if (!failed.empty()) if (!failed.empty())

View File

@ -150,7 +150,7 @@ public:
Paths importPaths(bool requireSignature, Source & source); Paths importPaths(bool requireSignature, Source & source);
void buildDerivations(const PathSet & drvPaths); void buildPaths(const PathSet & paths);
void ensurePath(const Path & path); void ensurePath(const Path & path);

View File

@ -397,10 +397,10 @@ Paths RemoteStore::importPaths(bool requireSignature, Source & source)
} }
void RemoteStore::buildDerivations(const PathSet & drvPaths) void RemoteStore::buildPaths(const PathSet & drvPaths)
{ {
openConnection(); openConnection();
writeInt(wopBuildDerivations, to); writeInt(wopBuildPaths, to);
writeStrings(drvPaths, to); writeStrings(drvPaths, to);
processStderr(); processStderr();
readInt(from); readInt(from);

View File

@ -60,7 +60,7 @@ public:
Paths importPaths(bool requireSignature, Source & source); Paths importPaths(bool requireSignature, Source & source);
void buildDerivations(const PathSet & drvPaths); void buildPaths(const PathSet & paths);
void ensurePath(const Path & path); void ensurePath(const Path & path);

View File

@ -172,13 +172,15 @@ public:
the Nix store. */ the Nix store. */
virtual Paths importPaths(bool requireSignature, Source & source) = 0; virtual Paths importPaths(bool requireSignature, Source & source) = 0;
/* Ensure that the output paths of the derivation are valid. If /* For each path, if it's a derivation, build it. Building a
derivation means ensuring that the output paths are valid. If
they are already valid, this is a no-op. Otherwise, validity they are already valid, this is a no-op. Otherwise, validity
can be reached in two ways. First, if the output paths is can be reached in two ways. First, if the output paths is
substitutable, then build the path that way. Second, the substitutable, then build the path that way. Second, the
output paths can be created by running the builder, after output paths can be created by running the builder, after
recursively building any sub-derivations. */ recursively building any sub-derivations. For inputs that are
virtual void buildDerivations(const PathSet & drvPaths) = 0; not derivations, substitute them. */
virtual void buildPaths(const PathSet & paths) = 0;
/* Ensure that a path is valid. If it is not currently valid, it /* Ensure that a path is valid. If it is not currently valid, it
may be made valid by running a substitute (if defined for the may be made valid by running a substitute (if defined for the

View File

@ -22,7 +22,7 @@ typedef enum {
wopQueryReferrers = 6, wopQueryReferrers = 6,
wopAddToStore = 7, wopAddToStore = 7,
wopAddTextToStore = 8, wopAddTextToStore = 8,
wopBuildDerivations = 9, wopBuildPaths = 9,
wopEnsurePath = 10, wopEnsurePath = 10,
wopAddTempRoot = 11, wopAddTempRoot = 11,
wopAddIndirectRoot = 12, wopAddIndirectRoot = 12,

View File

@ -695,7 +695,7 @@ static void opSet(Globals & globals,
PathSet paths = singleton<PathSet>(drv.queryDrvPath(globals.state)); PathSet paths = singleton<PathSet>(drv.queryDrvPath(globals.state));
printMissing(*store, paths); printMissing(*store, paths);
if (globals.dryRun) return; if (globals.dryRun) return;
store->buildDerivations(paths); store->buildPaths(paths);
} }
else { else {
printMissing(*store, singleton<PathSet>(drv.queryOutPath(globals.state))); printMissing(*store, singleton<PathSet>(drv.queryOutPath(globals.state)));

View File

@ -45,7 +45,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
drvsToBuild.insert(i->queryDrvPath(state)); drvsToBuild.insert(i->queryDrvPath(state));
debug(format("building user environment dependencies")); debug(format("building user environment dependencies"));
store->buildDerivations(drvsToBuild); store->buildPaths(drvsToBuild);
/* Construct the whole top level derivation. */ /* Construct the whole top level derivation. */
PathSet references; PathSet references;
@ -132,7 +132,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
/* Realise the resulting store expression. */ /* Realise the resulting store expression. */
debug("building user environment"); debug("building user environment");
store->buildDerivations(singleton<PathSet>(topLevelDrv.queryDrvPath(state))); store->buildPaths(singleton<PathSet>(topLevelDrv.queryDrvPath(state)));
/* Switch the current user environment to the output path. */ /* Switch the current user environment to the output path. */
PathLocks lock; PathLocks lock;

View File

@ -62,7 +62,7 @@ static Path useDeriver(Path path)
static PathSet realisePath(const Path & path) static PathSet realisePath(const Path & path)
{ {
if (isDerivation(path)) { if (isDerivation(path)) {
store->buildDerivations(singleton<PathSet>(path)); store->buildPaths(singleton<PathSet>(path));
Derivation drv = derivationFromPath(*store, path); Derivation drv = derivationFromPath(*store, path);
PathSet outputs; PathSet outputs;
@ -101,13 +101,11 @@ static void opRealise(Strings opFlags, Strings opArgs)
if (dryRun) return; if (dryRun) return;
/* Build all derivations at the same time to exploit parallelism. */ /* Build all paths at the same time to exploit parallelism. */
PathSet drvPaths; PathSet paths(opArgs.begin(), opArgs.end());
foreach (Strings::iterator, i, opArgs) store->buildPaths(paths);
if (isDerivation(*i)) drvPaths.insert(*i);
store->buildDerivations(drvPaths);
foreach (Strings::iterator, i, opArgs) { foreach (Paths::iterator, i, opArgs) {
PathSet paths = realisePath(*i); PathSet paths = realisePath(*i);
foreach (PathSet::iterator, j, paths) foreach (PathSet::iterator, j, paths)
cout << format("%1%\n") % *j; cout << format("%1%\n") % *j;

View File

@ -415,10 +415,10 @@ static void performOp(unsigned int clientVersion,
break; break;
} }
case wopBuildDerivations: { case wopBuildPaths: {
PathSet drvs = readStorePaths<PathSet>(from); PathSet drvs = readStorePaths<PathSet>(from);
startWork(); startWork();
store->buildDerivations(drvs); store->buildPaths(drvs);
stopWork(); stopWork();
writeInt(1, to); writeInt(1, to);
break; break;