diff --git a/doc/manual/opt-common-syn.xml b/doc/manual/opt-common-syn.xml index f900418635..f848ad2499 100644 --- a/doc/manual/opt-common-syn.xml +++ b/doc/manual/opt-common-syn.xml @@ -15,3 +15,4 @@ + diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml index 158e282fdb..db8a60ecad 100644 --- a/doc/manual/opt-common.xml +++ b/doc/manual/opt-common.xml @@ -147,3 +147,28 @@ + + + + + + Whenever Nix attempts to realise a derivation for which a + closure is already known, but this closure cannot be realised, + fall back on normalising the derivation. + + + + The most common scenario in which this is useful is when we have + registered substitutes in order to perform binary distribution + from, say, a network repository. If the repository is down, the + realisation of the derivation will fail. When this option is + specified, Nix will build the derivation instead. Thus, + binary installation falls back on a source installation. This + option is not the default since it is generally not desirable + for a transient failure in obtaining the substitutes to lead to + a full build from source (with the related consumption of + resources). + + + + diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 13ad4fedea..6aad03a37a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -139,6 +139,8 @@ static void initAndRun(int argc, char * * argv) keepFailed = true; else if (arg == "--keep-going" || arg == "-k") keepGoing = true; + else if (arg == "--fallback") + tryFallback = true; else if (arg == "--max-jobs" || arg == "-j") { ++i; if (i == args.end()) throw UsageError("`--max-jobs' requires an argument"); diff --git a/src/libstore/db.cc b/src/libstore/db.cc index e792a371b8..5c8e7edecc 100644 --- a/src/libstore/db.cc +++ b/src/libstore/db.cc @@ -366,9 +366,12 @@ void Database::setString(const Transaction & txn, TableId table, void Database::setStrings(const Transaction & txn, TableId table, - const string & key, const Strings & data) + const string & key, const Strings & data, bool deleteEmpty) { - setString(txn, table, key, packStrings(data)); + if (deleteEmpty && data.size() == 0) + delPair(txn, table, key); + else + setString(txn, table, key, packStrings(data)); } diff --git a/src/libstore/db.hh b/src/libstore/db.hh index 1c681b9b54..bbeabfc7df 100644 --- a/src/libstore/db.hh +++ b/src/libstore/db.hh @@ -76,7 +76,8 @@ public: const string & key, const string & data); void setStrings(const Transaction & txn, TableId table, - const string & key, const Strings & data); + const string & key, const Strings & data, + bool deleteEmpty = true); void delPair(const Transaction & txn, TableId table, const string & key); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index e7b32244b3..aad26501b7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -10,6 +10,8 @@ bool keepFailed = false; bool keepGoing = false; +bool tryFallback = false; + Verbosity buildVerbosity = lvlDebug; unsigned int maxBuildJobs = 1; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index cef4f704e6..7f88d5c53b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -33,6 +33,10 @@ extern bool keepFailed; of the same goal) fails. */ extern bool keepGoing; +/* Whether, if we cannot realise the known closure corresponding to a + derivation, we should try to normalise the derivation instead. */ +extern bool tryFallback; + /* Verbosity level for build output. */ extern Verbosity buildVerbosity; diff --git a/src/libstore/normalise.cc b/src/libstore/normalise.cc index a38bee60f3..6fc3bdfc33 100644 --- a/src/libstore/normalise.cc +++ b/src/libstore/normalise.cc @@ -201,10 +201,27 @@ void Goal::waiteeDone(GoalPtr waitee, bool success) { assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - assert(nrWaitees > 0); + if (!success) ++nrFailed; - if (!--nrWaitees || (!success && !keepGoing)) + + assert(nrWaitees > 0); + if (!--nrWaitees || (!success && !keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (Goals::iterator i = waitees.begin(); i != waitees.end(); ++i) { + GoalPtr goal = *i; + WeakGoals waiters2; + for (WeakGoals::iterator j = goal->waiters.begin(); + j != goal->waiters.end(); ++j) + if (j->lock() != shared_from_this()) + waiters2.insert(*j); + goal->waiters = waiters2; + } + waitees.clear(); + worker.wakeUp(shared_from_this()); + } } @@ -271,6 +288,17 @@ const char * * strings2CharPtrs(const Strings & ss) } +/* Should only be called after an expression has been normalised. */ +Path queryNormalForm(const Path & nePath) +{ + StoreExpr ne = storeExprFromPath(nePath); + if (ne.type == StoreExpr::neClosure) return nePath; + Path nfPath; + if (!querySuccessor(nePath, nfPath)) abort(); + return nfPath; +} + + ////////////////////////////////////////////////////////////////////// @@ -472,13 +500,7 @@ void NormalisationGoal::inputNormalised() /* Inputs must also be realised before we can build this goal. */ for (PathSet::iterator i = expr.derivation.inputs.begin(); i != expr.derivation.inputs.end(); ++i) - { - Path neInput = *i, nfInput; - if (querySuccessor(neInput, nfInput)) - neInput = nfInput; - /* Otherwise the input must be a closure. */ - addWaitee(worker.makeRealisationGoal(neInput)); - } + addWaitee(worker.makeRealisationGoal(queryNormalForm(*i))); resetWaitees(expr.derivation.inputs.size()); @@ -829,8 +851,8 @@ bool NormalisationGoal::prepareBuild() i != expr.derivation.inputs.end(); ++i) { checkInterrupt(); - Path nePath = *i, nfPath; - if (!querySuccessor(nePath, nfPath)) nfPath = nePath; + Path nePath = *i; + Path nfPath = queryNormalForm(nePath); inputNFs.insert(nfPath); if (nfPath != nePath) inputSucs[nePath] = nfPath; /* !!! nfPath should be a root of the garbage collector while @@ -1174,9 +1196,15 @@ string NormalisationGoal::name() class RealisationGoal : public Goal { private: - /* The path of the closure store expression. */ + /* The path of the store expression. */ Path nePath; + /* The normal form. */ + Path nfPath; + + /* Whether we should try to delete a broken successor mapping. */ + bool tryFallback; + /* The store expression stored at nePath. */ StoreExpr expr; @@ -1191,9 +1219,12 @@ public: /* The states. */ void init(); + void isNormalised(); void haveStoreExpr(); void elemFinished(); + void fallBack(const format & error); + string name(); }; @@ -1202,6 +1233,7 @@ RealisationGoal::RealisationGoal(const Path & _nePath, Worker & _worker) : Goal(_worker) { nePath = _nePath; + tryFallback = ::tryFallback; state = &RealisationGoal::init; } @@ -1221,11 +1253,36 @@ void RealisationGoal::init() { trace("init"); - /* The first thing to do is to make sure that the store expression - exists. If it doesn't, it may be created through a - substitute. */ + if (querySuccessor(nePath, nfPath)) { + isNormalised(); + return; + } + + /* First normalise the expression (which is a no-op if the + expression is already a closure). */ resetWaitees(1); - addWaitee(worker.makeSubstitutionGoal(nePath)); + addWaitee(worker.makeNormalisationGoal(nePath)); + + /* Since there is no successor right now, the normalisation goal + will perform an actual build. So there is no sense in trying a + fallback if the realisation of the closure fails (it can't + really fail). */ + tryFallback = false; + + state = &RealisationGoal::isNormalised; +} + + +void RealisationGoal::isNormalised() +{ + trace("has been normalised"); + + nfPath = queryNormalForm(nePath); + + /* Now make sure that the store expression exists. If it doesn't, + it may be created through a substitute. */ + resetWaitees(1); + addWaitee(worker.makeSubstitutionGoal(nfPath)); state = &RealisationGoal::haveStoreExpr; } @@ -1236,21 +1293,18 @@ void RealisationGoal::haveStoreExpr() trace("loading store expression"); if (nrFailed != 0) { - printMsg(lvlError, - format("cannot realise missing store expression `%1%'") - % nePath); - amDone(false); + fallBack(format("cannot realise closure `%1%' since that file is missing") % nfPath); return; } - assert(isValidPath(nePath)); + assert(isValidPath(nfPath)); /* Get the store expression. */ - expr = storeExprFromPath(nePath); + expr = storeExprFromPath(nfPath); /* If this is a normal form (i.e., a closure) we are also done. */ if (expr.type != StoreExpr::neClosure) - throw Error(format("expected closure in `%1%'") % nePath); + throw Error(format("expected closure in `%1%'") % nfPath); /* Each path in the closure should exist, or should be creatable through a substitute. */ @@ -1269,12 +1323,11 @@ void RealisationGoal::elemFinished() trace("all closure elements present"); if (nrFailed != 0) { - printMsg(lvlError, + fallBack( format("cannot realise closure `%1%': " "%2% closure element(s) are not present " "and could not be substituted") - % nePath % nrFailed); - amDone(false); + % nfPath % nrFailed); return; } @@ -1282,6 +1335,21 @@ void RealisationGoal::elemFinished() } +void RealisationGoal::fallBack(const format & error) +{ + if (tryFallback && nePath != nfPath) { + printMsg(lvlError, format("%1%; trying to normalise derivation instead") + % error); + tryFallback = false; + unregisterSuccessor(nePath); + init(); + } else { + printMsg(lvlError, format("%1%; maybe `--fallback' will help") % error); + amDone(false); + } +} + + string RealisationGoal::name() { return (format("realisation of `%1%'") % nePath).str(); @@ -1409,8 +1477,7 @@ void SubstitutionGoal::exprNormalised() } /* Realise the substitute store expression. */ - if (!querySuccessor(sub.storeExpr, nfSub)) - nfSub = sub.storeExpr; + nfSub = queryNormalForm(sub.storeExpr); addWaitee(worker.makeRealisationGoal(nfSub)); resetWaitees(1); @@ -1869,19 +1936,19 @@ Path normaliseStoreExpr(const Path & nePath) if (!worker.run(worker.makeNormalisationGoal(nePath))) throw Error(format("normalisation of store expression `%1%' failed") % nePath); - Path nfPath; - if (!querySuccessor(nePath, nfPath)) abort(); - return nfPath; + return queryNormalForm(nePath); } -void realiseClosure(const Path & nePath) +Path realiseStoreExpr(const Path & nePath) { - startNest(nest, lvlDebug, format("realising closure `%1%'") % nePath); + startNest(nest, lvlDebug, format("realising `%1%'") % nePath); Worker worker; if (!worker.run(worker.makeRealisationGoal(nePath))) - throw Error(format("realisation of closure `%1%' failed") % nePath); + throw Error(format("realisation of store expressions `%1%' failed") % nePath); + + return queryNormalForm(nePath); } diff --git a/src/libstore/normalise.hh b/src/libstore/normalise.hh index bbde545c40..43be136e5b 100644 --- a/src/libstore/normalise.hh +++ b/src/libstore/normalise.hh @@ -10,13 +10,13 @@ successor is known. */ Path normaliseStoreExpr(const Path & nePath); -/* Realise a closure store expression in the file system. - - The pending paths are those that are already being realised. This - prevents infinite recursion for paths realised through a substitute - (since when we build the substitute, we would first try to realise - its output paths through substitutes... kaboom!). */ -void realiseClosure(const Path & nePath); +/* Realise a store expression. If the expression is a derivation, it + is first normalised into a closure. The closure is then realised + in the file system (i.e., it is ensured that each path in the + closure exists in the file system, if necessary by using the + substitute mechanism). Returns the normal form of the expression + (i.e., its closure expression). */ +Path realiseStoreExpr(const Path & nePath); /* Ensure that a path exists, possibly by instantiating it by realising a substitute. */ diff --git a/src/libstore/store.cc b/src/libstore/store.cc index 9677f84223..44b3a29e34 100644 --- a/src/libstore/store.cc +++ b/src/libstore/store.cc @@ -237,6 +237,30 @@ void registerSuccessor(const Transaction & txn, } +void unregisterSuccessor(const Path & srcPath) +{ + assertStorePath(srcPath); + + Transaction txn(nixDB); + + Path sucPath; + if (!nixDB.queryString(txn, dbSuccessors, srcPath, sucPath)) { + txn.abort(); + return; + } + nixDB.delPair(txn, dbSuccessors, srcPath); + + Paths revs; + nixDB.queryStrings(txn, dbSuccessorsRev, sucPath, revs); + Paths::iterator i = find(revs.begin(), revs.end(), srcPath); + assert(i != revs.end()); + revs.erase(i); + nixDB.setStrings(txn, dbSuccessorsRev, sucPath, revs); + + txn.commit(); +} + + bool querySuccessor(const Path & srcPath, Path & sucPath) { return nixDB.queryString(noTxn, dbSuccessors, srcPath, sucPath); @@ -294,10 +318,7 @@ static void writeSubstitutes(const Transaction & txn, ss.push_back(packStrings(ss2)); } - if (ss.size() == 0) - nixDB.delPair(txn, dbSubstitutes, srcPath); - else - nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); + nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); } diff --git a/src/libstore/store.hh b/src/libstore/store.hh index 40d1859e53..68f7d61905 100644 --- a/src/libstore/store.hh +++ b/src/libstore/store.hh @@ -54,6 +54,9 @@ void copyPath(const Path & src, const Path & dst); void registerSuccessor(const Transaction & txn, const Path & srcPath, const Path & sucPath); +/* Remove a successor mapping. */ +void unregisterSuccessor(const Path & srcPath); + /* Return the predecessors of the Nix expression stored at the given path. */ bool querySuccessor(const Path & srcPath, Path & sucPath); diff --git a/src/nix-env/main.cc b/src/nix-env/main.cc index 50ff0b19e9..c57f03cce0 100644 --- a/src/nix-env/main.cc +++ b/src/nix-env/main.cc @@ -213,8 +213,7 @@ void createUserEnv(EvalState & state, const DrvInfos & drvs, /* Realise the resulting store expression. */ debug(format("realising user environment")); - Path nfPath = normaliseStoreExpr(topLevelDrv.drvPath); - realiseClosure(nfPath); + Path nfPath = realiseStoreExpr(topLevelDrv.drvPath); /* Switch the current user environment to the output path. */ debug(format("switching to new user environment")); diff --git a/src/nix-store/main.cc b/src/nix-store/main.cc index 2f38edf77b..e83f9133f6 100644 --- a/src/nix-store/main.cc +++ b/src/nix-store/main.cc @@ -26,8 +26,7 @@ static void opRealise(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) { - Path nfPath = normaliseStoreExpr(*i); - realiseClosure(nfPath); + Path nfPath = realiseStoreExpr(*i); cout << format("%1%\n") % (string) nfPath; } } @@ -58,8 +57,7 @@ static void opAdd(Strings opFlags, Strings opArgs) Path maybeNormalise(const Path & ne, bool normalise, bool realise) { if (realise) { - Path ne2 = normaliseStoreExpr(ne); - realiseClosure(ne2); + Path ne2 = realiseStoreExpr(ne); return normalise ? ne2 : ne; } else return normalise ? normaliseStoreExpr(ne) : ne; diff --git a/tests/Makefile.am b/tests/Makefile.am index 19122cd955..9e1d6a1983 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -19,10 +19,10 @@ parallel.sh: parallel.nix build-hook.sh: build-hook.nix substitutes.sh: substitutes.nix substituter.nix substitutes2.sh: substitutes2.nix substituter.nix substituter2.nix -fall-back.sh: fall-back.nix +fallback.sh: fallback.nix TESTS = init.sh simple.sh dependencies.sh locking.sh parallel.sh \ - build-hook.sh substitutes.sh substitutes2.sh + build-hook.sh substitutes.sh substitutes2.sh fallback.sh verify.sh XFAIL_TESTS = @@ -36,4 +36,4 @@ EXTRA_DIST = $(TESTS) \ build-hook.nix.in build-hook.hook.sh \ substitutes.nix.in substituter.nix.in substituter.builder.sh \ substitutes2.nix.in substituter2.nix.in substituter2.builder.sh \ - fall-back.nix.in + fallback.nix.in diff --git a/tests/fall-back.nix.in b/tests/fallback.nix.in similarity index 100% rename from tests/fall-back.nix.in rename to tests/fallback.nix.in diff --git a/tests/fall-back.sh b/tests/fallback.sh similarity index 88% rename from tests/fall-back.sh rename to tests/fallback.sh index e4a3942177..5799775eb2 100644 --- a/tests/fall-back.sh +++ b/tests/fallback.sh @@ -7,7 +7,7 @@ suc=$NIX_STORE_DIR/deadbeafdeadbeafdeadbeafdeadbeaf-s.store (echo $suc && echo $NIX_STORE_DIR/ffffffffffffffffffffffffffffffff.store && echo "/bla" && echo 0) | $TOP/src/nix-store/nix-store --substitute $TOP/src/nix-store/nix-store --successor $storeExpr $suc -outPath=$($TOP/src/nix-store/nix-store -qnf "$storeExpr") +outPath=$($TOP/src/nix-store/nix-store -qnf --fallback "$storeExpr") echo "output path is $outPath" diff --git a/tests/verify.sh b/tests/verify.sh new file mode 100644 index 0000000000..ede3e7d74b --- /dev/null +++ b/tests/verify.sh @@ -0,0 +1 @@ +$TOP/src/nix-store/nix-store --verify