From a3f205b24954c7f0983a937b0b9b3d64c22a2fa7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 3 Oct 2012 10:38:09 -0400 Subject: [PATCH] When repairing a derivation, check and repair the entire output closure If we find a corrupted path in the output closure, we rebuild the derivation that produced that particular path. --- src/libstore/build.cc | 68 +++++++++++++++++++++++++++++++++++-- src/libstore/local-store.cc | 24 ++++++++++--- src/libstore/local-store.hh | 5 +++ 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ddcd45a61b..6c40b6686d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -853,6 +853,7 @@ private: void init(); void haveDerivation(); void outputsSubstituted(); + void closureRepaired(); void inputsRealised(); void tryToBuild(); void buildDone(); @@ -896,6 +897,8 @@ private: void killChild(); Path addHashRewrite(const Path & path); + + void repairClosure(); }; @@ -1046,7 +1049,7 @@ void DerivationGoal::outputsSubstituted() nrFailed = nrNoSubstituters = 0; if (checkPathValidity(false, repair).size() == 0) { - amDone(ecSuccess); + if (repair) repairClosure(); else amDone(ecSuccess); return; } @@ -1055,7 +1058,7 @@ void DerivationGoal::outputsSubstituted() /* The inputs must be built before we can build this goal. */ foreach (DerivationInputs::iterator, i, drv.inputDrvs) - addWaitee(worker.makeDerivationGoal(i->first)); + addWaitee(worker.makeDerivationGoal(i->first, repair)); foreach (PathSet::iterator, i, drv.inputSrcs) addWaitee(worker.makeSubstitutionGoal(*i)); @@ -1067,6 +1070,63 @@ void DerivationGoal::outputsSubstituted() } +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + foreach (DerivationOutputs::iterator, i, drv.outputs) + computeFSClosure(worker.store, i->second.path, outputClosure); + + /* Filter out our own outputs (which we have already checked). */ + foreach (DerivationOutputs::iterator, i, drv.outputs) + outputClosure.erase(i->second.path); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + computeFSClosure(worker.store, drvPath, inputClosure); + std::map outputsToDrv; + foreach (PathSet::iterator, i, inputClosure) + if (isDerivation(*i)) { + Derivation drv = derivationFromPath(worker.store, *i); + foreach (DerivationOutputs::iterator, j, drv.outputs) + outputsToDrv[j->second.path] = *i; + } + + /* Check each path (slow!). */ + PathSet broken; + foreach (PathSet::iterator, i, outputClosure) { + if (worker.store.pathContentsGood(*i)) continue; + printMsg(lvlError, format("found corrupted or missing path `%1%' in the output closure of `%2%'") % *i % drvPath); + Path drvPath2 = outputsToDrv[*i]; + if (drvPath2 == "") throw Error(format("don't know how to repair corrupted or missing path `%1%'") % *i); + addWaitee(worker.makeDerivationGoal(drvPath2, true)); + } + + if (waitees.empty()) { + amDone(ecSuccess); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error(format("some paths in the output closure of derivation `%1%' could not be repaired") % drvPath); + amDone(ecSuccess); +} + + void DerivationGoal::inputsRealised() { trace("all inputs realised"); @@ -2197,6 +2257,8 @@ void DerivationGoal::computeClosure() } worker.store.optimisePath(path); // FIXME: combine with scanForReferences() + + worker.store.markContentsGood(path); } /* Register each output path as valid, and register the sets of @@ -2729,6 +2791,8 @@ void SubstitutionGoal::finished() outputLock->setDeletion(true); + worker.store.markContentsGood(storePath); + printMsg(lvlChatty, format("substitution of path `%1%' succeeded") % storePath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a4ad97331a..b55ab42842 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1673,11 +1673,27 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, bool LocalStore::pathContentsGood(const Path & path) { + std::map::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printMsg(lvlInfo, format("checking path `%1%'...") % path); ValidPathInfo info = queryPathInfo(path); - if (!pathExists(path)) return false; - HashResult current = hashPath(info.hash.type, path); - Hash nullHash(htSHA256); - return info.hash == nullHash || info.hash == current.first; + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info.hash.type, path); + Hash nullHash(htSHA256); + res = info.hash == nullHash || info.hash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path); + return res; +} + + +void LocalStore::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 00356bf9d7..b2f06d8118 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -206,6 +206,8 @@ public: contents. */ bool pathContentsGood(const Path & path); + void markContentsGood(const Path & path); + private: Path schemaPath; @@ -233,6 +235,9 @@ private: SQLiteStmt stmtQueryDerivationOutputs; SQLiteStmt stmtQueryPathFromHashPart; + /* Cache for pathContentsGood(). */ + std::map pathContentsGoodCache; + int getSchema(); void openDB(bool create);