From 9e975458b48d9eb041a6484c634be2ee8ee1649d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 12 Aug 2007 00:29:28 +0000 Subject: [PATCH] * Get rid of the substitutes database table (NIX-47). Instead, if we need any info on substitutable paths, we just call the substituters (such as download-using-manifests.pl) directly. This means that it's no longer necessary for nix-pull to register substitutes or for nix-channel to clear them, which makes those operations much faster (NIX-95). Also, we don't have to worry about keeping nix-pull manifests (in /nix/var/nix/manifests) and the database in sync with each other. The downside is that there is some overhead in calling an external program to get the substitutes info. For instance, "nix-env -qas" takes a bit longer. Abolishing the substitutes table also makes the logic in local-store.cc simpler, as we don't need to store info for invalid paths. On the downside, you cannot do things like "nix-store -qR" on a substitutable but invalid path (but nobody did that anyway). * Never catch interrupts (the Interrupted exception). --- doc/manual/nix-store.xml | 30 ---- scripts/download-using-manifests.pl.in | 58 ++++-- scripts/nix-channel.in | 4 - scripts/nix-pull.in | 38 ---- src/libmain/shared.cc | 8 +- src/libstore/build.cc | 200 ++++++++++++++++----- src/libstore/db.cc | 13 +- src/libstore/db.hh | 2 + src/libstore/gc.cc | 2 + src/libstore/globals.cc | 1 + src/libstore/globals.hh | 5 + src/libstore/local-store.cc | 233 +++++-------------------- src/libstore/local-store.hh | 25 +-- src/libstore/misc.cc | 7 +- src/libstore/remote-store.cc | 12 +- src/libstore/remote-store.hh | 8 +- src/libstore/store-api.cc | 21 ++- src/libstore/store-api.hh | 55 +++--- src/libstore/worker-protocol.hh | 1 - src/libutil/types.hh | 26 +-- src/libutil/util.cc | 7 +- src/libutil/util.hh | 2 +- src/nix-store/help.txt | 2 - src/nix-store/nix-store.cc | 66 +------ 24 files changed, 357 insertions(+), 469 deletions(-) diff --git a/doc/manual/nix-store.xml b/doc/manual/nix-store.xml index 038fd523f0..d787c337be 100644 --- a/doc/manual/nix-store.xml +++ b/doc/manual/nix-store.xml @@ -649,36 +649,6 @@ $ gv graph.ps - - - - - - Operation <option>--verify</option> diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index 65ab36084a..f963eead19 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -22,16 +22,6 @@ my $tmpNar2 = "$tmpDir/nar2"; END { unlink $tmpNar; unlink $tmpNar2; rmdir $tmpDir; } -# Check the arguments. -die unless scalar @ARGV == 1; -my $targetPath = $ARGV[0]; - -my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); -print LOGFILE "$$ get $targetPath $date\n"; - -print "\n*** Trying to download/patch `$targetPath'\n"; - - # Load all manifests. my %narFiles; my %localPaths; @@ -46,6 +36,54 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") { } +# Parse the arguments. + +if ($ARGV[0] eq "--query-paths") { + foreach my $storePath (keys %narFiles) { print "$storePath\n"; } + foreach my $storePath (keys %localPaths) { print "$storePath\n"; } + exit 0; +} + +elsif ($ARGV[0] eq "--query-info") { + shift @ARGV; + foreach my $storePath (@ARGV) { + my $info; + if (defined $narFiles{$storePath}) { + $info = @{$narFiles{$storePath}}[0]; + } + elsif (defined $localPaths{$storePath}) { + $info = @{$localPaths{$storePath}}[0]; + } + else { + next; # not an error + } + print "$storePath\n"; + print "$info->{deriver}\n"; + my @references = split " ", $info->{references}; + my $count = scalar @references; + print "$count\n"; + foreach my $reference (@references) { + print "$reference\n"; + } + } + exit 0; +} + +elsif ($ARGV[0] ne "--substitute") { + die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n"; +} + + +die unless scalar @ARGV == 2; +my $targetPath = $ARGV[1]; + + +my $date = strftime ("%F %H:%M:%S UTC", gmtime (time)); +print LOGFILE "$$ get $targetPath $date\n"; + +print "\n*** Trying to download/patch `$targetPath'\n"; + + # If we can copy from a local path, do that. my $localPathList = $localPaths{$targetPath}; foreach my $localPath (@{$localPathList}) { diff --git a/scripts/nix-channel.in b/scripts/nix-channel.in index 6a4603976f..1ef9c58447 100644 --- a/scripts/nix-channel.in +++ b/scripts/nix-channel.in @@ -76,10 +76,6 @@ sub removeChannel { sub update { readChannels; - # Get rid of all the old substitutes. - system("@bindir@/nix-store", "--clear-substitutes") == 0 - or die "cannot clear substitutes"; - # Remove all the old manifests. for my $manifest (glob "$stateDir/manifests/*.nixmanifest") { unlink $manifest or die "cannot remove `$manifest': $!"; diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index ff83f2857c..378d9a363e 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -108,41 +108,3 @@ while (@ARGV) { my $size = scalar (keys %narFiles) + scalar (keys %localPaths); print "$size store paths in manifest\n"; - - -# Register all substitutes. -print STDERR "registering substitutes...\n"; - -my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes") - or die "cannot run nix-store"; - -sub writeRegistration { - my $storePath = shift; - my $object = shift; - print WRITE "$storePath\n"; - print WRITE "$object->{deriver}\n"; - print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; - print WRITE "0\n"; - my @references = split " ", $object->{references}; - my $count = scalar @references; - print WRITE "$count\n"; - foreach my $reference (@references) { - print WRITE "$reference\n"; - } -} - -foreach my $storePath (keys %narFiles) { - my $narFileList = $narFiles{$storePath}; - foreach my $narFile (@{$narFileList}) { - writeRegistration $storePath, $narFile; - } -} - -foreach my $storePath (keys %localPaths) { - my $localPathList = $localPaths{$storePath}; - foreach my $localPath (@{$localPathList}) { - writeRegistration $storePath, $localPath; - } -} - -close WRITE or die "nix-store failed: $?"; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index ebdbeb11e1..a606f0dae3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -113,6 +113,12 @@ static void initAndRun(int argc, char * * argv) nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR)); nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR)); + string subs = getEnv("NIX_SUBSTITUTERS", "default"); + if (subs == "default") + substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl"); + else + substituters = tokenizeString(subs, ":"); + /* Get some settings from the configuration file. */ thisSystem = querySetting("system", SYSTEM); maxBuildJobs = queryIntSetting("build-max-jobs", 1); @@ -320,7 +326,7 @@ int main(int argc, char * * argv) "Try `%2% --help' for more information.") % e.what() % programId); return 1; - } catch (Error & e) { + } catch (BaseError & e) { printMsg(lvlError, format("error: %1%") % e.msg()); return 1; } catch (std::exception & e) { diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e300292e96..4a2affdf59 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -164,6 +164,11 @@ private: /* Goals waiting for a build slot. */ WeakGoals wantingToBuild; + /* Goals waiting for info from substituters (using --query-info), + and the info they're (collectively) waiting for. */ + WeakGoals waitingForInfo; + map requestedInfo; + /* Child processes currently running. */ Children children; @@ -212,12 +217,24 @@ public: /* Put `goal' to sleep until a child process terminates, i.e., a call is made to childTerminate(..., true). */ void waitForChildTermination(GoalPtr goal); + + /* Put `goal' to sleep until the top-level loop has run `sub' to + get info about `storePath' (with --query-info). We combine + substituter invocations to reduce overhead. */ + void waitForInfo(GoalPtr goal, Path sub, Path storePath); /* Loop until the specified top-level goals have finished. */ void run(const Goals & topGoals); /* Wait for input to become available. */ void waitForInput(); + +private: + + /* Process the pending paths in requestedInfo and wake up the + goals in waitingForInfo. */ + void getInfo(); + }; @@ -783,7 +800,7 @@ void DerivationGoal::haveDerivation() substitutes. */ if (store->hasSubstitutes(*i)) addWaitee(worker.makeSubstitutionGoal(*i)); - + if (waitees.empty()) /* to prevent hang (no wake-up event) */ outputsSubstituted(); else @@ -1829,18 +1846,22 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid) class SubstitutionGoal : public Goal { + friend class Worker; + private: /* The store path that should be realised through a substitute. */ Path storePath; - /* The remaining substitutes for this path. */ - Substitutes subs; + /* The remaining substituters. */ + Paths subs; - /* The current substitute. */ - Substitute sub; + /* The current substituter. */ + Path sub; - /* Outgoing references for this path. */ + /* Path info returned by the substituter's --query-info operation. */ + bool infoOkay; PathSet references; + Path deriver; /* Pipe for the substitute's standard output/error. */ Pipe logPipe; @@ -1864,8 +1885,9 @@ public: /* The states. */ void init(); - void referencesValid(); void tryNext(); + void gotInfo(); + void referencesValid(); void tryToRun(); void finished(); @@ -1923,17 +1945,46 @@ void SubstitutionGoal::init() return; } - /* !!! race condition; should get the substitutes and the - references in a transaction (in case a clearSubstitutes() is - done simultaneously). */ + subs = substituters; - /* Read the substitutes. */ - subs = store->querySubstitutes(storePath); + tryNext(); +} + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + printMsg(lvlError, + format("path `%1%' is required, but there is no substituter that can build it") + % storePath); + amDone(ecFailed); + return; + } + + sub = subs.front(); + subs.pop_front(); + + infoOkay = false; + state = &SubstitutionGoal::gotInfo; + worker.waitForInfo(shared_from_this(), sub, storePath); +} + + +void SubstitutionGoal::gotInfo() +{ + trace("got info"); + + if (!infoOkay) { + tryNext(); + return; + } + /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - store->queryReferences(storePath, references); - for (PathSet::iterator i = references.begin(); i != references.end(); ++i) if (*i != storePath) /* ignore self-references */ @@ -1948,7 +1999,7 @@ void SubstitutionGoal::init() void SubstitutionGoal::referencesValid() { - trace("all referenced realised"); + trace("all references realised"); if (nrFailed > 0) { printMsg(lvlError, @@ -1961,28 +2012,7 @@ void SubstitutionGoal::referencesValid() i != references.end(); ++i) if (*i != storePath) /* ignore self-references */ assert(store->isValidPath(*i)); - - tryNext(); -} - -void SubstitutionGoal::tryNext() -{ - trace("trying next substitute"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - printMsg(lvlError, - format("path `%1%' is required, but it has no (remaining) substitutes") - % storePath); - amDone(ecFailed); - return; - } - sub = subs.front(); - subs.pop_front(); - - /* Wait until we can run the substitute program. */ state = &SubstitutionGoal::tryToRun; worker.waitForBuildSlot(shared_from_this()); } @@ -2013,7 +2043,7 @@ void SubstitutionGoal::tryToRun() printMsg(lvlInfo, format("substituting path `%1%' using substituter `%2%'") - % storePath % sub.program); + % storePath % sub); logPipe.create(); @@ -2038,14 +2068,15 @@ void SubstitutionGoal::tryToRun() commonChildInit(logPipe); /* Fill in the arguments. */ - Strings args(sub.args); - args.push_front(storePath); - args.push_front(baseNameOf(sub.program)); + Strings args; + args.push_back(baseNameOf(sub)); + args.push_back("--substitute"); + args.push_back(storePath); const char * * argArr = strings2CharPtrs(args); - execv(sub.program.c_str(), (char * *) argArr); + execv(sub.c_str(), (char * *) argArr); - throw SysError(format("executing `%1%'") % sub.program); + throw SysError(format("executing `%1%'") % sub); } catch (std::exception & e) { std::cerr << format("substitute error: %1%\n") % e.what(); @@ -2098,7 +2129,7 @@ void SubstitutionGoal::finished() printMsg(lvlInfo, format("substitution of path `%1%' using substituter `%2%' failed: %3%") - % storePath % sub.program % e.msg()); + % storePath % sub % e.msg()); /* Try the next substitute. */ state = &SubstitutionGoal::tryNext; @@ -2113,7 +2144,7 @@ void SubstitutionGoal::finished() Transaction txn; createStoreTransaction(txn); registerValidPath(txn, storePath, contentHash, - references, sub.deriver); + references, deriver); txn.commit(); outputLock->setDeletion(true); @@ -2298,6 +2329,76 @@ void Worker::waitForChildTermination(GoalPtr goal) } +void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath) +{ + debug("wait for info"); + requestedInfo[sub].insert(storePath); + waitingForInfo.insert(goal); +} + + +void Worker::getInfo() +{ + for (map::iterator i = requestedInfo.begin(); + i != requestedInfo.end(); ++i) + { + Path sub = i->first; + PathSet paths = i->second; + + while (!paths.empty()) { + + /* Run the substituter for at most 100 paths at a time to + prevent command line overflows. */ + PathSet paths2; + while (!paths.empty() && paths2.size() < 100) { + paths2.insert(*paths.begin()); + paths.erase(paths.begin()); + } + + /* Ask the substituter for the references and deriver of + the paths. */ + debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2)); + Strings args; + args.push_back("--query-info"); + args.insert(args.end(), paths2.begin(), paths2.end()); + string res = runProgram(sub, false, args); + std::istringstream str(res); + + while (true) { + ValidPathInfo info = decodeValidPathInfo(str); + if (info.path == "") break; + + /* !!! inefficient */ + for (WeakGoals::iterator k = waitingForInfo.begin(); + k != waitingForInfo.end(); ++k) + { + GoalPtr goal = k->lock(); + if (goal) { + SubstitutionGoal * goal2 = dynamic_cast(goal.get()); + if (goal2->storePath == info.path) { + goal2->references = info.references; + goal2->deriver = info.deriver; + goal2->infoOkay = true; + wakeUp(goal); + } + } + } + } + } + } + + for (WeakGoals::iterator k = waitingForInfo.begin(); + k != waitingForInfo.end(); ++k) + { + GoalPtr goal = k->lock(); + if (goal) wakeUp(goal); + } + + requestedInfo.clear(); + waitingForInfo.clear(); // !!! have we done them all? +} + + void Worker::run(const Goals & _topGoals) { for (Goals::iterator i = _topGoals.begin(); @@ -2324,11 +2425,14 @@ void Worker::run(const Goals & _topGoals) if (topGoals.empty()) break; - /* !!! not when we're polling */ - assert(!children.empty()); - + getInfo(); + /* Wait for input. */ - waitForInput(); + if (!children.empty()) + waitForInput(); + else + /* !!! not when we're polling */ + assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal diff --git a/src/libstore/db.cc b/src/libstore/db.cc index 657f6b7b4b..59b9c0c8e3 100644 --- a/src/libstore/db.cc +++ b/src/libstore/db.cc @@ -194,7 +194,8 @@ void Database::open2(const string & path, bool removeOldEnv) env->set_errcall(errorPrinter); env->set_msgcall(messagePrinter); - //env->set_verbose(DB_VERB_REGISTER, 1); + if (getEnv("NIX_DEBUG_DB_REGISTER") == "1") + env->set_verbose(DB_VERB_REGISTER, 1); env->set_verbose(DB_VERB_RECOVERY, 1); /* Smaller log files. */ @@ -454,4 +455,14 @@ void Database::enumTable(const Transaction & txn, TableId table, } +void Database::clearTable(const Transaction & txn, TableId table) +{ + try { + Db * db = getDb(table); + u_int32_t count; + db->truncate(txn.txn, &count, 0); + } catch (DbException e) { rethrow(e); } +} + + } diff --git a/src/libstore/db.hh b/src/libstore/db.hh index 54f490f885..8672fbc863 100644 --- a/src/libstore/db.hh +++ b/src/libstore/db.hh @@ -89,6 +89,8 @@ public: void enumTable(const Transaction & txn, TableId table, Strings & keys, const string & keyPrefix = ""); + + void clearTable(const Transaction & txn, TableId table); }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3699b23440..8ce717aaff 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -602,6 +602,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete, } #endif + if (!pathExists(*i)) continue; + printMsg(lvlInfo, format("deleting `%1%'") % *i); /* Okay, it's safe to delete. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b0316f77c2..6c4944539d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -25,6 +25,7 @@ unsigned int maxBuildJobs = 1; bool readOnlyMode = false; string thisSystem = "unset"; unsigned int maxSilentTime = 0; +Paths substituters; static bool settingsRead = false; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 51fa685947..ea15da0e64 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -67,6 +67,11 @@ extern string thisSystem; infinity. */ extern unsigned int maxSilentTime; +/* The substituters. There are programs that can somehow realise a + store path without building, e.g., by downloading it or copying it + from a CD. */ +extern Paths substituters; + Strings querySetting(const string & name, const Strings & def); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1165335125..bffefbaa78 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -48,22 +48,6 @@ static TableId dbReferences = 0; referrer. */ static TableId dbReferrers = 0; -/* dbSubstitutes :: Path -> [[Path]] - - Each pair $(p, subs)$ tells Nix that it can use any of the - substitutes in $subs$ to build path $p$. Each substitute defines a - command-line invocation of a program (i.e., the first list element - is the full path to the program, the remaining elements are - arguments). - - The main purpose of this is for distributed caching of derivates. - One system can compute a derivate and put it on a website (as a Nix - archive), for instance, and then another system can register a - substitute for that derivate. The substitute in this case might be - a Nix derivation that fetches the Nix archive. -*/ -static TableId dbSubstitutes = 0; - /* dbDerivers :: Path -> [Path] This table lists the derivation used to build a path. There can @@ -72,13 +56,6 @@ static TableId dbSubstitutes = 0; static TableId dbDerivers = 0; -bool Substitute::operator == (const Substitute & sub) const -{ - return program == sub.program - && args == sub.args; -} - - static void upgradeStore07(); static void upgradeStore09(); @@ -103,6 +80,8 @@ void checkStoreNotSymlink() LocalStore::LocalStore(bool reserveSpace) { + substitutablePathsLoaded = false; + if (readOnlyMode) return; checkStoreNotSymlink(); @@ -133,7 +112,6 @@ LocalStore::LocalStore(bool reserveSpace) dbValidPaths = nixDB.openTable("validpaths"); dbReferences = nixDB.openTable("references"); dbReferrers = nixDB.openTable("referrers", true); /* must be sorted */ - dbSubstitutes = nixDB.openTable("substitutes"); dbDerivers = nixDB.openTable("derivers"); int curSchema = 0; @@ -280,17 +258,6 @@ bool LocalStore::isValidPath(const Path & path) } -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath); - - -static bool isRealisablePath(const Transaction & txn, const Path & path) -{ - return isValidPathTxn(txn, path) - || readSubstitutes(txn, path).size() > 0; -} - - static string addPrefix(const string & prefix, const string & s) { return prefix + string(1, (char) 0) + s; @@ -322,11 +289,10 @@ static PathSet getReferrers(const Transaction & txn, const Path & storePath) void setReferences(const Transaction & txn, const Path & storePath, const PathSet & references) { - /* For unrealisable paths, we can only clear the references. */ - if (references.size() > 0 && !isRealisablePath(txn, storePath)) + /* For invalid paths, we can only clear the references. */ + if (references.size() > 0 && !isValidPathTxn(txn, storePath)) throw Error( - format("cannot set references for path `%1%' which is invalid and has no substitutes") - % storePath); + format("cannot set references for invalid path `%1%'") % storePath); Paths oldReferences; nixDB.queryStrings(txn, dbReferences, storePath, oldReferences); @@ -356,7 +322,7 @@ void queryReferences(const Transaction & txn, const Path & storePath, PathSet & references) { Paths references2; - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); nixDB.queryStrings(txn, dbReferences, storePath, references2); references.insert(references2.begin(), references2.end()); @@ -373,7 +339,7 @@ void LocalStore::queryReferences(const Path & storePath, void queryReferrers(const Transaction & txn, const Path & storePath, PathSet & referrers) { - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); PathSet referrers2 = getReferrers(txn, storePath); referrers.insert(referrers2.begin(), referrers2.end()); @@ -393,7 +359,7 @@ void setDeriver(const Transaction & txn, const Path & storePath, assertStorePath(storePath); if (deriver == "") return; assertStorePath(deriver); - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); nixDB.setString(txn, dbDerivers, storePath, deriver); } @@ -401,7 +367,7 @@ void setDeriver(const Transaction & txn, const Path & storePath, static Path queryDeriver(const Transaction & txn, const Path & storePath) { - if (!isRealisablePath(txn, storePath)) + if (!isValidPathTxn(txn, storePath)) throw Error(format("path `%1%' is not valid") % storePath); Path deriver; if (nixDB.queryString(txn, dbDerivers, storePath, deriver)) @@ -417,119 +383,33 @@ Path LocalStore::queryDeriver(const Path & path) } -const int substituteVersion = 2; - - -static Substitutes readSubstitutes(const Transaction & txn, - const Path & srcPath) +PathSet LocalStore::querySubstitutablePaths() { - Strings ss; - nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss); - - Substitutes subs; - - for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - if (i->size() < 4 || (*i)[3] != 0) { - /* Old-style substitute. !!! remove this code - eventually? */ - break; + if (!substitutablePathsLoaded) { + for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) { + debug(format("running `%1%' to find out substitutable paths") % *i); + Strings args; + args.push_back("--query-paths"); + Strings ss = tokenizeString(runProgram(*i, false, args), "\n"); + for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) { + if (!isStorePath(*j)) + throw Error(format("`%1%' returned a bad substitutable path `%2%'") + % *i % *j); + substitutablePaths.insert(*j); + } } - Strings ss2 = unpackStrings(*i); - if (ss2.size() == 0) continue; - int version; - if (!string2Int(ss2.front(), version)) continue; - if (version != substituteVersion) continue; - if (ss2.size() != 4) throw Error("malformed substitute"); - Strings::iterator j = ss2.begin(); - j++; - Substitute sub; - sub.deriver = *j++; - sub.program = *j++; - sub.args = unpackStrings(*j++); - subs.push_back(sub); + substitutablePathsLoaded = true; } - return subs; + return substitutablePaths; } -static void writeSubstitutes(const Transaction & txn, - const Path & srcPath, const Substitutes & subs) +bool LocalStore::hasSubstitutes(const Path & path) { - Strings ss; - - for (Substitutes::const_iterator i = subs.begin(); - i != subs.end(); ++i) - { - Strings ss2; - ss2.push_back((format("%1%") % substituteVersion).str()); - ss2.push_back(i->deriver); - ss2.push_back(i->program); - ss2.push_back(packStrings(i->args)); - ss.push_back(packStrings(ss2)); - } - - nixDB.setStrings(txn, dbSubstitutes, srcPath, ss); -} - - -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub) -{ - assertStorePath(srcPath); - - Substitutes subs = readSubstitutes(txn, srcPath); - - if (find(subs.begin(), subs.end(), sub) != subs.end()) - return; - - /* New substitutes take precedence over old ones. If the - substitute is already present, it's moved to the front. */ - remove(subs.begin(), subs.end(), sub); - subs.push_front(sub); - - writeSubstitutes(txn, srcPath, subs); -} - - -Substitutes querySubstitutes(const Transaction & txn, const Path & path) -{ - return readSubstitutes(txn, path); -} - - -Substitutes LocalStore::querySubstitutes(const Path & path) -{ - return nix::querySubstitutes(noTxn, path); -} - - -static void invalidatePath(Transaction & txn, const Path & path); - - -void clearSubstitutes() -{ - Transaction txn(nixDB); - - /* Iterate over all paths for which there are substitutes. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - - /* Delete all substitutes for path *i. */ - nixDB.delPair(txn, dbSubstitutes, *i); - - /* Maintain the cleanup invariant. */ - if (!isValidPathTxn(txn, *i)) - invalidatePath(txn, *i); - } - - /* !!! there should be no referrers to any of the invalid - substitutable paths. This should be the case by construction - (the only referrers can be other invalid substitutable paths, - which have all been removed now). */ - - txn.commit(); + if (!substitutablePathsLoaded) + querySubstitutablePaths(); + return substitutablePaths.find(path) != substitutablePaths.end(); } @@ -615,17 +495,12 @@ void registerValidPaths(const Transaction & txn, there are no referrers. */ static void invalidatePath(Transaction & txn, const Path & path) { - debug(format("unregistering path `%1%'") % path); + debug(format("invalidating path `%1%'") % path); /* Clear the `references' entry for this path, as well as the - inverse `referrers' entries, and the `derivers' entry; but only - if there are no substitutes for this path. This maintains the - cleanup invariant. */ - if (querySubstitutes(txn, path).size() == 0) { - setReferences(txn, path, PathSet()); - nixDB.delPair(txn, dbDerivers, path); - } - + inverse `referrers' entries, and the `derivers' entry. */ + setReferences(txn, path, PathSet()); + nixDB.delPair(txn, dbDerivers, path); nixDB.delPair(txn, dbValidPaths, path); } @@ -967,30 +842,7 @@ void verifyStore(bool checkContents) } - printMsg(lvlInfo, "checking path realisability"); - - /* "Realisable" paths are those that are valid or have a - substitute. */ - PathSet realisablePaths(validPaths); - - /* Check that the values of the substitute mappings are valid - paths. */ - Paths subKeys; - nixDB.enumTable(txn, dbSubstitutes, subKeys); - for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) { - Substitutes subs = readSubstitutes(txn, *i); - if (!isStorePath(*i)) { - printMsg(lvlError, format("removing substitutes for non-store path `%1%'") % *i); - nixDB.delPair(txn, dbSubstitutes, *i); - } - else if (subs.size() == 0) - nixDB.delPair(txn, dbSubstitutes, *i); - else - realisablePaths.insert(*i); - } - - - /* Check the cleanup invariant: only realisable paths can have + /* Check the cleanup invariant: only valid paths can have `references', `referrers', or `derivers' entries. */ @@ -1001,8 +853,8 @@ void verifyStore(bool checkContents) for (Paths::iterator i = deriversKeys.begin(); i != deriversKeys.end(); ++i) { - if (realisablePaths.find(*i) == realisablePaths.end()) { - printMsg(lvlError, format("removing deriver entry for unrealisable path `%1%'") + if (validPaths.find(*i) == validPaths.end()) { + printMsg(lvlError, format("removing deriver entry for invalid path `%1%'") % *i); nixDB.delPair(txn, dbDerivers, *i); } @@ -1024,13 +876,12 @@ void verifyStore(bool checkContents) for (Paths::iterator i = referencesKeys.begin(); i != referencesKeys.end(); ++i) { - if (realisablePaths.find(*i) == realisablePaths.end()) { - printMsg(lvlError, format("removing references entry for unrealisable path `%1%'") + if (validPaths.find(*i) == validPaths.end()) { + printMsg(lvlError, format("removing references entry for invalid path `%1%'") % *i); setReferences(txn, *i, PathSet()); } else { - bool isValid = validPaths.find(*i) != validPaths.end(); PathSet references; queryReferences(txn, *i, references); for (PathSet::iterator j = references.begin(); @@ -1042,7 +893,7 @@ void verifyStore(bool checkContents) % *j % *i); nixDB.setString(txn, dbReferrers, addPrefix(*j, *i), ""); } - if (isValid && validPaths.find(*j) == validPaths.end()) { + if (validPaths.find(*j) == validPaths.end()) { printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") % *i % *j); } @@ -1066,14 +917,14 @@ void verifyStore(bool checkContents) Path to(*i, 0, nul); Path from(*i, nul + 1); - if (realisablePaths.find(to) == realisablePaths.end()) { - printMsg(lvlError, format("removing referrer entry from `%1%' to unrealisable `%2%'") + if (validPaths.find(to) == validPaths.end()) { + printMsg(lvlError, format("removing referrer entry from `%1%' to invalid `%2%'") % from % to); nixDB.delPair(txn, dbReferrers, *i); } - else if (realisablePaths.find(from) == realisablePaths.end()) { - printMsg(lvlError, format("removing referrer entry from unrealisable `%1%' to `%2%'") + else if (validPaths.find(from) == validPaths.end()) { + printMsg(lvlError, format("removing referrer entry from invalid `%1%' to `%2%'") % from % to); nixDB.delPair(txn, dbReferrers, *i); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7c5cd26684..ffcef60716 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -23,6 +23,10 @@ extern string drvsLogDir; class LocalStore : public StoreAPI { +private: + bool substitutablePathsLoaded; + PathSet substitutablePaths; + public: /* Open the database environment. If `reserveSpace' is true, make @@ -41,8 +45,6 @@ public: bool isValidPath(const Path & path); - Substitutes querySubstitutes(const Path & srcPath); - Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); @@ -51,6 +53,10 @@ public: Path queryDeriver(const Path & path); + PathSet querySubstitutablePaths(); + + bool hasSubstitutes(const Path & path); + Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", PathFilter & filter = defaultPathFilter); @@ -86,13 +92,6 @@ void createStoreTransaction(Transaction & txn); /* Copy a path recursively. */ void copyPath(const Path & src, const Path & dst); -/* Register a substitute. */ -void registerSubstitute(const Transaction & txn, - const Path & srcPath, const Substitute & sub); - -/* Deregister all substitutes. */ -void clearSubstitutes(); - /* Register the validity of a path, i.e., that `path' exists, that the paths referenced by it exists, and in the case of an output path of a derivation, that it has been produced by a succesful execution of @@ -103,14 +102,6 @@ void registerValidPath(const Transaction & txn, const Path & path, const Hash & hash, const PathSet & references, const Path & deriver); -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; -}; - typedef list ValidPathInfos; void registerValidPaths(const Transaction & txn, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1319245c05..25529d8bac 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -63,8 +63,7 @@ void queryMissing(const PathSet & targets, bool mustBuild = false; for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) - if (!store->isValidPath(i->second.path) && - !store->hasSubstitutes(i->second.path)) + if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path)) mustBuild = true; if (mustBuild) { @@ -83,8 +82,8 @@ void queryMissing(const PathSet & targets, if (store->isValidPath(p)) continue; if (store->hasSubstitutes(p)) willSubstitute.insert(p); - PathSet refs; - store->queryReferences(p, todo); + // XXX call the substituters + // store->queryReferences(p, todo); } } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 820bf66ad9..c5d13bf494 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -164,12 +164,6 @@ bool RemoteStore::isValidPath(const Path & path) } -Substitutes RemoteStore::querySubstitutes(const Path & path) -{ - throw Error("not implemented 2"); -} - - bool RemoteStore::hasSubstitutes(const Path & path) { writeInt(wopHasSubstitutes, to); @@ -221,6 +215,12 @@ Path RemoteStore::queryDeriver(const Path & path) } +PathSet RemoteStore::querySubstitutablePaths() +{ + throw Error("not implemented"); +} + + Path RemoteStore::addToStore(const Path & _srcPath, bool fixed, bool recursive, string hashAlgo, PathFilter & filter) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 29b4a88f9a..87850c4068 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -27,10 +27,6 @@ public: bool isValidPath(const Path & path); - Substitutes querySubstitutes(const Path & path); - - bool hasSubstitutes(const Path & path); - Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); @@ -39,6 +35,10 @@ public: Path queryDeriver(const Path & path); + PathSet querySubstitutablePaths(); + + bool hasSubstitutes(const Path & path); + Path addToStore(const Path & srcPath, bool fixed = false, bool recursive = false, string hashAlgo = "", PathFilter & filter = defaultPathFilter); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4d54413188..9eb313da01 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,7 +8,8 @@ namespace nix { bool StoreAPI::hasSubstitutes(const Path & path) { - return !querySubstitutes(path).empty(); + PathSet paths = querySubstitutablePaths(); + return paths.find(path) != paths.end(); } @@ -130,6 +131,24 @@ Path computeStorePathForText(const string & suffix, const string & s, } +ValidPathInfo decodeValidPathInfo(std::istream & str) +{ + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { info.path = ""; return info; } + getline(str, info.deriver); + string s; int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 01b561e9d4..f133302b2f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -13,28 +13,6 @@ namespace nix { -/* A substitute is a program invocation that constructs some store - path (typically by fetching it from somewhere, e.g., from the - network). */ -struct Substitute -{ - /* The derivation that built this store path (empty if none). */ - Path deriver; - - /* Program to be executed to create the store path. Must be in - the output path of `storeExpr'. */ - Path program; - - /* Extra arguments to be passed to the program (the first argument - is the store path to be substituted). */ - Strings args; - - bool operator == (const Substitute & sub) const; -}; - -typedef list Substitutes; - - typedef std::map Roots; @@ -57,13 +35,6 @@ public: /* Checks whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; - /* Return the substitutes for the given path. */ - virtual Substitutes querySubstitutes(const Path & path) = 0; - - /* More efficient variant if we just want to know if a path has - substitutes. */ - virtual bool hasSubstitutes(const Path & path); - /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -81,6 +52,13 @@ public: no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; + /* Query the set of substitutable paths. */ + virtual PathSet querySubstitutablePaths() = 0; + + /* More efficient variant if we just want to know if a path has + substitutes. */ + virtual bool hasSubstitutes(const Path & path); + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. If `fixed' is true, then the output of a fixed-output @@ -109,10 +87,10 @@ public: /* Ensure that the output paths of the derivation are valid. If they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths have - substitutes, then those can be used. Second, the output paths - can be created by running the builder, after recursively - building any sub-derivations. */ + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. */ virtual void buildDerivations(const PathSet & drvPaths) = 0; /* Ensure that a path is valid. If it is not currently valid, it @@ -261,6 +239,17 @@ extern boost::shared_ptr store; boost::shared_ptr openStore(bool reserveSpace = true); +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; +}; + +ValidPathInfo decodeValidPathInfo(std::istream & str); + + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 4a2e8b6942..3fdfa1807b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -12,7 +12,6 @@ namespace nix { typedef enum { wopQuit, wopIsValidPath, - wopQuerySubstitutes, wopHasSubstitutes, wopQueryPathHash, wopQueryReferences, diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 513aae0392..fd50de00cc 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -19,23 +19,18 @@ using std::vector; using boost::format; -class Error : public std::exception +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception { protected: string err; public: - Error(const format & f); - ~Error() throw () { }; + BaseError(const format & f); + ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } const string & msg() const throw () { return err; } - Error & addPrefix(const format & f); -}; - -class SysError : public Error -{ -public: - int errNo; - SysError(const format & f); + BaseError & addPrefix(const format & f); }; #define MakeError(newClass, superClass) \ @@ -45,6 +40,15 @@ public: newClass(const format & f) : superClass(f) { }; \ }; +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + SysError(const format & f); +}; + typedef list Strings; typedef set StringSet; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index d61b35bdfb..edfd26b0b7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -23,13 +23,13 @@ extern char * * environ; namespace nix { -Error::Error(const format & f) +BaseError::BaseError(const format & f) { err = f.str(); } -Error & Error::addPrefix(const format & f) +BaseError & BaseError::addPrefix(const format & f) { err = f.str() + err; return *this; @@ -491,6 +491,7 @@ string drainFD(int fd) string result; unsigned char buffer[4096]; while (1) { + checkInterrupt(); ssize_t rd = read(fd, buffer, sizeof buffer); if (rd == -1) { if (errno != EINTR) @@ -775,6 +776,8 @@ void killUser(uid_t uid) string runProgram(Path program, bool searchPath, const Strings & args) { + checkInterrupt(); + /* Create a pipe. */ Pipe pipe; pipe.create(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 63389867fe..f72a6f8201 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,7 +256,7 @@ void inline checkInterrupt() if (_isInterrupted) _interrupted(); } -MakeError(Interrupted, Error) +MakeError(Interrupted, BaseError) /* String packing / unpacking. */ diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index df84a2a763..0662f67962 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -11,8 +11,6 @@ Operations: --query / -q: query information --read-log / -l: print build log of given store paths - --register-substitutes: register a substitute expression (dangerous!) - --clear-substitutes: clear all substitutes --register-validity: register path validity (dangerous!) --check-validity: check path validity diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c600a5b9a7..176dc39f9b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -413,54 +413,6 @@ static void opReadLog(Strings opFlags, Strings opArgs) } -static void opRegisterSubstitutes(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) throw UsageError("no arguments expected"); - - Transaction txn; - createStoreTransaction(txn); - - while (1) { - Path srcPath; - Substitute sub; - PathSet references; - getline(cin, srcPath); - if (cin.eof()) break; - getline(cin, sub.deriver); - getline(cin, sub.program); - string s; int n; - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - sub.args.push_back(s); - } - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - references.insert(s); - } - if (!cin || cin.eof()) throw Error("missing input"); - registerSubstitute(txn, srcPath, sub); - setReferences(txn, srcPath, references); - } - - txn.commit(); -} - - -static void opClearSubstitutes(Strings opFlags, Strings opArgs) -{ - if (!opFlags.empty()) throw UsageError("unknown flag"); - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - - clearSubstitutes(); -} - - static void opRegisterValidity(Strings opFlags, Strings opArgs) { bool reregister = false; // !!! maybe this should be the default @@ -475,18 +427,8 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs) ValidPathInfos infos; while (1) { - ValidPathInfo info; - getline(cin, info.path); - if (cin.eof()) break; - getline(cin, info.deriver); - string s; int n; - getline(cin, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { - getline(cin, s); - info.references.insert(s); - } - if (!cin || cin.eof()) throw Error("missing input"); + ValidPathInfo info = decodeValidPathInfo(cin); + if (info.path == "") break; if (!store->isValidPath(info.path) || reregister) { /* !!! races */ canonicalisePathMetaData(info.path); @@ -699,10 +641,6 @@ void run(Strings args) op = opQuery; else if (arg == "--read-log" || arg == "-l") op = opReadLog; - else if (arg == "--register-substitutes") - op = opRegisterSubstitutes; - else if (arg == "--clear-substitutes") - op = opClearSubstitutes; else if (arg == "--register-validity") op = opRegisterValidity; else if (arg == "--check-validity")