From 6baa2c442035fb06652a7fad9d51df1ce41c05f9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Oct 2003 15:06:59 +0000 Subject: [PATCH] * Get rid of identifiers since they are redundant now. This greatly simplifies stuff. * The format of Nix expressions and the database schema changed because of this, so it's best to delete old Nix installations. --- src/archive.cc | 18 +-- src/archive.hh | 6 +- src/db.cc | 2 + src/dotgraph.cc | 50 ++++---- src/dotgraph.hh | 2 +- src/expr.cc | 75 ++++-------- src/expr.hh | 24 ++-- src/fix.cc | 109 +++++++++--------- src/globals.cc | 6 +- src/globals.hh | 40 +++---- src/hash.cc | 14 +-- src/hash.hh | 11 +- src/nix-help.txt | 5 - src/nix.cc | 86 +++++--------- src/normalise.cc | 285 +++++++++++++++++++++++++++++----------------- src/normalise.hh | 29 +++-- src/pathlocks.cc | 12 +- src/pathlocks.hh | 4 +- src/references.hh | 2 +- src/shared.cc | 2 - src/store.cc | 208 ++++++++------------------------- src/store.hh | 41 ++----- src/test.cc | 85 ++++++-------- 23 files changed, 485 insertions(+), 631 deletions(-) diff --git a/src/archive.cc b/src/archive.cc index 4a6211e00e..9039ad7db4 100644 --- a/src/archive.cc +++ b/src/archive.cc @@ -49,7 +49,7 @@ static void writeString(const string & s, DumpSink & sink) static void dump(const string & path, DumpSink & sink); -static void dumpEntries(const string & path, DumpSink & sink) +static void dumpEntries(const Path & path, DumpSink & sink) { DIR * dir = opendir(path.c_str()); if (!dir) throw SysError("opening directory " + path); @@ -82,7 +82,7 @@ static void dumpEntries(const string & path, DumpSink & sink) } -static void dumpContents(const string & path, unsigned int size, +static void dumpContents(const Path & path, unsigned int size, DumpSink & sink) { writeString("contents", sink); @@ -110,7 +110,7 @@ static void dumpContents(const string & path, unsigned int size, } -static void dump(const string & path, DumpSink & sink) +static void dump(const Path & path, DumpSink & sink) { struct stat st; if (lstat(path.c_str(), &st)) @@ -150,7 +150,7 @@ static void dump(const string & path, DumpSink & sink) } -void dumpPath(const string & path, DumpSink & sink) +void dumpPath(const Path & path, DumpSink & sink) { writeString(archiveVersion1, sink); dump(path, sink); @@ -207,10 +207,10 @@ static void skipGeneric(RestoreSource & source) } -static void restore(const string & path, RestoreSource & source); +static void restore(const Path & path, RestoreSource & source); -static void restoreEntry(const string & path, RestoreSource & source) +static void restoreEntry(const Path & path, RestoreSource & source) { string s, name; @@ -235,7 +235,7 @@ static void restoreEntry(const string & path, RestoreSource & source) } -static void restoreContents(int fd, const string & path, RestoreSource & source) +static void restoreContents(int fd, const Path & path, RestoreSource & source) { unsigned int size = readInt(source); unsigned int left = size; @@ -254,7 +254,7 @@ static void restoreContents(int fd, const string & path, RestoreSource & source) } -static void restore(const string & path, RestoreSource & source) +static void restore(const Path & path, RestoreSource & source) { string s; @@ -331,7 +331,7 @@ static void restore(const string & path, RestoreSource & source) } -void restorePath(const string & path, RestoreSource & source) +void restorePath(const Path & path, RestoreSource & source) { if (readString(source) != archiveVersion1) throw badArchive("expected Nix archive"); diff --git a/src/archive.hh b/src/archive.hh index e6006e4549..67e236668a 100644 --- a/src/archive.hh +++ b/src/archive.hh @@ -1,6 +1,6 @@ #include -using namespace std; +#include "util.hh" /* dumpPath creates a Nix archive of the specified path. The format @@ -45,7 +45,7 @@ struct DumpSink virtual void operator () (const unsigned char * data, unsigned int len) = 0; }; -void dumpPath(const string & path, DumpSink & sink); +void dumpPath(const Path & path, DumpSink & sink); struct RestoreSource @@ -57,4 +57,4 @@ struct RestoreSource virtual void operator () (unsigned char * data, unsigned int len) = 0; }; -void restorePath(const string & path, RestoreSource & source); +void restorePath(const Path & path, RestoreSource & source); diff --git a/src/db.cc b/src/db.cc index 61ecb203a5..ffb1999fe5 100644 --- a/src/db.cc +++ b/src/db.cc @@ -258,6 +258,8 @@ void Database::delPair(const Transaction & txn, TableId table, Db * db = getDb(table); Dbt kt((void *) key.c_str(), key.length()); db->del(txn.txn, &kt, 0); + /* Non-existence of a pair with the given key is not an + error. */ } catch (DbException e) { rethrow(e); } } diff --git a/src/dotgraph.cc b/src/dotgraph.cc index f9ff51b909..514fda3259 100644 --- a/src/dotgraph.cc +++ b/src/dotgraph.cc @@ -45,24 +45,24 @@ static string symbolicName(const string & path) } -string pathLabel(const FSId & id, const string & path) +string pathLabel(const Path & nePath, const string & elemPath) { - return (string) id + "-" + path; + return (string) nePath + "-" + elemPath; } -void printClosure(const FSId & id, const NixExpr & fs) +void printClosure(const Path & nePath, const NixExpr & fs) { - Strings workList(fs.closure.roots.begin(), fs.closure.roots.end()); - StringSet doneSet; + PathSet workList(fs.closure.roots); + PathSet doneSet; - for (Strings::iterator i = workList.begin(); i != workList.end(); i++) { - cout << makeEdge(pathLabel(id, *i), id); + for (PathSet::iterator i = workList.begin(); i != workList.end(); i++) { + cout << makeEdge(pathLabel(nePath, *i), nePath); } while (!workList.empty()) { - string path = workList.front(); - workList.pop_front(); + Path path = *(workList.begin()); + workList.erase(path); if (doneSet.find(path) == doneSet.end()) { doneSet.insert(path); @@ -74,41 +74,41 @@ void printClosure(const FSId & id, const NixExpr & fs) for (StringSet::const_iterator i = elem->second.refs.begin(); i != elem->second.refs.end(); i++) { - workList.push_back(*i); - cout << makeEdge(pathLabel(id, *i), pathLabel(id, path)); + workList.insert(*i); + cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); } - cout << makeNode(pathLabel(id, path), + cout << makeNode(pathLabel(nePath, path), symbolicName(path), "#ff0000"); } } } -void printDotGraph(const FSIds & roots) +void printDotGraph(const PathSet & roots) { - FSIds workList(roots.begin(), roots.end()); - FSIdSet doneSet; + PathSet workList(roots); + PathSet doneSet; cout << "digraph G {\n"; while (!workList.empty()) { - FSId id = workList.front(); - workList.pop_front(); + Path nePath = *(workList.begin()); + workList.erase(nePath); - if (doneSet.find(id) == doneSet.end()) { - doneSet.insert(id); + if (doneSet.find(nePath) == doneSet.end()) { + doneSet.insert(nePath); - NixExpr ne = parseNixExpr(termFromId(id)); + NixExpr ne = parseNixExpr(termFromPath(nePath)); string label, colour; if (ne.type == NixExpr::neDerivation) { - for (FSIdSet::iterator i = ne.derivation.inputs.begin(); + for (PathSet::iterator i = ne.derivation.inputs.begin(); i != ne.derivation.inputs.end(); i++) { - workList.push_back(*i); - cout << makeEdge(*i, id); + workList.insert(*i); + cout << makeEdge(*i, nePath); } label = "derivation"; @@ -121,12 +121,12 @@ void printDotGraph(const FSIds & roots) else if (ne.type == NixExpr::neClosure) { label = ""; colour = "#00ffff"; - printClosure(id, ne); + printClosure(nePath, ne); } else abort(); - cout << makeNode(id, label, colour); + cout << makeNode(nePath, label, colour); } } diff --git a/src/dotgraph.hh b/src/dotgraph.hh index e1678da967..a5c5248f14 100644 --- a/src/dotgraph.hh +++ b/src/dotgraph.hh @@ -3,6 +3,6 @@ #include "expr.hh" -void printDotGraph(const FSIds & roots); +void printDotGraph(const PathSet & roots); #endif /* !__DOTGRAPH_H */ diff --git a/src/expr.cc b/src/expr.cc index 7d79d1654f..2ed3e678b3 100644 --- a/src/expr.cc +++ b/src/expr.cc @@ -22,37 +22,33 @@ Hash hashTerm(ATerm t) } -ATerm termFromId(const FSId & id) +ATerm termFromPath(const Path & path) { - string path = expandId(id); ATerm t = ATreadFromNamedFile(path.c_str()); if (!t) throw Error(format("cannot read aterm from `%1%'") % path); return t; } -FSId writeTerm(ATerm t, const string & suffix, FSId id) +Path writeTerm(ATerm t, const string & suffix) { - /* By default, the id of a term is its hash. */ - if (id == FSId()) id = hashTerm(t); + /* The id of a term is its hash. */ + Hash h = hashTerm(t); - string path = canonPath(nixStore + "/" + - (string) id + suffix + ".nix"); + Path path = canonPath(nixStore + "/" + + (string) h + suffix + ".nix"); if (!ATwriteToNamedTextFile(t, path.c_str())) throw Error(format("cannot write aterm %1%") % path); -// debug(format("written term %1% = %2%") % (string) id % -// printTerm(t)); - Transaction txn(nixDB); - registerPath(txn, path, id); + registerValidPath(txn, path); txn.commit(); - return id; + return path; } -static void parsePaths(ATermList paths, StringSet & out) +static void parsePaths(ATermList paths, PathSet & out) { while (!ATisEmpty(paths)) { char * s; @@ -70,19 +66,19 @@ static void checkClosure(const Closure & closure) if (closure.elems.size() == 0) throw Error("empty closure"); - StringSet decl; + PathSet decl; for (ClosureElems::const_iterator i = closure.elems.begin(); i != closure.elems.end(); i++) decl.insert(i->first); - for (StringSet::const_iterator i = closure.roots.begin(); + for (PathSet::const_iterator i = closure.roots.begin(); i != closure.roots.end(); i++) if (decl.find(*i) == decl.end()) throw Error(format("undefined root path `%1%'") % *i); for (ClosureElems::const_iterator i = closure.elems.begin(); i != closure.elems.end(); i++) - for (StringSet::const_iterator j = i->second.refs.begin(); + for (PathSet::const_iterator j = i->second.refs.begin(); j != i->second.refs.end(); j++) if (decl.find(*j) == decl.end()) throw Error( @@ -102,13 +98,12 @@ static bool parseClosure(ATerm t, Closure & closure) parsePaths(roots, closure.roots); while (!ATisEmpty(elems)) { - char * s1, * s2; + char * s1; ATermList refs; ATerm t = ATgetFirst(elems); - if (!ATmatch(t, "(, , [])", &s1, &s2, &refs)) + if (!ATmatch(t, "(, [])", &s1, &refs)) throw badTerm("not a closure element", t); ClosureElem elem; - elem.id = parseHash(s2); parsePaths(refs, elem.refs); closure.elems[s1] = elem; elems = ATgetNext(elems); @@ -135,23 +130,8 @@ static bool parseDerivation(ATerm t, Derivation & derivation) args = ATempty; } - while (!ATisEmpty(outs)) { - char * s1, * s2; - ATerm t = ATgetFirst(outs); - if (!ATmatch(t, "(, )", &s1, &s2)) - throw badTerm("not a derivation output", t); - derivation.outputs[s1] = parseHash(s2); - outs = ATgetNext(outs); - } - - while (!ATisEmpty(ins)) { - char * s; - ATerm t = ATgetFirst(ins); - if (!ATmatch(t, "", &s)) - throw badTerm("not an id", t); - derivation.inputs.insert(parseHash(s)); - ins = ATgetNext(ins); - } + parsePaths(outs, derivation.outputs); + parsePaths(ins, derivation.inputs); derivation.builder = builder; derivation.platform = platform; @@ -190,10 +170,10 @@ NixExpr parseNixExpr(ATerm t) } -static ATermList unparsePaths(const StringSet & paths) +static ATermList unparsePaths(const PathSet & paths) { ATermList l = ATempty; - for (StringSet::const_iterator i = paths.begin(); + for (PathSet::const_iterator i = paths.begin(); i != paths.end(); i++) l = ATinsert(l, ATmake("", i->c_str())); return ATreverse(l); @@ -208,9 +188,8 @@ static ATerm unparseClosure(const Closure & closure) for (ClosureElems::const_iterator i = closure.elems.begin(); i != closure.elems.end(); i++) elems = ATinsert(elems, - ATmake("(, , )", + ATmake("(, )", i->first.c_str(), - ((string) i->second.id).c_str(), unparsePaths(i->second.refs))); return ATmake("Closure(, )", roots, elems); @@ -219,18 +198,6 @@ static ATerm unparseClosure(const Closure & closure) static ATerm unparseDerivation(const Derivation & derivation) { - ATermList outs = ATempty; - for (DerivationOutputs::const_iterator i = derivation.outputs.begin(); - i != derivation.outputs.end(); i++) - outs = ATinsert(outs, - ATmake("(, )", - i->first.c_str(), ((string) i->second).c_str())); - - ATermList ins = ATempty; - for (FSIdSet::const_iterator i = derivation.inputs.begin(); - i != derivation.inputs.end(); i++) - ins = ATinsert(ins, ATmake("", ((string) *i).c_str())); - ATermList args = ATempty; for (Strings::const_iterator i = derivation.args.begin(); i != derivation.args.end(); i++) @@ -244,8 +211,8 @@ static ATerm unparseDerivation(const Derivation & derivation) i->first.c_str(), i->second.c_str())); return ATmake("Derive(, , , , , )", - ATreverse(outs), - ATreverse(ins), + unparsePaths(derivation.outputs), + unparsePaths(derivation.inputs), derivation.platform.c_str(), derivation.builder.c_str(), ATreverse(args), diff --git a/src/expr.hh b/src/expr.hh index 2a6a06a0a9..b345643223 100644 --- a/src/expr.hh +++ b/src/expr.hh @@ -10,31 +10,27 @@ extern "C" { /* Abstract syntax of Nix expressions. */ -typedef list FSIds; - struct ClosureElem { - FSId id; - StringSet refs; + PathSet refs; }; -typedef map ClosureElems; +typedef map ClosureElems; struct Closure { - StringSet roots; + PathSet roots; ClosureElems elems; }; -typedef map DerivationOutputs; typedef map StringPairs; struct Derivation { - DerivationOutputs outputs; - FSIdSet inputs; + PathSet outputs; + PathSet inputs; /* Nix expressions, not actual inputs */ string platform; - string builder; + Path builder; Strings args; StringPairs env; }; @@ -57,11 +53,11 @@ Error badTerm(const format & f, ATerm t); /* Hash an aterm. */ Hash hashTerm(ATerm t); -/* Read an aterm from disk, given its id. */ -ATerm termFromId(const FSId & id); +/* Read an aterm from disk. */ +ATerm termFromPath(const Path & path); -/* Write an aterm to the Nix store directory, and return its hash. */ -FSId writeTerm(ATerm t, const string & suffix, FSId id = FSId()); +/* Write an aterm to the Nix store directory, and return its path. */ +Path writeTerm(ATerm t, const string & suffix); /* Parse a Nix expression. */ NixExpr parseNixExpr(ATerm t); diff --git a/src/fix.cc b/src/fix.cc index 66438464cd..71fd068775 100644 --- a/src/fix.cc +++ b/src/fix.cc @@ -9,12 +9,12 @@ typedef ATerm Expr; typedef map NormalForms; -typedef map PkgPaths; -typedef map PkgHashes; +typedef map PkgPaths; +typedef map PkgHashes; struct EvalState { - Strings searchDirs; + Paths searchDirs; NormalForms normalForms; PkgPaths pkgPaths; PkgHashes pkgHashes; /* normalised package hashes */ @@ -28,18 +28,18 @@ struct EvalState }; -static Expr evalFile(EvalState & state, string fileName); +static Expr evalFile(EvalState & state, const Path & path); static Expr evalExpr(EvalState & state, Expr e); -static string searchPath(const Strings & searchDirs, string relPath) +static Path searchPath(const Paths & searchDirs, const Path & relPath) { if (string(relPath, 0, 1) == "/") return relPath; - for (Strings::const_iterator i = searchDirs.begin(); + for (Paths::const_iterator i = searchDirs.begin(); i != searchDirs.end(); i++) { - string path = *i + "/" + relPath; + Path path = *i + "/" + relPath; if (pathExists(path)) return path; } @@ -121,14 +121,14 @@ static Expr substExprMany(ATermList formals, ATermList args, Expr body) } -static Strings nixExprPathsCached(EvalState & state, const FSId & id) +static PathSet nixExprRootsCached(EvalState & state, const Path & nePath) { - PkgPaths::iterator i = state.pkgPaths.find(id); + PkgPaths::iterator i = state.pkgPaths.find(nePath); if (i != state.pkgPaths.end()) return i->second; else { - Strings paths = nixExprPaths(id); - state.pkgPaths[id] = paths; + PathSet paths = nixExprRoots(nePath); + state.pkgPaths[nePath] = paths; return paths; } } @@ -137,13 +137,13 @@ static Strings nixExprPathsCached(EvalState & state, const FSId & id) static Hash hashPackage(EvalState & state, NixExpr ne) { if (ne.type == NixExpr::neDerivation) { - FSIdSet inputs2; - for (FSIdSet::iterator i = ne.derivation.inputs.begin(); + PathSet inputs2; + for (PathSet::iterator i = ne.derivation.inputs.begin(); i != ne.derivation.inputs.end(); i++) { PkgHashes::iterator j = state.pkgHashes.find(*i); if (j == state.pkgHashes.end()) - throw Error(format("unknown package id %1%") % (string) *i); + throw Error(format("don't know expression `%1%'") % (string) *i); inputs2.insert(j->second); } ne.derivation.inputs = inputs2; @@ -156,12 +156,12 @@ static string processBinding(EvalState & state, Expr e, NixExpr & ne) { char * s1; - if (ATmatch(e, "FSId()", &s1)) { - FSId id = parseHash(s1); - Strings paths = nixExprPathsCached(state, id); + if (ATmatch(e, "NixExpr()", &s1)) { + Path nePath(s1); + PathSet paths = nixExprRootsCached(state, nePath); if (paths.size() != 1) abort(); - string path = *(paths.begin()); - ne.derivation.inputs.insert(id); + Path path = *(paths.begin()); + ne.derivation.inputs.insert(nePath); return path; } @@ -200,14 +200,14 @@ static Expr evalExpr2(EvalState & state, Expr e) ATmatch(e, "True") || ATmatch(e, "False") || ATmatch(e, "Function([], )", &e1, &e2) || - ATmatch(e, "FSId()", &s1)) + ATmatch(e, "NixExpr()", &s1)) return e; try { Hash pkgHash = hashPackage(state, parseNixExpr(e)); - FSId pkgId = writeTerm(e, ""); - state.pkgHashes[pkgId] = pkgHash; - return ATmake("FSId()", ((string) pkgId).c_str()); + Path pkgPath = writeTerm(e, ""); + state.pkgHashes[pkgPath] = pkgHash; + return ATmake("NixExpr()", pkgPath.c_str()); } catch (...) { /* !!! catch parse errors only */ } @@ -254,32 +254,29 @@ static Expr evalExpr2(EvalState & state, Expr e) /* Fix inclusion. */ if (ATmatch(e, "IncludeFix()", &s1)) { - string fileName(s1); + Path fileName(s1); return evalFile(state, s1); } /* Relative files. */ if (ATmatch(e, "Relative()", &s1)) { - string srcPath = searchPath(state.searchDirs, s1); - string dstPath; - FSId id; - addToStore(srcPath, dstPath, id, true); + Path srcPath = searchPath(state.searchDirs, s1); + Path dstPath = addToStore(srcPath); ClosureElem elem; - elem.id = id; NixExpr ne; ne.type = NixExpr::neClosure; ne.closure.roots.insert(dstPath); ne.closure.elems[dstPath] = elem; Hash pkgHash = hashPackage(state, ne); - FSId pkgId = writeTerm(unparseNixExpr(ne), ""); - state.pkgHashes[pkgId] = pkgHash; + Path pkgPath = writeTerm(unparseNixExpr(ne), ""); + state.pkgHashes[pkgPath] = pkgHash; - msg(lvlChatty, format("copied `%1%' -> %2%") - % srcPath % (string) pkgId); + msg(lvlChatty, format("copied `%1%' -> closure `%2%'") + % srcPath % pkgPath); - return ATmake("FSId()", ((string) pkgId).c_str()); + return ATmake("NixExpr()", pkgPath.c_str()); } /* Packages are transformed into Nix derivation expressions. */ @@ -302,8 +299,8 @@ static Expr evalExpr2(EvalState & state, Expr e) ne.type = NixExpr::neDerivation; ne.derivation.platform = SYSTEM; string name; - FSId outId; - bool outIdGiven = false; + Hash outHash; + bool outHashGiven = false; bnds = ATempty; for (map::iterator it = bndMap.begin(); @@ -331,8 +328,8 @@ static Expr evalExpr2(EvalState & state, Expr e) if (key == "build") ne.derivation.builder = s; if (key == "name") name = s; if (key == "id") { - outId = parseHash(s); - outIdGiven = true; + outHash = parseHash(s); + outHashGiven = true; } } @@ -348,23 +345,23 @@ static Expr evalExpr2(EvalState & state, Expr e) /* Hash the Nix expression with no outputs to produce a unique but deterministic path name for this package. */ - if (!outIdGiven) outId = hashPackage(state, ne); - string outPath = - canonPath(nixStore + "/" + ((string) outId).c_str() + "-" + name); + if (!outHashGiven) outHash = hashPackage(state, ne); + Path outPath = + canonPath(nixStore + "/" + ((string) outHash).c_str() + "-" + name); ne.derivation.env["out"] = outPath; - ne.derivation.outputs[outPath] = outId; + ne.derivation.outputs.insert(outPath); /* Write the resulting term into the Nix store directory. */ - Hash pkgHash = outIdGiven - ? hashString((string) outId + outPath) + Hash pkgHash = outHashGiven + ? hashString((string) outHash + outPath) : hashPackage(state, ne); - FSId pkgId = writeTerm(unparseNixExpr(ne), "-d-" + name); - state.pkgHashes[pkgId] = pkgHash; + Path pkgPath = writeTerm(unparseNixExpr(ne), "-d-" + name); + state.pkgHashes[pkgPath] = pkgHash; - msg(lvlChatty, format("instantiated `%1%' -> %2%") - % name % (string) pkgId); + msg(lvlChatty, format("instantiated `%1%' -> `%2%'") + % name % pkgPath); - return ATmake("FSId()", ((string) pkgId).c_str()); + return ATmake("NixExpr()", pkgPath.c_str()); } /* BaseName primitive function. */ @@ -401,9 +398,9 @@ static Expr evalExpr(EvalState & state, Expr e) } -static Expr evalFile(EvalState & state, string relPath) +static Expr evalFile(EvalState & state, const Path & relPath) { - string path = searchPath(state.searchDirs, relPath); + Path path = searchPath(state.searchDirs, relPath); Nest nest(lvlTalkative, format("evaluating file `%1%'") % path); Expr e = ATreadFromNamedFile(path.c_str()); if (!e) @@ -422,16 +419,16 @@ static Expr evalStdin(EvalState & state) } -static void printFSId(EvalState & state, Expr e) +static void printNixExpr(EvalState & state, Expr e) { ATermList es; char * s; - if (ATmatch(e, "FSId()", &s)) { + if (ATmatch(e, "NixExpr()", &s)) { cout << format("%1%\n") % s; } else if (ATmatch(e, "[]", &es)) { while (!ATisEmpty(es)) { - printFSId(state, evalExpr(state, ATgetFirst(es))); + printNixExpr(state, evalExpr(state, ATgetFirst(es))); es = ATgetNext(es); } } @@ -472,14 +469,14 @@ void run(Strings args) if (readStdin) { Expr e = evalStdin(state); - printFSId(state, e); + printNixExpr(state, e); } for (Strings::iterator it = files.begin(); it != files.end(); it++) { Expr e = evalFile(state, *it); - printFSId(state, e); + printNixExpr(state, e); } } diff --git a/src/globals.cc b/src/globals.cc index f21820f597..f34f8f1029 100644 --- a/src/globals.cc +++ b/src/globals.cc @@ -5,8 +5,7 @@ Database nixDB; -TableId dbPath2Id; -TableId dbId2Paths; +TableId dbValidPaths; TableId dbSuccessors; TableId dbSubstitutes; @@ -23,8 +22,7 @@ bool keepFailed = false; void openDB() { nixDB.open(nixDBPath); - dbPath2Id = nixDB.openTable("path2id"); - dbId2Paths = nixDB.openTable("id2paths"); + dbValidPaths = nixDB.openTable("validpaths"); dbSuccessors = nixDB.openTable("successors"); dbSubstitutes = nixDB.openTable("substitutes"); } diff --git a/src/globals.hh b/src/globals.hh index b140f136c9..0ea2fca092 100644 --- a/src/globals.hh +++ b/src/globals.hh @@ -13,42 +13,36 @@ extern Database nixDB; /* Database tables. */ -/* dbPath2Id :: Path -> FSId - Each pair (p, id) records that path $p$ contains an expansion of - $id$. */ -extern TableId dbPath2Id; +/* dbValidPaths :: Path -> () + + The existence of a key $p$ indicates that path $p$ is valid (that + is, produced by a succesful build). */ +extern TableId dbValidPaths; -/* dbId2Paths :: FSId -> [Path] +/* dbSuccessors :: Path -> Path - A mapping from ids to lists of paths. */ -extern TableId dbId2Paths; + Each pair $(p_1, p_2)$ in this mapping records the fact that the + Nix expression stored at path $p_1$ has a successor expression + stored at path $p_2$. - -/* dbSuccessors :: FSId -> FSId - - Each pair $(id_1, id_2)$ in this mapping records the fact that a - successor of a Nix expression stored in a file with identifier - $id_1$ is stored in a file with identifier $id_2$. - - Note that a term $y$ is successor of $x$ iff there exists a + Note that a term $y$ is a successor of $x$ iff there exists a sequence of rewrite steps that rewrites $x$ into $y$. */ extern TableId dbSuccessors; -/* dbSubstitutes :: FSId -> [FSId] +/* dbSubstitutes :: Path -> [Path] - Each pair $(id, [ids])$ tells Nix that it can realise any of the - Nix expressions referenced by the identifiers in $ids$ to - generate a path with identifier $id$. + Each pair $(p, [ps])$ tells Nix that it can realise any of the + Nix expressions stored at paths $ps$ to produce a path $p$. The main purpose of this is for distributed caching of derivates. - One system can compute a derivate with hash $h$ and put it on a - website (as a Nix archive), for instance, and then another system - can register a substitute for that derivate. The substitute in - this case might be a Nix expression that fetches the Nix archive. + One system can compute a derivate and put it on a website (as a Nix + archive), for instance, and then another system can register a + substitute for that derivate. The substitute in this case might be + a Nix expression that fetches the Nix archive. */ extern TableId dbSubstitutes; diff --git a/src/hash.cc b/src/hash.cc index b59c4f2148..752b269125 100644 --- a/src/hash.cc +++ b/src/hash.cc @@ -54,11 +54,11 @@ Hash parseHash(const string & s) { Hash hash; if (s.length() != Hash::hashSize * 2) - throw BadRefError("invalid hash: " + s); + throw Error(format("invalid hash `%1%'") % s); for (unsigned int i = 0; i < Hash::hashSize; i++) { string s2(s, i * 2, 2); if (!isxdigit(s2[0]) || !isxdigit(s2[1])) - throw BadRefError("invalid hash: " + s); + throw Error(format("invalid hash `%1%'") % s); istringstream str(s2); int n; str >> hex >> n; @@ -89,15 +89,15 @@ Hash hashString(const string & s) } -Hash hashFile(const string & fileName) +Hash hashFile(const Path & path) { Hash hash; - FILE * file = fopen(fileName.c_str(), "rb"); + FILE * file = fopen(path.c_str(), "rb"); if (!file) - throw SysError("file `" + fileName + "' does not exist"); + throw SysError(format("file `%1%' does not exist") % path); int err = md5_stream(file, hash.hash); fclose(file); - if (err) throw SysError("cannot hash file " + fileName); + if (err) throw SysError(format("cannot hash file `%1%'") % path); return hash; } @@ -113,7 +113,7 @@ struct HashSink : DumpSink }; -Hash hashPath(const string & path) +Hash hashPath(const Path & path) { Hash hash; HashSink sink; diff --git a/src/hash.hh b/src/hash.hh index 387939e931..0062f987c0 100644 --- a/src/hash.hh +++ b/src/hash.hh @@ -30,13 +30,6 @@ struct Hash }; -class BadRefError : public Error -{ -public: - BadRefError(string _err) : Error(_err) { }; -}; - - /* Parse a hexadecimal representation of a hash code. */ Hash parseHash(const string & s); @@ -47,12 +40,12 @@ bool isHash(const string & s); Hash hashString(const string & s); /* Compute the hash of the given file. */ -Hash hashFile(const string & fileName); +Hash hashFile(const Path & path); /* Compute the hash of the given path. The hash is defined as md5(dump(path)). */ -Hash hashPath(const string & path); +Hash hashPath(const Path & path); #endif /* !__HASH_H */ diff --git a/src/nix-help.txt b/src/nix-help.txt index be8ecf0a15..a51018ea19 100644 --- a/src/nix-help.txt +++ b/src/nix-help.txt @@ -19,16 +19,11 @@ Operations: --version: output version information --help: display help -Source selection for --install, --dump: - - --path / -p: by file name !!! -> path - Query flags: --list / -l: query the output paths (roots) of a Nix expression (default) --requisites / -r: print all paths necessary to realise expression --generators / -g: find expressions producing a subset of given ids - --expansion / -e: print a path containing id --graph: print a dot graph rooted at given ids Options: diff --git a/src/nix.cc b/src/nix.cc index 87553de2d6..9907f5c74d 100644 --- a/src/nix.cc +++ b/src/nix.cc @@ -11,9 +11,6 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs); -static bool pathArgs = false; - - static void printHelp() { cout << @@ -24,16 +21,9 @@ static void printHelp() -static FSId argToId(const string & arg) +static Path checkPath(const Path & arg) { - if (!pathArgs) - return parseHash(arg); - else { - FSId id; - if (!queryPathId(arg, id)) - throw Error(format("don't know id of `%1%'") % arg); - return id; - } + return arg; /* !!! check that arg is in the store */ } @@ -42,12 +32,12 @@ static void opInstall(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - for (Strings::iterator it = opArgs.begin(); - it != opArgs.end(); it++) + for (Strings::iterator i = opArgs.begin(); + i != opArgs.end(); i++) { - FSId id = normaliseNixExpr(argToId(*it)); - realiseClosure(id); - cout << format("%1%\n") % (string) id; + Path nfPath = normaliseNixExpr(checkPath(*i)); + realiseClosure(nfPath); + cout << format("%1%\n") % (string) nfPath; } } @@ -59,7 +49,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (Strings::iterator it = opArgs.begin(); it != opArgs.end(); it++) - deleteFromStore(absPath(*it)); + deleteFromStore(checkPath(*it)); } @@ -69,27 +59,21 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - for (Strings::iterator it = opArgs.begin(); - it != opArgs.end(); it++) - { - string path; - FSId id; - addToStore(*it, path, id); - cout << format("%1% %2%\n") % (string) id % path; - } + for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) + cout << format("%1%\n") % addToStore(*i); } -FSId maybeNormalise(const FSId & id, bool normalise) +Path maybeNormalise(const Path & ne, bool normalise) { - return normalise ? normaliseNixExpr(id) : id; + return normalise ? normaliseNixExpr(ne) : ne; } /* Perform various sorts of queries. */ static void opQuery(Strings opFlags, Strings opArgs) { - enum { qList, qRequisites, qGenerators, qExpansion, qGraph + enum { qList, qRequisites, qGenerators, qGraph } query = qList; bool normalise = false; bool includeExprs = true; @@ -100,7 +84,6 @@ static void opQuery(Strings opFlags, Strings opArgs) if (*i == "--list" || *i == "-l") query = qList; else if (*i == "--requisites" || *i == "-r") query = qRequisites; else if (*i == "--generators" || *i == "-g") query = qGenerators; - else if (*i == "--expansion" || *i == "-e") query = qExpansion; else if (*i == "--graph") query = qGraph; else if (*i == "--normalise" || *i == "-n") normalise = true; else if (*i == "--exclude-exprs") includeExprs = false; @@ -110,12 +93,12 @@ static void opQuery(Strings opFlags, Strings opArgs) switch (query) { case qList: { - StringSet paths; + PathSet paths; for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) { - Strings paths2 = nixExprPaths( - maybeNormalise(argToId(*i), normalise)); + StringSet paths2 = nixExprRoots( + maybeNormalise(checkPath(*i), normalise)); paths.insert(paths2.begin(), paths2.end()); } for (StringSet::iterator i = paths.begin(); @@ -129,8 +112,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) { - Strings paths2 = nixExprRequisites( - maybeNormalise(argToId(*i), normalise), + StringSet paths2 = nixExprRequisites( + maybeNormalise(checkPath(*i), normalise), includeExprs, includeSuccessors); paths.insert(paths2.begin(), paths2.end()); } @@ -140,11 +123,12 @@ static void opQuery(Strings opFlags, Strings opArgs) break; } +#if 0 case qGenerators: { FSIds outIds; for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) - outIds.push_back(argToId(*i)); + outIds.push_back(checkPath(*i)); FSIds genIds = findGenerators(outIds); @@ -153,21 +137,13 @@ static void opQuery(Strings opFlags, Strings opArgs) cout << format("%s\n") % expandId(*i); break; } - - case qExpansion: { - for (Strings::iterator i = opArgs.begin(); - i != opArgs.end(); i++) - /* !!! should not use substitutes; this is a query, - it should not have side-effects */ - cout << format("%s\n") % expandId(parseHash(*i)); - break; - } +#endif case qGraph: { - FSIds roots; + PathSet roots; for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); i++) - roots.push_back(maybeNormalise(argToId(*i), normalise)); + roots.insert(maybeNormalise(checkPath(*i), normalise)); printDotGraph(roots); break; } @@ -187,9 +163,9 @@ static void opSuccessor(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ) { - FSId id1 = parseHash(*i++); - FSId id2 = parseHash(*i++); - registerSuccessor(txn, id1, id2); + Path path1 = checkPath(*i++); + Path path2 = checkPath(*i++); + registerSuccessor(txn, path1, path2); } txn.commit(); } @@ -203,8 +179,8 @@ static void opSubstitute(Strings opFlags, Strings opArgs) for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ) { - FSId src = parseHash(*i++); - FSId sub = parseHash(*i++); + Path src = checkPath(*i++); + Path sub = checkPath(*i++); registerSubstitute(src, sub); } } @@ -229,9 +205,7 @@ static void opDump(Strings opFlags, Strings opArgs) if (opArgs.size() != 1) throw UsageError("only one argument allowed"); StdoutSink sink; - string arg = *opArgs.begin(); - string path = pathArgs ? arg : expandId(parseHash(arg)); - + string path = *opArgs.begin(); dumpPath(path, sink); } @@ -311,8 +285,6 @@ void run(Strings args) op = opInit; else if (arg == "--verify") op = opVerify; - else if (arg == "--path" || arg == "-p") - pathArgs = true; else if (arg == "--verbose" || arg == "-v") verbosity = (Verbosity) ((int) verbosity + 1); else if (arg == "--keep-failed" || arg == "-K") diff --git a/src/normalise.cc b/src/normalise.cc index 994a07df1c..834276f1be 100644 --- a/src/normalise.cc +++ b/src/normalise.cc @@ -9,47 +9,131 @@ void registerSuccessor(const Transaction & txn, - const FSId & id1, const FSId & id2) + const Path & path1, const Path & path2) { - nixDB.setString(txn, dbSuccessors, id1, id2); + nixDB.setString(txn, dbSuccessors, path1, path2); } -static FSId useSuccessor(const FSId & id) +static Path useSuccessor(const Path & path) { - string idSucc; - if (nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) { - debug(format("successor %1% -> %2%") % (string) id % idSucc); - return parseHash(idSucc); + string pathSucc; + if (nixDB.queryString(noTxn, dbSuccessors, path, pathSucc)) { + debug(format("successor %1% -> %2%") % (string) path % pathSucc); + return pathSucc; } else - return id; + return path; } -Strings pathsFromOutputs(const DerivationOutputs & ps) +#if 0 +/* Return a path whose contents have the given hash. If target is + not empty, ensure that such a path is realised in target (if + necessary by copying from another location). If prefix is not + empty, only return a path that is an descendent of prefix. */ + +string expandId(const FSId & id, const string & target = "", + const string & prefix = "/", FSIdSet pending = FSIdSet(), + bool ignoreSubstitutes = false) { - Strings ss; - for (DerivationOutputs::const_iterator i = ps.begin(); - i != ps.end(); i++) - ss.push_back(i->first); - return ss; + xxx } -FSId normaliseNixExpr(FSId id, FSIdSet pending) +string expandId(const FSId & id, const string & target, + const string & prefix, FSIdSet pending, bool ignoreSubstitutes) +{ + Nest nest(lvlDebug, format("expanding %1%") % (string) id); + + Strings paths; + + if (!target.empty() && !isInPrefix(target, prefix)) + abort(); + + nixDB.queryStrings(noTxn, dbId2Paths, id, paths); + + /* Pick one equal to `target'. */ + if (!target.empty()) { + + for (Strings::iterator i = paths.begin(); + i != paths.end(); i++) + { + string path = *i; + if (path == target && pathExists(path)) + return path; + } + + } + + /* Arbitrarily pick the first one that exists and isn't stale. */ + for (Strings::iterator it = paths.begin(); + it != paths.end(); it++) + { + string path = *it; + if (isInPrefix(path, prefix) && pathExists(path)) { + if (target.empty()) + return path; + else { + /* Acquire a lock on the target path. */ + Strings lockPaths; + lockPaths.push_back(target); + PathLocks outputLock(lockPaths); + + /* Copy. */ + copyPath(path, target); + + /* Register the target path. */ + Transaction txn(nixDB); + registerPath(txn, target, id); + txn.commit(); + + return target; + } + } + } + + if (!ignoreSubstitutes) { + + if (pending.find(id) != pending.end()) + throw Error(format("id %1% already being expanded") % (string) id); + pending.insert(id); + + /* Try to realise the substitutes, but only if this id is not + already being realised by a substitute. */ + Strings subs; + nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */ + + for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { + FSId subId = parseHash(*it); + + debug(format("trying substitute %1%") % (string) subId); + + realiseClosure(normaliseNixExpr(subId, pending), pending); + + return expandId(id, target, prefix, pending); + } + + } + + throw Error(format("cannot expand id `%1%'") % (string) id); +} +#endif + + +Path normaliseNixExpr(const Path & _nePath, PathSet pending) { Nest nest(lvlTalkative, - format("normalising nix expression %1%") % (string) id); + format("normalising expression in `%1%'") % (string) _nePath); - /* Try to substitute $id$ by any known successors in order to - speed up the rewrite process. */ - id = useSuccessor(id); + /* Try to substitute the expression by any known successors in + order to speed up the rewrite process. */ + Path nePath = useSuccessor(_nePath); /* Get the Nix expression. */ - NixExpr ne = parseNixExpr(termFromId(id)); + NixExpr ne = parseNixExpr(termFromPath(nePath)); /* If this is a normal form (i.e., a closure) we are done. */ - if (ne.type == NixExpr::neClosure) return id; + if (ne.type == NixExpr::neClosure) return nePath; if (ne.type != NixExpr::neDerivation) abort(); @@ -62,8 +146,8 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) /* Input paths, with their closure elements. */ ClosureElems inClosures; - /* Referencable paths (i.e., input and output paths). */ - StringSet allPaths; + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; /* The environment to be passed to the builder. */ Environment env; @@ -73,17 +157,17 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) nf.type = NixExpr::neClosure; - /* Parse the outputs. */ - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); + /* The outputs are referenceable paths. */ + for (PathSet::iterator i = ne.derivation.outputs.begin(); i != ne.derivation.outputs.end(); i++) { - debug(format("building %1% in `%2%'") % (string) i->second % i->first); - allPaths.insert(i->first); + debug(format("building path `%1%'") % *i); + allPaths.insert(*i); } /* Obtain locks on all output paths. The locks are automatically released when we exit this function or Nix crashes. */ - PathLocks outputLocks(pathsFromOutputs(ne.derivation.outputs)); + PathLocks outputLocks(ne.derivation.outputs); /* Now check again whether there is a successor. This is because another process may have started building in parallel. After @@ -94,13 +178,13 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) other process can build this expression, so no further checks are necessary. */ { - FSId id2 = useSuccessor(id); - if (id2 != id) { - NixExpr ne = parseNixExpr(termFromId(id2)); - debug(format("skipping build of %1%, someone beat us to it") - % (string) id); + Path nePath2 = useSuccessor(nePath); + if (nePath != nePath2) { + NixExpr ne = parseNixExpr(termFromPath(nePath2)); + debug(format("skipping build of expression `%1%', someone beat us to it") + % (string) nePath); if (ne.type != NixExpr::neClosure) abort(); - return id2; + return nePath2; } } @@ -110,14 +194,14 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) % ne.derivation.platform % thisSystem); /* Realise inputs (and remember all input paths). */ - for (FSIdSet::iterator i = ne.derivation.inputs.begin(); + for (PathSet::iterator i = ne.derivation.inputs.begin(); i != ne.derivation.inputs.end(); i++) { - FSId nf = normaliseNixExpr(*i, pending); - realiseClosure(nf, pending); - /* !!! nf should be a root of the garbage collector while we - are building */ - NixExpr ne = parseNixExpr(termFromId(nf)); + Path nfPath = normaliseNixExpr(*i, pending); + realiseClosure(nfPath, pending); + /* !!! nfPath should be a root of the garbage collector while + we are building */ + NixExpr ne = parseNixExpr(termFromPath(nfPath)); if (ne.type != NixExpr::neClosure) abort(); for (ClosureElems::iterator j = ne.closure.elems.begin(); j != ne.closure.elems.end(); j++) @@ -147,8 +231,10 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) /* We can skip running the builder if we can expand all output paths from their ids. */ + bool fastBuild = false; +#if 0 bool fastBuild = true; - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); + for (PathSet::iterator i = ne.derivation.outputs.begin(); i != ne.derivation.outputs.end(); i++) { try { @@ -160,17 +246,17 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) break; } } +#endif if (!fastBuild) { /* If any of the outputs already exist but are not registered, delete them. */ - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); + for (PathSet::iterator i = ne.derivation.outputs.begin(); i != ne.derivation.outputs.end(); i++) { - string path = i->first; - FSId id; - if (queryPathId(path, id)) + Path path = *i; + if (isValidPath(path)) throw Error(format("obstructed build: path `%1%' exists") % path); if (pathExists(path)) { debug(format("removing unregistered path `%1%'") % path); @@ -189,11 +275,11 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ - StringSet usedPaths; - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); + PathSet usedPaths; + for (PathSet::iterator i = ne.derivation.outputs.begin(); i != ne.derivation.outputs.end(); i++) { - string path = i->first; + Path path = *i; if (!pathExists(path)) throw Error(format("path `%1%' does not exist") % path); nf.closure.roots.insert(path); @@ -207,15 +293,14 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) /* Construct a closure element for this output path. */ ClosureElem elem; - elem.id = i->second; /* For each path referenced by this output path, add its id to the closure element and add the id to the `usedPaths' set (so that the elements referenced by *its* closure are added below). */ - for (Strings::iterator j = refPaths.begin(); + for (Paths::iterator j = refPaths.begin(); j != refPaths.end(); j++) { - string path = *j; + Path path = *j; elem.refs.insert(path); if (inClosures.find(path) != inClosures.end()) usedPaths.insert(path); @@ -228,11 +313,11 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) /* Close the closure. That is, for any referenced path, add the paths referenced by it. */ - StringSet donePaths; + PathSet donePaths; while (!usedPaths.empty()) { - StringSet::iterator i = usedPaths.begin(); - string path = *i; + PathSet::iterator i = usedPaths.begin(); + Path path = *i; usedPaths.erase(i); if (donePaths.find(path) != donePaths.end()) continue; @@ -243,7 +328,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) nf.closure.elems[path] = j->second; - for (StringSet::iterator k = j->second.refs.begin(); + for (PathSet::iterator k = j->second.refs.begin(); k != j->second.refs.end(); k++) usedPaths.insert(*k); } @@ -252,7 +337,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) for (ClosureElems::iterator i = inClosures.begin(); i != inClosures.end(); i++) { - StringSet::iterator j = donePaths.find(i->first); + PathSet::iterator j = donePaths.find(i->first); if (j == donePaths.end()) debug(format("NOT referenced: `%1%'") % i->first); else @@ -263,7 +348,7 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) transaction below because writing terms is idem-potent. */ ATerm nfTerm = unparseNixExpr(nf); msg(lvlVomit, format("normal form: %1%") % printTerm(nfTerm)); - FSId idNF = writeTerm(nfTerm, "-s-" + (string) id); + Path nfPath = writeTerm(nfTerm, "-s"); /* Register each outpat path, and register the normal form. This is wrapped in one database transaction to ensure that if we @@ -272,63 +357,58 @@ FSId normaliseNixExpr(FSId id, FSIdSet pending) deleted arbitrarily, while registered paths can only be deleted by running the garbage collector. */ Transaction txn(nixDB); - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); + for (PathSet::iterator i = ne.derivation.outputs.begin(); i != ne.derivation.outputs.end(); i++) - registerPath(txn, i->first, i->second); - registerSuccessor(txn, id, idNF); + registerValidPath(txn, *i); + registerSuccessor(txn, nePath, nfPath); txn.commit(); - return idNF; + return nfPath; } -void realiseClosure(const FSId & id, FSIdSet pending) +void realiseClosure(const Path & nePath, PathSet pending) { - Nest nest(lvlDebug, - format("realising closure %1%") % (string) id); + Nest nest(lvlDebug, format("realising closure `%1%'") % nePath); - NixExpr ne = parseNixExpr(termFromId(id)); + NixExpr ne = parseNixExpr(termFromPath(nePath)); if (ne.type != NixExpr::neClosure) - throw Error(format("expected closure in %1%") % (string) id); + throw Error(format("expected closure in `%1%'") % nePath); for (ClosureElems::const_iterator i = ne.closure.elems.begin(); i != ne.closure.elems.end(); i++) + assert(isValidPath(i->first)); +#if 0 expandId(i->second.id, i->first, "/", pending); +#endif } -Strings nixExprPaths(const FSId & id) +PathSet nixExprRoots(const Path & nePath) { - Strings paths; + PathSet paths; - NixExpr ne = parseNixExpr(termFromId(id)); + NixExpr ne = parseNixExpr(termFromPath(nePath)); - if (ne.type == NixExpr::neClosure) { - for (StringSet::const_iterator i = ne.closure.roots.begin(); - i != ne.closure.roots.end(); i++) - paths.push_back(*i); - } - - else if (ne.type == NixExpr::neDerivation) { - for (DerivationOutputs::iterator i = ne.derivation.outputs.begin(); - i != ne.derivation.outputs.end(); i++) - paths.push_back(i->first); - } - + if (ne.type == NixExpr::neClosure) + paths.insert(ne.closure.roots.begin(), ne.closure.roots.end()); + else if (ne.type == NixExpr::neDerivation) + paths.insert(ne.derivation.outputs.begin(), + ne.derivation.outputs.end()); else abort(); return paths; } -static void nixExprRequisitesSet(const FSId & id, - bool includeExprs, bool includeSuccessors, StringSet & paths, - FSIdSet & doneSet) +static void requisitesWorker(const Path & nePath, + bool includeExprs, bool includeSuccessors, + PathSet & paths, PathSet & doneSet) { - if (doneSet.find(id) != doneSet.end()) return; - doneSet.insert(id); + if (doneSet.find(nePath) != doneSet.end()) return; + doneSet.insert(nePath); - NixExpr ne = parseNixExpr(termFromId(id)); + NixExpr ne = parseNixExpr(termFromPath(nePath)); if (ne.type == NixExpr::neClosure) for (ClosureElems::iterator i = ne.closure.elems.begin(); @@ -336,35 +416,35 @@ static void nixExprRequisitesSet(const FSId & id, paths.insert(i->first); else if (ne.type == NixExpr::neDerivation) - for (FSIdSet::iterator i = ne.derivation.inputs.begin(); + for (PathSet::iterator i = ne.derivation.inputs.begin(); i != ne.derivation.inputs.end(); i++) - nixExprRequisitesSet(*i, + requisitesWorker(*i, includeExprs, includeSuccessors, paths, doneSet); else abort(); - if (includeExprs) - paths.insert(expandId(id)); + if (includeExprs) paths.insert(nePath); - string idSucc; - if (includeSuccessors && - nixDB.queryString(noTxn, dbSuccessors, id, idSucc)) - nixExprRequisitesSet(parseHash(idSucc), - includeExprs, includeSuccessors, paths, doneSet); + string nfPath; + if (includeSuccessors && (nfPath = useSuccessor(nePath)) != nePath) + requisitesWorker(nfPath, includeExprs, includeSuccessors, + paths, doneSet); } -Strings nixExprRequisites(const FSId & id, +PathSet nixExprRequisites(const Path & nePath, bool includeExprs, bool includeSuccessors) { - StringSet paths; - FSIdSet doneSet; - nixExprRequisitesSet(id, includeExprs, includeSuccessors, paths, doneSet); - return Strings(paths.begin(), paths.end()); + PathSet paths; + PathSet doneSet; + requisitesWorker(nePath, includeExprs, includeSuccessors, + paths, doneSet); + return paths; } -FSIds findGenerators(const FSIds & _ids) +#if 0 +PathSet findGenerators(const PathSet & outputs) { FSIdSet ids(_ids.begin(), _ids.end()); FSIds generators; @@ -407,3 +487,4 @@ FSIds findGenerators(const FSIds & _ids) return generators; } +#endif diff --git a/src/normalise.hh b/src/normalise.hh index 9b82746819..4b4db4ee2a 100644 --- a/src/normalise.hh +++ b/src/normalise.hh @@ -4,15 +4,22 @@ #include "expr.hh" -/* Normalise a Nix expression, that is, return an equivalent - closure. (For the meaning of `pending', see expandId()). */ -FSId normaliseNixExpr(FSId id, FSIdSet pending = FSIdSet()); +/* Normalise a Nix expression. That is, if the expression is a + derivation, a path containing an equivalent closure expression is + returned. This requires that the derivation is performed, unless a + successor is known. */ +Path normaliseNixExpr(const Path & nePath, PathSet pending = PathSet()); -/* Realise a Closure in the file system. */ -void realiseClosure(const FSId & id, FSIdSet pending = FSIdSet()); +/* Realise a closure expression in the file system. + + The pending paths are those that are already being realised. This + prevents infinite recursion for paths realised through a substitute + (since when we build the substitute, we would first try to realise + its output paths through substitutes... kaboom!). */ +void realiseClosure(const Path & nePath, PathSet pending = PathSet()); /* Get the list of root (output) paths of the given Nix expression. */ -Strings nixExprPaths(const FSId & id); +PathSet nixExprRoots(const Path & nePath); /* Get the list of paths that are required to realise the given expression. For a derive expression, this is the union of @@ -20,16 +27,16 @@ Strings nixExprPaths(const FSId & id); each element in the closure. If `includeExprs' is true, include the paths of the Nix expressions themselves. If `includeSuccessors' is true, include the requisites of successors. */ -Strings nixExprRequisites(const FSId & id, +PathSet nixExprRequisites(const Path & nePath, bool includeExprs, bool includeSuccessors); -/* Return the list of the ids of all known Nix expressions whose - output ids are completely contained in `ids'. */ -FSIds findGenerators(const FSIds & ids); +/* Return the list of the paths of all known Nix expressions whose + output paths are completely contained in the set `outputs'. */ +PathSet findGenerators(const PathSet & outputs); /* Register a successor. */ void registerSuccessor(const Transaction & txn, - const FSId & id1, const FSId & id2); + const Path & path1, const Path & path2); #endif /* !__NORMALISE_H */ diff --git a/src/pathlocks.cc b/src/pathlocks.cc index 93f456ace6..ff0226c84e 100644 --- a/src/pathlocks.cc +++ b/src/pathlocks.cc @@ -13,20 +13,20 @@ static StringSet lockedPaths; /* !!! not thread-safe */ -PathLocks::PathLocks(const Strings & _paths) +PathLocks::PathLocks(const PathSet & _paths) { /* Note that `fds' is built incrementally so that the destructor will only release those locks that we have already acquired. */ /* Sort the paths. This assures that locks are always acquired in the same order, thus preventing deadlocks. */ - Strings paths(_paths); + Paths paths(_paths.begin(), _paths.end()); paths.sort(); /* Acquire the lock for each path. */ - for (Strings::iterator i = paths.begin(); i != paths.end(); i++) { - string path = *i; - string lockPath = path + ".lock"; + for (Paths::iterator i = paths.begin(); i != paths.end(); i++) { + Path path = *i; + Path lockPath = path + ".lock"; debug(format("locking path `%1%'") % path); @@ -64,6 +64,6 @@ PathLocks::~PathLocks() for (list::iterator i = fds.begin(); i != fds.end(); i++) close(*i); - for (Strings::iterator i = paths.begin(); i != paths.end(); i++) + for (Paths::iterator i = paths.begin(); i != paths.end(); i++) lockedPaths.erase(*i); } diff --git a/src/pathlocks.hh b/src/pathlocks.hh index 03a62e65a7..6c36b9b1d1 100644 --- a/src/pathlocks.hh +++ b/src/pathlocks.hh @@ -8,10 +8,10 @@ class PathLocks { private: list fds; - Strings paths; + Paths paths; public: - PathLocks(const Strings & _paths); + PathLocks(const PathSet & _paths); ~PathLocks(); }; diff --git a/src/references.hh b/src/references.hh index b19fbf72c6..d009453d6a 100644 --- a/src/references.hh +++ b/src/references.hh @@ -4,7 +4,7 @@ #include "util.hh" -Strings filterReferences(const string & path, const Strings & refs); +Strings filterReferences(const Path & path, const Strings & refs); #endif /* !__VALUES_H */ diff --git a/src/shared.cc b/src/shared.cc index c0f07e955c..32e8916175 100644 --- a/src/shared.cc +++ b/src/shared.cc @@ -72,5 +72,3 @@ int main(int argc, char * * argv) return 0; } - - diff --git a/src/store.cc b/src/store.cc index f05cdf3ba4..b2b479e0dd 100644 --- a/src/store.cc +++ b/src/store.cc @@ -8,7 +8,6 @@ #include "db.hh" #include "archive.hh" #include "pathlocks.hh" -#include "normalise.hh" struct CopySink : DumpSink @@ -31,7 +30,7 @@ struct CopySource : RestoreSource }; -void copyPath(string src, string dst) +void copyPath(const Path & src, const Path & dst) { debug(format("copying `%1%' to `%2%'") % src % dst); @@ -82,7 +81,7 @@ void copyPath(string src, string dst) } -void registerSubstitute(const FSId & srcId, const FSId & subId) +void registerSubstitute(const Path & srcPath, const Path & subPath) { #if 0 Strings subs; @@ -98,202 +97,89 @@ void registerSubstitute(const FSId & srcId, const FSId & subId) /* For now, accept only one substitute per id. */ Strings subs; - subs.push_back(subId); + subs.push_back(subPath); Transaction txn(nixDB); - nixDB.setStrings(txn, dbSubstitutes, srcId, subs); + nixDB.setStrings(txn, dbSubstitutes, srcPath, subs); txn.commit(); } -void registerPath(const Transaction & txn, - const string & _path, const FSId & id) +void registerValidPath(const Transaction & txn, const Path & _path) { - string path(canonPath(_path)); - - debug(format("registering path `%1%' with id %2%") - % path % (string) id); - - string oldId; - if (nixDB.queryString(txn, dbPath2Id, path, oldId)) { - if (id != parseHash(oldId)) - throw Error(format("path `%1%' already contains id %2%") - % path % oldId); - return; - } - - nixDB.setString(txn, dbPath2Id, path, id); - - Strings paths; - nixDB.queryStrings(txn, dbId2Paths, id, paths); /* non-existence = ok */ - - paths.push_back(path); - - nixDB.setStrings(txn, dbId2Paths, id, paths); + Path path(canonPath(_path)); + debug(format("registering path `%1%'") % path); + nixDB.setString(txn, dbValidPaths, path, ""); } -void unregisterPath(const string & _path) +bool isValidPath(const Path & path) { - string path(canonPath(_path)); + string s; + return nixDB.queryString(noTxn, dbValidPaths, path, s); +} + + +void unregisterValidPath(const Path & _path) +{ + Path path(canonPath(_path)); Transaction txn(nixDB); debug(format("unregistering path `%1%'") % path); - string _id; - if (!nixDB.queryString(txn, dbPath2Id, path, _id)) { - txn.abort(); - return; - } - FSId id(parseHash(_id)); - - nixDB.delPair(txn, dbPath2Id, path); - - Strings paths, paths2; - nixDB.queryStrings(txn, dbId2Paths, id, paths); /* non-existence = ok */ - - for (Strings::iterator it = paths.begin(); - it != paths.end(); it++) - if (*it != path) paths2.push_back(*it); - - nixDB.setStrings(txn, dbId2Paths, id, paths2); + nixDB.delPair(txn, dbValidPaths, path); txn.commit(); } -bool queryPathId(const string & path, FSId & id) -{ - string s; - if (!nixDB.queryString(noTxn, dbPath2Id, absPath(path), s)) return false; - id = parseHash(s); - return true; -} - - -bool isInPrefix(const string & path, const string & _prefix) +static bool isInPrefix(const string & path, const string & _prefix) { string prefix = canonPath(_prefix + "/"); return string(path, 0, prefix.size()) == prefix; } -string expandId(const FSId & id, const string & target, - const string & prefix, FSIdSet pending, bool ignoreSubstitutes) -{ - Nest nest(lvlDebug, format("expanding %1%") % (string) id); - - Strings paths; - - if (!target.empty() && !isInPrefix(target, prefix)) - abort(); - - nixDB.queryStrings(noTxn, dbId2Paths, id, paths); - - /* Pick one equal to `target'. */ - if (!target.empty()) { - - for (Strings::iterator i = paths.begin(); - i != paths.end(); i++) - { - string path = *i; - if (path == target && pathExists(path)) - return path; - } - - } - - /* Arbitrarily pick the first one that exists and isn't stale. */ - for (Strings::iterator it = paths.begin(); - it != paths.end(); it++) - { - string path = *it; - if (isInPrefix(path, prefix) && pathExists(path)) { - if (target.empty()) - return path; - else { - /* Acquire a lock on the target path. */ - Strings lockPaths; - lockPaths.push_back(target); - PathLocks outputLock(lockPaths); - - /* Copy. */ - copyPath(path, target); - - /* Register the target path. */ - Transaction txn(nixDB); - registerPath(txn, target, id); - txn.commit(); - - return target; - } - } - } - - if (!ignoreSubstitutes) { - - if (pending.find(id) != pending.end()) - throw Error(format("id %1% already being expanded") % (string) id); - pending.insert(id); - - /* Try to realise the substitutes, but only if this id is not - already being realised by a substitute. */ - Strings subs; - nixDB.queryStrings(noTxn, dbSubstitutes, id, subs); /* non-existence = ok */ - - for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { - FSId subId = parseHash(*it); - - debug(format("trying substitute %1%") % (string) subId); - - realiseClosure(normaliseNixExpr(subId, pending), pending); - - return expandId(id, target, prefix, pending); - } - - } - - throw Error(format("cannot expand id `%1%'") % (string) id); -} - - -void addToStore(string srcPath, string & dstPath, FSId & id, - bool deterministicName) +Path addToStore(const Path & _srcPath) { + Path srcPath(absPath(_srcPath)); debug(format("adding `%1%' to the store") % srcPath); - srcPath = absPath(srcPath); - id = hashPath(srcPath); + Hash h = hashPath(srcPath); string baseName = baseNameOf(srcPath); - dstPath = canonPath(nixStore + "/" + (string) id + "-" + baseName); + Path dstPath = canonPath(nixStore + "/" + (string) h + "-" + baseName); - try { - dstPath = expandId(id, deterministicName ? dstPath : "", - nixStore, FSIdSet(), true); - return; - } catch (...) { + if (!isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + PathSet lockPaths; + lockPaths.insert(dstPath); + PathLocks outputLock(lockPaths); + + if (!isValidPath(dstPath)) { + copyPath(srcPath, dstPath); + + Transaction txn(nixDB); + registerValidPath(txn, dstPath); + txn.commit(); + } } - - Strings lockPaths; - lockPaths.push_back(dstPath); - PathLocks outputLock(lockPaths); - copyPath(srcPath, dstPath); - - Transaction txn(nixDB); - registerPath(txn, dstPath, id); - txn.commit(); + return dstPath; } -void deleteFromStore(const string & path) +void deleteFromStore(const Path & _path) { - string prefix = + "/"; - if (!isInPrefix(path, nixStore)) - throw Error(format("path %1% is not in the store") % path); + Path path(canonPath(_path)); - unregisterPath(path); + if (!isInPrefix(path, nixStore)) + throw Error(format("path `%1%' is not in the store") % path); + + unregisterValidPath(path); deletePath(path); } @@ -305,6 +191,7 @@ void verifyStore() /* !!! verify that the result is consistent */ +#if 0 Strings paths; nixDB.enumTable(txn, dbPath2Id, paths); @@ -421,6 +308,7 @@ void verifyStore() } } } +#endif txn.commit(); } diff --git a/src/store.hh b/src/store.hh index 7f6b245690..69de824786 100644 --- a/src/store.hh +++ b/src/store.hh @@ -9,44 +9,27 @@ using namespace std; -typedef Hash FSId; - -typedef set FSIdSet; - - /* Copy a path recursively. */ -void copyPath(string src, string dst); +void copyPath(const Path & src, const Path & dst); /* Register a substitute. */ -void registerSubstitute(const FSId & srcId, const FSId & subId); +void registerSubstitute(const Path & srcPath, const Path & subPath); -/* Register a path keyed on its id. */ -void registerPath(const Transaction & txn, - const string & path, const FSId & id); +/* Register the validity of a path. */ +void registerValidPath(const Transaction & txn, const Path & path); -/* Query the id of a path. */ -bool queryPathId(const string & path, FSId & id); +/* Unregister the validity of a path. */ +void unregisterValidPath(const Path & path); -/* Return a path whose contents have the given hash. If target is - not empty, ensure that such a path is realised in target (if - necessary by copying from another location). If prefix is not - empty, only return a path that is an descendent of prefix. +/* Checks whether a path is valid. */ +bool isValidPath(const Path & path); - The list of pending ids are those that already being expanded. - This prevents infinite recursion for ids realised through a - substitute (since when we build the substitute, we would first try - to expand the id... kaboom!). */ -string expandId(const FSId & id, const string & target = "", - const string & prefix = "/", FSIdSet pending = FSIdSet(), - bool ignoreSubstitutes = false); - -/* Copy a file to the nixStore directory and register it in dbRefs. - Return the hash code of the value. */ -void addToStore(string srcPath, string & dstPath, FSId & id, - bool deterministicName = false); +/* Copy the contents of a path to the store and register the validity + the resulting path. The resulting path is returned. */ +Path addToStore(const Path & srcPath); /* Delete a value from the nixStore directory. */ -void deleteFromStore(const string & path); +void deleteFromStore(const Path & path); void verifyStore(); diff --git a/src/test.cc b/src/test.cc index fb1e62eb38..457fecf245 100644 --- a/src/test.cc +++ b/src/test.cc @@ -10,10 +10,10 @@ #include "globals.hh" -void realise(FSId id) +void realise(Path nePath) { - Nest nest(lvlDebug, format("TEST: realising %1%") % (string) id); - realiseClosure(normaliseNixExpr(id)); + Nest nest(lvlDebug, format("TEST: realising `%1%'") % nePath); + realiseClosure(normaliseNixExpr(nePath)); } @@ -48,12 +48,12 @@ void runTests() try { h = parseHash("blah blah"); abort(); - } catch (BadRefError err) { }; + } catch (Error err) { }; try { h = parseHash("0b0ffd0538622bfe20b92c4aa57254d99"); abort(); - } catch (BadRefError err) { }; + } catch (Error err) { }; /* Path canonicalisation. */ cout << canonPath("/./../././//") << endl; @@ -97,76 +97,59 @@ void runTests() /* Expression evaluation. */ - FSId builder1id; - string builder1fn; - addToStore("./test-builder-1.sh", builder1fn, builder1id); + Path builder1fn; + builder1fn = addToStore("./test-builder-1.sh"); ATerm fs1 = ATmake( - "Closure([], [(, , [])])", + "Closure([], [(, [])])", builder1fn.c_str(), - builder1fn.c_str(), - ((string) builder1id).c_str()); - FSId fs1id = writeTerm(fs1, ""); + builder1fn.c_str()); + Path fs1ne = writeTerm(fs1, "-c"); - realise(fs1id); - realise(fs1id); + realise(fs1ne); + realise(fs1ne); - ATerm fs2 = ATmake( - "Closure([], [(, , [])])", - (builder1fn + "_bla").c_str(), - (builder1fn + "_bla").c_str(), - ((string) builder1id).c_str()); - FSId fs2id = writeTerm(fs2, ""); - - realise(fs2id); - realise(fs2id); - - string out1id = hashString("foo"); /* !!! bad */ - string out1fn = nixStore + "/" + (string) out1id + "-hello.txt"; + string out1h = hashString("foo"); /* !!! bad */ + Path out1fn = nixStore + "/" + (string) out1h + "-hello.txt"; ATerm fs3 = ATmake( - "Derive([(, )], [], , , [], [(\"out\", )])", + "Derive([], [], , , [], [(\"out\", )])", out1fn.c_str(), - ((string) out1id).c_str(), - ((string) fs1id).c_str(), + fs1ne.c_str(), thisSystem.c_str(), - ((string) builder1fn).c_str(), + builder1fn.c_str(), out1fn.c_str()); debug(printTerm(fs3)); - FSId fs3id = writeTerm(fs3, ""); + Path fs3ne = writeTerm(fs3, "-d"); - realise(fs3id); - realise(fs3id); + realise(fs3ne); + realise(fs3ne); - FSId builder4id; - string builder4fn; - addToStore("./test-builder-2.sh", builder4fn, builder4id); + Path builder4fn = addToStore("./test-builder-2.sh"); ATerm fs4 = ATmake( - "Closure([], [(, , [])])", + "Closure([], [(, [])])", builder4fn.c_str(), - builder4fn.c_str(), - ((string) builder4id).c_str()); - FSId fs4id = writeTerm(fs4, ""); + builder4fn.c_str()); + Path fs4ne = writeTerm(fs4, "-c"); - realise(fs4id); + realise(fs4ne); - string out5id = hashString("bar"); /* !!! bad */ - string out5fn = nixStore + "/" + (string) out5id + "-hello2"; + string out5h = hashString("bar"); /* !!! bad */ + Path out5fn = nixStore + "/" + (string) out5h + "-hello2"; ATerm fs5 = ATmake( - "Derive([(, )], [], , , [], [(\"out\", ), (\"builder\", )])", + "Derive([], [], , , [], [(\"out\", ), (\"builder\", )])", out5fn.c_str(), - ((string) out5id).c_str(), - ((string) fs4id).c_str(), + fs4ne.c_str(), thisSystem.c_str(), - ((string) builder4fn).c_str(), + builder4fn.c_str(), out5fn.c_str(), - ((string) builder4fn).c_str()); + builder4fn.c_str()); debug(printTerm(fs5)); - FSId fs5id = writeTerm(fs5, ""); + Path fs5ne = writeTerm(fs5, "-d"); - realise(fs5id); - realise(fs5id); + realise(fs5ne); + realise(fs5ne); }