diff --git a/src/fstate.cc b/src/fstate.cc index 36f7482acf..596d63a5f4 100644 --- a/src/fstate.cc +++ b/src/fstate.cc @@ -154,55 +154,47 @@ FState hash2fstate(Hash hash) } -ATerm termFromHash(const Hash & hash, string * p) +ATerm termFromId(const FSId & id, string * p) { - string path = expandHash(hash); + string path = expandId(id); if (p) *p = path; ATerm t = ATreadFromNamedFile(path.c_str()); - if (!t) throw Error(format("cannot read aterm %1%") % path); + if (!t) throw Error(format("cannot read aterm from `%1%'") % path); return t; } -Hash writeTerm(ATerm t, const string & suffix, string * p) +FSId writeTerm(ATerm t, const string & suffix, string * p) { - string path = nixStore + "/tmp.nix"; /* !!! */ + FSId id = hashTerm(t); + + string path = canonPath(nixStore + "/" + + (string) id + suffix + ".nix"); if (!ATwriteToNamedTextFile(t, path.c_str())) throw Error(format("cannot write aterm %1%") % path); - Hash hash = hashPath(path); - string path2 = canonPath(nixStore + "/" + - (string) hash + suffix + ".nix"); - if (rename(path.c_str(), path2.c_str()) == -1) - throw SysError(format("renaming %1% to %2%") % path % path2); - registerPath(path2, hash); - if (p) *p = path2; - return hash; + + registerPath(path, id); + if (p) *p = path; + + return id; } -void registerSuccessor(const Hash & fsHash, const Hash & scHash) +void registerSuccessor(const FSId & id1, const FSId & id2) { - setDB(nixDB, dbSuccessors, fsHash, scHash); + setDB(nixDB, dbSuccessors, id1, id2); } -FState storeSuccessor(FState fs, FState sc, StringSet & paths) +static FSId storeSuccessor(const FSId & id1, FState sc) { - if (fs == sc) return sc; - - string path; - Hash fsHash = hashTerm(fs); - Hash scHash = writeTerm(sc, "-s-" + (string) fsHash, &path); - registerSuccessor(fsHash, scHash); - paths.insert(path); + FSId id2 = writeTerm(sc, "-s-" + (string) id1, 0); + registerSuccessor(id1, id2); + return id2; +} + #if 0 - return ATmake("Include()", ((string) scHash).c_str()); -#endif - return sc; -} - - static FState realise(FState fs, StringSet & paths) { char * s1, * s2, * s3; @@ -421,3 +413,193 @@ void fstateRefs(FState fs, StringSet & paths) { fstateRefs2(fs, paths); } +#endif + + +static void parseIds(ATermList ids, FSIds & out) +{ + while (!ATisEmpty(ids)) { + char * s; + ATerm id = ATgetFirst(ids); + if (!ATmatch(id, "", &s)) + throw badTerm("not an id", id); + out.push_back(parseHash(s)); + debug(s); + ids = ATgetNext(ids); + } +} + + +/* Parse a slice. */ +static Slice parseSlice(FState fs) +{ + Slice slice; + ATermList roots, elems; + + if (!ATmatch(fs, "Slice([], [])", &roots, &elems)) + throw badTerm("not a slice", fs); + + parseIds(roots, slice.roots); + + while (!ATisEmpty(elems)) { + char * s1, * s2; + ATermList refs; + ATerm t = ATgetFirst(elems); + if (!ATmatch(t, "(, , [])", &s1, &s2, &refs)) + throw badTerm("not a slice element", t); + SliceElem elem; + elem.path = s1; + elem.id = parseHash(s2); + parseIds(refs, elem.refs); + slice.elems.push_back(elem); + elems = ATgetNext(elems); + } + + return slice; +} + + +Slice normaliseFState(FSId id) +{ + debug(format("normalising fstate")); + Nest nest(true); + + /* Try to substitute $id$ by any known successors in order to + speed up the rewrite process. */ + string idSucc; + while (queryDB(nixDB, dbSuccessors, id, idSucc)) { + debug(format("successor %1% -> %2%") % (string) id % idSucc); + id = parseHash(idSucc); + } + + /* Get the fstate expression. */ + FState fs = termFromId(id); + + /* Already in normal form (i.e., a slice)? */ + if (ATgetType(fs) == AT_APPL && + (string) ATgetName(ATgetAFun(fs)) == "Slice") + return parseSlice(fs); + + /* Then we it's a Derive node. */ + ATermList outs, ins, bnds; + char * builder; + char * platform; + if (!ATmatch(fs, "Derive([], [], , , [])", + &outs, &ins, &builder, &platform, &bnds)) + throw badTerm("not a derive", fs); + + /* Right platform? */ + checkPlatform(platform); + + /* Realise inputs (and remember all input paths). */ + FSIds inIds; + parseIds(ins, inIds); + + SliceElems inElems; /* !!! duplicates */ + StringSet inPathsSet; + for (FSIds::iterator i = inIds.begin(); i != inIds.end(); i++) { + Slice slice = normaliseFState(*i); + realiseSlice(slice); + + for (SliceElems::iterator j = slice.elems.begin(); + j != slice.elems.end(); j++) + { + inElems.push_back(*j); + inPathsSet.insert(j->path); + } + } + + Strings inPaths; + copy(inPathsSet.begin(), inPathsSet.end(), + inserter(inPaths, inPaths.begin())); + + /* Build the environment. */ + Environment env; + while (!ATisEmpty(bnds)) { + char * s1, * s2; + ATerm bnd = ATgetFirst(bnds); + if (!ATmatch(bnd, "(, )", &s1, &s2)) + throw badTerm("tuple of strings expected", bnd); + env[s1] = s2; + bnds = ATgetNext(bnds); + } + + /* Check that none of the output paths exist. */ + typedef pair OutPath; + list outPaths; + while (!ATisEmpty(outs)) { + ATerm t = ATgetFirst(outs); + char * s1, * s2; + if (!ATmatch(t, "(, )", &s1, &s2)) + throw badTerm("string expected", t); + outPaths.push_back(OutPath(s1, parseHash(s2))); + outs = ATgetNext(outs); + } + + for (list::iterator i = outPaths.begin(); + i != outPaths.end(); i++) + if (pathExists(i->first)) + throw Error(format("path `%1%' exists") % i->first); + + /* Run the builder. */ + runProgram(builder, env); + + Slice slice; + + /* Check whether the output paths were created, and register each + one. */ + for (list::iterator i = outPaths.begin(); + i != outPaths.end(); i++) + { + string path = i->first; + if (!pathExists(path)) + throw Error(format("path `%1%' does not exist") % path); + registerPath(path, i->second); + slice.roots.push_back(i->second); + + Strings outPaths = filterReferences(path, inPaths); + } + + return slice; +} + + +void realiseSlice(Slice slice) +{ + debug(format("realising slice")); + Nest nest(true); + + if (slice.elems.size() == 0) + throw Error("empty slice"); + + /* Perhaps all paths already contain the right id? */ + + bool missing = false; + for (SliceElems::iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + { + SliceElem elem = *i; + string id; + if (!queryDB(nixDB, dbPath2Id, elem.path, id)) { + if (pathExists(elem.path)) + throw Error(format("path `%1%' obstructed") % elem.path); + missing = true; + break; + } + if (parseHash(id) != elem.id) + throw Error(format("path `%1%' obstructed") % elem.path); + } + + if (!missing) { + debug(format("already installed")); + return; + } + + /* For each element, expand its id at its path. */ + for (SliceElems::iterator i = slice.elems.begin(); + i != slice.elems.end(); i++) + { + SliceElem elem = *i; + expandId(elem.id, elem.path); + } +} diff --git a/src/fstate.hh b/src/fstate.hh index 9d789c8346..f06a4807ef 100644 --- a/src/fstate.hh +++ b/src/fstate.hh @@ -8,6 +8,7 @@ extern "C" { } #include "hash.hh" +#include "store.hh" using namespace std; @@ -17,33 +18,24 @@ using namespace std; A Nix file system state expression, or FState, describes a (partial) state of the file system. - Path : Path * Content * [FState] -> FState + Slice : [Id] * [(Path, Id, [Id])] -> FState + (update) Path(path, content, refs) specifies a file object (its full path and contents), along with all file objects referenced by it (that is, that it has pointers to). We assume that all files are self-referential. This prevents us from having to deal with cycles. - Derive : String * Path * [FState] * Path * [(String, String)] -> FState + Derive : [(Path, Id)] * [FStateId] * Path * [(String, String)] -> FState + (update) Derive(platform, builder, ins, outs, env) specifies the creation of new file objects (in paths declared by `outs') by the execution of a program `builder' on a platform `platform'. This execution takes place in a file system state given by `ins'. `env' specifies a mapping of strings to strings. - [ !!! NOT IMPLEMENTED - Regular : String -> Content - Directory : [(String, Content)] -> Content - (this complicates unambiguous normalisation) - ] - CHash : Hash -> Content - - File content, given either in situ, or through an external reference - to the file system or url-space decorated with a hash to preserve - purity. - A FState expression is in {\em $f$-normal form} if all Derive nodes have been reduced to File nodes. @@ -63,7 +55,26 @@ typedef ATerm Content; typedef set StringSet; +typedef list FSIds; + +struct SliceElem +{ + string path; + FSId id; + FSIds refs; +}; + +typedef list SliceElems; + +struct Slice +{ + FSIds roots; + SliceElems elems; +}; + + +#if 0 /* Realise an fstate expression in the file system. This requires execution of all Derive() nodes. */ FState realiseFState(FState fs, StringSet & paths); @@ -74,6 +85,8 @@ string fstatePath(FState fs); /* Return the paths referenced by fstate expression. */ void fstateRefs(FState fs, StringSet & paths); +#endif + /* Return a canonical textual representation of an expression. */ string printTerm(ATerm t); @@ -85,16 +98,22 @@ Error badTerm(const format & f, ATerm t); /* Hash an aterm. */ Hash hashTerm(ATerm t); -FState hash2fstate(Hash hash); - -/* Read an aterm from disk, given its hash. */ -ATerm termFromHash(const Hash & hash, string * p = 0); +/* Read an aterm from disk, given its id. */ +ATerm termFromId(const FSId & id, string * p = 0); /* Write an aterm to the Nix store directory, and return its hash. */ -Hash writeTerm(ATerm t, const string & suffix, string * p = 0); +FSId writeTerm(ATerm t, const string & suffix, string * p = 0); /* Register a successor. */ -void registerSuccessor(const Hash & fsHash, const Hash & scHash); +void registerSuccessor(const FSId & id1, const FSId & id2); + + +/* Normalise an fstate-expression, that is, return an equivalent + Slice. */ +Slice normaliseFState(FSId id); + +/* Realise a Slice in the file system. */ +void realiseSlice(Slice slice); #endif /* !__FSTATE_H */ diff --git a/src/globals.cc b/src/globals.cc index 9893d7ad22..1edb38f748 100644 --- a/src/globals.cc +++ b/src/globals.cc @@ -2,7 +2,8 @@ #include "db.hh" -string dbHash2Paths = "hash2paths"; +string dbPath2Id = "path2id"; +string dbId2Paths = "id2paths"; string dbSuccessors = "successors"; string dbSubstitutes = "substitutes"; @@ -15,7 +16,8 @@ string nixDB = "/UNINIT"; void initDB() { - createDB(nixDB, dbHash2Paths); + createDB(nixDB, dbPath2Id); + createDB(nixDB, dbId2Paths); createDB(nixDB, dbSuccessors); createDB(nixDB, dbSubstitutes); } diff --git a/src/globals.hh b/src/globals.hh index 0668ac40e5..fbb020df7c 100644 --- a/src/globals.hh +++ b/src/globals.hh @@ -8,33 +8,36 @@ using namespace std; /* Database names. */ -/* dbHash2Paths :: Hash -> [Path] +/* dbPath2Id :: Path -> FSId - Maintains a mapping from hashes to lists of paths. This is what we - use to resolve Hash(hash) content descriptors. */ -extern string dbHash2Paths; + Each pair (p, id) records that path $p$ contains an expansion of + $id$. */ +extern string dbPath2Id; -/* dbSuccessors :: Hash -> Hash - Each pair (h1, h2) in this mapping records the fact that a - successor of an fstate expression with hash h1 is stored in a file - with hash h2. +/* dbId2Paths :: FSId -> [Path] + + A mapping from ids to lists of paths. */ +extern string dbId2Paths; + + +/* dbSuccessors :: FSId -> FSId + + Each pair $(id_1, id_2)$ in this mapping records the fact that a + successor of an fstate expression stored in a file with identifier + $id_1$ is stored in a file with identifier $id_2$. Note that a term $y$ is successor of $x$ iff there exists a sequence of rewrite steps that rewrites $x$ into $y$. - - Also note that instead of a successor, $y$ can be any term - equivalent to $x$, that is, reducing to the same result, as long as - $x$ is equal to or a successor of $y$. (This is useful, e.g., for - shared derivate caching over the network). */ extern string dbSuccessors; -/* dbSubstitutes :: Hash -> [Hash] - Each pair $(h, [hs])$ tells Nix that it can realise any of the - fstate expressions referenced by the hashes in $hs$ to obtain a Nix - archive that, when unpacked, will produce a path with hash $h$. +/* dbSubstitutes :: FSId -> [FSId] + + Each pair $(id, [ids])$ tells Nix that it can realise any of the + fstate expressions referenced by the identifiers in $ids$ to + generate a path with identifier $id$. The main purpose of this is for distributed caching of derivates. One system can compute a derivate with hash $h$ and put it on a diff --git a/src/store.cc b/src/store.cc index 5feb80c9de..c8b3bd37f4 100644 --- a/src/store.cc +++ b/src/store.cc @@ -84,39 +84,36 @@ void copyPath(string src, string dst) } -void registerSubstitute(const Hash & srcHash, const Hash & subHash) +void registerSubstitute(const FSId & srcId, const FSId & subId) { Strings subs; - queryListDB(nixDB, dbSubstitutes, srcHash, subs); /* non-existence = ok */ + queryListDB(nixDB, dbSubstitutes, srcId, subs); /* non-existence = ok */ for (Strings::iterator it = subs.begin(); it != subs.end(); it++) - if (parseHash(*it) == subHash) return; + if (parseHash(*it) == subId) return; - subs.push_back(subHash); + subs.push_back(subId); - setListDB(nixDB, dbSubstitutes, srcHash, subs); + setListDB(nixDB, dbSubstitutes, srcId, subs); } -Hash registerPath(const string & _path, Hash hash) +void registerPath(const string & _path, const FSId & id) { string path(canonPath(_path)); - if (hash == Hash()) hash = hashPath(path); + setDB(nixDB, dbPath2Id, path, id); Strings paths; - queryListDB(nixDB, dbHash2Paths, hash, paths); /* non-existence = ok */ + queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */ for (Strings::iterator it = paths.begin(); it != paths.end(); it++) - if (*it == path) goto exists; + if (*it == path) return; paths.push_back(path); - setListDB(nixDB, dbHash2Paths, hash, paths); - - exists: - return hash; + setListDB(nixDB, dbId2Paths, id, paths); } @@ -124,10 +121,15 @@ void unregisterPath(const string & _path) { string path(canonPath(_path)); - Hash hash = hashPath(path); + string _id; + if (!queryDB(nixDB, dbPath2Id, path, _id)) + return; + FSId id(parseHash(_id)); + + /* begin transaction */ Strings paths, paths2; - queryListDB(nixDB, dbHash2Paths, hash, paths); /* non-existence = ok */ + queryListDB(nixDB, dbId2Paths, id, paths); /* non-existence = ok */ bool changed = false; for (Strings::iterator it = paths.begin(); @@ -135,7 +137,9 @@ void unregisterPath(const string & _path) if (*it != path) paths2.push_back(*it); else changed = true; if (changed) - setListDB(nixDB, dbHash2Paths, hash, paths2); + setListDB(nixDB, dbId2Paths, id, paths2); + + /* end transaction */ } @@ -146,7 +150,7 @@ bool isInPrefix(const string & path, const string & _prefix) } -string expandHash(const Hash & hash, const string & target, +string expandId(const FSId & id, const string & target, const string & prefix) { Strings paths; @@ -154,9 +158,7 @@ string expandHash(const Hash & hash, const string & target, if (!target.empty() && !isInPrefix(target, prefix)) abort(); - queryListDB(nixDB, dbHash2Paths, hash, paths); - - /* !!! we shouldn't check for staleness by default --- too slow */ + queryListDB(nixDB, dbId2Paths, id, paths); /* Pick one equal to `target'. */ if (!target.empty()) { @@ -165,16 +167,8 @@ string expandHash(const Hash & hash, const string & target, i != paths.end(); i++) { string path = *i; - try { -#if 0 - if (path == target && hashPath(path) == hash) -#endif - if (path == target && pathExists(path)) - return path; - } catch (Error & e) { - debug(format("stale path: %1%") % e.msg()); - /* try next one */ - } + if (path == target && pathExists(path)) + return path; } } @@ -184,28 +178,26 @@ string expandHash(const Hash & hash, const string & target, it != paths.end(); it++) { string path = *it; - try { - if (isInPrefix(path, prefix) && hashPath(path) == hash) { - if (target.empty()) - return path; - else { - copyPath(path, target); - return target; - } + if (isInPrefix(path, prefix) && pathExists(path)) { + if (target.empty()) + return path; + else { + copyPath(path, target); + registerPath(target, id); + return target; } - } catch (Error & e) { - debug(format("stale path: %1%") % e.msg()); - /* try next one */ } } +#if 0 /* Try to realise the substitutes. */ Strings subs; - queryListDB(nixDB, dbSubstitutes, hash, subs); /* non-existence = ok */ + queryListDB(nixDB, dbSubstitutes, id, subs); /* non-existence = ok */ for (Strings::iterator it = subs.begin(); it != subs.end(); it++) { - StringSet dummy; + realiseSlice(normaliseFState(*it)); + FState nf = realiseFState(hash2fstate(parseHash(*it)), dummy); string path = fstatePath(nf); @@ -222,29 +214,30 @@ string expandHash(const Hash & hash, const string & target, return target; } } +#endif - throw Error(format("cannot expand hash `%1%'") % (string) hash); + throw Error(format("cannot expand id `%1%'") % (string) id); } -void addToStore(string srcPath, string & dstPath, Hash & hash, +void addToStore(string srcPath, string & dstPath, FSId & id, bool deterministicName) { srcPath = absPath(srcPath); - hash = hashPath(srcPath); + id = hashPath(srcPath); string baseName = baseNameOf(srcPath); - dstPath = canonPath(nixStore + "/" + (string) hash + "-" + baseName); + dstPath = canonPath(nixStore + "/" + (string) id + "-" + baseName); try { /* !!! should not use the substitutes! */ - dstPath = expandHash(hash, deterministicName ? dstPath : "", nixStore); + dstPath = expandId(id, deterministicName ? dstPath : "", nixStore); return; } catch (...) { } copyPath(srcPath, dstPath); - registerPath(dstPath, hash); + registerPath(dstPath, id); } diff --git a/src/store.hh b/src/store.hh index b6ed43ff67..7dd0f72e6f 100644 --- a/src/store.hh +++ b/src/store.hh @@ -8,29 +8,28 @@ using namespace std; +typedef Hash FSId; + + /* Copy a path recursively. */ void copyPath(string src, string dst); /* Register a substitute. */ -void registerSubstitute(const Hash & srcHash, const Hash & subHash); +void registerSubstitute(const FSId & srcId, const FSId & subId); -/* Register a path keyed on its hash. */ -Hash registerPath(const string & path, Hash hash = Hash()); +/* Register a path keyed on its id. */ +void registerPath(const string & path, const FSId & id); /* Return a path whose contents have the given hash. If target is not empty, ensure that such a path is realised in target (if necessary by copying from another location). If prefix is not - empty, only return a path that is an descendent of prefix. - - If no path with the given hash is known to exist in the file - system, -*/ -string expandHash(const Hash & hash, const string & target = "", + empty, only return a path that is an descendent of prefix. */ +string expandId(const FSId & id, const string & target = "", const string & prefix = "/"); /* Copy a file to the nixStore directory and register it in dbRefs. Return the hash code of the value. */ -void addToStore(string srcPath, string & dstPath, Hash & hash, +void addToStore(string srcPath, string & dstPath, FSId & id, bool deterministicName = false); /* Delete a value from the nixStore directory. */ diff --git a/src/test.cc b/src/test.cc index c2a1cd3bfd..3437650ac2 100644 --- a/src/test.cc +++ b/src/test.cc @@ -11,14 +11,15 @@ #include "globals.hh" -void realise(FState fs) +void realise(FSId id) { - cout << format("%1% => %2%\n") - % printTerm(fs) - % printTerm(realiseFState(fs)); + cout << format("realising %1%\n") % (string) id; + Slice slice = normaliseFState(id); + realiseSlice(slice); } +#if 0 void realiseFail(FState fs) { try { @@ -28,6 +29,7 @@ void realiseFail(FState fs) cout << "error (expected): " << e.what() << endl; } } +#endif struct MySink : DumpSink @@ -111,54 +113,47 @@ void runTests() /* Expression evaluation. */ -#if 0 - eval(whNormalise, - ATmake("Str(\"Hello World\")")); - eval(whNormalise, - ATmake("Bool(True)")); - eval(whNormalise, - ATmake("Bool(False)")); - eval(whNormalise, - ATmake("App(Lam(\"x\", Var(\"x\")), Str(\"Hello World\"))")); - eval(whNormalise, - ATmake("App(App(Lam(\"x\", Lam(\"y\", Var(\"x\"))), Str(\"Hello World\")), Str(\"Hallo Wereld\"))")); - eval(whNormalise, - ATmake("App(Lam(\"sys\", Lam(\"x\", [Var(\"x\"), Var(\"sys\")])), Str(\"i686-suse-linux\"))")); - - evalFail(whNormalise, - ATmake("Foo(123)")); - - string builder1fn = absPath("./test-builder-1.sh"); - Hash builder1h = hashPath(builder1fn); - - string fn1 = nixValues + "/builder-1.sh"; - Expr e1 = ATmake("Path(, ExtFile(, ), [])", - fn1.c_str(), - builder1h.c_str(), - builder1fn.c_str()); - eval(fNormalise, e1); - - string fn2 = nixValues + "/refer.txt"; - Expr e2 = ATmake("Path(, Regular(), [])", - fn2.c_str(), - ("I refer to " + fn1).c_str(), - e1); - eval(fNormalise, e2); - - realise(e2); -#endif - - Hash builder1h; + FSId builder1id; string builder1fn; - addToStore("./test-builder-1.sh", builder1fn, builder1h); + addToStore("./test-builder-1.sh", builder1fn, builder1id); FState fs1 = ATmake( - "Path(, Hash(), [])", + "Slice([], [(, , [])])", + ((string) builder1id).c_str(), builder1fn.c_str(), - ((string) builder1h).c_str()); - realise(fs1); - realise(fs1); + ((string) builder1id).c_str()); + FSId fs1id = writeTerm(fs1, "", 0); + realise(fs1id); + realise(fs1id); + + FState fs2 = ATmake( + "Slice([], [(, , [])])", + ((string) builder1id).c_str(), + (builder1fn + "_bla").c_str(), + ((string) builder1id).c_str()); + FSId fs2id = writeTerm(fs2, "", 0); + + realise(fs2id); + realise(fs2id); + + string out1fn = nixStore + "/hello.txt"; + string out1id = hashString("foo"); /* !!! bad */ + FState fs3 = ATmake( + "Derive([(, )], [], , , [(\"out\", )])", + out1fn.c_str(), + ((string) out1id).c_str(), + ((string) fs1id).c_str(), + ((string) builder1fn).c_str(), + thisSystem.c_str(), + out1fn.c_str()); + debug(printTerm(fs3)); + FSId fs3id = writeTerm(fs3, "", 0); + + realise(fs3id); + realise(fs3id); + +#if 0 FState fs2 = ATmake( "Path(, Hash(), [])", (builder1fn + "_bla").c_str(), @@ -175,28 +170,8 @@ void runTests() out1fn.c_str(), out1fn.c_str()); realise(fs3); - -#if 0 - Expr e1 = ATmake("Exec(Str(), Hash(), [])", - thisSystem.c_str(), ((string) builder1).c_str()); - - eval(e1); - - Hash builder2 = addValue("./test-builder-2.sh"); - - Expr e2 = ATmake( - "Exec(Str(), Hash(), [Tup(Str(\"src\"), )])", - thisSystem.c_str(), ((string) builder2).c_str(), e1); - - eval(e2); - - Hash h3 = addValue("./test-expr-1.nix"); - Expr e3 = ATmake("Deref(Hash())", ((string) h3).c_str()); - - eval(e3); - - deleteValue(h3); #endif + }