diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index 2e2f92fadf..f38cd29940 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -13,6 +13,7 @@ #include #include +#include #include #include #include diff --git a/nix/libstore/derivations.cc b/nix/libstore/derivations.cc index d91e42784c..b452aa2caf 100644 --- a/nix/libstore/derivations.cc +++ b/nix/libstore/derivations.cc @@ -48,7 +48,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw Error(format("bad path `%1%' in derivation") % s); + throw FormatError(format("bad path `%1%' in derivation") % s); return s; } @@ -62,7 +62,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -Derivation parseDerivation(const string & s) +static Derivation parseDerivation(const string & s) { Derivation drv; std::istringstream str(s); @@ -112,6 +112,16 @@ Derivation parseDerivation(const string & s) } +Derivation readDerivation(const Path & drvPath) +{ + try { + return parseDerivation(readFile(drvPath)); + } catch (FormatError & e) { + throw Error(format("error parsing derivation `%1%': %2%") % drvPath % e.msg()); + } +} + + static void printString(string & res, const string & s) { res += '"'; @@ -240,7 +250,7 @@ Hash hashDerivationModulo(StoreAPI & store, Derivation drv) Hash h = drvHashes[i->first]; if (h.type == htUnknown) { assert(store.isValidPath(i->first)); - Derivation drv2 = parseDerivation(readFile(i->first)); + Derivation drv2 = readDerivation(i->first); h = hashDerivationModulo(store, drv2); drvHashes[i->first] = h; } diff --git a/nix/libstore/derivations.hh b/nix/libstore/derivations.hh index 703410b925..04b64dfc88 100644 --- a/nix/libstore/derivations.hh +++ b/nix/libstore/derivations.hh @@ -59,8 +59,8 @@ class StoreAPI; Path writeDerivation(StoreAPI & store, const Derivation & drv, const string & name, bool repair = false); -/* Parse a derivation. */ -Derivation parseDerivation(const string & s); +/* Read a derivation from a file. */ +Derivation readDerivation(const Path & drvPath); /* Print a derivation. */ string unparseDerivation(const Derivation & drv); diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 1293a6e8f2..5d210ae017 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -20,6 +20,7 @@ #include #include #include +#include #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H #include @@ -237,7 +238,7 @@ LocalStore::LocalStore(bool reserveSpace) makeStoreWritable(); createDirs(linksDir = settings.nixStore + "/.links"); Path profilesDir = settings.nixStateDir + "/profiles"; - createDirs(settings.nixStateDir + "/profiles"); + createDirs(profilesDir); createDirs(settings.nixStateDir + "/temproots"); createDirs(settings.nixDBPath); Path gcRootsDir = settings.nixStateDir + "/gcroots"; @@ -246,6 +247,32 @@ LocalStore::LocalStore(bool reserveSpace) createSymlink(profilesDir, gcRootsDir + "/profiles"); } + /* Optionally, create directories and set permissions for a + multi-user install. */ + if (getuid() == 0 && settings.buildUsersGroup != "") { + + Path perUserDir = profilesDir + "/per-user"; + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 01777) == -1) + throw SysError(format("could not set permissions on `%1%' to 1777") % perUserDir); + + struct group * gr = getgrnam(settings.buildUsersGroup.c_str()); + if (!gr) + throw Error(format("the group `%1%' specified in `build-users-group' does not exist") + % settings.buildUsersGroup); + + struct stat st; + if (stat(settings.nixStore.c_str(), &st)) + throw SysError(format("getting attributes of path `%1%'") % settings.nixStore); + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != 01775) { + if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1) + throw SysError(format("changing ownership of path `%1%'") % settings.nixStore); + if (chmod(settings.nixStore.c_str(), 01775) == -1) + throw SysError(format("changing permissions on path `%1%'") % settings.nixStore); + } + } + checkStoreNotSymlink(); /* We can't open a SQLite database if the disk is full. Since @@ -661,7 +688,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che efficiently query whether a path is an output of some derivation. */ if (isDerivation(info.path)) { - Derivation drv = parseDerivation(readFile(info.path)); + Derivation drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1290,7 +1317,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) if (isDerivation(i->path)) { // FIXME: inefficient; we already loaded the // derivation in addValidPath(). - Derivation drv = parseDerivation(readFile(i->path)); + Derivation drv = readDerivation(i->path); checkDerivationOutputs(i->path, drv); } diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 09639e74cf..54331e448a 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -6,6 +6,11 @@ #include "util.hh" #include "pathlocks.hh" +#if HAVE_TR1_UNORDERED_SET +#include +#endif + + class sqlite3; class sqlite3_stmt; @@ -29,14 +34,12 @@ struct Derivation; struct OptimiseStats { - unsigned long totalFiles; - unsigned long sameContents; unsigned long filesLinked; unsigned long long bytesFreed; unsigned long long blocksFreed; OptimiseStats() { - totalFiles = sameContents = filesLinked = 0; + filesLinked = 0; bytesFreed = blocksFreed = 0; } }; @@ -303,7 +306,15 @@ private: void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); - void optimisePath_(OptimiseStats & stats, const Path & path); +#if HAVE_TR1_UNORDERED_SET + typedef std::tr1::unordered_set InodeHash; +#else + typedef std::set InodeHash; +#endif + + InodeHash loadInodeHash(); + Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); + void optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(const Path & path); diff --git a/nix/libstore/misc.cc b/nix/libstore/misc.cc index 1bf3f93782..6ecf8787cf 100644 --- a/nix/libstore/misc.cc +++ b/nix/libstore/misc.cc @@ -11,7 +11,7 @@ Derivation derivationFromPath(StoreAPI & store, const Path & drvPath) { assertStorePath(drvPath); store.ensurePath(drvPath); - return parseDerivation(readFile(drvPath)); + return readDerivation(drvPath); } diff --git a/nix/libstore/optimise-store.cc b/nix/libstore/optimise-store.cc index d833f3aa05..67ee94a4bd 100644 --- a/nix/libstore/optimise-store.cc +++ b/nix/libstore/optimise-store.cc @@ -40,18 +40,66 @@ struct MakeReadOnly }; -void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) +LocalStore::InodeHash LocalStore::loadInodeHash() +{ + printMsg(lvlDebug, "loading hash inodes in memory"); + InodeHash inodeHash; + + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + // We don't care if we hit non-hash files, anything goes + inodeHash.insert(dirent->d_ino); + } + if (errno) throw SysError(format("reading directory `%1%'") % linksDir); + + printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + + return inodeHash; +} + + +Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) +{ + Strings names; + + AutoCloseDir dir = opendir(path.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + + if (inodeHash.count(dirent->d_ino)) { + printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name); + continue; + } + + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory `%1%'") % path); + + return names; +} + + +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash) { checkInterrupt(); - + struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); + Strings names = readDirectoryIgnoringInodes(path, inodeHash); foreach (Strings::iterator, i, names) - optimisePath_(stats, path + "/" + *i); + optimisePath_(stats, path + "/" + *i, inodeHash); return; } @@ -71,6 +119,12 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) return; } + /* This can still happen on top-level files */ + if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { + printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2)); + return; + } + /* Hash the file. Note that hashPath() returns the hash over the NAR serialisation, which includes the execute bit on the file. Thus, executable and non-executable files with the same @@ -81,7 +135,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); /* Check if this is a known hash. */ @@ -89,7 +142,10 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ - if (link(path.c_str(), linkPath.c_str()) == 0) return; + if (link(path.c_str(), linkPath.c_str()) == 0) { + inodeHash.insert(st.st_ino); + return; + } if (errno != EEXIST) throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); /* Fall through if another process created ‘linkPath’ before @@ -102,7 +158,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) if (lstat(linkPath.c_str(), &stLink)) throw SysError(format("getting attributes of path `%1%'") % linkPath); - stats.sameContents++; if (st.st_ino == stLink.st_ino) { printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); return; @@ -160,12 +215,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) void LocalStore::optimiseStore(OptimiseStats & stats) { PathSet paths = queryAllValidPaths(); + InodeHash inodeHash = loadInodeHash(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - optimisePath_(stats, *i); + optimisePath_(stats, *i, inodeHash); } } @@ -173,7 +229,9 @@ void LocalStore::optimiseStore(OptimiseStats & stats) void LocalStore::optimisePath(const Path & path) { OptimiseStats stats; - if (settings.autoOptimiseStore) optimisePath_(stats, path); + InodeHash inodeHash; + + if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash); } diff --git a/nix/libutil/archive.cc b/nix/libutil/archive.cc index ab4cd47351..70a1c580dd 100644 --- a/nix/libutil/archive.cc +++ b/nix/libutil/archive.cc @@ -104,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) writeString(readLink(path), sink); } - else throw Error(format("file `%1%' has an unknown type") % path); + else throw Error(format("file `%1%' has an unsupported type") % path); writeString(")", sink); } diff --git a/nix/libutil/util.cc b/nix/libutil/util.cc index 15c462ce4e..846674a29d 100644 --- a/nix/libutil/util.cc +++ b/nix/libutil/util.cc @@ -1041,7 +1041,7 @@ void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw Error(format("expected string `%1%'") % s); + throw FormatError(format("expected string `%1%'") % s); } diff --git a/nix/libutil/util.hh b/nix/libutil/util.hh index 8bedfea9a0..ce2d77c19a 100644 --- a/nix/libutil/util.hh +++ b/nix/libutil/util.hh @@ -326,6 +326,8 @@ bool hasSuffix(const string & s, const string & suffix); /* Read string `s' from stream `str'. */ void expect(std::istream & str, const string & s); +MakeError(FormatError, Error) + /* Read a C-style string from stream `str'. */ string parseString(std::istream & str);