Merge branch 'readonly-store'

This commit is contained in:
Eelco Dolstra 2012-09-25 15:38:00 -04:00
commit e464b0247d
14 changed files with 105 additions and 134 deletions

View File

@ -10,7 +10,7 @@ namespace nix {
void findAlongAttrPath(EvalState & state, const string & attrPath, void findAlongAttrPath(EvalState & state, const string & attrPath,
Bindings & autoArgs, Expr * e, Value & v) Bindings & autoArgs, Expr * e, Value & v)
{ {
Strings tokens = tokenizeString(attrPath, "."); Strings tokens = tokenizeString<Strings>(attrPath, ".");
Error attrError = Error attrError =
Error(format("attribute selection path `%1%' does not match expression") % attrPath); Error(format("attribute selection path `%1%' does not match expression") % attrPath);

View File

@ -179,7 +179,7 @@ EvalState::EvalState()
/* Initialise the Nix expression search path. */ /* Initialise the Nix expression search path. */
searchPathInsertionPoint = searchPath.end(); searchPathInsertionPoint = searchPath.end();
Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":"); Strings paths = tokenizeString<Strings>(getEnv("NIX_PATH", ""), ":");
foreach (Strings::iterator, i, paths) addToSearchPath(*i); foreach (Strings::iterator, i, paths) addToSearchPath(*i);
addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs");
searchPathInsertionPoint = searchPath.begin(); searchPathInsertionPoint = searchPath.begin();

View File

@ -363,7 +363,7 @@ static void prim_derivationStrict(EvalState & state, Value * * args, Value & v)
else throw EvalError(format("invalid value `%1%' for `outputHashMode' attribute") % s); else throw EvalError(format("invalid value `%1%' for `outputHashMode' attribute") % s);
} }
else if (key == "outputs") { else if (key == "outputs") {
Strings tmp = tokenizeString(s); Strings tmp = tokenizeString<Strings>(s);
outputs.clear(); outputs.clear();
foreach (Strings::iterator, j, tmp) { foreach (Strings::iterator, j, tmp) {
if (outputs.find(*j) != outputs.end()) if (outputs.find(*j) != outputs.end())

View File

@ -1355,11 +1355,6 @@ void DerivationGoal::buildDone()
/* Delete the chroot (if we were using one). */ /* Delete the chroot (if we were using one). */
autoDelChroot.reset(); /* this runs the destructor */ autoDelChroot.reset(); /* this runs the destructor */
/* Deleting the chroot will have caused the immutable bits on
hard-linked inputs to be cleared. So set them again. */
foreach (PathSet::iterator, i, regularInputPaths)
makeImmutable(*i);
/* Delete redirected outputs (when doing hash rewriting). */ /* Delete redirected outputs (when doing hash rewriting). */
foreach (PathSet::iterator, i, redirectedOutputs) foreach (PathSet::iterator, i, redirectedOutputs)
deletePath(*i); deletePath(*i);
@ -1435,7 +1430,7 @@ HookReply DerivationGoal::tryBuildHook()
/* Tell the hook about system features (beyond the system type) /* Tell the hook about system features (beyond the system type)
required from the build machine. (The hook could parse the required from the build machine. (The hook could parse the
drv file itself, but this is easier.) */ drv file itself, but this is easier.) */
Strings features = tokenizeString(drv.env["requiredSystemFeatures"]); Strings features = tokenizeString<Strings>(drv.env["requiredSystemFeatures"]);
foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
/* Send the request to the hook. */ /* Send the request to the hook. */
@ -1594,7 +1589,7 @@ void DerivationGoal::startBuilder()
fixed-output derivations is by definition pure (since we fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */ already know the cryptographic hash of the output). */
if (fixedOutput) { if (fixedOutput) {
Strings varNames = tokenizeString(drv.env["impureEnvVars"]); Strings varNames = tokenizeString<Strings>(drv.env["impureEnvVars"]);
foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i); foreach (Strings::iterator, i, varNames) env[*i] = getEnv(*i);
} }
@ -1606,7 +1601,7 @@ void DerivationGoal::startBuilder()
by `nix-store --register-validity'. However, the deriver by `nix-store --register-validity'. However, the deriver
fields are left empty. */ fields are left empty. */
string s = drv.env["exportReferencesGraph"]; string s = drv.env["exportReferencesGraph"];
Strings ss = tokenizeString(s); Strings ss = tokenizeString<Strings>(s);
if (ss.size() % 2 != 0) if (ss.size() % 2 != 0)
throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s); throw BuildError(format("odd number of tokens in `exportReferencesGraph': `%1%'") % s);
for (Strings::iterator i = ss.begin(); i != ss.end(); ) { for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
@ -1766,11 +1761,8 @@ void DerivationGoal::startBuilder()
/* Hard-linking fails if we exceed the maximum /* Hard-linking fails if we exceed the maximum
link count on a file (e.g. 32000 of ext3), link count on a file (e.g. 32000 of ext3),
which is quite possible after a `nix-store which is quite possible after a `nix-store
--optimise'. It can also fail if another --optimise'. */
process called makeImmutable() on *i after we if (errno != EMLINK)
did makeMutable(). In those cases, make a copy
instead. */
if (errno != EMLINK && errno != EPERM)
throw SysError(format("linking `%1%' to `%2%'") % p % *i); throw SysError(format("linking `%1%' to `%2%'") % p % *i);
StringSink sink; StringSink sink;
dumpPath(*i, sink); dumpPath(*i, sink);
@ -1778,7 +1770,6 @@ void DerivationGoal::startBuilder()
restorePath(p, source); restorePath(p, source);
} }
makeImmutable(*i);
regularInputPaths.insert(*i); regularInputPaths.insert(*i);
} }
} }
@ -1911,14 +1902,11 @@ void DerivationGoal::initChild()
outside of the namespace. Making a subtree private is outside of the namespace. Making a subtree private is
local to the namespace, though, so setting MS_PRIVATE local to the namespace, though, so setting MS_PRIVATE
does not affect the outside world. */ does not affect the outside world. */
Strings mounts = tokenizeString(readFile("/proc/self/mountinfo", true), "\n"); Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
foreach (Strings::iterator, i, mounts) { foreach (Strings::iterator, i, mounts) {
Strings fields = tokenizeString(*i, " "); vector<string> fields = tokenizeString<vector<string> >(*i, " ");
assert(fields.size() >= 5); if (mount(0, fields.at(4).c_str(), 0, MS_PRIVATE, 0) == -1)
Strings::iterator j = fields.begin(); throw SysError(format("unable to make filesystem `%1%' private") % fields.at(4));
std::advance(j, 4);
if (mount(0, j->c_str(), 0, MS_PRIVATE, 0) == -1)
throw SysError(format("unable to make filesystem `%1%' private") % *j);
} }
/* Bind-mount all the directories from the "host" /* Bind-mount all the directories from the "host"
@ -2053,7 +2041,7 @@ void DerivationGoal::initChild()
PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) PathSet parseReferenceSpecifiers(const Derivation & drv, string attr)
{ {
PathSet result; PathSet result;
Paths paths = tokenizeString(attr); Paths paths = tokenizeString<Paths>(attr);
foreach (Strings::iterator, i, paths) { foreach (Strings::iterator, i, paths) {
if (isStorePath(*i)) if (isStorePath(*i))
result.insert(*i); result.insert(*i);

View File

@ -373,7 +373,7 @@ static void addAdditionalRoots(StoreAPI & store, PathSet & roots)
string result = runProgram(rootFinder); string result = runProgram(rootFinder);
Strings paths = tokenizeString(result, "\n"); Strings paths = tokenizeString<Strings>(result, "\n");
foreach (Strings::iterator, i, paths) { foreach (Strings::iterator, i, paths) {
if (isInStore(*i)) { if (isInStore(*i)) {

View File

@ -65,15 +65,7 @@ void Settings::processEnvironment()
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
} else } else
substituters = tokenizeString(subs, ":"); substituters = tokenizeString<Strings>(subs, ":");
}
string & at(Strings & ss, unsigned int n)
{
Strings::iterator i = ss.begin();
advance(i, n);
return *i;
} }
@ -95,15 +87,15 @@ void Settings::loadConfFile()
if (hash != string::npos) if (hash != string::npos)
line = string(line, 0, hash); line = string(line, 0, hash);
Strings tokens = tokenizeString(line); vector<string> tokens = tokenizeString<vector<string> >(line);
if (tokens.empty()) continue; if (tokens.empty()) continue;
if (tokens.size() < 2 || at(tokens, 1) != "=") if (tokens.size() < 2 || tokens[1] != "=")
throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile); throw Error(format("illegal configuration line `%1%' in `%2%'") % line % settingsFile);
string name = at(tokens, 0); string name = tokens[0];
Strings::iterator i = tokens.begin(); vector<string>::iterator i = tokens.begin();
advance(i, 2); advance(i, 2);
settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow settings[name] = concatStringsSep(" ", Strings(i, tokens.end())); // FIXME: slow
}; };
@ -170,7 +162,7 @@ void Settings::get(PathSet & res, const string & name)
SettingsMap::iterator i = settings.find(name); SettingsMap::iterator i = settings.find(name);
if (i == settings.end()) return; if (i == settings.end()) return;
res.clear(); res.clear();
Strings ss = tokenizeString(i->second); Strings ss = tokenizeString<Strings>(i->second);
res.insert(ss.begin(), ss.end()); res.insert(ss.begin(), ss.end());
} }

View File

@ -20,6 +20,11 @@
#include <stdio.h> #include <stdio.h>
#include <time.h> #include <time.h>
#if HAVE_UNSHARE
#include <sched.h>
#include <sys/mount.h>
#endif
#include <sqlite3.h> #include <sqlite3.h>
@ -292,6 +297,8 @@ LocalStore::LocalStore(bool reserveSpace)
} }
else openDB(false); else openDB(false);
makeStoreWritable();
} }
@ -411,6 +418,38 @@ void LocalStore::openDB(bool create)
} }
/* To improve purity, users may want to make the Nix store a read-only
bind mount. So make the Nix store writable for this process. */
void LocalStore::makeStoreWritable()
{
#if HAVE_UNSHARE
if (getuid() != 0) return;
if (!pathExists("/proc/self/mountinfo")) return;
/* Check if /nix/store is a read-only bind mount. */
bool found = false;
Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
foreach (Strings::iterator, i, mounts) {
vector<string> fields = tokenizeString<vector<string> >(*i, " ");
if (fields.at(3) == "/" || fields.at(4) != settings.nixStore) continue;
Strings options = tokenizeString<Strings>(fields.at(5), ",");
if (std::find(options.begin(), options.end(), "ro") == options.end()) continue;
found = true;
break;
}
if (!found) return;
if (unshare(CLONE_NEWNS) == -1)
throw SysError("setting up a private mount namespace");
if (mount(0, settings.nixStore.c_str(), 0, MS_REMOUNT | MS_BIND, 0) == -1)
throw SysError(format("remounting %1% writable") % settings.nixStore);
#endif
}
const time_t mtimeStore = 1; /* 1 second into the epoch */ const time_t mtimeStore = 1; /* 1 second into the epoch */
@ -478,8 +517,6 @@ void canonicalisePathMetaData(const Path & path, bool recurse)
foreach (Strings::iterator, i, names) foreach (Strings::iterator, i, names)
canonicalisePathMetaData(path + "/" + *i, true); canonicalisePathMetaData(path + "/" + *i, true);
} }
makeImmutable(path);
} }
@ -1435,7 +1472,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source)
/* Lock the output path. But don't lock if we're being called /* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a from a build hook (whose parent process already acquired a
lock on this path). */ lock on this path). */
Strings locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS")); Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end()) if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
outputLock.lockPaths(singleton<PathSet, Path>(dstPath)); outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
@ -1645,7 +1682,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
string info = readFile(infoFile); string info = readFile(infoFile);
/* Parse it. */ /* Parse it. */
Strings lines = tokenizeString(info, "\n"); Strings lines = tokenizeString<Strings>(info, "\n");
foreach (Strings::iterator, i, lines) { foreach (Strings::iterator, i, lines) {
string::size_type p = i->find(':'); string::size_type p = i->find(':');
@ -1654,7 +1691,7 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
string name(*i, 0, p); string name(*i, 0, p);
string value(*i, p + 2); string value(*i, p + 2);
if (name == "References") { if (name == "References") {
Strings refs = tokenizeString(value, " "); Strings refs = tokenizeString<Strings>(value, " ");
res.references = PathSet(refs.begin(), refs.end()); res.references = PathSet(refs.begin(), refs.end());
} else if (name == "Deriver") { } else if (name == "Deriver") {
res.deriver = value; res.deriver = value;

View File

@ -228,6 +228,8 @@ private:
void openDB(bool create); void openDB(bool create);
void makeStoreWritable();
unsigned long long queryValidPathId(const Path & path); unsigned long long queryValidPathId(const Path & path);
unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true); unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true);

View File

@ -33,8 +33,7 @@ struct MakeReadOnly
~MakeReadOnly() ~MakeReadOnly()
{ {
try { try {
/* This will make the path read-only (and restore the /* This will make the path read-only. */
immutable bit on platforms that support it). */
if (path != "") canonicalisePathMetaData(path, false); if (path != "") canonicalisePathMetaData(path, false);
} catch (...) { } catch (...) {
ignoreException(); ignoreException();
@ -43,14 +42,6 @@ struct MakeReadOnly
}; };
struct MakeImmutable
{
Path path;
MakeImmutable(const Path & path) : path(path) { }
~MakeImmutable() { makeImmutable(path); }
};
void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
{ {
checkInterrupt(); checkInterrupt();
@ -101,7 +92,6 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
if (!pathExists(linkPath)) { if (!pathExists(linkPath)) {
/* Nope, create a hard link in the links directory. */ /* Nope, create a hard link in the links directory. */
makeMutable(path); makeMutable(path);
MakeImmutable mk1(path);
if (link(path.c_str(), linkPath.c_str()) == 0) return; if (link(path.c_str(), linkPath.c_str()) == 0) return;
if (errno != EEXIST) if (errno != EEXIST)
throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
@ -134,57 +124,41 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path)
MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
/* If linkPath is immutable, we can't create hard links to it, /* If linkPath is immutable, we can't create hard links to it,
so make it mutable first (and make it immutable again when so make it mutable first. We also have to make path mutable,
we're done). We also have to make path mutable, otherwise otherwise rename() will fail to delete it. */
rename() will fail to delete it. */
makeMutable(path); makeMutable(path);
MakeImmutable mk2(path); makeMutable(linkPath);
/* Another process might be doing the same thing (creating a new Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
link to linkPath) and make linkPath immutable before we're % settings.nixStore % getpid() % rand()).str();
done. In that case, just retry. */
unsigned int retries = 1024;
while (--retries > 0) {
makeMutable(linkPath);
MakeImmutable mk1(linkPath);
Path tempLink = (format("%1%/.tmp-link-%2%-%3%") if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
% settings.nixStore % getpid() % rand()).str(); if (errno == EMLINK) {
/* Too many links to the same file (>= 32000 on most file
if (link(linkPath.c_str(), tempLink.c_str()) == -1) { systems). This is likely to happen with empty files.
if (errno == EMLINK) { Just shrug and ignore. */
/* Too many links to the same file (>= 32000 on most if (st.st_size)
file systems). This is likely to happen with empty printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
files. Just shrug and ignore. */ return;
if (st.st_size)
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
if (errno == EPERM) continue;
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
} }
throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
if (errno == EMLINK) {
/* Some filesystems generate too many links on the
rename, rather than on the original link.
(Probably it temporarily increases the st_nlink
field before decreasing it again.) */
if (st.st_size)
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
if (errno == EPERM) continue;
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
}
break;
} }
if (retries == 0) throw Error(format("cannot link `%1%' to `%2%'") % path % linkPath); /* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1)
printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
temporarily increases the st_nlink field before
decreasing it again.) */
if (st.st_size)
printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
return;
}
throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
}
stats.filesLinked++; stats.filesLinked++;
stats.bytesFreed += st.st_size; stats.bytesFreed += st.st_size;

View File

@ -16,7 +16,7 @@
namespace nix { namespace nix {
void changeMutable(const Path & path, bool mut) void makeMutable(const Path & path)
{ {
#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL)
@ -38,30 +38,12 @@ void changeMutable(const Path & path, bool mut)
/* Silently ignore errors getting/setting the immutable flag so /* Silently ignore errors getting/setting the immutable flag so
that we work correctly on filesystems that don't support it. */ that we work correctly on filesystems that don't support it. */
if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return;
old = flags; old = flags;
flags &= ~FS_IMMUTABLE_FL;
if (mut) flags &= ~FS_IMMUTABLE_FL;
else flags |= FS_IMMUTABLE_FL;
if (old == flags) return; if (old == flags) return;
if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return;
#endif #endif
} }
void makeImmutable(const Path & path)
{
changeMutable(path, false);
}
void makeMutable(const Path & path)
{
changeMutable(path, true);
}
} }

View File

@ -4,12 +4,6 @@
namespace nix { namespace nix {
/* Make the given path immutable, i.e., prevent it from being modified
in any way, even by root. This is a no-op on platforms that do not
support this, or if the calling user is not privileged. On Linux,
this is implemented by doing the equivalent of chattr +i path. */
void makeImmutable(const Path & path);
/* Make the given path mutable. */ /* Make the given path mutable. */
void makeMutable(const Path & path); void makeMutable(const Path & path);

View File

@ -984,9 +984,9 @@ void _interrupted()
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////
Strings tokenizeString(const string & s, const string & separators) template<class C> C tokenizeString(const string & s, const string & separators)
{ {
Strings result; C result;
string::size_type pos = s.find_first_not_of(separators, 0); string::size_type pos = s.find_first_not_of(separators, 0);
while (pos != string::npos) { while (pos != string::npos) {
string::size_type end = s.find_first_of(separators, pos + 1); string::size_type end = s.find_first_of(separators, pos + 1);
@ -998,6 +998,9 @@ Strings tokenizeString(const string & s, const string & separators)
return result; return result;
} }
template Strings tokenizeString(const string & s, const string & separators);
template vector<string> tokenizeString(const string & s, const string & separators);
string concatStringsSep(const string & sep, const Strings & ss) string concatStringsSep(const string & sep, const Strings & ss)
{ {

View File

@ -286,7 +286,7 @@ MakeError(Interrupted, BaseError)
/* String tokenizer. */ /* String tokenizer. */
Strings tokenizeString(const string & s, const string & separators = " \t\n\r"); template<class C> C tokenizeString(const string & s, const string & separators = " \t\n\r");
/* Concatenate the given strings with a separator between the /* Concatenate the given strings with a separator between the

View File

@ -193,16 +193,15 @@ static void run(int argc, char * * argv)
if (st.st_mode & (S_IWGRP | S_IWOTH)) if (st.st_mode & (S_IWGRP | S_IWOTH))
throw Error(format("`%1%' should not be group or world-writable") % configFile); throw Error(format("`%1%' should not be group or world-writable") % configFile);
Strings tokens = tokenizeString(readFile(fdConfig)); vector<string> tokens = tokenizeString<vector<string> >(readFile(fdConfig));
fdConfig.close(); fdConfig.close();
if (tokens.size() != 2) if (tokens.size() != 2)
throw Error(format("parse error in `%1%'") % configFile); throw Error(format("parse error in `%1%'") % configFile);
Strings::iterator i = tokens.begin(); string nixUser = tokens[0];
string nixUser = *i++; string buildUsersGroup = tokens[1];
string buildUsersGroup = *i++;
/* Check that the caller (real uid) is the one allowed to call /* Check that the caller (real uid) is the one allowed to call