diff --git a/nix/libstore/build.cc b/nix/libstore/build.cc index f38cd29940..d3184507ce 100644 --- a/nix/libstore/build.cc +++ b/nix/libstore/build.cc @@ -602,42 +602,29 @@ HookInstance::HookInstance() builderOut.create(); /* Fork the hook. */ - pid = maybeVfork(); - switch (pid) { + pid = startProcess([&]() { - case -1: - throw SysError("unable to fork"); + commonChildInit(fromHook); - case 0: - try { /* child */ + if (chdir("/") == -1) throw SysError("changing into `/"); - commonChildInit(fromHook); + /* Dup the communication pipes. */ + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); - if (chdir("/") == -1) throw SysError("changing into `/"); + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); - /* Dup the communication pipes. */ - if (dup2(toHook.readSide, STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); + execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), + (format("%1%") % settings.maxSilentTime).str().c_str(), + (format("%1%") % settings.printBuildTrace).str().c_str(), + (format("%1%") % settings.buildTimeout).str().c_str(), + NULL); - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide, 4) == -1) - throw SysError("dupping builder's stdout/stderr"); + throw SysError(format("executing `%1%'") % buildHook); + }); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (std::exception & e) { - writeToStderr("build hook error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); fromHook.writeSide.close(); @@ -2781,32 +2768,18 @@ void SubstitutionGoal::tryToRun() const char * * argArr = strings2CharPtrs(args); /* Fork the substitute program. */ - pid = maybeVfork(); + pid = startProcess([&]() { - switch (pid) { + commonChildInit(logPipe); - case -1: - throw SysError("unable to fork"); + if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("cannot dup output pipe into stdout"); - case 0: - try { /* child */ + execv(sub.c_str(), (char * *) argArr); - commonChildInit(logPipe); + throw SysError(format("executing `%1%'") % sub); + }); - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); - - execv(sub.c_str(), (char * *) argArr); - - throw SysError(format("executing `%1%'") % sub); - - } catch (std::exception & e) { - writeToStderr("substitute error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); outPipe.writeSide.close(); diff --git a/nix/libstore/globals.cc b/nix/libstore/globals.cc index 86fa56739c..62c2fd6290 100644 --- a/nix/libstore/globals.cc +++ b/nix/libstore/globals.cc @@ -55,6 +55,7 @@ Settings::Settings() envKeepDerivations = false; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; showTrace = false; + enableImportNative = false; } @@ -141,6 +142,8 @@ void Settings::update() get(envKeepDerivations, "env-keep-derivations"); get(sshSubstituterHosts, "ssh-substituter-hosts"); get(useSshSubstituter, "use-ssh-substituter"); + get(logServers, "log-servers"); + get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/nix/libstore/globals.hh b/nix/libstore/globals.hh index 711c365294..8dd59a9c79 100644 --- a/nix/libstore/globals.hh +++ b/nix/libstore/globals.hh @@ -197,6 +197,12 @@ struct Settings { /* Whether to show a stack trace if Nix evaluation fails. */ bool showTrace; + /* A list of URL prefixes that can return Nix build logs. */ + Strings logServers; + + /* Whether the importNative primop should be enabled */ + bool enableImportNative; + private: SettingsMap settings, overrides; diff --git a/nix/libstore/local-store.cc b/nix/libstore/local-store.cc index 5d210ae017..e66042c57f 100644 --- a/nix/libstore/local-store.cc +++ b/nix/libstore/local-store.cc @@ -1083,31 +1083,16 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & setSubstituterEnv(); - run.pid = maybeVfork(); - - switch (run.pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - restoreAffinity(); - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing `%1%'") % substituter); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } - - /* Parent. */ + run.pid = startProcess([&]() { + if (dup2(toPipe.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); + execl(substituter.c_str(), substituter.c_str(), "--query", NULL); + throw SysError(format("executing `%1%'") % substituter); + }); run.program = baseNameOf(substituter); run.to = toPipe.writeSide.borrow(); diff --git a/nix/libstore/local-store.hh b/nix/libstore/local-store.hh index 54331e448a..e58e6563f1 100644 --- a/nix/libstore/local-store.hh +++ b/nix/libstore/local-store.hh @@ -1,16 +1,12 @@ #pragma once #include +#include #include "store-api.hh" #include "util.hh" #include "pathlocks.hh" -#if HAVE_TR1_UNORDERED_SET -#include -#endif - - class sqlite3; class sqlite3_stmt; @@ -306,11 +302,7 @@ private: void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); -#if HAVE_TR1_UNORDERED_SET - typedef std::tr1::unordered_set InodeHash; -#else - typedef std::set InodeHash; -#endif + typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); diff --git a/nix/libstore/remote-store.cc b/nix/libstore/remote-store.cc index 4619206932..3b021bb2a5 100644 --- a/nix/libstore/remote-store.cc +++ b/nix/libstore/remote-store.cc @@ -402,8 +402,23 @@ Path RemoteStore::addToStore(const Path & _srcPath, writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); writeInt(recursive ? 1 : 0, to); writeString(printHashType(hashAlgo), to); - dumpPath(srcPath, to, filter); - processStderr(); + + try { + to.written = 0; + to.warn = true; + dumpPath(srcPath, to, filter); + to.warn = false; + processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + processStderr(); + } catch (EndOfFile & e) { } + throw; + } + return readStorePath(from); } diff --git a/nix/libutil/serialise.cc b/nix/libutil/serialise.cc index 6b71f52c15..9241750750 100644 --- a/nix/libutil/serialise.cc +++ b/nix/libutil/serialise.cc @@ -54,8 +54,24 @@ FdSink::~FdSink() } +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() +{ + printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory"); +} + + void FdSink::write(const unsigned char * data, size_t len) { + static bool warned = false; + if (warn && !warned) { + written += len; + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } writeFull(fd, data, len); } @@ -256,4 +272,15 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +void StringSink::operator () (const unsigned char * data, size_t len) +{ + static bool warned = false; + if (!warned && s.size() > threshold) { + warnLargeDump(); + warned = true; + } + s.append((const char *) data, len); +} + + } diff --git a/nix/libutil/serialise.hh b/nix/libutil/serialise.hh index e5a9df1d05..6a6f028aa6 100644 --- a/nix/libutil/serialise.hh +++ b/nix/libutil/serialise.hh @@ -72,9 +72,11 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; + bool warn; + size_t written; - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } + FdSink() : fd(-1), warn(false), written(0) { } + FdSink(int fd) : fd(fd), warn(false), written(0) { } ~FdSink(); void write(const unsigned char * data, size_t len); @@ -95,10 +97,7 @@ struct FdSource : BufferedSource struct StringSink : Sink { string s; - void operator () (const unsigned char * data, size_t len) - { - s.append((const char *) data, len); - } + void operator () (const unsigned char * data, size_t len); }; diff --git a/nix/libutil/types.hh b/nix/libutil/types.hh index 4b5ce9a78c..906a959e30 100644 --- a/nix/libutil/types.hh +++ b/nix/libutil/types.hh @@ -41,8 +41,8 @@ public: BaseError(const FormatOrString & fs, unsigned int status = 1); ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } - const string & msg() const throw () { return err; } - const string & prefix() const throw () { return prefix_; } + const string & msg() const { return err; } + const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); }; diff --git a/nix/libutil/util.cc b/nix/libutil/util.cc index 846674a29d..faa2b83c37 100644 --- a/nix/libutil/util.cc +++ b/nix/libutil/util.cc @@ -1,5 +1,8 @@ #include "config.h" +#include "util.hh" +#include "affinity.hh" + #include #include #include @@ -16,8 +19,6 @@ #include #endif -#include "util.hh" - extern char * * environ; @@ -714,6 +715,13 @@ Pid::Pid() } +Pid::Pid(pid_t pid) +{ + Pid(); + *this = pid; +} + + Pid::~Pid() { kill(); @@ -801,43 +809,30 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - Pid pid; - pid = fork(); - switch (pid) { + Pid pid = startProcess([&]() { - case -1: - throw SysError("unable to fork"); + if (setuid(uid) == -1) + throw SysError("setting uid"); - case 0: - try { /* child */ - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { + while (true) { #ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among other - things, determines if kill(-1, signo) affects the calling - process. In the OSX libc, it's set to true, which means - "follow POSIX", which we don't want here + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; #else - if (kill(-1, SIGKILL) == 0) break; + if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ - if (errno != EINTR) - throw SysError(format("cannot kill processes for uid `%1%'") % uid); - } - - } catch (std::exception & e) { - writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); - _exit(1); + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); } - _exit(0); - } - /* parent */ + _exit(0); + }); + int status = pid.wait(true); if (status != 0) throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); @@ -852,6 +847,25 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// +pid_t startProcess(std::function fun, const string & errorPrefix) +{ + pid_t pid = fork(); + if (pid == -1) throw SysError("unable to fork"); + + if (pid == 0) { + try { + restoreAffinity(); + fun(); + } catch (std::exception & e) { + writeToStderr(errorPrefix + string(e.what()) + "\n"); + } + _exit(1); + } + + return pid; +} + + string runProgram(Path program, bool searchPath, const Strings & args) { checkInterrupt(); @@ -867,32 +881,17 @@ string runProgram(Path program, bool searchPath, const Strings & args) pipe.create(); /* Fork. */ - Pid pid; - pid = maybeVfork(); + Pid pid = startProcess([&]() { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); - switch (pid) { + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - - if (searchPath) - execvp(program.c_str(), (char * *) &cargs[0]); - else - execv(program.c_str(), (char * *) &cargs[0]); - throw SysError(format("executing `%1%'") % program); - - } catch (std::exception & e) { - writeToStderr("error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* Parent. */ + throw SysError(format("executing `%1%'") % program); + }); pipe.writeSide.close(); @@ -901,7 +900,7 @@ string runProgram(Path program, bool searchPath, const Strings & args) /* Wait for the child to finish. */ int status = pid.wait(true); if (!statusOk(status)) - throw Error(format("program `%1%' %2%") + throw ExecError(format("program `%1%' %2%") % program % statusToString(status)); return result; @@ -928,13 +927,6 @@ void closeOnExec(int fd) } -#if HAVE_VFORK -pid_t (*maybeVfork)() = vfork; -#else -pid_t (*maybeVfork)() = fork; -#endif - - ////////////////////////////////////////////////////////////////////// diff --git a/nix/libutil/util.hh b/nix/libutil/util.hh index ce2d77c19a..ad0d377a4f 100644 --- a/nix/libutil/util.hh +++ b/nix/libutil/util.hh @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -237,6 +238,7 @@ class Pid int killSignal; public: Pid(); + Pid(pid_t pid); ~Pid(); void operator =(pid_t pid); operator pid_t(); @@ -252,11 +254,18 @@ public: void killUser(uid_t uid); +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +pid_t startProcess(std::function fun, const string & errorPrefix = "error: "); + + /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, const Strings & args = Strings()); +MakeError(ExecError, Error) + /* Close all file descriptors except stdin, stdout, stderr, and those listed in the given set. Good practice in child processes. */ void closeMostFDs(const set & exceptions); @@ -264,9 +273,6 @@ void closeMostFDs(const set & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); -/* Call vfork() if available, otherwise fork(). */ -extern pid_t (*maybeVfork)(); - /* User interruption. */ diff --git a/nix/nix-daemon/nix-daemon.cc b/nix/nix-daemon/nix-daemon.cc index 8814fe3155..265131c613 100644 --- a/nix/nix-daemon/nix-daemon.cc +++ b/nix/nix-daemon/nix-daemon.cc @@ -735,12 +735,10 @@ static void processConnection(bool trusted) during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = canSendStderr; - if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg()); stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); - if (!errorAllowed) break; + if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - if (canSendStderr) - stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); + stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); throw; } @@ -874,40 +872,27 @@ static void daemonLoop() printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); /* Fork a child to handle the connection. */ - pid_t child; - child = fork(); + startProcess([&]() { + /* Background the daemon. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); - switch (child) { + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - /* Background the daemon. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Restore normal handling of SIGCHLD. */ - setSigChldAction(false); - - /* For debugging, stuff the pid into argv[1]. */ - if (clientPid != -1 && argvSaved[1]) { - string processName = int2String(clientPid); - strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); - } - - /* Handle the connection. */ - from.fd = remote; - to.fd = remote; - processConnection(trusted); - - } catch (std::exception & e) { - writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n"); + /* For debugging, stuff the pid into argv[1]. */ + if (clientPid != -1 && argvSaved[1]) { + string processName = int2String(clientPid); + strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); } - exit(0); - } + + /* Handle the connection. */ + from.fd = remote; + to.fd = remote; + processConnection(trusted); + + _exit(0); + }, "unexpected Nix daemon error: "); } catch (Interrupted & e) { throw;