Merge commit '8e9140cfdef9dbd1eb61e4c75c91d452ab5e4a74' into nix

Conflicts:
	Makefile.config.in
	configure.ac
	dev-shell
	doc/manual/builtins.xml
	doc/manual/conf-file.xml
	doc/manual/local.mk
	doc/manual/nix-instantiate.xml
	doc/manual/nix-store.xml
	doc/manual/writing-nix-expressions.xml
	misc/emacs/nix-mode.el
	perl/lib/Nix/CopyClosure.pm
	release.nix
	scripts/nix-build.in
	scripts/nix-copy-closure.in
	src/download-via-ssh/download-via-ssh.cc
	src/libexpr/common-opts.cc
	src/libexpr/common-opts.hh
	src/libexpr/eval.cc
	src/libexpr/eval.hh
	src/libexpr/local.mk
	src/libexpr/nixexpr.cc
	src/libexpr/nixexpr.hh
	src/libexpr/parser.y
	src/libexpr/primops.cc
	src/libexpr/symbol-table.hh
	src/libmain/shared.cc
	src/libstore/local.mk
	src/nix-env/nix-env.cc
	src/nix-instantiate/nix-instantiate.cc
	src/nix-store/local.mk
	src/nix-store/nix-store.cc
	src/nix-store/serve-protocol.hh
	tests/lang.sh
	tests/lang/eval-okay-context.nix
	tests/lang/eval-okay-search-path.exp
	tests/lang/eval-okay-search-path.nix
	tests/local.mk
	tests/nix-copy-closure.nix

nix/libstore/build.cc

@@ -602,42 +602,29 @@ HookInstance::HookInstance()
     builderOut.create();
 
     /* Fork the hook. */
-    pid = maybeVfork();
+    pid = startProcess([&]() {
-    switch (pid) {
 
-    case -1:
+        commonChildInit(fromHook);
-        throw SysError("unable to fork");
 
-    case 0:
+        if (chdir("/") == -1) throw SysError("changing into `/");
-        try { /* child */
 
-            commonChildInit(fromHook);
+        /* Dup the communication pipes. */
+        if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+            throw SysError("dupping to-hook read side");
 
-            if (chdir("/") == -1) throw SysError("changing into `/");
+        /* Use fd 4 for the builder's stdout/stderr. */
+        if (dup2(builderOut.writeSide, 4) == -1)
+            throw SysError("dupping builder's stdout/stderr");
 
-            /* Dup the communication pipes. */
+        execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
-            if (dup2(toHook.readSide, STDIN_FILENO) == -1)
+            (format("%1%") % settings.maxSilentTime).str().c_str(),
-                throw SysError("dupping to-hook read side");
+            (format("%1%") % settings.printBuildTrace).str().c_str(),
+            (format("%1%") % settings.buildTimeout).str().c_str(),
+            NULL);
 
-            /* Use fd 4 for the builder's stdout/stderr. */
+        throw SysError(format("executing `%1%'") % buildHook);
-            if (dup2(builderOut.writeSide, 4) == -1)
+    });
-                throw SysError("dupping builder's stdout/stderr");
 
-            execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
-                (format("%1%") % settings.maxSilentTime).str().c_str(),
-                (format("%1%") % settings.printBuildTrace).str().c_str(),
-                (format("%1%") % settings.buildTimeout).str().c_str(),
-                NULL);
-
-            throw SysError(format("executing `%1%'") % buildHook);
-
-        } catch (std::exception & e) {
-            writeToStderr("build hook error: " + string(e.what()) + "\n");
-        }
-        _exit(1);
-    }
-
-    /* parent */
     pid.setSeparatePG(true);
     pid.setKillSignal(SIGTERM);
     fromHook.writeSide.close();
@@ -2781,32 +2768,18 @@ void SubstitutionGoal::tryToRun()
     const char * * argArr = strings2CharPtrs(args);
 
     /* Fork the substitute program. */
-    pid = maybeVfork();
+    pid = startProcess([&]() {
 
-    switch (pid) {
+        commonChildInit(logPipe);
 
-    case -1:
+        if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
-        throw SysError("unable to fork");
+            throw SysError("cannot dup output pipe into stdout");
 
-    case 0:
+        execv(sub.c_str(), (char * *) argArr);
-        try { /* child */
 
-            commonChildInit(logPipe);
+        throw SysError(format("executing `%1%'") % sub);
+    });
 
-            if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
-                throw SysError("cannot dup output pipe into stdout");
-
-            execv(sub.c_str(), (char * *) argArr);
-
-            throw SysError(format("executing `%1%'") % sub);
-
-        } catch (std::exception & e) {
-            writeToStderr("substitute error: " + string(e.what()) + "\n");
-        }
-        _exit(1);
-    }
-
-    /* parent */
     pid.setSeparatePG(true);
     pid.setKillSignal(SIGTERM);
     outPipe.writeSide.close();

nix/libstore/globals.cc

@@ -55,6 +55,7 @@ Settings::Settings()
     envKeepDerivations = false;
     lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1";
     showTrace = false;
+    enableImportNative = false;
 }
 
 
@@ -141,6 +142,8 @@ void Settings::update()
     get(envKeepDerivations, "env-keep-derivations");
     get(sshSubstituterHosts, "ssh-substituter-hosts");
     get(useSshSubstituter, "use-ssh-substituter");
+    get(logServers, "log-servers");
+    get(enableImportNative, "allow-unsafe-native-code-during-evaluation");
 
     string subs = getEnv("NIX_SUBSTITUTERS", "default");
     if (subs == "default") {

nix/libstore/globals.hh

@@ -197,6 +197,12 @@ struct Settings {
     /* Whether to show a stack trace if Nix evaluation fails. */
     bool showTrace;
 
+    /* A list of URL prefixes that can return Nix build logs. */
+    Strings logServers;
+
+    /* Whether the importNative primop should be enabled */
+    bool enableImportNative;
+
 private:
     SettingsMap settings, overrides;
 

nix/libstore/local-store.cc

@@ -1083,31 +1083,16 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
 
     setSubstituterEnv();
 
-    run.pid = maybeVfork();
+    run.pid = startProcess([&]() {
-
+        if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
-    switch (run.pid) {
+            throw SysError("dupping stdin");
-
+        if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
-    case -1:
+            throw SysError("dupping stdout");
-        throw SysError("unable to fork");
+        if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
-
+            throw SysError("dupping stderr");
-    case 0: /* child */
+        execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
-        try {
+        throw SysError(format("executing `%1%'") % substituter);
-            restoreAffinity();
+    });
-            if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
-                throw SysError("dupping stdin");
-            if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
-                throw SysError("dupping stdout");
-            if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
-                throw SysError("dupping stderr");
-            execl(substituter.c_str(), substituter.c_str(), "--query", NULL);
-            throw SysError(format("executing `%1%'") % substituter);
-        } catch (std::exception & e) {
-            std::cerr << "error: " << e.what() << std::endl;
-        }
-        _exit(1);
-    }
-
-    /* Parent. */
 
     run.program = baseNameOf(substituter);
     run.to = toPipe.writeSide.borrow();

nix/libstore/local-store.hh

@@ -1,16 +1,12 @@
 #pragma once
 
 #include <string>
+#include <unordered_set>
 
 #include "store-api.hh"
 #include "util.hh"
 #include "pathlocks.hh"
 
-#if HAVE_TR1_UNORDERED_SET
-#include <tr1/unordered_set>
-#endif
-
-
 
 class sqlite3;
 class sqlite3_stmt;
@@ -306,11 +302,7 @@ private:
 
     void checkDerivationOutputs(const Path & drvPath, const Derivation & drv);
 
-#if HAVE_TR1_UNORDERED_SET
+    typedef std::unordered_set<ino_t> InodeHash;
-    typedef std::tr1::unordered_set<ino_t> InodeHash;
-#else
-    typedef std::set<ino_t> InodeHash;
-#endif
 
     InodeHash loadInodeHash();
     Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);

nix/libstore/remote-store.cc

@@ -402,8 +402,23 @@ Path RemoteStore::addToStore(const Path & _srcPath,
     writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to);
     writeInt(recursive ? 1 : 0, to);
     writeString(printHashType(hashAlgo), to);
-    dumpPath(srcPath, to, filter);
+
-    processStderr();
+    try {
+        to.written = 0;
+        to.warn = true;
+        dumpPath(srcPath, to, filter);
+        to.warn = false;
+        processStderr();
+    } catch (SysError & e) {
+        /* Daemon closed while we were sending the path. Probably OOM
+           or I/O error. */
+        if (e.errNo == EPIPE)
+            try {
+                processStderr();
+            } catch (EndOfFile & e) { }
+        throw;
+    }
+
     return readStorePath(from);
 }
 

nix/libutil/serialise.cc

@@ -54,8 +54,24 @@ FdSink::~FdSink()
 }
 
 
+size_t threshold = 256 * 1024 * 1024;
+
+static void warnLargeDump()
+{
+    printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory");
+}
+
+
 void FdSink::write(const unsigned char * data, size_t len)
 {
+    static bool warned = false;
+    if (warn && !warned) {
+        written += len;
+        if (written > threshold) {
+            warnLargeDump();
+            warned = true;
+        }
+    }
     writeFull(fd, data, len);
 }
 
@@ -256,4 +272,15 @@ template Paths readStrings(Source & source);
 template PathSet readStrings(Source & source);
 
 
+void StringSink::operator () (const unsigned char * data, size_t len)
+{
+    static bool warned = false;
+    if (!warned && s.size() > threshold) {
+        warnLargeDump();
+        warned = true;
+    }
+    s.append((const char *) data, len);
+}
+
+
 }

nix/libutil/serialise.hh

@@ -72,9 +72,11 @@ struct BufferedSource : Source
 struct FdSink : BufferedSink
 {
     int fd;
+    bool warn;
+    size_t written;
 
-    FdSink() : fd(-1) { }
+    FdSink() : fd(-1), warn(false), written(0) { }
-    FdSink(int fd) : fd(fd) { }
+    FdSink(int fd) : fd(fd), warn(false), written(0) { }
     ~FdSink();
     
     void write(const unsigned char * data, size_t len);
@@ -95,10 +97,7 @@ struct FdSource : BufferedSource
 struct StringSink : Sink
 {
     string s;
-    void operator () (const unsigned char * data, size_t len)
+    void operator () (const unsigned char * data, size_t len);
-    {
-        s.append((const char *) data, len);
-    }
 };
 
 

nix/libutil/types.hh

@@ -41,8 +41,8 @@ public:
     BaseError(const FormatOrString & fs, unsigned int status = 1);
     ~BaseError() throw () { };
     const char * what() const throw () { return err.c_str(); }
-    const string & msg() const throw () { return err; }
+    const string & msg() const { return err; }
-    const string & prefix() const throw () { return prefix_; }
+    const string & prefix() const { return prefix_; }
     BaseError & addPrefix(const FormatOrString & fs);
 };
 

nix/libutil/util.cc

@@ -1,5 +1,8 @@
 #include "config.h"
 
+#include "util.hh"
+#include "affinity.hh"
+
 #include <iostream>
 #include <cerrno>
 #include <cstdio>
@@ -16,8 +19,6 @@
 #include <sys/syscall.h>
 #endif
 
-#include "util.hh"
-
 
 extern char * * environ;
 
@@ -714,6 +715,13 @@ Pid::Pid()
 }
 
 
+Pid::Pid(pid_t pid)
+{
+    Pid();
+    *this = pid;
+}
+
+
 Pid::~Pid()
 {
     kill();
@@ -801,43 +809,30 @@ void killUser(uid_t uid)
        users to which the current process can send signals.  So we
        fork a process, switch to uid, and send a mass kill. */
 
-    Pid pid;
+    Pid pid = startProcess([&]() {
-    pid = fork();
-    switch (pid) {
 
-    case -1:
+        if (setuid(uid) == -1)
-        throw SysError("unable to fork");
+            throw SysError("setting uid");
 
-    case 0:
+        while (true) {
-        try { /* child */
-
-            if (setuid(uid) == -1)
-                throw SysError("setting uid");
-
-            while (true) {
 #ifdef __APPLE__
-                /* OSX's kill syscall takes a third parameter that, among other
+            /* OSX's kill syscall takes a third parameter that, among
-                   things, determines if kill(-1, signo) affects the calling
+               other things, determines if kill(-1, signo) affects the
-                   process. In the OSX libc, it's set to true, which means
+               calling process. In the OSX libc, it's set to true,
-                   "follow POSIX", which we don't want here
+               which means "follow POSIX", which we don't want here
                  */
-                if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
+            if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break;
 #else
-                if (kill(-1, SIGKILL) == 0) break;
+            if (kill(-1, SIGKILL) == 0) break;
 #endif
-                if (errno == ESRCH) break; /* no more processes */
+            if (errno == ESRCH) break; /* no more processes */
-                if (errno != EINTR)
+            if (errno != EINTR)
-                    throw SysError(format("cannot kill processes for uid `%1%'") % uid);
+                throw SysError(format("cannot kill processes for uid `%1%'") % uid);
-            }
-
-        } catch (std::exception & e) {
-            writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str());
-            _exit(1);
         }
-        _exit(0);
-    }
 
-    /* parent */
+        _exit(0);
+    });
+
     int status = pid.wait(true);
     if (status != 0)
         throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status));
@@ -852,6 +847,25 @@ void killUser(uid_t uid)
 //////////////////////////////////////////////////////////////////////
 
 
+pid_t startProcess(std::function<void()> fun, const string & errorPrefix)
+{
+    pid_t pid = fork();
+    if (pid == -1) throw SysError("unable to fork");
+
+    if (pid == 0) {
+        try {
+            restoreAffinity();
+            fun();
+        } catch (std::exception & e) {
+            writeToStderr(errorPrefix + string(e.what()) + "\n");
+        }
+        _exit(1);
+    }
+
+    return pid;
+}
+
+
 string runProgram(Path program, bool searchPath, const Strings & args)
 {
     checkInterrupt();
@@ -867,32 +881,17 @@ string runProgram(Path program, bool searchPath, const Strings & args)
     pipe.create();
 
     /* Fork. */
-    Pid pid;
+    Pid pid = startProcess([&]() {
-    pid = maybeVfork();
+        if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
+            throw SysError("dupping stdout");
 
-    switch (pid) {
+        if (searchPath)
+            execvp(program.c_str(), (char * *) &cargs[0]);
+        else
+            execv(program.c_str(), (char * *) &cargs[0]);
 
-    case -1:
+        throw SysError(format("executing `%1%'") % program);
-        throw SysError("unable to fork");
+    });
-
-    case 0: /* child */
-        try {
-            if (dup2(pipe.writeSide, STDOUT_FILENO) == -1)
-                throw SysError("dupping stdout");
-
-            if (searchPath)
-                execvp(program.c_str(), (char * *) &cargs[0]);
-            else
-                execv(program.c_str(), (char * *) &cargs[0]);
-            throw SysError(format("executing `%1%'") % program);
-
-        } catch (std::exception & e) {
-            writeToStderr("error: " + string(e.what()) + "\n");
-        }
-        _exit(1);
-    }
-
-    /* Parent. */
 
     pipe.writeSide.close();
 
@@ -901,7 +900,7 @@ string runProgram(Path program, bool searchPath, const Strings & args)
     /* Wait for the child to finish. */
     int status = pid.wait(true);
     if (!statusOk(status))
-        throw Error(format("program `%1%' %2%")
+        throw ExecError(format("program `%1%' %2%")
             % program % statusToString(status));
 
     return result;
@@ -928,13 +927,6 @@ void closeOnExec(int fd)
 }
 
 
-#if HAVE_VFORK
-pid_t (*maybeVfork)() = vfork;
-#else
-pid_t (*maybeVfork)() = fork;
-#endif
-
-
 //////////////////////////////////////////////////////////////////////
 
 

nix/libutil/util.hh

@@ -7,6 +7,7 @@
 #include <dirent.h>
 #include <unistd.h>
 #include <signal.h>
+#include <functional>
 
 #include <cstdio>
 
@@ -237,6 +238,7 @@ class Pid
     int killSignal;
 public:
     Pid();
+    Pid(pid_t pid);
     ~Pid();
     void operator =(pid_t pid);
     operator pid_t();
@@ -252,11 +254,18 @@ public:
 void killUser(uid_t uid);
 
 
+/* Fork a process that runs the given function, and return the child
+   pid to the caller. */
+pid_t startProcess(std::function<void()> fun, const string & errorPrefix = "error: ");
+
+
 /* Run a program and return its stdout in a string (i.e., like the
    shell backtick operator). */
 string runProgram(Path program, bool searchPath = false,
     const Strings & args = Strings());
 
+MakeError(ExecError, Error)
+
 /* Close all file descriptors except stdin, stdout, stderr, and those
    listed in the given set.  Good practice in child processes. */
 void closeMostFDs(const set<int> & exceptions);
@@ -264,9 +273,6 @@ void closeMostFDs(const set<int> & exceptions);
 /* Set the close-on-exec flag for the given file descriptor. */
 void closeOnExec(int fd);
 
-/* Call vfork() if available, otherwise fork(). */
-extern pid_t (*maybeVfork)();
-
 
 /* User interruption. */
 

nix/nix-daemon/nix-daemon.cc

@@ -735,12 +735,10 @@ static void processConnection(bool trusted)
                during addTextToStore() / importPath().  If that
                happens, just send the error message and exit. */
             bool errorAllowed = canSendStderr;
-            if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg());
             stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0);
-            if (!errorAllowed) break;
+            if (!errorAllowed) throw;
         } catch (std::bad_alloc & e) {
-            if (canSendStderr)
+            stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
-                stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0);
             throw;
         }
 
@@ -874,40 +872,27 @@ static void daemonLoop()
             printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid);
 
             /* Fork a child to handle the connection. */
-            pid_t child;
+            startProcess([&]() {
-            child = fork();
+                /* Background the daemon. */
+                if (setsid() == -1)
+                    throw SysError(format("creating a new session"));
 
-            switch (child) {
+                /* Restore normal handling of SIGCHLD. */
+                setSigChldAction(false);
 
-            case -1:
+                /* For debugging, stuff the pid into argv[1]. */
-                throw SysError("unable to fork");
+                if (clientPid != -1 && argvSaved[1]) {
-
+                    string processName = int2String(clientPid);
-            case 0:
+                    strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
-                try { /* child */
-
-                    /* Background the daemon. */
-                    if (setsid() == -1)
-                        throw SysError(format("creating a new session"));
-
-                    /* Restore normal handling of SIGCHLD. */
-                    setSigChldAction(false);
-
-                    /* For debugging, stuff the pid into argv[1]. */
-                    if (clientPid != -1 && argvSaved[1]) {
-                        string processName = int2String(clientPid);
-                        strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1]));
-                    }
-
-                    /* Handle the connection. */
-                    from.fd = remote;
-                    to.fd = remote;
-                    processConnection(trusted);
-
-                } catch (std::exception & e) {
-                    writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n");
                 }
-                exit(0);
+
-            }
+                /* Handle the connection. */
+                from.fd = remote;
+                to.fd = remote;
+                processConnection(trusted);
+
+                _exit(0);
+            }, "unexpected Nix daemon error: ");
 
         } catch (Interrupted & e) {
             throw;