From 43b64f503844a66c344780a11289678a001572db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 17:32:21 +0200 Subject: [PATCH 01/31] Remove tabs --- src/libstore/build.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d3184507ce..1870abead4 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2075,9 +2075,9 @@ void DerivationGoal::initChild() throw SysError("mounting /dev/pts"); createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); - /* Make sure /dev/pts/ptmx is world-writable. With some - Linux versions, it is created with permissions 0. */ - chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); } /* Do the chroot(). Below we do a chdir() to the From 7c3a5090bff4e9cfe70f1d89619563b55af13d89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 17:44:18 +0200 Subject: [PATCH 02/31] nix-copy-closure: Fix --dry-run --- perl/lib/Nix/CopyClosure.pm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 5085ec075b..53da72038d 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -78,6 +78,13 @@ sub copyTo { my @missing = grep { !$present{$_} } @closure; return if !@missing; + my $missingSize = 0; + $missingSize += (queryPathInfo($_, 1))[3] foreach @missing; + + printf STDERR "copying %d missing paths (%.2f MiB) to ‘$sshHost’...\n", + scalar(@missing), $missingSize / (1024.0^2); + return if $dryRun; + # Send the "import paths" command. syswrite($to, pack("L Date: Thu, 10 Jul 2014 20:43:04 +0200 Subject: [PATCH 03/31] =?UTF-8?q?nix-copy-closure=20-s:=20Do=20substitutio?= =?UTF-8?q?ns=20via=20=E2=80=98nix-store=20--serve=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This means we no longer need an SSH master connection, since we only execute a single command on the remote host. --- perl/lib/Nix/CopyClosure.pm | 19 +++++++++------- scripts/nix-copy-closure.in | 5 ++-- src/download-via-ssh/download-via-ssh.cc | 1 + src/nix-store/nix-store.cc | 29 ++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 53da72038d..779d7439f6 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -29,16 +29,12 @@ sub copyTo { my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign, $progressViewer, $useSubstitutes) = @_; + $useSubstitutes = 0 if $dryRun; + # Get the closure of this path. my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @{$storePaths}))); - # Optionally use substitutes on the remote host. - if (!$dryRun && $useSubstitutes) { - system "ssh $sshHost @{$sshOpts} nix-store -r --ignore-unknown @closure"; - # Ignore exit status because this is just an optimisation. - } - # Start ‘nix-store --serve’ on the remote host. my ($from, $to); my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write"); @@ -60,8 +56,9 @@ sub copyTo { # Send the "query valid paths" command with the "lock" option # enabled. This prevents a race where the remote host - # garbage-collect paths that are already there. - syswrite($to, pack("L & pipes) if (cmd == "have") { writeInt(cmdQueryValidPaths, pipes.first); writeInt(0, pipes.first); // don't lock + writeInt(0, pipes.first); // don't substitute writeStrings(tokenized, pipes.first); pipes.first.flush(); PathSet paths = readStrings(pipes.second); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 28b205b1fd..f31eb0e29a 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -896,16 +896,42 @@ static void opServe(Strings opFlags, Strings opArgs) } switch (cmd) { + case cmdQueryValidPaths: { bool lock = readInt(in); + bool substitute = readInt(in); PathSet paths = readStorePaths(in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); + + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + if (substitute && writeAllowed) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto & path : paths) + if (!isDerivation(path)) paths2.insert(path); + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + queryMissing(*store, PathSet(paths2.begin(), paths2.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) + try { + store->buildPaths(willSubstitute); + } catch (Error & e) { + printMsg(lvlError, format("warning: %1%") % e.msg()); + } + } + writeStrings(store->queryValidPaths(paths), out); out.flush(); break; } + case cmdQueryPathInfos: { PathSet paths = readStorePaths(in); // !!! Maybe we want a queryPathInfos? @@ -924,10 +950,12 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); break; } + case cmdDumpStorePath: dumpPath(readStorePath(in), out); out.flush(); break; + case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths not allowed"); string compression = readString(in); @@ -966,6 +994,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } + default: throw Error(format("unknown serve command %1%") % cmd); } From 42d91b079c5d0b468663511e7b2a8e2f4048c475 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 21:14:34 +0200 Subject: [PATCH 04/31] Fix use of sysread --- perl/lib/Nix/CopyClosure.pm | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 779d7439f6..8be4ead763 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -7,11 +7,24 @@ use List::Util qw(sum); use IPC::Open2; +sub readN { + my ($bytes, $from) = @_; + my $res = ""; + while ($bytes > 0) { + my $s; + my $n = sysread($from, $s, $bytes); + die "I/O error reading from remote side\n" if !defined $n; + die "got EOF while expecting $bytes bytes from remote side\n" if !$n; + $bytes -= $n; + $res .= $s; + } + return $res; +} + + sub readInt { my ($from) = @_; - my $resp; - sysread($from, $resp, 8) == 8 or die "did not receive valid reply from remote host\n"; - return unpack("L Date: Thu, 10 Jul 2014 21:30:22 +0200 Subject: [PATCH 05/31] Replace message "importing path <...>" with "exporting path <...>" This causes nix-copy-closure to show what it's doing before rather than after. --- src/libstore/local-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e66042c57f..7d78ab7ecf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1486,6 +1486,8 @@ void LocalStore::exportPath(const Path & path, bool sign, { assertStorePath(path); + printMsg(lvlInfo, format("exporting path `%1%'") % path); + addTempRoot(path); if (!isValidPath(path)) throw Error(format("path `%1%' is not valid") % path); @@ -1596,8 +1598,6 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path dstPath = readStorePath(hashAndReadSource); - printMsg(lvlInfo, format("importing path `%1%'") % dstPath); - PathSet references = readStorePaths(hashAndReadSource); Path deriver = readString(hashAndReadSource); From d0eb970fb4d3b5c347506b77f9657fc5eb6229e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 21:48:21 +0200 Subject: [PATCH 06/31] Fix broken Pid constructor --- src/libutil/util.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index faa2b83c37..6df93f12a0 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -708,17 +708,14 @@ void AutoCloseDir::close() Pid::Pid() + : pid(-1), separatePG(false), killSignal(SIGKILL) { - pid = -1; - separatePG = false; - killSignal = SIGKILL; } Pid::Pid(pid_t pid) + : pid(pid), separatePG(false), killSignal(SIGKILL) { - Pid(); - *this = pid; } From e196eecbe6552d5afed89ad480544c90cf959922 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Jul 2014 13:55:06 +0200 Subject: [PATCH 07/31] Allow $NIX_BUILD_HOOK to be relative to Nix libexec directory --- src/libstore/build.cc | 6 ++++-- src/libstore/globals.cc | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1870abead4..d594706032 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -590,7 +590,9 @@ HookInstance::HookInstance() { debug("starting build hook"); - Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); + Path buildHook = getEnv("NIX_BUILD_HOOK"); + if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook; + buildHook = canonPath(buildHook); /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -1503,7 +1505,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; + if (!settings.useBuildHook) return rpDecline; if (!worker.hook) worker.hook = std::shared_ptr(new HookInstance); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8bffdd73f7..b9d028be96 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -38,7 +38,7 @@ Settings::Settings() thisSystem = SYSTEM; maxSilentTime = 0; buildTimeout = 0; - useBuildHook = true; + useBuildHook = getEnv("NIX_BUILD_HOOK") != ""; printBuildTrace = false; reservedSize = 1024 * 1024; fsyncMetadata = true; From b8f24f253527e1cb071785c3b2d677ed2f734ab1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Jul 2014 14:27:17 +0200 Subject: [PATCH 08/31] Fix closure size display --- perl/lib/Nix/CopyClosure.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 8be4ead763..131f0b5a45 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -91,7 +91,7 @@ sub copyTo { $missingSize += (queryPathInfo($_, 1))[3] foreach @missing; printf STDERR "copying %d missing paths (%.2f MiB) to ‘$sshHost’...\n", - scalar(@missing), $missingSize / (1024.0^2); + scalar(@missing), $missingSize / (1024**2); return if $dryRun; # Send the "import paths" command. From a5c6347ff06ba09530fdf0e01828aaec89f6ceb6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Jul 2014 16:02:19 +0200 Subject: [PATCH 09/31] =?UTF-8?q?build-remote.pl:=20Use=20=E2=80=98nix-sto?= =?UTF-8?q?re=20--serve=E2=80=99=20on=20the=20remote=20side?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes things more efficient (we don't need to use an SSH master connection, and we only start a single remote process) and gets rid of locking issues (the remote nix-store process will keep inputs and outputs locked as long as they're needed). It also makes it more or less secure to connect directly to the root account on the build machine, using a forced command (e.g. ‘command="nix-store --serve --write"’). This bypasses the Nix daemon and is therefore more efficient. Also, don't call nix-store to import the output paths. --- perl/lib/Nix/CopyClosure.pm | 96 +++++++++++++-------------------- perl/lib/Nix/SSH.pm | 81 +++++++++++++++++++++++++++- perl/lib/Nix/Store.pm | 2 +- perl/lib/Nix/Store.xs | 11 ++++ perl/local.mk | 2 +- scripts/build-remote.pl.in | 68 ++++++++--------------- src/libstore/remote-store.cc | 1 + src/nix-store/nix-store.cc | 37 +++++++++++-- src/nix-store/serve-protocol.hh | 2 + 9 files changed, 185 insertions(+), 115 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 131f0b5a45..f701a7c8a0 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -3,76 +3,27 @@ package Nix::CopyClosure; use strict; use Nix::Config; use Nix::Store; +use Nix::SSH; use List::Util qw(sum); use IPC::Open2; -sub readN { - my ($bytes, $from) = @_; - my $res = ""; - while ($bytes > 0) { - my $s; - my $n = sysread($from, $s, $bytes); - die "I/O error reading from remote side\n" if !defined $n; - die "got EOF while expecting $bytes bytes from remote side\n" if !$n; - $bytes -= $n; - $res .= $s; - } - return $res; -} - - -sub readInt { - my ($from) = @_; - return unpack("L= 0x300; - }; - if ($@) { - chomp $@; - warn "$@; falling back to old closure copying method\n"; - return oldCopyTo(\@closure, @_); - } - # Send the "query valid paths" command with the "lock" option # enabled. This prevents a race where the remote host # garbage-collect paths that are already there. Optionally, ask # the remote host to substitute missing paths. - syswrite($to, pack("L 0) { - my @ps = splice(@$closure, 0, 1500); + while (scalar(@closure) > 0) { + my @ps = splice(@closure, 0, 1500); open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|"); while () { chomp; diff --git a/perl/lib/Nix/SSH.pm b/perl/lib/Nix/SSH.pm index 584c445009..c8792043c2 100644 --- a/perl/lib/Nix/SSH.pm +++ b/perl/lib/Nix/SSH.pm @@ -1,5 +1,16 @@ +package Nix::SSH; + use strict; use File::Temp qw(tempdir); +use IPC::Open2; + +our @ISA = qw(Exporter); +our @EXPORT = qw( + sshOpts openSSHConnection closeSSHConnection + readN readInt writeInt writeString writeStrings + connectToRemoteNix +); + our @sshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); @@ -8,6 +19,7 @@ push @sshOpts, "-x"; my $sshStarted = 0; my $sshHost; + # Open a master SSH connection to `host', unless there already is a # running master connection (as determined by `-O check'). sub openSSHConnection { @@ -18,7 +30,7 @@ sub openSSHConnection { my $tmpDir = tempdir("nix-ssh.XXXXXX", CLEANUP => 1, TMPDIR => 1) or die "cannot create a temporary directory"; - + push @sshOpts, "-S", "$tmpDir/control"; # Start the master. We can't use the `-f' flag (fork into @@ -39,6 +51,7 @@ sub openSSHConnection { return 0; } + # Tell the master SSH client to exit. sub closeSSHConnection { if ($sshStarted) { @@ -48,6 +61,70 @@ sub closeSSHConnection { } } + +sub readN { + my ($bytes, $from) = @_; + my $res = ""; + while ($bytes > 0) { + my $s; + my $n = sysread($from, $s, $bytes); + die "I/O error reading from remote side\n" if !defined $n; + die "got EOF while expecting $bytes bytes from remote side\n" if !$n; + $bytes -= $n; + $res .= $s; + } + return $res; +} + + +sub readInt { + my ($from) = @_; + return unpack("L= 0x300; + + return ($from, $to, $pid); +} + + END { my $saved = $?; closeSSHConnection; $? = $saved; } -return 1; +1; diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index 191116ee56..89cfaefa5f 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -15,7 +15,7 @@ our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); our @EXPORT = qw( isValidPath queryReferences queryPathInfo queryDeriver queryPathHash queryPathFromHashPart - topoSortPaths computeFSClosure followLinksToStorePath exportPaths + topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths hashPath hashFile hashString addToStore makeFixedOutputPath derivationFromPath diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 07ccebe62f..ff90616d37 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -179,6 +179,17 @@ void exportPaths(int fd, int sign, ...) } +void importPaths(int fd) + PPCODE: + try { + doInit(); + FdSource source(fd); + store->importPaths(false, source); + } catch (Error & e) { + croak(e.what()); + } + + SV * hashPath(char * algo, int base32, char * path) PPCODE: try { diff --git a/perl/local.mk b/perl/local.mk index 74c054e713..564683dffe 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -27,7 +27,7 @@ ifeq ($(perlbindings), yes) Store_CXXFLAGS = \ -I$(shell $(perl) -e 'use Config; print $$Config{archlibexp};')/CORE \ - -D_FILE_OFFSET_BITS=64 + -D_FILE_OFFSET_BITS=64 -Wno-unused-variable -Wno-literal-suffix Store_ALLOW_UNDEFINED = 1 diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 6dfa16d5cb..687b0e1310 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -4,7 +4,7 @@ use Fcntl qw(:DEFAULT :flock); use English '-no_match_vars'; use IO::Handle; use Nix::Config; -use Nix::SSH qw/sshOpts openSSHConnection/; +use Nix::SSH; use Nix::CopyClosure; use Nix::Store; no warnings('once'); @@ -90,6 +90,7 @@ if (defined $conf && -e $conf) { # Wait for the calling process to ask us whether we can build some derivation. my ($drvPath, $hostName, $slotLock); +my ($from, $to); REQ: while (1) { $_ = || exit 0; @@ -195,13 +196,15 @@ REQ: while (1) { # Connect to the selected machine. @sshOpts = ("-i", $machine->{sshKeys}, "-x"); $hostName = $machine->{hostName}; - if (openSSHConnection($hostName)) { - last REQ if system("ssh $hostName @sshOpts nix-builds-inhibited < /dev/null > /dev/null 2>&1") != 0; - warn "machine `$hostName' is refusing builds, trying other available machines...\n"; - closeSSHConnection; - } else { - warn "unable to open SSH connection to `$hostName', trying other available machines...\n"; - } + eval { + ($from, $to) = connectToRemoteNix($hostName, \@sshOpts); + # FIXME: check if builds are inhibited. + }; + last REQ unless $@; + print STDERR "$@"; + warn "unable to open SSH connection to `$hostName', trying other available machines...\n"; + $from = undef; + $to = undef; $machine->{enabled} = 0; } } @@ -220,18 +223,6 @@ my $maybeSign = ""; $maybeSign = "--sign" if -e "$Nix::Config::confDir/signing-key.sec"; -# Register the derivation as a temporary GC root. Note that $PPID is -# the PID of the remote SSH process, which, due to the use of a -# persistant SSH connection, should be the same across all remote -# command invocations for this session. -my $rootsDir = "@localstatedir@/nix/gcroots/tmp"; -system("ssh $hostName @sshOpts 'mkdir -m 1777 -p $rootsDir; ln -sfn $drvPath $rootsDir/\$PPID.drv'"); - -sub removeRoots { - system("ssh $hostName @sshOpts 'rm -f $rootsDir/\$PPID.drv $rootsDir/\$PPID.out'"); -} - - # Copy the derivation and its dependencies to the build machine. This # is guarded by an exclusive lock per machine to prevent multiple # build-remote instances from copying to a machine simultaneously. @@ -255,48 +246,33 @@ if ($@) { print STDERR "somebody is hogging $uploadLock, continuing...\n"; unlink $uploadLock; } -Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "", ""); +Nix::CopyClosure::copyToOpen($from, $to, $hostName, [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne "", ""); close UPLOADLOCK; # Perform the build. -my $buildFlags = - "--max-silent-time $maxSilentTime --option build-timeout $buildTimeout" - . " --fallback --add-root $rootsDir/\$PPID.out --quiet" - . " --option build-keep-log false --option build-use-substitutes false"; - -# We let the remote side kill its process group when the connection is -# closed unexpectedly. This is necessary to ensure that no processes -# are left running on the remote system if the local Nix process is -# killed. (SSH itself doesn't kill child processes if the connection -# is interrupted unless the `-tt' flag is used to force a pseudo-tty, -# in which case every child receives SIGHUP; however, `-tt' doesn't -# work on some platforms when connection sharing is used.) print STDERR "building `$drvPath' on `$hostName'\n"; -pipe STDIN, DUMMY; # make sure we have a readable STDIN -if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & exec nix-store -r $drvPath $buildFlags > /dev/null' 2>&4") != 0) { +writeInt(6, $to) or die; # == cmdBuildPaths +writeStrings([$drvPath], $to); +writeInt($maxSilentTime, $to); +writeInt($buildTimeout, $to); +my $res = readInt($from); +if ($res != 0) { # Note that if we get exit code 100 from `nix-store -r', it # denotes a permanent build failure (as opposed to an SSH problem # or a temporary Nix problem). We propagate this to the caller to # allow it to distinguish between transient and permanent # failures. - my $res = $? >> 8; print STDERR "build of `$drvPath' on `$hostName' failed with exit code $res\n"; - removeRoots; exit $res; } -#print "build of `$drvPath' on `$hostName' succeeded\n"; - # Copy the output from the build machine. my @outputs2 = grep { !isValidPath($_) } @outputs; if (scalar @outputs2 > 0) { - system("exec ssh $hostName @sshOpts 'nix-store --export @outputs2'" . - "| NIX_HELD_LOCKS='@outputs2' @bindir@/nix-store --import > /dev/null") == 0 - or die("cannot copy paths " . join(", ", @outputs) . " from `$hostName': $?"); + writeInt(5, $to) or die; # == cmdExportPaths + writeStrings(\@outputs2, $to); + $ENV{'NIX_HELD_LOCKS'} = "@outputs2"; # FIXME: ugly + importPaths(fileno($from)); } - - -# Get rid of the temporary GC roots. -removeRoots; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3b021bb2a5..f566ccf531 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -35,6 +35,7 @@ template T readStorePaths(Source & from) } template PathSet readStorePaths(Source & from); +template Paths readStorePaths(Source & from); RemoteStore::RemoteStore() diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f31eb0e29a..0196c2fc18 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -928,7 +928,6 @@ static void opServe(Strings opFlags, Strings opArgs) } writeStrings(store->queryValidPaths(paths), out); - out.flush(); break; } @@ -947,17 +946,15 @@ static void opServe(Strings opFlags, Strings opArgs) writeLongLong(info.narSize, out); } writeString("", out); - out.flush(); break; } case cmdDumpStorePath: dumpPath(readStorePath(in), out); - out.flush(); break; case cmdImportPaths: { - if (!writeAllowed) throw Error("importing paths not allowed"); + if (!writeAllowed) throw Error("importing paths is not allowed"); string compression = readString(in); if (compression != "") { @@ -986,7 +983,6 @@ static void opServe(Strings opFlags, Strings opArgs) store->importPaths(false, in); writeInt(1, out); // indicate success - out.flush(); /* The decompressor will have left stdin in an undefined state, so we can't continue. */ @@ -995,9 +991,40 @@ static void opServe(Strings opFlags, Strings opArgs) break; } + case cmdExportPaths: { + exportPaths(*store, readStorePaths(in), false, out); + break; + } + + case cmdBuildPaths: { + /* Used by build-remote.pl. */ + if (!writeAllowed) throw Error("building paths is not allowed"); + PathSet paths = readStorePaths(in); + + // FIXME: changing options here doesn't work if we're + // building through the daemon. + verbosity = lvlError; + settings.keepLog = false; + settings.useSubstitutes = false; + settings.maxSilentTime = readInt(in); + settings.buildTimeout = readInt(in); + + int res = 0; + try { + store->buildPaths(paths); + } catch (Error & e) { + printMsg(lvlError, format("error: %1%") % e.msg()); + res = e.status; + } + writeInt(res, out); + break; + } + default: throw Error(format("unknown serve command %1%") % cmd); } + + out.flush(); } } diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh index 07ff4f7a7c..eb13b46e51 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/nix-store/serve-protocol.hh @@ -14,6 +14,8 @@ typedef enum { cmdQueryPathInfos = 2, cmdDumpStorePath = 3, cmdImportPaths = 4, + cmdExportPaths = 5, + cmdBuildPaths = 6, } ServeCommand; } From 838138c5c4d21a207f3579c4f743698bd6dbb6b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 11 Jul 2014 16:20:12 +0200 Subject: [PATCH 10/31] Fix test --- tests/build-hook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/build-hook.sh b/tests/build-hook.sh index 681f65cc3b..ef77a3ae52 100644 --- a/tests/build-hook.sh +++ b/tests/build-hook.sh @@ -1,6 +1,6 @@ source common.sh -export NIX_BUILD_HOOK="build-hook.hook.sh" +export NIX_BUILD_HOOK="$(pwd)/build-hook.hook.sh" outPath=$(nix-build build-hook.nix --no-out-link) From a00a98548e994d1ea258e14793c7bcd8ea56cfdf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 12 Jul 2014 00:09:43 +0200 Subject: [PATCH 11/31] build-remote.pl: Fix build log --- perl/lib/Nix/SSH.pm | 6 ++++-- scripts/build-remote.pl.in | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/perl/lib/Nix/SSH.pm b/perl/lib/Nix/SSH.pm index c8792043c2..84bff5c785 100644 --- a/perl/lib/Nix/SSH.pm +++ b/perl/lib/Nix/SSH.pm @@ -107,11 +107,13 @@ sub writeStrings { sub connectToRemoteNix { - my ($sshHost, $sshOpts) = @_; + my ($sshHost, $sshOpts, $extraFlags) = @_; + + $extraFlags ||= ""; # Start ‘nix-store --serve’ on the remote host. my ($from, $to); - my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write"); + my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write $extraFlags"); # Do the handshake. my $SERVE_MAGIC_1 = 0x390c9deb; # FIXME diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 687b0e1310..c2f1fecd87 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -197,7 +197,7 @@ REQ: while (1) { @sshOpts = ("-i", $machine->{sshKeys}, "-x"); $hostName = $machine->{hostName}; eval { - ($from, $to) = connectToRemoteNix($hostName, \@sshOpts); + ($from, $to) = connectToRemoteNix($hostName, \@sshOpts, "2>&4"); # FIXME: check if builds are inhibited. }; last REQ unless $@; From b2e0293f022123b11759dfd498d4eff72233d3f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 12 Jul 2014 00:43:28 +0200 Subject: [PATCH 12/31] build-remote.pl: Don't keep a shell process around --- perl/lib/Nix/SSH.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perl/lib/Nix/SSH.pm b/perl/lib/Nix/SSH.pm index 84bff5c785..3bcbabe981 100644 --- a/perl/lib/Nix/SSH.pm +++ b/perl/lib/Nix/SSH.pm @@ -113,7 +113,8 @@ sub connectToRemoteNix { # Start ‘nix-store --serve’ on the remote host. my ($from, $to); - my $pid = open2($from, $to, "ssh $sshHost @{$sshOpts} nix-store --serve --write $extraFlags"); + # FIXME: don't start a shell, start ssh directly. + my $pid = open2($from, $to, "exec ssh $sshHost @{$sshOpts} nix-store --serve --write $extraFlags"); # Do the handshake. my $SERVE_MAGIC_1 = 0x390c9deb; # FIXME From fa13d3f4f3d8fb6dc3e3fc87ac5a2e26d8b32d84 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Jul 2014 12:19:27 +0200 Subject: [PATCH 13/31] build-remote.pl: Fix building multiple output derivations We were importing paths without sorting them topologically, leading to "path is not valid" errors. See e.g. http://hydra.nixos.org/build/12451761 --- src/libstore/remote-store.cc | 1 - src/nix-store/nix-store.cc | 4 +++- tests/remote-builds.nix | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f566ccf531..3b021bb2a5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -35,7 +35,6 @@ template T readStorePaths(Source & from) } template PathSet readStorePaths(Source & from); -template Paths readStorePaths(Source & from); RemoteStore::RemoteStore() diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 0196c2fc18..048f5c6030 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -992,7 +992,9 @@ static void opServe(Strings opFlags, Strings opArgs) } case cmdExportPaths: { - exportPaths(*store, readStorePaths(in), false, out); + Paths sorted = topoSortPaths(*store, readStorePaths(in)); + reverse(sorted.begin(), sorted.end()); + exportPaths(*store, sorted, false, out); break; } diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix index 571cdfbdd2..eb80a7d7e9 100644 --- a/tests/remote-builds.nix +++ b/tests/remote-builds.nix @@ -25,7 +25,8 @@ let system = "i686-linux"; PATH = "''${utils}/bin"; builder = "''${utils}/bin/sh"; - args = [ "-c" "echo Hello; mkdir $out; cat /proc/sys/kernel/hostname > $out/host; sleep 3" ]; + args = [ "-c" "echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 5" ]; + outputs = [ "out" "foo" ]; } ''; @@ -86,7 +87,7 @@ in # And a parallel build. my ($out1, $out2) = split /\s/, - $client->succeed("nix-store -r \$(nix-instantiate ${expr nodes.client.config 2} ${expr nodes.client.config 3})"); + $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); $slave1->succeed("test -e $out1 -o -e $out2"); $slave2->succeed("test -e $out1 -o -e $out2"); From 5bcb98271103c6c2ca3b993d8b1b0eb9eadcbc1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 14 Jul 2014 12:39:33 +0200 Subject: [PATCH 14/31] Remove cruft --- scripts/copying-collector.pl | 111 ----------------------------------- scripts/remove-patches.pl | 16 ----- 2 files changed, 127 deletions(-) delete mode 100755 scripts/copying-collector.pl delete mode 100755 scripts/remove-patches.pl diff --git a/scripts/copying-collector.pl b/scripts/copying-collector.pl deleted file mode 100755 index 2e8be2d41f..0000000000 --- a/scripts/copying-collector.pl +++ /dev/null @@ -1,111 +0,0 @@ -#! /usr/bin/perl -w - -use strict; - -my @paths = `nix-store -qR /home/eelco/.nix-profile/bin/firefox`; - -my %copyMap; -my %rewriteMap; - - -my $counter = 0; - -foreach my $path (@paths) { - chomp $path; - - $path =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$path'"; - my $hash = $2; - -# my $newHash = "deadbeef" . (sprintf "%024d", $counter++); - my $newHash = "deadbeef" . substr($hash, 0, 24); - my $newPath = "/home/eelco/chroot/$1/$newHash-$3"; - - die unless length $newHash == length $hash; - - $copyMap{$path} = $newPath; - $rewriteMap{$hash} = $newHash; -} - - -my %rewriteMap2; - - -sub rewrite; -sub rewrite { - my $src = shift; - my $dst = shift; - - if (-l $dst) { - - my $target = readlink $dst or die; - - foreach my $srcHash (keys %rewriteMap2) { - my $dstHash = $rewriteMap{$srcHash}; - print " $srcHash -> $dstHash\n"; - $target =~ s/$srcHash/$dstHash/g; - } - - unlink $dst or die; - - symlink $target, $dst; - - } - - elsif (-f $dst) { - - print "$dst\n"; - - foreach my $srcHash (keys %rewriteMap2) { - my $dstHash = $rewriteMap{$srcHash}; - print " $srcHash -> $dstHash\n"; - - my @stats = lstat $dst or die; - - system "sed s/$srcHash/$dstHash/g < '$dst' > '$dst.tmp'"; - die if $? != 0; - rename "$dst.tmp", $dst or die; - - chmod $stats[2], $dst or die; - } - - } - - elsif (-d $dst) { - - chmod 0755, $dst; - - opendir(DIR, "$dst") or die "cannot open `$dst': $!"; - my @files = readdir DIR; - closedir DIR; - - foreach my $file (@files) { - next if $file eq "." || $file eq ".."; - rewrite "$src/$file", "$dst/$file"; - } - } -} - - -foreach my $src (keys %copyMap) { - my $dst = $copyMap{$src}; - print "$src -> $dst\n"; - - if (!-e $dst) { - system "cp -prd $src $dst"; - die if $? != 0; - - my @refs = `nix-store -q --references $src`; - - %rewriteMap2 = (); - foreach my $ref (@refs) { - chomp $ref; - - $ref =~ /^(.*)\/([^-]+)-(.*)$/ or die "invalid store path `$ref'"; - my $hash = $2; - - $rewriteMap2{$hash} = $rewriteMap{$hash}; - } - - rewrite $src, $dst; - } -} diff --git a/scripts/remove-patches.pl b/scripts/remove-patches.pl deleted file mode 100755 index 401771a27d..0000000000 --- a/scripts/remove-patches.pl +++ /dev/null @@ -1,16 +0,0 @@ -#! /usr/bin/perl -w -I/home/eelco/nix/scripts - -use strict; -use readmanifest; - -for my $p (@ARGV) { - - my %narFiles; - my %patches; - - readManifest $p, \%narFiles, \%patches; - - %patches = (); - - writeManifest $p, \%narFiles, \%patches; -} From a2c85b2ef85a34bf8e5238c315a4ca73606f5ae6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Jul 2014 11:09:01 +0200 Subject: [PATCH 15/31] Manual: Typo --- doc/manual/hacking.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/hacking.xml b/doc/manual/hacking.xml index 1228285e12..11af0998f9 100644 --- a/doc/manual/hacking.xml +++ b/doc/manual/hacking.xml @@ -26,7 +26,7 @@ $ ./dev-shell To build Nix itself in this shell: -[nix-shell]$ ./bootstrap +[nix-shell]$ ./bootstrap.sh [nix-shell]$ configurePhase [nix-shell]$ make From 048be62484537633e2523dd4d200619649ff860d Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 16 Jul 2014 01:11:24 -0400 Subject: [PATCH 16/31] Pass *_proxy vars to bootstrap fetchurl --- corepkgs/fetchurl.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/corepkgs/fetchurl.nix b/corepkgs/fetchurl.nix index 39b9dd5082..4faedb1406 100644 --- a/corepkgs/fetchurl.nix +++ b/corepkgs/fetchurl.nix @@ -34,4 +34,12 @@ derivation { # Don't build in a chroot because Nix's dependencies may not be there. __noChroot = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; } From de8be7c3e06b52c313e0b452b641ad5f90dca2fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Jul 2014 11:16:54 +0200 Subject: [PATCH 17/31] Install systemd and Upstart stuff only on Linux --- misc/systemd/local.mk | 6 +++++- misc/upstart/local.mk | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 5555818c52..004549fd27 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -1 +1,5 @@ -$(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) +ifeq ($(OS), Linux) + + $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) + +endif diff --git a/misc/upstart/local.mk b/misc/upstart/local.mk index 249262d4f7..a73dc061e4 100644 --- a/misc/upstart/local.mk +++ b/misc/upstart/local.mk @@ -1 +1,5 @@ -$(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) +ifeq ($(OS), Linux) + + $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) + +endif From bb65460feb265be4d938c7dc724a76ef41a8bfaf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Jul 2014 11:19:12 +0200 Subject: [PATCH 18/31] Make dev-shell script work on Darwin --- dev-shell | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dev-shell b/dev-shell index 2fe62a4696..eae9246f47 100755 --- a/dev-shell +++ b/dev-shell @@ -6,11 +6,12 @@ fi s=$(type -p nix-shell) exec $s release.nix -A tarball --command " - export NIX_REMOTE=daemon + unset http_proxy + export NIX_REMOTE=$NIX_REMOTE export NIX_PATH='$NIX_PATH' export NIX_BUILD_SHELL=$(type -p bash) export c=\$configureFlags - exec $s release.nix -A build.x86_64-linux --exclude tarball --command ' + exec $s release.nix -A build.$(if [ $(uname -s) = Darwin ]; then echo x86_64-darwin; else echo x86_64-linux; fi) --exclude tarball --command ' configureFlags+=\" \$c --prefix=$(pwd)/inst --sysconfdir=$(pwd)/inst/etc\" return '" \ From 276a40b31f631c188d6dcbdf603a738e1380ff74 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Jul 2014 16:02:05 +0200 Subject: [PATCH 19/31] Handle case collisions on case-insensitive systems When running NixOps under Mac OS X, we need to be able to import store paths built on Linux into the local Nix store. However, HFS+ is usually case-insensitive, so if there are directories with file names that differ only in case, then importing will fail. The solution is to add a suffix ("~nix~case~hack~") to colliding files. For instance, if we have a directory containing xt_CONNMARK.h and xt_connmark.h, then the latter will be renamed to "xt_connmark.h~nix~case~hack~1". If a store path is dumped as a NAR, the suffixes are removed. Thus, importing and exporting via a case-insensitive Nix store is round-tripping. So when NixOps calls nix-copy-closure to copy the path to a Linux machine, you get the original file names back. Closes #119. --- src/libstore/globals.cc | 2 + src/libutil/archive.cc | 163 +++++++++++++++++++++++----------------- src/libutil/archive.hh | 12 ++- tests/case-hack.sh | 19 +++++ tests/case.nar | Bin 0 -> 2416 bytes tests/local.mk | 2 +- 6 files changed, 122 insertions(+), 76 deletions(-) create mode 100644 tests/case-hack.sh create mode 100644 tests/case.nar diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b9d028be96..60bc1dba13 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" +#include "archive.hh" #include #include @@ -150,6 +151,7 @@ void Settings::update() get(useSshSubstituter, "use-ssh-substituter"); get(logServers, "log-servers"); get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); + get(useCaseHack, "use-case-hack"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 70a1c580dd..dfe9653d17 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -3,6 +3,8 @@ #include #include #include +#include +#include #define _XOPEN_SOURCE 600 #include @@ -18,39 +20,21 @@ namespace nix { +bool useCaseHack = +#if __APPLE__ + true; +#else + false; +#endif + static string archiveVersion1 = "nix-archive-1"; +static string caseHackSuffix = "~nix~case~hack~"; PathFilter defaultPathFilter; -static void dump(const string & path, Sink & sink, PathFilter & filter); - - -static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) -{ - Strings names = readDirectory(path); - vector names2(names.begin(), names.end()); - sort(names2.begin(), names2.end()); - - for (vector::iterator i = names2.begin(); - i != names2.end(); ++i) - { - Path entry = path + "/" + *i; - if (filter(entry)) { - writeString("entry", sink); - writeString("(", sink); - writeString("name", sink); - writeString(*i, sink); - writeString("node", sink); - dump(entry, sink, filter); - writeString(")", sink); - } - } -} - - -static void dumpContents(const Path & path, size_t size, +static void dumpContents(const Path & path, size_t size, Sink & sink) { writeString("contents", sink); @@ -58,7 +42,7 @@ static void dumpContents(const Path & path, size_t size, AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file `%1%'") % path); - + unsigned char buf[65536]; size_t left = size; @@ -89,12 +73,41 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) writeString("", sink); } dumpContents(path, (size_t) st.st_size, sink); - } + } else if (S_ISDIR(st.st_mode)) { writeString("type", sink); writeString("directory", sink); - dumpEntries(path, sink, filter); + + /* If we're on a case-insensitive system like Mac OS X, undo + the case hack applied by restorePath(). */ + Strings names = readDirectory(path); + std::map unhacked; + for (auto & i : names) + if (useCaseHack) { + string name(i); + size_t pos = i.find(caseHackSuffix); + if (pos != string::npos) { + printMsg(lvlDebug, format("removing case hack suffix from `%1%'") % (path + "/" + i)); + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) + throw Error(format("file name collision in between `%1%' and `%2%'") + % (path + "/" + unhacked[name]) % (path + "/" + i)); + unhacked[name] = i; + } else + unhacked[i] = i; + + for (auto & i : unhacked) + if (filter(path + "/" + i.first)) { + writeString("entry", sink); + writeString("(", sink); + writeString("name", sink); + writeString(i.first, sink); + writeString("node", sink); + dump(path + "/" + i.second, sink, filter); + writeString(")", sink); + } } else if (S_ISLNK(st.st_mode)) { @@ -123,6 +136,7 @@ static SerialisationError badArchive(string s) } +#if 0 static void skipGeneric(Source & source) { if (readString(source) == "(") { @@ -130,43 +144,13 @@ static void skipGeneric(Source & source) skipGeneric(source); } } - - -static void parse(ParseSink & sink, Source & source, const Path & path); - - - -static void parseEntry(ParseSink & sink, Source & source, const Path & path) -{ - string s, name; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - checkInterrupt(); - - s = readString(source); - - if (s == ")") { - break; - } else if (s == "name") { - name = readString(source); - } else if (s == "node") { - if (s == "") throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else { - throw badArchive("unknown field " + s); - skipGeneric(source); - } - } -} +#endif static void parseContents(ParseSink & sink, Source & source, const Path & path) { unsigned long long size = readLongLong(source); - + sink.preallocateContents(size); unsigned long long left = size; @@ -185,6 +169,15 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) } +struct CaseInsensitiveCompare +{ + bool operator() (const string & a, const string & b) const + { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + + static void parse(ParseSink & sink, Source & source, const Path & path) { string s; @@ -194,6 +187,8 @@ static void parse(ParseSink & sink, Source & source, const Path & path) enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + std::map names; + while (1) { checkInterrupt(); @@ -221,9 +216,9 @@ static void parse(ParseSink & sink, Source & source, const Path & path) else if (t == "symlink") { type = tpSymlink; } - + else throw badArchive("unknown file type " + t); - + } else if (s == "contents" && type == tpRegular) { @@ -236,7 +231,35 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "entry" && type == tpDirectory) { - parseEntry(sink, source, path); + string name; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + if (useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + printMsg(lvlDebug, format("case collision between `%1%' and `%2%'") % i->first % name); + name += caseHackSuffix; + name += int2String(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (s.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } } else if (s == "target" && type == tpSymlink) { @@ -244,17 +267,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path) sink.createSymlink(path, target); } - else { + else throw badArchive("unknown field " + s); - skipGeneric(source); - } } } void parseDump(ParseSink & sink, Source & source) { - string version; + string version; try { version = readString(source); } catch (SerialisationError & e) { @@ -323,7 +344,7 @@ struct RestoreSink : ParseSink } }; - + void restorePath(const Path & path, Source & source) { RestoreSink sink; @@ -331,5 +352,5 @@ void restorePath(const Path & path, Source & source) parseDump(sink, source); } - + } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index ccac92074d..c216e9768f 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -28,7 +28,7 @@ namespace nix { where: - attrs(as) = concat(map(attr, as)) + encN(0) + attrs(as) = concat(map(attr, as)) + encN(0) attrs((a, b)) = encS(a) + encS(b) encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) @@ -58,7 +58,7 @@ void dumpPath(const Path & path, Sink & sink, struct ParseSink { virtual void createDirectory(const Path & path) { }; - + virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; @@ -66,10 +66,14 @@ struct ParseSink virtual void createSymlink(const Path & path, const string & target) { }; }; - + void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); - + +// FIXME: global variables are bad m'kay. +extern bool useCaseHack; + + } diff --git a/tests/case-hack.sh b/tests/case-hack.sh new file mode 100644 index 0000000000..ebc7cb1d5d --- /dev/null +++ b/tests/case-hack.sh @@ -0,0 +1,19 @@ +source common.sh + +clearStore + +rm -rf $TEST_ROOT/case + +opts="--option use-case-hack true" + +# Check whether restoring and dumping a NAR that contains case +# collisions is round-tripping, even on a case-insensitive system. +nix-store $opts --restore $TEST_ROOT/case < case.nar +nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar +cmp case.nar $TEST_ROOT/case.nar +[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] + +# Check whether we detect true collisions (e.g. those remaining after +# removal of the suffix). +touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" +! nix-store $opts --dump $TEST_ROOT/case > /dev/null diff --git a/tests/case.nar b/tests/case.nar new file mode 100644 index 0000000000000000000000000000000000000000..22ff26db5afd0f7f713ce0cc70420782a219d13f GIT binary patch literal 2416 zcmeH{Q44}F5XX&*>cQ_(4@P|i33`bX`T{X@m6Tho75eR)>rP0JTr5P$m+f}$_S^q% z=Q^*CC7DM=bV?r7TdJiVtWRMaz$o)8Ev6H&<3!L%a*=~Qgpo3-*XrUhlIl3<&tM;( z!+8AW|20P+&R>V2p4sBS`^_MJ)i%|uJ9a%^PAoa Date: Wed, 16 Jul 2014 16:30:50 +0200 Subject: [PATCH 20/31] Be more strict about file names in NARs --- src/libutil/archive.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index dfe9653d17..9dc2ebeeb9 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -231,7 +231,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) } else if (s == "entry" && type == tpDirectory) { - string name; + string name, prevName; s = readString(source); if (s != "(") throw badArchive("expected open tag"); @@ -245,6 +245,11 @@ static void parse(ParseSink & sink, Source & source, const Path & path) break; } else if (s == "name") { name = readString(source); + if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) + throw Error(format("NAR contains invalid file name `%1%'") % name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; if (useCaseHack) { auto i = names.find(name); if (i != names.end()) { From 2304a7dd21639959dc4bcafa3e17374cc087cd0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 16 Jul 2014 16:32:26 +0200 Subject: [PATCH 21/31] Get rid of a compiler warning --- src/libutil/archive.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 9dc2ebeeb9..00c9ae3c7c 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -1,3 +1,5 @@ +#define _XOPEN_SOURCE 600 + #include "config.h" #include @@ -6,7 +8,6 @@ #include #include -#define _XOPEN_SOURCE 600 #include #include #include From 8f72e702a114458e92f644160950344a7bf7166a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jul 2014 15:23:31 +0200 Subject: [PATCH 22/31] nix-daemon: Fix compat with older clients --- src/nix-daemon/nix-daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 265131c613..ab9e4b9684 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -548,7 +548,7 @@ static void performOp(bool trusted, unsigned int clientVersion, settings.set("build-max-jobs", int2String(readInt(from))); settings.set("build-max-silent-time", int2String(readInt(from))); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.useBuildHook = readInt(from) != 0; + if (readInt(from) != 0) settings.useBuildHook = true; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { settings.buildVerbosity = (Verbosity) readInt(from); logType = (LogType) readInt(from); From 77c972c898b325997fa2f527264a9706f1e414a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jul 2014 15:41:11 +0200 Subject: [PATCH 23/31] nix-daemon: Only print connection info if we have SO_PEERCRED --- src/nix-daemon/nix-daemon.cc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index ab9e4b9684..f486806353 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -854,22 +854,25 @@ static void daemonLoop() closeOnExec(remote); - /* Get the identity of the caller, if possible. */ - uid_t clientUid = -1; - pid_t clientPid = -1; bool trusted = false; + pid_t clientPid = -1; + #if defined(SO_PEERCRED) + /* Get the identity of the caller, if possible. */ + uid_t clientUid = -1; + ucred cred; socklen_t credLen = sizeof(cred); - if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) != -1) { - clientPid = cred.pid; - clientUid = cred.uid; - if (clientUid == 0) trusted = true; - } -#endif + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + + clientPid = cred.pid; + clientUid = cred.uid; + if (clientUid == 0) trusted = true; printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); +#endif /* Fork a child to handle the connection. */ startProcess([&]() { From 0c730887c4ec4a03fb854490e422c134a1bf8139 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jul 2014 15:49:33 +0200 Subject: [PATCH 24/31] nix-daemon: Show name of connecting user --- src/nix-daemon/nix-daemon.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index f486806353..fd030fe476 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -17,6 +17,7 @@ #include #include #include +#include using namespace nix; @@ -855,23 +856,23 @@ static void daemonLoop() closeOnExec(remote); bool trusted = false; - pid_t clientPid = -1; #if defined(SO_PEERCRED) /* Get the identity of the caller, if possible. */ - uid_t clientUid = -1; - ucred cred; socklen_t credLen = sizeof(cred); if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) throw SysError("getting peer credentials"); clientPid = cred.pid; - clientUid = cred.uid; - if (clientUid == 0) trusted = true; - printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); + struct passwd * pw = getpwuid(cred.uid); + string user = pw ? pw->pw_name : int2String(cred.uid); + + if (cred.uid == 0) trusted = true; + + printMsg(lvlInfo, format("accepted connection from pid %1%, user %2%") % clientPid % user); #endif /* Fork a child to handle the connection. */ From 049c0eb49c621ae50f49c8a06dc6c3a9839ef388 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jul 2014 16:57:07 +0200 Subject: [PATCH 25/31] nix-daemon: Add trusted-users and allowed-users options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ‘trusted-users’ is a list of users and groups that have elevated rights, such as the ability to specify binary caches. It defaults to ‘root’. A typical value would be ‘@wheel’ to specify all users in the wheel group. ‘allowed-users’ is a list of users and groups that are allowed to connect to the daemon. It defaults to ‘*’. A typical value would be ‘@users’ to specify the ‘users’ group. --- doc/manual/conf-file.xml | 42 ++++++++++++++++++++++++++++++++++++ src/libstore/globals.cc | 4 ++++ src/libstore/globals.hh | 9 ++++++++ src/nix-daemon/nix-daemon.cc | 38 +++++++++++++++++++++++++++++--- 4 files changed, 90 insertions(+), 3 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 29f7f9c51a..6af4c77654 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -479,6 +479,48 @@ flag, e.g. --option gc-keep-outputs false. + trusted-users + + + + A list of names of users (separated by whitespace) that + have additional rights when connecting to the Nix daemon, such + as the ability to specify additional binary caches, or to import + unsigned NARs. You can also specify groups by prefixing them + with @; for instance, + @wheel means all users in the + wheel group. The default is + root. + + The users listed here have the ability to + compromise the security of a multi-user Nix store. For instance, + they could install Trojan horses subsequently executed by other + users. So you should consider carefully whether to add users to + this list. + + + + + + + allowed-users + + + + A list of names of users (separated by whitespace) that + are allowed to connect to the Nix daemon. As with the + option, you can specify groups by + prefixing them with @. Also, you can allow + all users by specifying *. The default is + *. + + Note that trusted users are always allowed to connect. + + + + + + diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 60bc1dba13..2bfebb77a1 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -63,6 +63,8 @@ Settings::Settings() lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; showTrace = false; enableImportNative = false; + trustedUsers = Strings({"root"}); + allowedUsers = Strings({"*"}); } @@ -152,6 +154,8 @@ void Settings::update() get(logServers, "log-servers"); get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); get(useCaseHack, "use-case-hack"); + get(trustedUsers, "trusted-users"); + get(allowedUsers, "allowed-users"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8dd59a9c79..f1748336fd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -203,6 +203,15 @@ struct Settings { /* Whether the importNative primop should be enabled */ bool enableImportNative; + /* List of users that have elevated rights in the Nix daemon, such + as the ability to specify additional binary caches, or to + import unsigned NARs. */ + Strings trustedUsers; + + /* List of users that are allowed to connect to the daemon, in + addition to the trusted users. These have normal rights. */ + Strings allowedUsers; + private: SettingsMap settings, overrides; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index fd030fe476..dde501d309 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -7,6 +7,8 @@ #include "affinity.hh" #include "globals.hh" +#include + #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include using namespace nix; @@ -451,7 +454,7 @@ static void performOp(bool trusted, unsigned int clientVersion, case wopImportPaths: { startWork(); TunnelSource source(from); - Paths paths = store->importPaths(true, source); + Paths paths = store->importPaths(!trusted, source); stopWork(); writeStrings(paths, to); break; @@ -770,6 +773,27 @@ static void setSigChldAction(bool autoReap) } +bool matchUser(const string & user, const string & group, const Strings & users) +{ + if (find(users.begin(), users.end(), "*") != users.end()) + return true; + + if (find(users.begin(), users.end(), user) != users.end()) + return true; + + for (auto & i : users) + if (string(i, 0, 1) == "@") { + if (group == string(i, 1)) return true; + struct group * gr = getgrnam(i.c_str() + 1); + if (!gr) continue; + for (char * * mem = gr->gr_mem; *mem; mem++) + if (user == string(*mem)) return true; + } + + return false; +} + + #define SD_LISTEN_FDS_START 3 @@ -870,9 +894,17 @@ static void daemonLoop() struct passwd * pw = getpwuid(cred.uid); string user = pw ? pw->pw_name : int2String(cred.uid); - if (cred.uid == 0) trusted = true; + struct group * gr = getgrgid(cred.gid); + string group = gr ? gr->gr_name : int2String(cred.gid); - printMsg(lvlInfo, format("accepted connection from pid %1%, user %2%") % clientPid % user); + if (matchUser(user, group, settings.trustedUsers)) + trusted = true; + + if (!trusted && !matchUser(user, group, settings.allowedUsers)) + throw Error(format("user `%1%' is not allowed to connect to the Nix daemon") % user); + + printMsg(lvlInfo, format((string) "accepted connection from pid %1%, user %2%" + + (trusted ? " (trusted)" : "")) % clientPid % user); #endif /* Fork a child to handle the connection. */ From 8ddffe7aac414756809f43732effb8951858243b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Jul 2014 23:57:17 +0200 Subject: [PATCH 26/31] Ugly hack to fix building on old Darwin http://hydra.nixos.org/build/12580878 --- src/libutil/archive.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 00c9ae3c7c..bef0f4d578 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,10 @@ #include #include #include + +#define _DARWIN_C_SOURCE // hack to get strcasecmp on old Darwin versions #include +#undef _DARWIN_C_SOURCE #include #include From f609eec71a25a9bb8c32dd9620b7deb88633a22a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Jul 2014 00:01:06 +0200 Subject: [PATCH 27/31] Bump --- doc/manual/release-notes.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index fd5061cc01..426078b829 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -5,6 +5,15 @@ Nix Release Notes + + +
Release 1.8 (TBA) + +TODO + +
+ +
Release 1.7 (April 11, 2014) From 2e77bd70faee34cb2518529318a54b39f2d9143e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Jul 2014 12:54:30 +0200 Subject: [PATCH 28/31] Better fix for strcasecmp on Darwin --- src/libutil/archive.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index bef0f4d578..5450fd2f71 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -7,9 +7,7 @@ #include #include -#define _DARWIN_C_SOURCE // hack to get strcasecmp on old Darwin versions -#include -#undef _DARWIN_C_SOURCE +#include // for strcasecmp #include #include From ee3c5d7916b48d0c3b1cc08044e27209c14acfdc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 19 Jul 2014 02:25:47 +0200 Subject: [PATCH 29/31] Revert old useBuildHook behaviour --- src/libstore/build.cc | 2 +- src/libstore/globals.cc | 2 +- src/nix-daemon/nix-daemon.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index d594706032..4376a8322c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1505,7 +1505,7 @@ void DerivationGoal::buildDone() HookReply DerivationGoal::tryBuildHook() { - if (!settings.useBuildHook) return rpDecline; + if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; if (!worker.hook) worker.hook = std::shared_ptr(new HookInstance); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 2bfebb77a1..28982ec150 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -39,7 +39,7 @@ Settings::Settings() thisSystem = SYSTEM; maxSilentTime = 0; buildTimeout = 0; - useBuildHook = getEnv("NIX_BUILD_HOOK") != ""; + useBuildHook = true; printBuildTrace = false; reservedSize = 1024 * 1024; fsyncMetadata = true; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index dde501d309..f2141ee536 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -552,7 +552,7 @@ static void performOp(bool trusted, unsigned int clientVersion, settings.set("build-max-jobs", int2String(readInt(from))); settings.set("build-max-silent-time", int2String(readInt(from))); if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - if (readInt(from) != 0) settings.useBuildHook = true; + settings.useBuildHook = readInt(from) != 0; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { settings.buildVerbosity = (Verbosity) readInt(from); logType = (LogType) readInt(from); From 5989966ed3bd58cd362aed8cba6cd5c90b380a32 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Jul 2014 14:46:28 +0200 Subject: [PATCH 30/31] Remove dead code --- src/libstore/remote-store.cc | 2 -- src/libstore/remote-store.hh | 1 - 2 files changed, 3 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3b021bb2a5..b3967bb241 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -133,8 +133,6 @@ RemoteStore::~RemoteStore() try { to.flush(); fdSocket.close(); - if (child != -1) - child.wait(true); } catch (...) { ignoreException(); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 04b60fce4b..b010147643 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,7 +87,6 @@ private: AutoCloseFD fdSocket; FdSink to; FdSource from; - Pid child; unsigned int daemonVersion; bool initialised; From fdee1ced43fb495d612a29e955141cdf6b9a95ba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Jul 2014 19:11:26 +0200 Subject: [PATCH 31/31] startProcess: Make writing error messages from the child more robust --- src/libutil/util.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6df93f12a0..32244b2185 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -854,8 +854,10 @@ pid_t startProcess(std::function fun, const string & errorPrefix) restoreAffinity(); fun(); } catch (std::exception & e) { - writeToStderr(errorPrefix + string(e.what()) + "\n"); - } + try { + std::cerr << errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } _exit(1); }