From a2785b739169832f09285c81695c90a3aac3f731 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 11 Sep 2012 16:59:59 -0400 Subject: [PATCH] Support building a derivation if some outputs are already valid This handles the chroot and build hook cases, which are easy. Supporting the non-chroot-build case will require more work (hash rewriting!). Issue #21. --- scripts/build-remote.pl.in | 23 ++++++++++++----------- src/libstore/build.cc | 28 ++++++++++++++++------------ 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index e54386d424..631aba6808 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -6,6 +6,7 @@ use IO::Handle; use Nix::Config; use Nix::SSH qw/sshOpts openSSHConnection/; use Nix::CopyClosure; +use Nix::Store; no warnings('once'); @@ -18,7 +19,7 @@ no warnings('once'); # less than the maximum load for that machine, we try to get an # exclusive lock on $currentLoad/$machine-$slot (without blocking). # If we get such a lock, we send "accept" to the caller. Otherwise, -# we send "postpone" and exit. +# we send "postpone" and exit. # - We release the exclusive lock on $currentLoad/main-lock. # - We perform the build on $neededSystem. # - We release the exclusive lock on $currentLoad/$machine-$slot. @@ -102,14 +103,14 @@ REQ: while (1) { sendReply "decline"; next; } - + # Acquire the exclusive lock on $currentLoad/main-lock. mkdir $currentLoad, 0777 or die unless -d $currentLoad; my $mainLock = "$currentLoad/main-lock"; sysopen MAINLOCK, "$mainLock", O_RDWR|O_CREAT, 0600 or die; flock(MAINLOCK, LOCK_EX) or die; - - + + while (1) { # Find all machine that can execute this build, i.e., that # support builds for the given platform and features, and are @@ -141,7 +142,7 @@ REQ: while (1) { close $slotLock; $slot++; } - + push @available, { machine => $cur, load => $load, free => $free } if $load < $cur->{maxJobs}; } @@ -160,7 +161,7 @@ REQ: while (1) { if ($rightType && !$canBuildLocally) { sendReply "postpone"; } else { - sendReply "decline"; + sendReply "decline"; } close MAINLOCK; next REQ; @@ -182,9 +183,9 @@ REQ: while (1) { # Select the best available machine and lock a free slot. - my $selected = $available[0]; + my $selected = $available[0]; my $machine = $selected->{machine}; - + $slotLock = openSlotLock($machine, $selected->{free}); flock($slotLock, LOCK_EX | LOCK_NB) or die; utime undef, undef, $slotLock; @@ -196,7 +197,7 @@ REQ: while (1) { @sshOpts = ("-i", $machine->{sshKeys}, "-x"); $hostName = $machine->{hostName}; last REQ if openSSHConnection $hostName; - + warn "unable to open SSH connection to $hostName, trying other available machines...\n"; $machine->{enabled} = 0; } @@ -286,10 +287,10 @@ if (system("exec ssh $hostName @sshOpts '(read; kill -INT -\$\$) <&0 & nix-store foreach my $output (@outputs) { my $maybeSignRemote = ""; $maybeSignRemote = "--sign" if $UID != 0; - + next if isValidPath($output); system("exec ssh $hostName @sshOpts 'nix-store --export $maybeSignRemote $output'" . "| NIX_HELD_LOCKS=$output @bindir@/nix-store --import > /dev/null") == 0 - or die "cannot copy $output from $hostName: $?"; + or die "cannot copy $output from $hostName: $?"; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f93dd9b84a..391db9e233 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -756,6 +756,9 @@ private: /* Referenceable paths (i.e., input and output paths). */ PathSet allPaths; + /* Outputs that are already valid. */ + PathSet validPaths; + /* User selected for running the builder. */ UserLock buildUser; @@ -1141,7 +1144,7 @@ void DerivationGoal::tryToBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - PathSet validPaths = checkPathValidity(true); + validPaths = checkPathValidity(true); if (validPaths.size() == drv.outputs.size()) { debug(format("skipping build of derivation `%1%', someone beat us to it") % drvPath); @@ -1150,22 +1153,16 @@ void DerivationGoal::tryToBuild() return; } - if (validPaths.size() > 0) - /* !!! fix this; try to delete valid paths */ - throw Error( - format("derivation `%1%' is blocked by its output paths") - % drvPath); + printMsg(lvlError, format("BLOCKERS: %1%") % showPaths(validPaths)); /* If any of the outputs already exist but are not valid, delete them. */ foreach (DerivationOutputs::iterator, i, drv.outputs) { Path path = i->second.path; - if (worker.store.isValidPath(path)) - throw Error(format("obstructed build: path `%1%' exists") % path); - if (pathExists(path)) { - debug(format("removing unregistered path `%1%'") % path); - deletePathWrapped(path); - } + if (worker.store.isValidPath(path)) continue; + if (!pathExists(path)) continue; + debug(format("removing unregistered path `%1%'") % path); + deletePathWrapped(path); } /* Check again whether any output previously failed to build, @@ -1283,6 +1280,10 @@ void DerivationGoal::buildDone() Path path = i->second.path; if (useChroot && pathExists(chrootRootDir + path)) { + /* Move output paths from the chroot to the Nix store. + If the output was already valid, just skip + (discard) it. */ + if (validPaths.find(path) != validPaths.end()) continue; if (rename((chrootRootDir + path).c_str(), path.c_str()) == -1) throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path); } @@ -1747,6 +1748,9 @@ void DerivationGoal::startBuilder() #else throw Error("chroot builds are not supported on this platform"); #endif + } else { // !useChroot + if (validPaths.size() > 0) + throw Error(format("derivation `%1%' is blocked by its output paths %2%") % drvPath % showPaths(validPaths)); }