Don't keep "disabled" substituters running

For instance, it's pointless to keep copy-from-other-stores running if
there are no other stores, or download-using-manifests if there are no
manifests.  This also speeds things up because we don't send queries
to those substituters.
This commit is contained in:
Eelco Dolstra 2013-06-20 11:55:15 +02:00
parent 2b29e4b852
commit 22144afa8d
9 changed files with 46 additions and 7 deletions

View file

@ -227,6 +227,9 @@ sub writeManifest {
sub updateManifestDB { sub updateManifestDB {
my $manifestDir = $Nix::Config::manifestDir; my $manifestDir = $Nix::Config::manifestDir;
my @manifests = glob "$manifestDir/*.nixmanifest";
return undef if scalar @manifests == 0;
mkpath($manifestDir); mkpath($manifestDir);
unlink "$manifestDir/cache.sqlite"; # remove obsolete cache unlink "$manifestDir/cache.sqlite"; # remove obsolete cache
@ -311,7 +314,7 @@ EOF
# unless we've already done so on a previous run. # unless we've already done so on a previous run.
my %seen; my %seen;
for my $manifestLink (glob "$manifestDir/*.nixmanifest") { for my $manifestLink (@manifests) {
my $manifest = Cwd::abs_path($manifestLink); my $manifest = Cwd::abs_path($manifestLink);
next unless -f $manifest; next unless -f $manifest;
my $timestamp = lstat($manifest)->mtime; my $timestamp = lstat($manifest)->mtime;

View file

@ -16,6 +16,9 @@ foreach my $dir (@remoteStoresAll) {
push @remoteStores, glob($dir); push @remoteStores, glob($dir);
} }
exit if scalar @remoteStores == 0;
print "\n";
$ENV{"NIX_REMOTE"} = ""; $ENV{"NIX_REMOTE"} = "";

View file

@ -199,10 +199,6 @@ sub getAvailableCaches {
return if $gotCaches; return if $gotCaches;
$gotCaches = 1; $gotCaches = 1;
return if
($Nix::Config::config{"use-binary-caches"} // "true") eq "false" ||
($Nix::Config::config{"untrusted-use-binary-caches"} // "true") eq "false";
sub strToList { sub strToList {
my ($s) = @_; my ($s) = @_;
return map { s/\/+$//; $_ } split(/ /, $s); return map { s/\/+$//; $_ } split(/ /, $s);
@ -543,6 +539,13 @@ sub downloadBinary {
} }
# Bail out right away if binary caches are disabled.
exit 0 if
($Nix::Config::config{"use-binary-caches"} // "true") eq "false" ||
($Nix::Config::config{"untrusted-use-binary-caches"} // "true") eq "false";
print "\n";
flush STDOUT;
initCache(); initCache();

View file

@ -22,6 +22,8 @@ my $curl = "$Nix::Config::curl --fail --location --insecure";
# Open the manifest cache and update it if necessary. # Open the manifest cache and update it if necessary.
my $dbh = updateManifestDB(); my $dbh = updateManifestDB();
exit 0 unless defined $dbh; # exit if there are no manifests
print "\n";
# $hashCache->{$algo}->{$path} yields the $algo-hash of $path. # $hashCache->{$algo}->{$path} yields the $algo-hash of $path.

View file

@ -317,8 +317,10 @@ LocalStore::~LocalStore()
{ {
try { try {
foreach (RunningSubstituters::iterator, i, runningSubstituters) { foreach (RunningSubstituters::iterator, i, runningSubstituters) {
if (i->second.disabled) continue;
i->second.to.close(); i->second.to.close();
i->second.from.close(); i->second.from.close();
i->second.error.close();
i->second.pid.wait(true); i->second.pid.wait(true);
} }
} catch (...) { } catch (...) {
@ -998,7 +1000,7 @@ void LocalStore::setSubstituterEnv()
void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run)
{ {
if (run.pid != -1) return; if (run.disabled || run.pid != -1) return;
debug(format("starting substituter program `%1%'") % substituter); debug(format("starting substituter program `%1%'") % substituter);
@ -1039,6 +1041,23 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter &
run.to = toPipe.writeSide.borrow(); run.to = toPipe.writeSide.borrow();
run.from = run.fromBuf.fd = fromPipe.readSide.borrow(); run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
run.error = errorPipe.readSide.borrow(); run.error = errorPipe.readSide.borrow();
toPipe.readSide.close();
fromPipe.writeSide.close();
errorPipe.writeSide.close();
/* The substituter may exit right away if it's disabled in any way
(e.g. copy-from-other-stores.pl will exit if no other stores
are configured). */
try {
getLineFromSubstituter(run);
} catch (EndOfFile & e) {
run.to.close();
run.from.close();
run.error.close();
run.disabled = true;
if (run.pid.wait(true) != 0) throw;
}
} }
@ -1052,6 +1071,8 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
if (run.fromBuf.hasData()) goto haveData; if (run.fromBuf.hasData()) goto haveData;
while (1) { while (1) {
checkInterrupt();
fd_set fds; fd_set fds;
FD_ZERO(&fds); FD_ZERO(&fds);
FD_SET(run.from, &fds); FD_SET(run.from, &fds);
@ -1072,7 +1093,7 @@ string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
if (errno == EINTR) continue; if (errno == EINTR) continue;
throw SysError("reading from substituter's stderr"); throw SysError("reading from substituter's stderr");
} }
if (n == 0) throw Error(format("substituter `%1%' died unexpectedly") % run.program); if (n == 0) throw EndOfFile(format("substituter `%1%' died unexpectedly") % run.program);
err.append(buf, n); err.append(buf, n);
string::size_type p; string::size_type p;
while ((p = err.find('\n')) != string::npos) { while ((p = err.find('\n')) != string::npos) {
@ -1114,6 +1135,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
if (res.size() == paths.size()) break; if (res.size() == paths.size()) break;
RunningSubstituter & run(runningSubstituters[*i]); RunningSubstituter & run(runningSubstituters[*i]);
startSubstituter(*i, run); startSubstituter(*i, run);
if (run.disabled) continue;
string s = "have "; string s = "have ";
foreach (PathSet::const_iterator, j, paths) foreach (PathSet::const_iterator, j, paths)
if (res.find(*j) == res.end()) { s += *j; s += " "; } if (res.find(*j) == res.end()) { s += *j; s += " "; }
@ -1137,6 +1159,7 @@ void LocalStore::querySubstitutablePathInfos(const Path & substituter,
{ {
RunningSubstituter & run(runningSubstituters[substituter]); RunningSubstituter & run(runningSubstituters[substituter]);
startSubstituter(substituter, run); startSubstituter(substituter, run);
if (run.disabled) return;
string s = "info "; string s = "info ";
foreach (PathSet::const_iterator, i, paths) foreach (PathSet::const_iterator, i, paths)

View file

@ -48,6 +48,8 @@ struct RunningSubstituter
Pid pid; Pid pid;
AutoCloseFD to, from, error; AutoCloseFD to, from, error;
FdSource fromBuf; FdSource fromBuf;
bool disabled;
RunningSubstituter() : disabled(false) { };
}; };

View file

@ -790,6 +790,7 @@ void Pid::kill()
int Pid::wait(bool block) int Pid::wait(bool block)
{ {
assert(pid != -1);
while (1) { while (1) {
int status; int status;
int res = waitpid(pid, &status, block ? 0 : WNOHANG); int res = waitpid(pid, &status, block ? 0 : WNOHANG);

View file

@ -1,4 +1,5 @@
#! /bin/sh -e #! /bin/sh -e
echo
echo substituter args: $* >&2 echo substituter args: $* >&2
if test $1 = "--query"; then if test $1 = "--query"; then

View file

@ -1,4 +1,5 @@
#! /bin/sh -e #! /bin/sh -e
echo
echo substituter2 args: $* >&2 echo substituter2 args: $* >&2
if test $1 = "--query"; then if test $1 = "--query"; then