* Sync with the trunk.

This commit is contained in:
Eelco Dolstra 2010-08-04 17:48:29 +00:00
commit 587dc8aa00
16 changed files with 106 additions and 42 deletions

View File

@ -10,8 +10,8 @@ AC_CANONICAL_HOST
# Construct a Nix system name (like "i686-linux"). # Construct a Nix system name (like "i686-linux").
AC_MSG_CHECKING([for the canonical Nix system name]) AC_MSG_CHECKING([for the canonical Nix system name])
cpu_name=$(uname -p | tr 'A-Z ' 'a-z_') cpu_name=$(uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
machine_name=$(uname -m | tr 'A-Z ' 'a-z_') machine_name=$(uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
case $machine_name in case $machine_name in
i*86) i*86)
@ -30,7 +30,7 @@ case $machine_name in
;; ;;
esac esac
sys_name=$(uname -s | tr 'A-Z ' 'a-z_') sys_name=$(uname -s | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ ' 'abcdefghijklmnopqrstuvwxyz_')
case $sys_name in case $sys_name in
cygwin*) cygwin*)
@ -128,6 +128,10 @@ AC_CHECK_HEADERS([locale], [], [], [])
AC_LANG_POP(C++) AC_LANG_POP(C++)
# Check for <err.h>.
AC_CHECK_HEADER([err.h], [], [bsddiff_compat_include="-Icompat-include"])
AC_SUBST([bsddiff_compat_include])
# Check whether we have the personality() syscall, which allows us to # Check whether we have the personality() syscall, which allows us to
# do i686-linux builds on x86_64-linux machines. # do i686-linux builds on x86_64-linux machines.
AC_CHECK_HEADERS([sys/personality.h]) AC_CHECK_HEADERS([sys/personality.h])

View File

@ -59,6 +59,17 @@
#build-max-jobs = 1 #build-max-jobs = 1
### Option `build-cores'
#
# This option defines the number of CPU cores to utilize in parallel
# within a build job, i.e. by passing an appropriate `-jN' flag to GNU
# make. The default is 1, meaning that parallel building within jobs
# is disabled. Passing the special value `0' causes Nix to try and
# auto-detect the number of available cores on the local host. This
# setting can be overridden using the `--cores' command line switch.
#build-cores = 1
### Option `build-max-silent-time' ### Option `build-max-silent-time'
# #
# This option defines the maximum number of seconds that a builder can # This option defines the maximum number of seconds that a builder can

View File

@ -20,11 +20,11 @@ die unless scalar @ARGV == 5;
my $hashAlgo = "sha256"; my $hashAlgo = "sha256";
my $cacheDir = $ARGV[0]; my $narDir = $ARGV[0];
my $patchesDir = $ARGV[1]; my $patchesDir = $ARGV[1];
my $patchesURL = $ARGV[2]; my $patchesURL = $ARGV[2];
my $srcDir = $ARGV[3]; my $srcManifest = $ARGV[3];
my $dstDir = $ARGV[4]; my $dstManifest = $ARGV[4];
my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1) my $tmpDir = tempdir("nix-generate-patches.XXXXXX", CLEANUP => 1, TMPDIR => 1)
or die "cannot create a temporary directory"; or die "cannot create a temporary directory";
@ -41,10 +41,10 @@ my %dstNarFiles;
my %dstLocalPaths; my %dstLocalPaths;
my %dstPatches; my %dstPatches;
readManifest "$srcDir/MANIFEST", readManifest "$srcManifest",
\%srcNarFiles, \%srcLocalPaths, \%srcPatches; \%srcNarFiles, \%srcLocalPaths, \%srcPatches;
readManifest "$dstDir/MANIFEST", readManifest "$dstManifest",
\%dstNarFiles, \%dstLocalPaths, \%dstPatches; \%dstNarFiles, \%dstLocalPaths, \%dstPatches;
@ -55,8 +55,7 @@ sub findOutputPaths {
foreach my $p (keys %{$narFiles}) { foreach my $p (keys %{$narFiles}) {
# Ignore store expressions. # Ignore derivations.
next if ($p =~ /\.store$/);
next if ($p =~ /\.drv$/); next if ($p =~ /\.drv$/);
# Ignore builders (too much ambiguity -- they're all called # Ignore builders (too much ambiguity -- they're all called
@ -65,7 +64,7 @@ sub findOutputPaths {
next if ($p =~ /\.patch$/); next if ($p =~ /\.patch$/);
# Don't bother including tar files etc. # Don't bother including tar files etc.
next if ($p =~ /\.tar\.(gz|bz2)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/); next if ($p =~ /\.tar$/ || $p =~ /\.tar\.(gz|bz2|Z|lzma|xz)$/ || $p =~ /\.zip$/ || $p =~ /\.bin$/ || $p =~ /\.tgz$/ || $p =~ /\.rpm$/ || $p =~ /cvs-export$/ || $p =~ /fetchhg$/);
$outPaths{$p} = 1; $outPaths{$p} = 1;
} }
@ -85,6 +84,7 @@ sub getNameVersion {
$p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/; $p =~ /\/[0-9a-z]+((?:-[a-zA-Z][^\/-]*)+)([^\/]*)$/;
my $name = $1; my $name = $1;
my $version = $2; my $version = $2;
return undef unless defined $name && defined $version;
$name =~ s/^-//; $name =~ s/^-//;
$version =~ s/^-//; $version =~ s/^-//;
return ($name, $version); return ($name, $version);
@ -112,14 +112,14 @@ sub getNarBz2 {
my $storePath = shift; my $storePath = shift;
my $narFileList = $$narFiles{$storePath}; my $narFileList = $$narFiles{$storePath};
die "missing store expression $storePath" unless defined $narFileList; die "missing path $storePath" unless defined $narFileList;
my $narFile = @{$narFileList}[0]; my $narFile = @{$narFileList}[0];
die unless defined $narFile; die unless defined $narFile;
$narFile->{url} =~ /\/([^\/]+)$/; $narFile->{url} =~ /\/([^\/]+)$/;
die unless defined $1; die unless defined $1;
return "$cacheDir/$1"; return "$narDir/$1";
} }
@ -213,6 +213,7 @@ foreach my $p (keys %dstOutPaths) {
# this path. # this path.
(my $name, my $version) = getNameVersion $p; (my $name, my $version) = getNameVersion $p;
next unless defined $name && defined $version;
my @closest = (); my @closest = ();
my $closestVersion; my $closestVersion;
@ -222,6 +223,8 @@ foreach my $p (keys %dstOutPaths) {
foreach my $q (keys %srcOutPaths) { foreach my $q (keys %srcOutPaths) {
(my $name2, my $version2) = getNameVersion $q; (my $name2, my $version2) = getNameVersion $q;
next unless defined $name2 && defined $version2;
if ($name eq $name2) { if ($name eq $name2) {
# If the sizes differ too much, then skip. This # If the sizes differ too much, then skip. This
@ -241,11 +244,11 @@ foreach my $p (keys %dstOutPaths) {
# If the numbers of weighted uses differ too much, then # If the numbers of weighted uses differ too much, then
# skip. This disambiguates between, e.g., the bootstrap # skip. This disambiguates between, e.g., the bootstrap
# GCC and the final GCC in Nixpkgs. # GCC and the final GCC in Nixpkgs.
my $srcUses = computeUses \%srcNarFiles, $q; # my $srcUses = computeUses \%srcNarFiles, $q;
my $dstUses = computeUses \%dstNarFiles, $p; # my $dstUses = computeUses \%dstNarFiles, $p;
$ratio = $srcUses / $dstUses; # $ratio = $srcUses / $dstUses;
$ratio = 1 / $ratio if $ratio < 1; # $ratio = 1 / $ratio if $ratio < 1;
print " USE $srcUses $dstUses $ratio $q\n"; # print " USE $srcUses $dstUses $ratio $q\n";
# if ($ratio >= 2) { # if ($ratio >= 2) {
# print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n"; # print " SKIPPING $q due to use ratio $ratio ($srcUses $dstUses)\n";
@ -288,6 +291,11 @@ foreach my $p (keys %dstOutPaths) {
my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest; my $srcNarBz2 = getNarBz2 \%srcNarFiles, $closest;
my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p; my $dstNarBz2 = getNarBz2 \%dstNarFiles, $p;
if (! -f $srcNarBz2) {
warn "patch source archive $srcNarBz2 is missing\n";
next;
}
system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0 system("@bunzip2@ < $srcNarBz2 > $tmpDir/A") == 0
or die "cannot unpack $srcNarBz2"; or die "cannot unpack $srcNarBz2";
@ -404,5 +412,5 @@ do {
# Rewrite the manifest of the destination (with the new patches). # Rewrite the manifest of the destination (with the new patches).
writeManifest "$dstDir/MANIFEST", writeManifest "${dstManifest}",
\%dstNarFiles, \%dstPatches; \%dstNarFiles, \%dstPatches;

View File

@ -1,3 +1,5 @@
EXTRA_DIST = compat-include
libexec_PROGRAMS = bsdiff bspatch libexec_PROGRAMS = bsdiff bspatch
bsdiff_SOURCES = bsdiff.c bsdiff_SOURCES = bsdiff.c
@ -8,4 +10,4 @@ bspatch_SOURCES = bspatch.c
bspatch_LDADD = ${bzip2_lib} bspatch_LDADD = ${bzip2_lib}
AM_CFLAGS = -O3 ${bzip2_include} AM_CFLAGS = -O3 ${bzip2_include} ${bsddiff_compat_include}

View File

@ -0,0 +1,12 @@
/* Simulate BSD's <err.h> functionality. */
#ifndef COMPAT_ERR_H_INCLUDED
#define COMPAT_ERR_H_INCLUDED 1
#include <stdio.h>
#include <stdlib.h>
#define err(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
#define errx(rc,...) do { fprintf(stderr,__VA_ARGS__); exit(rc); } while(0)
#endif

View File

@ -750,13 +750,15 @@ void ExprOpImpl::eval(EvalState & state, Env & env, Value & v)
void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v)
{ {
Value v2; Value v1, v2;
state.evalAttrs(env, e1, v2); state.evalAttrs(env, e1, v1);
state.cloneAttrs(v2, v);
state.evalAttrs(env, e2, v2); state.evalAttrs(env, e2, v2);
if (v1.attrs->size() == 0) { v = v2; return; }
if (v2.attrs->size() == 0) { v = v1; return; }
state.cloneAttrs(v1, v);
foreach (Bindings::iterator, i, *v2.attrs) { foreach (Bindings::iterator, i, *v2.attrs) {
Attr & a = (*v.attrs)[i->first]; Attr & a = (*v.attrs)[i->first];
mkCopy(a.value, i->second.value); mkCopy(a.value, i->second.value);

View File

@ -788,13 +788,13 @@ static void prim_intersectAttrs(EvalState & state, Value * * args, Value & v)
state.forceAttrs(*args[1]); state.forceAttrs(*args[1]);
state.mkAttrs(v); state.mkAttrs(v);
foreach (Bindings::iterator, i, *args[1]->attrs) { foreach (Bindings::iterator, i, *args[0]->attrs) {
Bindings::iterator j = args[0]->attrs->find(i->first); Bindings::iterator j = args[1]->attrs->find(i->first);
if (j != args[0]->attrs->end()) { if (j != args[1]->attrs->end()) {
Attr & a = (*v.attrs)[i->first]; Attr & a = (*v.attrs)[j->first];
mkCopy(a.value, i->second.value); mkCopy(a.value, j->second.value);
a.pos = i->second.pos; a.pos = j->second.pos;
} }
} }
} }

View File

@ -135,6 +135,7 @@ static void initAndRun(int argc, char * * argv)
/* Get some settings from the configuration file. */ /* Get some settings from the configuration file. */
thisSystem = querySetting("system", SYSTEM); thisSystem = querySetting("system", SYSTEM);
maxBuildJobs = queryIntSetting("build-max-jobs", 1); maxBuildJobs = queryIntSetting("build-max-jobs", 1);
buildCores = queryIntSetting("build-cores", 1);
maxSilentTime = queryIntSetting("build-max-silent-time", 0); maxSilentTime = queryIntSetting("build-max-silent-time", 0);
/* Catch SIGINT. */ /* Catch SIGINT. */
@ -226,6 +227,8 @@ static void initAndRun(int argc, char * * argv)
tryFallback = true; tryFallback = true;
else if (arg == "--max-jobs" || arg == "-j") else if (arg == "--max-jobs" || arg == "-j")
maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end()); maxBuildJobs = getIntArg<unsigned int>(arg, i, args.end());
else if (arg == "--cores")
buildCores = getIntArg<unsigned int>(arg, i, args.end());
else if (arg == "--readonly-mode") else if (arg == "--readonly-mode")
readOnlyMode = true; readOnlyMode = true;
else if (arg == "--max-silent-time") else if (arg == "--max-silent-time")

View File

@ -25,6 +25,7 @@
#include <unistd.h> #include <unistd.h>
#include <errno.h> #include <errno.h>
#include <stdio.h> #include <stdio.h>
#include <cstring>
#include <pwd.h> #include <pwd.h>
#include <grp.h> #include <grp.h>
@ -1427,6 +1428,9 @@ void DerivationGoal::startBuilder()
in the store or in the build directory). */ in the store or in the build directory). */
env["NIX_STORE"] = nixStore; env["NIX_STORE"] = nixStore;
/* The maximum number of cores to utilize for parallel building. */
env["NIX_BUILD_CORES"] = (format("%d") % buildCores).str();
/* Add all bindings specified in the derivation. */ /* Add all bindings specified in the derivation. */
foreach (StringPairs::iterator, i, drv.env) foreach (StringPairs::iterator, i, drv.env)
env[i->first] = i->second; env[i->first] = i->second;
@ -2654,6 +2658,7 @@ void Worker::waitForInput()
timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before); timeout.tv_sec = std::max((time_t) 0, lastWokenUp + wakeUpInterval - before);
} else lastWokenUp = 0; } else lastWokenUp = 0;
using namespace std;
/* Use select() to wait for the input side of any logger pipe to /* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking) become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */ includes EOF. */

View File

@ -22,6 +22,7 @@ bool keepGoing = false;
bool tryFallback = false; bool tryFallback = false;
Verbosity buildVerbosity = lvlInfo; Verbosity buildVerbosity = lvlInfo;
unsigned int maxBuildJobs = 1; unsigned int maxBuildJobs = 1;
unsigned int buildCores = 1;
bool readOnlyMode = false; bool readOnlyMode = false;
string thisSystem = "unset"; string thisSystem = "unset";
time_t maxSilentTime = 0; time_t maxSilentTime = 0;

View File

@ -55,6 +55,11 @@ extern Verbosity buildVerbosity;
/* Maximum number of parallel build jobs. 0 means unlimited. */ /* Maximum number of parallel build jobs. 0 means unlimited. */
extern unsigned int maxBuildJobs; extern unsigned int maxBuildJobs;
/* Number of CPU cores to utilize in parallel within a build, i.e. by passing
this number to Make via '-j'. 0 means that the number of actual CPU cores on
the local host ought to be auto-detected. */
extern unsigned int buildCores;
/* Read-only mode. Don't copy stuff to the store, don't change the /* Read-only mode. Don't copy stuff to the store, don't change the
database. */ database. */
extern bool readOnlyMode; extern bool readOnlyMode;

View File

@ -13,6 +13,7 @@
#include <iostream> #include <iostream>
#include <unistd.h> #include <unistd.h>
#include <cstring>
namespace nix { namespace nix {
@ -158,6 +159,7 @@ void RemoteStore::connectToDaemon()
addr.sun_family = AF_UNIX; addr.sun_family = AF_UNIX;
if (socketPathRel.size() >= sizeof(addr.sun_path)) if (socketPathRel.size() >= sizeof(addr.sun_path))
throw Error(format("socket path `%1%' is too long") % socketPathRel); throw Error(format("socket path `%1%' is too long") % socketPathRel);
using namespace std;
strcpy(addr.sun_path, socketPathRel.c_str()); strcpy(addr.sun_path, socketPathRel.c_str());
if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1) if (connect(fdSocket, (struct sockaddr *) &addr, sizeof(addr)) == -1)

View File

@ -12,6 +12,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <fcntl.h> #include <fcntl.h>
#include <unistd.h> #include <unistd.h>
#include <limits.h>
#include "util.hh" #include "util.hh"

View File

@ -247,11 +247,12 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
} }
/* If `newestOnly', if a selector matches multiple derivations /* If `newestOnly', if a selector matches multiple derivations
with the same name, pick the one with the highest priority. with the same name, pick the one matching the current
If there are multiple derivations with the same priority, system. If there are still multiple derivations, pick the
pick the one with the highest version. If there are one with the highest priority. If there are still multiple
multiple derivations with the same priority and name and derivations, pick the one with the highest version.
version, then pick the first one. */ Finally, if there are still multiple derivations,
arbitrarily pick the first one. */
if (newestOnly) { if (newestOnly) {
/* Map from package names to derivations. */ /* Map from package names to derivations. */
@ -266,7 +267,12 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems,
Newest::iterator k = newest.find(drvName.name); Newest::iterator k = newest.find(drvName.name);
if (k != newest.end()) { if (k != newest.end()) {
d = comparePriorities(state, j->first, k->second.first); d = j->first.system == k->second.first.system ? 0 :
j->first.system == thisSystem ? 1 :
k->second.first.system == thisSystem ? -1 : 0;
printMsg(lvlError, format("%1% %2% %3% %4%") % j->first.system % k->second.first.system % thisSystem % d);
if (d == 0)
d = comparePriorities(state, j->first, k->second.first);
if (d == 0) if (d == 0)
d = compareVersions(drvName.version, DrvName(k->second.first.name).version); d = compareVersions(drvName.version, DrvName(k->second.first.name).version);
} }
@ -1230,7 +1236,7 @@ void run(Strings args)
globals.instSource.type = srcUnknown; globals.instSource.type = srcUnknown;
globals.instSource.nixExprPath = getDefNixExprPath(); globals.instSource.nixExprPath = getDefNixExprPath();
globals.instSource.systemFilter = thisSystem; globals.instSource.systemFilter = "*";
globals.dryRun = false; globals.dryRun = false;
globals.preserveInstalled = false; globals.preserveInstalled = false;

View File

@ -7,6 +7,7 @@
#include "globals.hh" #include "globals.hh"
#include <iostream> #include <iostream>
#include <cstring>
#include <unistd.h> #include <unistd.h>
#include <signal.h> #include <signal.h>
#include <sys/types.h> #include <sys/types.h>
@ -111,6 +112,7 @@ static bool isFarSideClosed(int socket)
time and wouldn't have to worry about races. */ time and wouldn't have to worry about races. */
static void sigPollHandler(int sigNo) static void sigPollHandler(int sigNo)
{ {
using namespace std;
try { try {
/* Check that the far side actually closed. We're still /* Check that the far side actually closed. We're still
getting spurious signals every once in a while. I.e., getting spurious signals every once in a while. I.e.,

View File

@ -6,11 +6,11 @@ drv=$4
echo "HOOK for $drv" >&2 echo "HOOK for $drv" >&2
outPath=$(sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv) outPath=`sed 's/Derive(\[("out",\"\([^\"]*\)\".*/\1/' $drv`
echo "output path is $outPath" >&2 echo "output path is $outPath" >&2
if $(echo $outPath | grep -q input-1); then if `echo $outPath | grep -q input-1`; then
echo "# accept" >&2 echo "# accept" >&2
read x read x
echo "got $x" echo "got $x"