From 6846ed8b442c20430b816a6b2ec926c841e38e0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Aug 2010 09:21:50 +0000 Subject: [PATCH 01/10] * Make --cores work when building through the Nix daemon. --- src/libstore/remote-store.cc | 3 +++ src/libstore/worker-protocol.hh | 2 +- src/nix-worker/nix-worker.cc | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9db90c872a..3d8b2a0c42 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -198,6 +198,9 @@ void RemoteStore::setOptions() writeInt(logType, to); writeInt(printBuildTrace, to); } + if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) { + writeInt(buildCores, to); + } processStderr(); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e44c1e36b5..5504773b8f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x105 +#define PROTOCOL_VERSION 0x106 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 0052fbb7d5..13cecf654e 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -500,6 +500,9 @@ static void performOp(unsigned int clientVersion, logType = (LogType) readInt(from); printBuildTrace = readInt(from) != 0; } + if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { + buildCores = readInt(from); + } startWork(); stopWork(); break; From 5fb824e896630741b6ca2c1c27d5f293f7d1f84e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Aug 2010 13:36:56 +0000 Subject: [PATCH 02/10] * Urgh, this was supposed to go in the trunk... --- configure.ac | 3 --- 1 file changed, 3 deletions(-) diff --git a/configure.ac b/configure.ac index 94a92c7ce5..094e6bd09c 100644 --- a/configure.ac +++ b/configure.ac @@ -243,9 +243,6 @@ AC_SUBST(bzip2_bin) AC_SUBST(bzip2_bin_test) -AC_CHECK_LIB(pthread, pthread_mutex_init) - - AC_ARG_ENABLE(init-state, AC_HELP_STRING([--disable-init-state], [do not initialise DB etc. in `make install']), init_state=$enableval, init_state=yes) From 12721a3a9a5b6f64b9150833a2e682daf9c03e4d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Aug 2010 12:38:32 +0000 Subject: [PATCH 03/10] * Nix 0.16 release notes. --- doc/manual/release-notes.xml | 53 ++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index e795634404..b3a324c97b 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -6,6 +6,59 @@ + + +
Release 0.16 (August 16, 2010) + +This release has the following improvements: + + + + + The Nix expression evaluator is now much faster in most + cases: typically, 3 + to 8 times compared to the old implementation. It also + uses less memory. It no longer depends on the ATerm + library. + + + + + Support for configurable parallelism inside builders. Build + scripts have always had the ability to perform multiple build + actions in parallel (for instance, by running make -j + 2), but this was not desirable because the number of + actions to be performed in parallel was not configurable. Nix + now has an option as well as a configuration + setting build-cores = + N that causes the + environment variable NIX_BUILD_CORES to be set to + N when the builder is invoked. The + builder can use this at its discretion to perform a parallel + build, e.g., by calling make -j + N. In Nixpkgs, this can be + enabled on a per-package basis by setting + enableParallelBuilding to + true. + + + + + nix-store -q now supports XML output + through the flag. + + + + Several bug fixes. + + + + +
+ +
Release 0.15 (March 17, 2010) From 8ec6594d6d77d73c7f961881f6575a56d81ca77f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Aug 2010 13:01:31 +0000 Subject: [PATCH 04/10] * Remove the "tarball" jobset argument. --- release.nix | 73 +++++++++++++++-------------------------------------- 1 file changed, 20 insertions(+), 53 deletions(-) diff --git a/release.nix b/release.nix index 3343f86580..6cc0d56a77 100644 --- a/release.nix +++ b/release.nix @@ -1,4 +1,7 @@ -{ nixpkgs ? ../nixpkgs }: +{ nixpkgs ? ../nixpkgs +, nix ? { outPath = ./.; rev = 1234; } +, officialRelease ? false +}: let @@ -6,10 +9,6 @@ let tarball = - { nix ? {outPath = ./.; rev = 1234;} - , officialRelease ? false - }: - with import nixpkgs {}; releaseTools.sourceTarball { @@ -60,17 +59,15 @@ let build = - { tarball ? jobs.tarball {} - , system ? "i686-linux" - }: + { system ? "i686-linux" }: - with import nixpkgs {inherit system;}; + with import nixpkgs { inherit system; }; releaseTools.nixBuild { name = "nix"; src = tarball; - buildInputs = [curl perl bzip2 openssl]; + buildInputs = [ curl perl bzip2 openssl ]; configureFlags = '' --disable-init-state @@ -79,44 +76,18 @@ let }; - /* - static = - { tarball ? jobs.tarball {} - , system ? "i686-linux" - }: - - with import nixpkgs {inherit system;}; - - releaseTools.binaryTarball { - name = "nix-static-tarball"; - src = tarball; - - buildInputs = [curl perl bzip2]; - - configureFlags = '' - --disable-init-state - --with-bzip2=${bzip2} - --enable-static-nix - ''; - }; - */ - - coverage = - { tarball ? jobs.tarball {} - }: - - with import nixpkgs {}; + with import nixpkgs { system = "x86_64-linux"; }; releaseTools.coverageAnalysis { name = "nix-build"; src = tarball; - buildInputs = [ - curl perl bzip2 openssl - # These are for "make check" only: - graphviz libxml2 libxslt - ]; + buildInputs = + [ curl perl bzip2 openssl + # These are for "make check" only: + graphviz libxml2 libxslt + ]; configureFlags = '' --disable-init-state --disable-shared @@ -168,17 +139,15 @@ let makeRPM = system: diskImageFun: prio: - { tarball ? jobs.tarball {} - }: - with import nixpkgs {inherit system;}; + with import nixpkgs { inherit system; }; releaseTools.rpmBuild rec { name = "nix-rpm-${diskImage.name}"; - src = tarball; + src = jobs.tarball; diskImage = diskImageFun vmTools.diskImages; memSize = 1024; - meta = { schedulingPriority = toString prio; }; + meta = { schedulingPriority = prio; }; }; @@ -187,19 +156,17 @@ let makeDeb = system: diskImageFun: prio: - { tarball ? jobs.tarball {} - }: - with import nixpkgs {inherit system;}; + with import nixpkgs { inherit system; }; releaseTools.debBuild { name = "nix-deb"; - src = tarball; + src = jobs.tarball; diskImage = diskImageFun vmTools.diskImages; memSize = 1024; - meta = { schedulingPriority = toString prio; }; + meta = { schedulingPriority = prio; }; configureFlags = "--sysconfdir=/etc"; - debRequires = ["curl"]; + debRequires = [ "curl" ]; }; From b75e1043a3a4bd72086e4007a53f0a1c3211c9fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 16 Aug 2010 13:23:45 +0000 Subject: [PATCH 05/10] * Typo. --- doc/manual/nix-push.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/nix-push.xml b/doc/manual/nix-push.xml index 1435c55949..d2a7e063e5 100644 --- a/doc/manual/nix-push.xml +++ b/doc/manual/nix-push.xml @@ -120,7 +120,7 @@ dependencies used in the build, such as compilers). dependencies, we can do: -$ nix-push urls $(nix-instantiate $(nix-store -r foo.nix)) +$ nix-push urls $(nix-store -r $(nix-instantiate foo.nix)) From 86f65edf4e519224eaa35ebc6bcf98bffe04b3d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Aug 2010 07:22:05 +0000 Subject: [PATCH 06/10] * Document --cores in the manual. --- doc/manual/conf-file.xml | 19 +++++++++++++++++++ doc/manual/opt-common-syn.xml | 4 ++++ doc/manual/opt-common.xml | 20 +++++++++++++++++++- doc/manual/release-notes.xml | 6 +++--- nix.conf.example | 2 +- 5 files changed, 46 insertions(+), 5 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 2ee2680970..ec64e8b114 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -97,6 +97,25 @@ env-keep-derivations = false + build-cores + + Sets the value of the + NIX_BUILD_CORES environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + enableParallelBuilding is set to + true, the builder passes the + flag to GNU Make. + It can be overriden using the command line switch and + defaults to 1. The value 0 + means that the builder should use all available CPU cores in the + system. + + + + build-max-silent-time diff --git a/doc/manual/opt-common-syn.xml b/doc/manual/opt-common-syn.xml index 0412fcb6ab..da60288b2a 100644 --- a/doc/manual/opt-common-syn.xml +++ b/doc/manual/opt-common-syn.xml @@ -13,6 +13,10 @@ number + + + number + number diff --git a/doc/manual/opt-common.xml b/doc/manual/opt-common.xml index 3be9f5dd5f..2d67162d7c 100644 --- a/doc/manual/opt-common.xml +++ b/doc/manual/opt-common.xml @@ -98,7 +98,25 @@ linkend='conf-build-max-jobs'>build-max-jobs configuration setting, which itself defaults to 1. A higher value is useful on SMP systems or to - exploit I/O latency. + exploit I/O latency. + + + + + + + Sets the value of the NIX_BUILD_CORES + environment variable in the invocation of builders. Builders can + use this variable at their discretion to control the maximum amount + of parallelism. For instance, in Nixpkgs, if the derivation + attribute enableParallelBuilding is set to + true, the builder passes the + flag to GNU Make. + It defaults to the value of the build-cores + configuration setting, if set, or 1 otherwise. + The value 0 means that the builder should use all + available CPU cores in the system. diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index b3a324c97b..5b1c30bf82 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -8,7 +8,7 @@ -
Release 0.16 (August 16, 2010) +
Release 0.16 (August 17, 2010) This release has the following improvements: @@ -39,8 +39,8 @@ builder can use this at its discretion to perform a parallel build, e.g., by calling make -j N. In Nixpkgs, this can be - enabled on a per-package basis by setting - enableParallelBuilding to + enabled on a per-package basis by setting the derivation + attribute enableParallelBuilding to true. diff --git a/nix.conf.example b/nix.conf.example index 7063bed54c..3c55b570e6 100644 --- a/nix.conf.example +++ b/nix.conf.example @@ -63,7 +63,7 @@ # # This option defines the number of CPU cores to utilize in parallel # within a build job, i.e. by passing an appropriate `-jN' flag to GNU -# make. The default is 1, meaning that parallel building within jobs +# Make. The default is 1, meaning that parallel building within jobs # is disabled. Passing the special value `0' causes Nix to try and # auto-detect the number of available cores on the local host. This # setting can be overridden using the `--cores' command line switch. From 2de17f4edcd7ee66b1094d777cdb62ec4a786a8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Aug 2010 10:06:26 +0000 Subject: [PATCH 07/10] * Update date. --- doc/manual/manual.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/manual.xml b/doc/manual/manual.xml index df926841aa..090a92ce6b 100644 --- a/doc/manual/manual.xml +++ b/doc/manual/manual.xml @@ -27,10 +27,11 @@ 2007 2008 2009 + 2010 Eelco Dolstra - September 2009 + August 2010 From ed133e6e6443646a21d87cf9a8eb0263f7c3f3fc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Aug 2010 14:08:44 +0000 Subject: [PATCH 08/10] * rpmBuild already includes the disk image in the output name. --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 6cc0d56a77..695521d398 100644 --- a/release.nix +++ b/release.nix @@ -143,7 +143,7 @@ let with import nixpkgs { inherit system; }; releaseTools.rpmBuild rec { - name = "nix-rpm-${diskImage.name}"; + name = "nix-rpm"; src = jobs.tarball; diskImage = diskImageFun vmTools.diskImages; memSize = 1024; From 2c8e070e5d75e1d1f610c8493b359a194c30e3b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Aug 2010 15:39:35 +0000 Subject: [PATCH 09/10] * Bump the version number. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index a59ac46083..9f8e9b69a3 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.16 \ No newline at end of file +1.0 \ No newline at end of file From c67eccc26d42bce9496024ac77eaa7e4ff725d01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 09:21:59 +0000 Subject: [PATCH 10/10] * nix-build: pass --cores. --- scripts/nix-build.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index d27eb08e22..ed85d57121 100644 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -102,7 +102,7 @@ EOF $n += 2; } - elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type") { + elsif ($arg eq "--max-jobs" or $arg eq "-j" or $arg eq "--max-silent-time" or $arg eq "--log-type" or $arg eq "--cores") { $n++; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]);