From eac5841970737b799c55ec78f6ace6d80762ff04 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 15 May 2014 11:30:46 -0400 Subject: [PATCH 01/41] Provide a more useful error message when a dynamic attr lookup fails --- src/libexpr/eval.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0f7e8e385a..81abe5b618 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -757,8 +757,16 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } } else { state.forceAttrs(*vAttrs, pos); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - throwEvalError("attribute `%1%' missing, at %2%", showAttrPath(attrPath), pos); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + AttrPath staticPath; + AttrPath::const_iterator j; + for (j = attrPath.begin(); j != i; ++j) + staticPath.push_back(AttrName(getName(*j, state, env))); + staticPath.push_back(AttrName(getName(*j, state, env))); + for (j = j + 1; j != attrPath.end(); ++j) + staticPath.push_back(*j); + throwEvalError("attribute `%1%' missing, at %2%", showAttrPath(staticPath), pos); + } } vAttrs = j->value; pos2 = j->pos; From 9f9080e2c019f188ba679a7a89284d7eaf629710 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 21 May 2014 17:19:36 +0200 Subject: [PATCH 02/41] nix-store -l: Fetch build logs from the Internet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a build log is not available locally, then ‘nix-store -l’ will now try to download it from the servers listed in the ‘log-servers’ option in nix.conf. For instance, if you have: log-servers = http://hydra.nixos.org/log then it will try to get logs from http://hydra.nixos.org/log/. So you can do things like: $ nix-store -l $(which xterm) and get a log even if xterm wasn't built locally. --- Makefile.config.in | 1 + doc/manual/conf-file.xml | 14 +++ doc/manual/nix-store.xml | 183 +++++++++++++++++++------------------ src/libstore/globals.cc | 1 + src/libstore/globals.hh | 3 + src/libutil/util.cc | 2 +- src/libutil/util.hh | 2 + src/nix-store/local.mk | 2 + src/nix-store/nix-store.cc | 29 +++++- 9 files changed, 144 insertions(+), 93 deletions(-) diff --git a/Makefile.config.in b/Makefile.config.in index 0937483322..53bdbbf610 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -10,6 +10,7 @@ PACKAGE_VERSION = @PACKAGE_VERSION@ bash = @bash@ bindir = @bindir@ bsddiff_compat_include = @bsddiff_compat_include@ +curl = @curl@ datadir = @datadir@ datarootdir = @datarootdir@ dblatex = @dblatex@ diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 327d22c4a1..29f7f9c51a 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -465,6 +465,20 @@ flag, e.g. --option gc-keep-outputs false. + log-servers + + + + A list of URL prefixes (such as + http://hydra.nixos.org/log) from which + nix-store -l will try to fetch build logs if + they’re not available locally. + + + + + + diff --git a/doc/manual/nix-store.xml b/doc/manual/nix-store.xml index 87cbc307a9..c9a912ff0e 100644 --- a/doc/manual/nix-store.xml +++ b/doc/manual/nix-store.xml @@ -111,7 +111,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 - + @@ -120,7 +120,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 - + @@ -141,7 +141,7 @@ lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r1134 Description - + The operation essentially “builds” the specified store paths. Realisation is a somewhat overloaded term: @@ -196,14 +196,14 @@ printed.) - + Examples This operation is typically used to build store derivations produced by nix-instantiate: - + $ nix-store -r $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 @@ -216,7 +216,7 @@ linkend="sec-nix-build">nix-build does. - + @@ -239,7 +239,7 @@ linkend="sec-nix-build">nix-build does. Description - + Without additional flags, the operation performs a garbage collection on the Nix store. That is, all paths in the Nix store not reachable via file system references from a set of @@ -250,40 +250,40 @@ the Nix store not reachable via file system references from a set of - + This operation prints on standard output the set of roots used by the garbage collector. What constitutes a root is described in . - + - + This operation prints on standard output the set of “live” store paths, which are all the store paths reachable from the roots. Live paths should never be deleted, since that would break consistency — it would become possible that applications are installed that reference things that are no longer present in the store. - + - + This operation prints out on standard output the set of “dead” store paths, which is just the opposite of the set of live paths: any path in the store that is not live (with respect to the roots) is dead. - + - + This operation performs an actual garbage collection. All dead paths are removed from the store. This is the default. - + @@ -294,7 +294,7 @@ options control what gets deleted and in what order: bytes - + Keep deleting paths until at least bytes bytes have been deleted, then stop. The argument bytes can be @@ -302,7 +302,7 @@ options control what gets deleted and in what order: M, G or T, denoting KiB, MiB, GiB or TiB units. - + @@ -326,7 +326,7 @@ would be freed. Examples To delete all unreachable paths, just do: - + $ nix-store --gc deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' @@ -348,7 +348,7 @@ $ nix-store --gc --max-freed $((100 * 1024 * 1024)) - + Operation <option>--delete</option> @@ -433,7 +433,7 @@ error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' Description - + The operation displays various bits of information about the store paths . The queries are described below. At most one query can be specified. The default query is @@ -453,16 +453,16 @@ query is applied to the target of the symlink. - + For each argument to the query that is a store derivation, apply the query to the output path of the derivation instead. - + - + Realise each argument to the query first (see nix-store --realise). @@ -470,12 +470,12 @@ query is applied to the target of the symlink. - + - + Queries - + @@ -485,7 +485,7 @@ query is applied to the target of the symlink. derivations paths. These are the paths that will be produced when the derivation is built. - + @@ -500,10 +500,10 @@ query is applied to the target of the symlink. - + Also include the output path of store derivations, and their closures. - + @@ -517,13 +517,13 @@ query is applied to the target of the symlink. including binaries of build-time-only dependencies) is obtained by distributing the closure of a store derivation and specifying the option . - + - + Prints the set of references of the store paths paths, that is, their immediate @@ -531,9 +531,9 @@ query is applied to the target of the symlink. .) - + - + Prints the set of referrers of the store paths paths, that is, the store paths currently existing in the Nix store that refer to one @@ -542,9 +542,9 @@ query is applied to the target of the symlink. store paths are added or removed. - + - + Prints the closure of the set of store paths paths under the referrers relation; that is, all store paths that directly or indirectly refer to one of @@ -555,7 +555,7 @@ query is applied to the target of the symlink. - + Prints the deriver of the store paths paths. If the path has no deriver @@ -566,7 +566,7 @@ query is applied to the target of the symlink. - + Prints the references graph of the store paths paths in the format of the dot tool of AT&T's - + Prints the references graph of the store paths paths as a nested ASCII tree. References are ordered by descending closure size; this tends to @@ -591,7 +591,7 @@ query is applied to the target of the symlink. name - + Prints the value of the attribute name (i.e., environment variable) of the store derivations paths. It is an @@ -601,7 +601,7 @@ query is applied to the target of the symlink. - + Prints the SHA-256 hash of the contents of the store paths paths (that is, the hash of the output of nix-store --dump on the given @@ -611,7 +611,7 @@ query is applied to the target of the symlink. - + Prints the size in bytes of the contents of the store paths paths — to be precise, the size of the output of nix-store --dump on the @@ -622,7 +622,7 @@ query is applied to the target of the symlink. - + Prints the garbage collector roots that point, directly or indirectly, at the store paths paths. @@ -638,7 +638,7 @@ query is applied to the target of the symlink. Print the closure (runtime dependencies) of the svn program in the current user environment: - + $ nix-store -qR $(which svn) /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 @@ -723,7 +723,7 @@ $ nix-store -q --roots $(which svn) - + @@ -739,11 +739,11 @@ $ nix-store -q --roots $(which svn) Description - + TODO - + --> @@ -798,7 +798,7 @@ $ nix-store --add ./foo.c Description - + The operation verifies the internal consistency of the Nix database, and the consistency between the Nix database and the Nix store. Any inconsistencies encountered are @@ -811,32 +811,32 @@ in Nix itself. - + Checks that the contents of every valid store path has not been altered by computing a SHA-256 hash of the contents and comparing it with the hash stored in the Nix database at build time. Paths that have been modified are printed out. For large stores, is obviously quite slow. - + - + - + If any valid path is missing from the store, or (if is given) the contents of a valid path has been modified, then try to repair the path by redownloading it. See nix-store --repair-path for details. - + - + - + @@ -855,7 +855,7 @@ in Nix itself. Description - + The operation compares the contents of the given store paths to their cryptographic hashes stored in Nix’s database. For every changed path, it prints a warning @@ -863,7 +863,7 @@ message. The exit status is 0 if no path has changed, and 1 otherwise. - + Example To verify the integrity of the svn command and all its dependencies: @@ -875,7 +875,7 @@ $ nix-store --verify-path $(nix-store -qR $(which svn)) - + @@ -893,7 +893,7 @@ $ nix-store --verify-path $(nix-store -qR $(which svn)) Description - + The operation attempts to “repair” the specified paths by redownloading them using the available substituters. If no substitutes are available, then repair is not @@ -906,7 +906,7 @@ system may be left in a broken state (e.g., if the path contains a critical system component like the GNU C Library). - + Example @@ -921,7 +921,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... - + @@ -939,7 +939,7 @@ fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... Description - + The operation produces a NAR (Nix ARchive) file containing the contents of the file system tree rooted at path. The archive is written to @@ -970,7 +970,7 @@ links, but not other types of files (such as device nodes). --restore. - + @@ -989,13 +989,13 @@ links, but not other types of files (such as device nodes). Description - + The operation unpacks a NAR archive to path, which must not already exist. The archive is read from standard input. - + @@ -1014,7 +1014,7 @@ archive is read from standard input. Description - + The operation writes a serialisation of the specified store paths to standard output in a format that can be imported into another Nix store with nix-copy-closure command. - + @@ -1059,7 +1059,7 @@ command. Description - + The operation reads a serialisation of a set of store paths produced by nix-store --export from @@ -1069,7 +1069,7 @@ another path that doesn’t exist in the Nix store, the import fails. - + @@ -1087,7 +1087,7 @@ fails. Description - + The operation reduces Nix store disk space usage by finding identical files in the store and hard-linking them to each other. It typically reduces the size of the store by @@ -1104,7 +1104,7 @@ on the achieved savings is printed on standard error. progress indication. - + Example @@ -1138,7 +1138,7 @@ there are 114486 files with equal contents out of 215894 files in total Description - + The operation prints the build log of the specified store paths on standard output. The build log is whatever the builder of a derivation wrote to standard output and @@ -1147,12 +1147,17 @@ the store path is used. Build logs are kept in /nix/var/log/nix/drvs. However, there is no -guarantee that a build log is available for any particular store -path. For instance, if the path was downloaded as a pre-built binary -through a substitute, then the log is unavailable. +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through +a substitute, then the log is unavailable. If the log is not available +locally, then nix-store will try to download the +log from the servers specified in the Nix option +. For example, if it’s set to +http://hydra.nixos.org/log, then Nix will check +http://hydra.nixos.org/log/base-name. - + Example @@ -1184,14 +1189,14 @@ ktorrent-2.2.1/NEWS Description - + The operation writes a dump of the Nix database to standard output. It can be loaded into an empty Nix store using . This is useful for making backups and when migrating to different database schemas. - + @@ -1208,13 +1213,13 @@ backups and when migrating to different database schemas. Description - + The operation reads a dump of the Nix database created by from standard input and loads it into the Nix database. - + @@ -1232,14 +1237,14 @@ loads it into the Nix database. Description - + The operation prints out the environment of a derivation in a format that can be evaluated by a shell. The command line arguments of the builder are placed in the variable _args. - + Example @@ -1252,7 +1257,7 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil - + @@ -1269,14 +1274,14 @@ export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-buil Description - + If build failure caching is enabled through the build-cache-failures configuration option, the operation will print out all store paths that have failed to build. - + Example @@ -1288,7 +1293,7 @@ $ nix-store --query-failed-paths - + @@ -1306,7 +1311,7 @@ $ nix-store --query-failed-paths Description - + If build failure caching is enabled through the build-cache-failures configuration option, the operation clears the “failed” @@ -1319,7 +1324,7 @@ You can provide the argument * to clear all store paths. - + Example @@ -1328,7 +1333,7 @@ $ nix-store --clear-failed-paths * - + @@ -1341,6 +1346,6 @@ $ nix-store --clear-failed-paths * - + diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 739199d48e..180344e336 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -147,6 +147,7 @@ void Settings::update() get(envKeepDerivations, "env-keep-derivations"); get(sshSubstituterHosts, "ssh-substituter-hosts"); get(useSshSubstituter, "use-ssh-substituter"); + get(logServers, "log-servers"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 711c365294..65a6c388b8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -197,6 +197,9 @@ struct Settings { /* Whether to show a stack trace if Nix evaluation fails. */ bool showTrace; + /* A list of URL prefixes that can return Nix build logs. */ + Strings logServers; + private: SettingsMap settings, overrides; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 846674a29d..8fc78b1463 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -901,7 +901,7 @@ string runProgram(Path program, bool searchPath, const Strings & args) /* Wait for the child to finish. */ int status = pid.wait(true); if (!statusOk(status)) - throw Error(format("program `%1%' %2%") + throw ExecError(format("program `%1%' %2%") % program % statusToString(status)); return result; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ce2d77c19a..1e9ffcf51b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -257,6 +257,8 @@ void killUser(uid_t uid); string runProgram(Path program, bool searchPath = false, const Strings & args = Strings()); +MakeError(ExecError, Error) + /* Close all file descriptors except stdin, stdout, stderr, and those listed in the given set. Good practice in child processes. */ void closeMostFDs(const set & exceptions); diff --git a/src/nix-store/local.mk b/src/nix-store/local.mk index 7f93e4c191..dc049f348f 100644 --- a/src/nix-store/local.mk +++ b/src/nix-store/local.mk @@ -7,3 +7,5 @@ nix-store_SOURCES := $(wildcard $(d)/*.cc) nix-store_LIBS = libmain libstore libutil libformat nix-store_LDFLAGS = -lbz2 + +nix-store_CXXFLAGS = -DCURL=\"$(curl)\" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5da401c77f..4fee7258cb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -467,10 +467,11 @@ static void opReadLog(Strings opFlags, Strings opArgs) foreach (Strings::iterator, i, opArgs) { Path path = useDeriver(followLinksToStorePath(*i)); - for (int j = 0; j <= 2; j++) { - if (j == 2) throw Error(format("build log of derivation `%1%' is not available") % path); + string baseName = baseNameOf(path); + bool found = false; + + for (int j = 0; j < 2; j++) { - string baseName = baseNameOf(path); Path logPath = j == 0 ? (format("%1%/%2%/%3%/%4%") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2) % string(baseName, 2)).str() @@ -481,6 +482,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) /* !!! Make this run in O(1) memory. */ string log = readFile(logPath); writeFull(STDOUT_FILENO, (const unsigned char *) log.data(), log.size()); + found = true; break; } @@ -500,9 +502,30 @@ static void opReadLog(Strings opFlags, Strings opArgs) writeFull(STDOUT_FILENO, buf, n); } while (err != BZ_STREAM_END); BZ2_bzReadClose(&err, bz); + found = true; break; } } + + if (!found) { + for (auto & i : settings.logServers) { + string prefix = i; + if (!prefix.empty() && prefix.back() != '/') prefix += '/'; + string url = prefix + baseName; + try { + string log = runProgram(CURL, true, {"--fail", "--location", "--silent", "--", url}); + std::cout << "(using build log from " << url << ")" << std::endl; + std::cout << log; + found = true; + break; + } catch (ExecError & e) { + /* Ignore errors from curl. FIXME: actually, might be + nice to print a warning on HTTP status != 404. */ + } + } + } + + if (!found) throw Error(format("build log of derivation `%1%' is not available") % path); } } From 3064a8215608eca391fcb9d492735a662f48242e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 May 2014 11:38:50 +0200 Subject: [PATCH 03/41] Disable parallel.sh test It breaks randomly: http://hydra.nixos.org/build/11152871 --- tests/local.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/local.mk b/tests/local.mk index ab170dfe51..add6d7b7b4 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -3,7 +3,7 @@ check: nix_tests = \ init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ - parallel.sh build-hook.sh substitutes.sh substitutes2.sh \ + build-hook.sh substitutes.sh substitutes2.sh \ fallback.sh nix-push.sh gc.sh gc-concurrent.sh nix-pull.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ @@ -11,6 +11,7 @@ nix_tests = \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh nix-profile.sh repair.sh dump-db.sh + # parallel.sh install-tests += $(foreach x, $(nix_tests), tests/$(x)) From 0321ef9bb261958fe4d63210e9a9d3350737ef18 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 May 2014 14:43:55 +0200 Subject: [PATCH 04/41] Ugly hack to allow --argstr values starting with a dash Fixes #265. --- src/libmain/shared.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a390654452..14263446fe 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -220,6 +220,13 @@ static void initAndRun(int argc, char * * argv) string value = *i; settings.set(name, value); } + else if (arg == "--arg" || arg == "--argstr") { + remaining.push_back(arg); + ++i; if (i == args.end()) throw UsageError(format("`%1%' requires two arguments") % arg); + remaining.push_back(*i); + ++i; if (i == args.end()) throw UsageError(format("`%1%' requires two arguments") % arg); + remaining.push_back(*i); + } else remaining.push_back(arg); } From f0fdbd0897ce63c138ec663ed89a94709a8441a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 12:34:15 +0200 Subject: [PATCH 05/41] Shut up some signedness warnings --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c69d520d3b..f402478dd9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -944,7 +944,7 @@ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Valu static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) { state.forceList(list, pos); - if (n < 0 || n >= list.list.length) + if (n < 0 || (unsigned int) n >= list.list.length) throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); state.forceValue(*list.list.elems[n]); v = *list.list.elems[n]; @@ -1123,7 +1123,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V if (start < 0) throw EvalError(format("negative start position in `substring', at %1%") % pos); - mkString(v, start >= s.size() ? "" : string(s, start, len), context); + mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } From c273c15cb13bb86420dda1e5341a4e19517532b5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 13:46:11 +0200 Subject: [PATCH 06/41] =?UTF-8?q?Add=20primop=20=E2=80=98scopedImport?= =?UTF-8?q?=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ‘scopedImport’ works like ‘import’, except that it takes a set of attributes to be added to the lexical scope of the expression, essentially extending or overriding the builtin variables. For instance, the expression scopedImport { x = 1; } ./foo.nix where foo.nix contains ‘x’, will evaluate to 1. This has a few applications: * It allows getting rid of function argument specifications in package expressions. For instance, a package expression like: { stdenv, fetchurl, libfoo }: stdenv.mkDerivation { ... buildInputs = [ libfoo ]; } can now we written as just stdenv.mkDerivation { ... buildInputs = [ libfoo ]; } and imported in all-packages.nix as: bar = scopedImport pkgs ./bar.nix; So whereas we once had dependencies listed in three places (buildInputs, the function, and the call site), they now only need to appear in one place. * It allows overriding builtin functions. For instance, to trace all calls to ‘map’: let overrides = { map = f: xs: builtins.trace "map called!" (map f xs); # Ensure that our override gets propagated by calls to # import/scopedImport. import = fn: scopedImport overrides fn; scopedImport = attrs: fn: scopedImport (overrides // attrs) fn; # Also update ‘builtins’. builtins = builtins // overrides; }; in scopedImport overrides ./bla.nix * Similarly, it allows extending the set of builtin functions. For instance, during Nixpkgs/NixOS evaluation, the Nixpkgs library functions could be added to the default scope. There is a downside: calls to scopedImport are not memoized, unlike import. So importing a file multiple times leads to multiple parsings / evaluations. It would be possible to construct the AST only once, but that would require careful handling of variables/environments. --- src/libexpr/eval.hh | 1 + src/libexpr/nixexpr.hh | 1 - src/libexpr/parser.y | 10 ++++++++-- src/libexpr/primops.cc | 25 +++++++++++++++++++++++++ tests/lang/eval-okay-import.exp | 1 + tests/lang/eval-okay-import.nix | 11 +++++++++++ tests/lang/imported.nix | 3 +++ tests/lang/imported2.nix | 1 + 8 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 tests/lang/eval-okay-import.exp create mode 100644 tests/lang/eval-okay-import.nix create mode 100644 tests/lang/imported.nix create mode 100644 tests/lang/imported2.nix diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 6d4cb8abe8..ad4c6a4b5f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -124,6 +124,7 @@ public: /* Parse a Nix expression from the specified file. */ Expr * parseExprFromFile(const Path & path); + Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); /* Parse a Nix expression from the specified string. */ Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 813efbe21a..fbd5bad81e 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -336,5 +336,4 @@ struct StaticEnv }; - } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index dbcffff996..06d6d643f6 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -592,7 +592,13 @@ Path resolveExprPath(Path path) Expr * EvalState::parseExprFromFile(const Path & path) { - return parse(readFile(path).c_str(), path, dirOf(path), staticBaseEnv); + return parseExprFromFile(path, staticBaseEnv); +} + + +Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) +{ + return parse(readFile(path).c_str(), path, dirOf(path), staticEnv); } @@ -608,7 +614,7 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) } - void EvalState::addToSearchPath(const string & s, bool warn) +void EvalState::addToSearchPath(const string & s, bool warn) { size_t pos = s.find('='); string prefix; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f402478dd9..533ae37684 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -93,6 +93,30 @@ static void prim_import(EvalState & state, const Pos & pos, Value * * args, Valu } +static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceAttrs(*args[0]); + Path path = resolveExprPath(state.coerceToPath(pos, *args[1], context)); + + Env * env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; + + StaticEnv staticEnv(false, &state.staticBaseEnv); + + unsigned int displ = 0; + for (auto & attr : *args[0]->attrs) { + staticEnv.vars[attr.name] = displ; + env->values[displ++] = attr.value; + } + + startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path); + Expr * e = state.parseExprFromFile(path, staticEnv); + + e->eval(state, *env, v); +} + + /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1247,6 +1271,7 @@ void EvalState::createBaseEnv() // Miscellaneous addPrimOp("import", 1, prim_import); + addPrimOp("scopedImport", 2, prim_scopedImport); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); diff --git a/tests/lang/eval-okay-import.exp b/tests/lang/eval-okay-import.exp new file mode 100644 index 0000000000..c508125b55 --- /dev/null +++ b/tests/lang/eval-okay-import.exp @@ -0,0 +1 @@ +[ 1 2 3 4 5 6 7 8 9 10 ] diff --git a/tests/lang/eval-okay-import.nix b/tests/lang/eval-okay-import.nix new file mode 100644 index 0000000000..0b18d94131 --- /dev/null +++ b/tests/lang/eval-okay-import.nix @@ -0,0 +1,11 @@ +let + + overrides = { + import = fn: scopedImport overrides fn; + + scopedImport = attrs: fn: scopedImport (overrides // attrs) fn; + + builtins = builtins // overrides; + } // import ./lib.nix; + +in scopedImport overrides ./imported.nix diff --git a/tests/lang/imported.nix b/tests/lang/imported.nix new file mode 100644 index 0000000000..fb39ee4efa --- /dev/null +++ b/tests/lang/imported.nix @@ -0,0 +1,3 @@ +# The function ‘range’ comes from lib.nix and was added to the lexical +# scope by scopedImport. +range 1 5 ++ import ./imported2.nix diff --git a/tests/lang/imported2.nix b/tests/lang/imported2.nix new file mode 100644 index 0000000000..6d0a2992b7 --- /dev/null +++ b/tests/lang/imported2.nix @@ -0,0 +1 @@ +range 6 10 From a8edf185a9e1677088c8c30acc9d281c8350bca7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 14:55:47 +0200 Subject: [PATCH 07/41] =?UTF-8?q?Add=20constant=20=E2=80=98nixPath?= =?UTF-8?q?=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It contains the Nix expression search path as a list of { prefix, path } sets, e.g. [ { path = "/nix/var/nix/profiles/per-user/root/channels/nixos"; prefix = ""; } { path = "/etc/nixos/configuration.nix"; prefix = "nixos-config"; } { path = "/home/eelco/Dev/nix/inst/share/nix/corepkgs"; prefix = "nix"; } ] --- src/libexpr/primops.cc | 11 +++++++++++ tests/lang.sh | 2 +- tests/lang/eval-okay-search-path.nix | 9 ++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 533ae37684..e492ff683a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1352,6 +1352,17 @@ void EvalState::createBaseEnv() evalFile(path, v); addConstant("derivation", v); + /* Add a value containing the current Nix expression search path. */ + mkList(v, searchPath.size()); + int n = 0; + for (auto & i : searchPath) { + Value * v2 = v.list.elems[n++] = allocValue(); + mkAttrs(*v2, 2); + mkString(*allocAttr(*v2, symbols.create("path")), i.second); + mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); + } + addConstant("nixPath", v); + /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); diff --git a/tests/lang.sh b/tests/lang.sh index bb3b9ca775..7157a68c5c 100644 --- a/tests/lang.sh +++ b/tests/lang.sh @@ -46,7 +46,7 @@ for i in lang/eval-okay-*.nix; do if test -e lang/$i.flags; then flags=$(cat lang/$i.flags) fi - if ! NIX_PATH=lang/dir3:lang/dir4_PATH nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then echo "FAIL: $i should evaluate" fail=1 elif ! diff lang/$i.out lang/$i.exp; then diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index 9b71150233..c9ea768a41 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -1,3 +1,10 @@ -assert builtins.pathExists ; +with import ./lib.nix; +with builtins; + +assert pathExists ; + +assert length nixPath == 3; +assert length (filter (x: x.prefix == "nix") nixPath) == 1; +assert length (filter (x: baseNameOf x.path == "dir4") nixPath) == 1; import + import + import + import From 39d72640c2459dc2fa689bfe8b756ee193f7b98a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 16:50:36 +0200 Subject: [PATCH 08/41] Ensure that -I flags get included in nixPath Also fixes #261. --- src/libexpr/common-opts.cc | 4 ++-- src/libexpr/common-opts.hh | 4 ++-- src/libexpr/eval.cc | 7 +++---- src/libexpr/eval.hh | 3 +-- src/libexpr/parser.y | 2 +- src/nix-env/nix-env.cc | 15 ++++++++++++--- src/nix-instantiate/nix-instantiate.cc | 14 +++++++++++--- tests/lang/eval-okay-search-path.nix | 2 +- 8 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 9b3421f6c4..14a75f7b6f 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -35,11 +35,11 @@ bool parseOptionArg(const string & arg, Strings::iterator & i, bool parseSearchPathArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state) + const Strings::iterator & argsEnd, Strings & searchPath) { if (arg != "-I") return false; if (i == argsEnd) throw UsageError(format("`%1%' requires an argument") % arg);; - state.addToSearchPath(*i++, true); + searchPath.push_back(*i++); return true; } diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index e2e3fe7717..759358950f 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -8,9 +8,9 @@ namespace nix { bool parseOptionArg(const string & arg, Strings::iterator & i, const Strings::iterator & argsEnd, EvalState & state, Bindings & autoArgs); - + bool parseSearchPathArg(const string & arg, Strings::iterator & i, - const Strings::iterator & argsEnd, EvalState & state); + const Strings::iterator & argsEnd, Strings & searchPath); Path lookupFileArg(EvalState & state, string s); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81abe5b618..b6b69c2bdb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -153,7 +153,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } -EvalState::EvalState() +EvalState::EvalState(const Strings & _searchPath) : sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -219,11 +219,10 @@ EvalState::EvalState() #endif /* Initialise the Nix expression search path. */ - searchPathInsertionPoint = searchPath.end(); Strings paths = tokenizeString(getEnv("NIX_PATH", ""), ":"); - foreach (Strings::iterator, i, paths) addToSearchPath(*i); + for (auto & i : _searchPath) addToSearchPath(i); + for (auto & i : paths) addToSearchPath(i); addToSearchPath("nix=" + settings.nixDataDir + "/nix/corepkgs"); - searchPathInsertionPoint = searchPath.begin(); createBaseEnv(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index ad4c6a4b5f..200ec75e0f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -113,11 +113,10 @@ private: typedef list > SearchPath; SearchPath searchPath; - SearchPath::iterator searchPathInsertionPoint; public: - EvalState(); + EvalState(const Strings & _searchPath); ~EvalState(); void addToSearchPath(const string & s, bool warn = false); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 06d6d643f6..698e8ce3ff 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -629,7 +629,7 @@ void EvalState::addToSearchPath(const string & s, bool warn) path = absPath(path); if (pathExists(path)) { debug(format("adding path `%1%' to the search path") % path); - searchPath.insert(searchPathInsertionPoint, std::pair(prefix, path)); + searchPath.push_back(std::pair(prefix, path)); } else if (warn) printMsg(lvlError, format("warning: Nix search path entry `%1%' does not exist, ignoring") % path); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3db84ff5c7..2d38f2aea7 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -58,6 +58,7 @@ struct Globals bool removeAll; string forceName; bool prebuiltOnly; + Globals(const Strings & searchPath) : state(searchPath) { } }; @@ -1351,7 +1352,17 @@ void run(Strings args) Strings opFlags, opArgs, remaining; Operation op = 0; - Globals globals; + /* FIXME: hack. */ + Strings searchPath; + Strings args2; + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + if (!parseSearchPathArg(arg, i, args.end(), searchPath)) + args2.push_back(arg); + } + args = args2; + + Globals globals(searchPath); globals.instSource.type = srcUnknown; globals.instSource.nixExprPath = getDefNixExprPath(); @@ -1372,8 +1383,6 @@ void run(Strings args) else if (parseOptionArg(arg, i, args.end(), globals.state, globals.instSource.autoArgs)) ; - else if (parseSearchPathArg(arg, i, args.end(), globals.state)) - ; else if (arg == "--force-name") // undocumented flag for nix-install-package globals.forceName = needArg(i, args, arg); else if (arg == "--uninstall" || arg == "-e") diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 7cdabcb924..cdd74523ca 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -91,7 +91,17 @@ void processExpr(EvalState & state, const Strings & attrPaths, void run(Strings args) { - EvalState state; + /* FIXME: hack. */ + Strings searchPath; + Strings args2; + for (Strings::iterator i = args.begin(); i != args.end(); ) { + string arg = *i++; + if (!parseSearchPathArg(arg, i, args.end(), searchPath)) + args2.push_back(arg); + } + args = args2; + + EvalState state(searchPath); Strings files; bool readStdin = false; bool fromArgs = false; @@ -127,8 +137,6 @@ void run(Strings args) } else if (parseOptionArg(arg, i, args.end(), state, autoArgs)) ; - else if (parseSearchPathArg(arg, i, args.end(), state)) - ; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root' requires an argument"); diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index c9ea768a41..501d5f3f48 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -3,7 +3,7 @@ with builtins; assert pathExists ; -assert length nixPath == 3; +assert length nixPath == 6; assert length (filter (x: x.prefix == "nix") nixPath) == 1; assert length (filter (x: baseNameOf x.path == "dir4") nixPath) == 1; From 62a6eeb1f3da0a5954ad2da54c454eb7fc1c6e5d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 17:02:22 +0200 Subject: [PATCH 09/41] Make the Nix search path declarative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nix search path lookups like are now desugared to ‘findFile nixPath ’, where ‘findFile’ is a new primop. Thus you can override the search path simply by saying let nixPath = [ { prefix = "nixpkgs"; path = "/my-nixpkgs"; } ]; in ... ... In conjunction with ‘scopedImport’ (commit c273c15cb13bb86420dda1e5341a4e19517532b5), the Nix search path can be propagated across imports, e.g. let overrides = { nixPath = [ ... ] ++ builtins.nixPath; import = fn: scopedImport overrides fn; scopedImport = attrs: fn: scopedImport (overrides // attrs) fn; builtins = builtins // overrides; }; in scopedImport overrides ./nixos --- src/libexpr/common-opts.cc | 4 +--- src/libexpr/eval.hh | 5 ++++- src/libexpr/nixexpr.hh | 1 + src/libexpr/parser.y | 23 ++++++++++---------- src/libexpr/primops.cc | 32 ++++++++++++++++++++++++++++ tests/lang/eval-okay-search-path.exp | 2 +- tests/lang/eval-okay-search-path.nix | 1 + 7 files changed, 51 insertions(+), 17 deletions(-) diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 14a75f7b6f..a3ea202e88 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -48,9 +48,7 @@ Path lookupFileArg(EvalState & state, string s) { if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); - Path p2 = state.findFile(p); - if (p2 == "") throw Error(format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % p); - return p2; + return state.findFile(p); } else return absPath(s); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 200ec75e0f..aa706cf983 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -86,6 +86,9 @@ typedef std::map SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); +typedef list > SearchPath; + + class EvalState { public: @@ -111,7 +114,6 @@ private: #endif FileEvalCache fileEvalCache; - typedef list > SearchPath; SearchPath searchPath; public: @@ -137,6 +139,7 @@ public: /* Look up a file in the search path. */ Path findFile(const string & path); + Path findFile(SearchPath & searchPath, const string & path); /* Evaluate an expression to normal form, storing the result in value `v'. */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index fbd5bad81e..8826ad2323 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -142,6 +142,7 @@ struct ExprVar : Expr unsigned int level; unsigned int displ; + ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 698e8ce3ff..134d68d6e1 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -386,17 +386,10 @@ expr_simple | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } | SPATH { string path($1 + 1, strlen($1) - 2); - Path path2 = data->state.findFile(path); - /* The file wasn't found in the search path. However, we can't - throw an error here, because the expression might never be - evaluated. So return an expression that lazily calls - ‘throw’. */ - $$ = path2 == "" - ? (Expr * ) new ExprApp( - new ExprBuiltin(data->symbols.create("throw")), - new ExprString(data->symbols.create( - (format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % path).str()))) - : (Expr * ) new ExprPath(path2); + $$ = new ExprApp(CUR_POS, + new ExprApp(new ExprVar(data->symbols.create("__findFile")), + new ExprVar(data->symbols.create("nixPath"))), + new ExprString(data->symbols.create(path))); } | URI { $$ = new ExprString(data->symbols.create($1)); } | '(' expr ')' { $$ = $2; } @@ -636,6 +629,12 @@ void EvalState::addToSearchPath(const string & s, bool warn) Path EvalState::findFile(const string & path) +{ + return findFile(searchPath, path); +} + + +Path EvalState::findFile(SearchPath & searchPath, const string & path) { foreach (SearchPath::iterator, i, searchPath) { Path res; @@ -650,7 +649,7 @@ Path EvalState::findFile(const string & path) } if (pathExists(res)) return canonPath(res); } - return ""; + throw ThrownError(format("file `%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)") % path); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e492ff683a..333748973d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -654,6 +654,37 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va } +/* Find a file in the Nix search path. Used to implement paths, + which are desugared to ‘findFile nixPath "x"’. */ +static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + + SearchPath searchPath; + + for (unsigned int n = 0; n < args[0]->list.length; ++n) { + Value & v2(*args[0]->list.elems[n]); + state.forceAttrs(v2, pos); + + string prefix; + Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); + if (i != v2.attrs->end()) + prefix = state.forceStringNoCtx(*i->value, pos); + + i = v2.attrs->find(state.symbols.create("path")); + if (i == v2.attrs->end()) + throw EvalError(format("attribute `path' missing, at %1%") % pos); + PathSet context; + string path = state.coerceToPath(pos, *i->value, context); + + searchPath.push_back(std::pair(prefix, path)); + } + + string path = state.forceStringNoCtx(*args[1], pos); + mkPath(v, state.findFile(searchPath, path).c_str()); +} + + /************************************************************* * Creating files *************************************************************/ @@ -1293,6 +1324,7 @@ void EvalState::createBaseEnv() addPrimOp("baseNameOf", 1, prim_baseNameOf); addPrimOp("dirOf", 1, prim_dirOf); addPrimOp("__readFile", 1, prim_readFile); + addPrimOp("__findFile", 2, prim_findFile); // Creating files addPrimOp("__toXML", 1, prim_toXML); diff --git a/tests/lang/eval-okay-search-path.exp b/tests/lang/eval-okay-search-path.exp index d0bc8c5e86..ad05904ba1 100644 --- a/tests/lang/eval-okay-search-path.exp +++ b/tests/lang/eval-okay-search-path.exp @@ -1 +1 @@ -"abcc" +"abcca" diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index 501d5f3f48..e0f433fec6 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -8,3 +8,4 @@ assert length (filter (x: x.prefix == "nix") nixPath) == 1; assert length (filter (x: baseNameOf x.path == "dir4") nixPath) == 1; import + import + import + import + + (let nixPath = [ { path = ./dir1; } { path = ./dir2; } ]; in import ) From d8c061e044a07f7516d76df12bc6920f4f04e5ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 17:14:28 +0200 Subject: [PATCH 10/41] Remove ExprBuiltin It's slower than ExprVar since it doesn't compute a static displacement. Since we're not using the throw primop in the implementation of <...> anymore, it's also not really needed. --- src/libexpr/eval.cc | 11 ----------- src/libexpr/nixexpr.cc | 9 --------- src/libexpr/nixexpr.hh | 7 ------- src/libexpr/parser.y | 16 ++++++++-------- 4 files changed, 8 insertions(+), 35 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b6b69c2bdb..81ce7d9a30 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1004,17 +1004,6 @@ void ExprOpNot::eval(EvalState & state, Env & env, Value & v) } -void ExprBuiltin::eval(EvalState & state, Env & env, Value & v) -{ - // Not a hot path at all, but would be nice to access state.baseEnv directly - Env *baseEnv = &env; - while (baseEnv->up) baseEnv = baseEnv->up; - Bindings::iterator binding = baseEnv->values[0]->attrs->find(name); - assert(binding != baseEnv->values[0]->attrs->end()); - v = *binding->value; -} - - void ExprOpEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d40250d877..b916a26d21 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -123,11 +123,6 @@ void ExprOpNot::show(std::ostream & str) str << "! " << *e; } -void ExprBuiltin::show(std::ostream & str) -{ - str << "builtins." << name; -} - void ExprConcatStrings::show(std::ostream & str) { bool first = true; @@ -342,10 +337,6 @@ void ExprOpNot::bindVars(const StaticEnv & env) e->bindVars(env); } -void ExprBuiltin::bindVars(const StaticEnv & env) -{ -} - void ExprConcatStrings::bindVars(const StaticEnv & env) { foreach (vector::iterator, i, *es) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 8826ad2323..3df9dd47f0 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -272,13 +272,6 @@ struct ExprOpNot : Expr COMMON_METHODS }; -struct ExprBuiltin : Expr -{ - Symbol name; - ExprBuiltin(const Symbol & name) : name(name) { }; - COMMON_METHODS -}; - #define MakeBinOp(name, s) \ struct Expr##name : Expr \ { \ diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 134d68d6e1..a27c53043c 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -328,13 +328,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } -| '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } + | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } @@ -346,9 +346,9 @@ expr_op l->push_back($3); $$ = new ExprConcatStrings(CUR_POS, false, l); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprBuiltin(data->symbols.create("div")), $1), $3); } + | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } + | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } + | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } | expr_app ; From 8ea9fd7aa6b2152f95724e504ac61c57d90b113c Mon Sep 17 00:00:00 2001 From: Adam Szkoda Date: Sun, 25 May 2014 10:54:54 +0200 Subject: [PATCH 11/41] Rephrase @ operator description --- doc/manual/writing-nix-expressions.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/doc/manual/writing-nix-expressions.xml b/doc/manual/writing-nix-expressions.xml index 5585e89fe5..6db2adcfa0 100644 --- a/doc/manual/writing-nix-expressions.xml +++ b/doc/manual/writing-nix-expressions.xml @@ -1060,15 +1060,14 @@ map (concat "foo") [ "bar" "bla" "abc" ] and z. - An @-pattern requires that the - argument matches with the patterns on the left- and right-hand side - of the @-sign. For example: + An @-pattern provides a means of referring + to the whole value being matched: args@{ x, y, z, ... }: z + y + x + args.a Here args is bound to the entire argument, which - is further matches against the pattern { x, y, z, + is further matched against the pattern { x, y, z, ... }. From b1d39d476544644b2de8addb5ad3289fede2f95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B6nke=20Hahn?= Date: Fri, 23 May 2014 11:41:09 +0800 Subject: [PATCH 12/41] dev-shell is a bash script, not sh 'type -p' does not work in e.g. dash --- dev-shell | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-shell b/dev-shell index a4fdc6814b..2fe62a4696 100755 --- a/dev-shell +++ b/dev-shell @@ -1,4 +1,4 @@ -#! /bin/sh +#!/usr/bin/env bash if [ -e tests/test-tmp ]; then chmod -R u+w tests/test-tmp rm -rf tests/test-tmp From a457d5ad4d7f6cd4f817581de1b4f70cdad9c617 Mon Sep 17 00:00:00 2001 From: Aristid Breitkreuz Date: Sun, 4 May 2014 14:26:41 +0200 Subject: [PATCH 13/41] nix-build: --add-root also takes 1 parameter --- scripts/nix-build.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 2f95e74755..74307a5baf 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -93,7 +93,7 @@ for (my $n = 0; $n < scalar @ARGV; $n++) { $n += 2; } - elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout") { + elsif ($arg eq "--max-jobs" || $arg eq "-j" || $arg eq "--max-silent-time" || $arg eq "--log-type" || $arg eq "--cores" || $arg eq "--timeout" || $arg eq '--add-root') { $n++; die "$0: `$arg' requires an argument\n" unless $n < scalar @ARGV; push @buildArgs, ($arg, $ARGV[$n]); From 54a34119f349d531557af9e90d21d04d689ee817 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 May 2014 17:53:17 +0200 Subject: [PATCH 14/41] Use std::unordered_set --- configure.ac | 6 ------ src/libexpr/symbol-table.hh | 11 ++--------- src/libstore/local-store.hh | 12 ++---------- src/libstore/local.mk | 3 +-- 4 files changed, 5 insertions(+), 27 deletions(-) diff --git a/configure.ac b/configure.ac index 043aed5b36..55e6191cfa 100644 --- a/configure.ac +++ b/configure.ac @@ -134,12 +134,6 @@ AC_CHECK_HEADERS([sys/personality.h]) AC_CHECK_HEADERS([linux/fs.h]) -# Check for tr1/unordered_set. -AC_LANG_PUSH(C++) -AC_CHECK_HEADERS([tr1/unordered_set]) -AC_LANG_POP(C++) - - AC_DEFUN([NEED_PROG], [ AC_PATH_PROG($1, $2) diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 08e31d4965..140662b515 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -3,10 +3,7 @@ #include "config.h" #include - -#if HAVE_TR1_UNORDERED_SET -#include -#endif +#include #include "types.hh" @@ -70,11 +67,7 @@ inline std::ostream & operator << (std::ostream & str, const Symbol & sym) class SymbolTable { private: -#if HAVE_TR1_UNORDERED_SET - typedef std::tr1::unordered_set Symbols; -#else - typedef std::set Symbols; -#endif + typedef std::unordered_set Symbols; Symbols symbols; public: diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 54331e448a..e58e6563f1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,16 +1,12 @@ #pragma once #include +#include #include "store-api.hh" #include "util.hh" #include "pathlocks.hh" -#if HAVE_TR1_UNORDERED_SET -#include -#endif - - class sqlite3; class sqlite3_stmt; @@ -306,11 +302,7 @@ private: void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); -#if HAVE_TR1_UNORDERED_SET - typedef std::tr1::unordered_set InodeHash; -#else - typedef std::set InodeHash; -#endif + typedef std::unordered_set InodeHash; InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 40cb25dc5f..64dbfa3c5b 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -21,8 +21,7 @@ libstore_CXXFLAGS = \ -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ - -DNIX_BIN_DIR=\"$(bindir)\" \ - -DPACKAGE_VERSION=\"$(PACKAGE_VERSION)\" + -DNIX_BIN_DIR=\"$(bindir)\" $(d)/local-store.cc: $(d)/schema.sql.hh From becc2b01678c5742b3fe6c379430606a5ef8e34d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 May 2014 19:02:14 +0200 Subject: [PATCH 15/41] Sort nixPath attributes --- src/libexpr/primops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 333748973d..993d290eb3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1392,6 +1392,7 @@ void EvalState::createBaseEnv() mkAttrs(*v2, 2); mkString(*allocAttr(*v2, symbols.create("path")), i.second); mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); + v2->attrs->sort(); } addConstant("nixPath", v); From ceed8192848760430c9c23659f9cb979aad1f9c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 May 2014 19:04:27 +0200 Subject: [PATCH 16/41] Fix test --- tests/lang/eval-okay-search-path.exp | 2 +- tests/lang/eval-okay-search-path.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/lang/eval-okay-search-path.exp b/tests/lang/eval-okay-search-path.exp index ad05904ba1..4519bc406d 100644 --- a/tests/lang/eval-okay-search-path.exp +++ b/tests/lang/eval-okay-search-path.exp @@ -1 +1 @@ -"abcca" +"abccX" diff --git a/tests/lang/eval-okay-search-path.nix b/tests/lang/eval-okay-search-path.nix index e0f433fec6..f48bf3fad2 100644 --- a/tests/lang/eval-okay-search-path.nix +++ b/tests/lang/eval-okay-search-path.nix @@ -8,4 +8,4 @@ assert length (filter (x: x.prefix == "nix") nixPath) == 1; assert length (filter (x: baseNameOf x.path == "dir4") nixPath) == 1; import + import + import + import - + (let nixPath = [ { path = ./dir1; } { path = ./dir2; } ]; in import ) + + (let nixPath = [ { path = ./dir2; } { path = ./dir1; } ]; in import ) From 3c6b8a521561f5341652ffe37b869d5ab457227b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Jun 2014 17:58:43 +0200 Subject: [PATCH 17/41] nix-env -qa --json: Generate valid JSON even if there are invalid meta attrs --- src/nix-env/nix-env.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 2d38f2aea7..05f6aa3549 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -882,9 +882,10 @@ static void queryJSON(Globals & globals, vector & elems) foreach (StringSet::iterator, j, metaNames) { metaObj.attr(*j); Value * v = i->queryMeta(*j); - if (!v) + if (!v) { printMsg(lvlError, format("derivation `%1%' has invalid meta attribute `%2%'") % i->name % *j); - else { + cout << "null"; + } else { PathSet context; printValueAsJSON(globals.state, true, *v, cout, context); } From 829af22759b8a99c3b44697365390a945f3acc04 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jun 2014 13:30:09 +0200 Subject: [PATCH 18/41] Print a warning when loading a large path into memory I.e. if you have a derivation with src = ./huge-directory; you'll get a warning that this is not a good idea. --- src/libstore/remote-store.cc | 3 +++ src/libutil/serialise.cc | 27 +++++++++++++++++++++++++++ src/libutil/serialise.hh | 7 +++---- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4619206932..177b8e7afd 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,10 @@ Path RemoteStore::addToStore(const Path & _srcPath, writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); writeInt(recursive ? 1 : 0, to); writeString(printHashType(hashAlgo), to); + to.written = 0; + to.warn = true; dumpPath(srcPath, to, filter); + to.warn = false; processStderr(); return readStorePath(from); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6b71f52c15..9241750750 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -54,8 +54,24 @@ FdSink::~FdSink() } +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() +{ + printMsg(lvlError, "warning: dumping very large path (> 256 MiB); this may run out of memory"); +} + + void FdSink::write(const unsigned char * data, size_t len) { + static bool warned = false; + if (warn && !warned) { + written += len; + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } writeFull(fd, data, len); } @@ -256,4 +272,15 @@ template Paths readStrings(Source & source); template PathSet readStrings(Source & source); +void StringSink::operator () (const unsigned char * data, size_t len) +{ + static bool warned = false; + if (!warned && s.size() > threshold) { + warnLargeDump(); + warned = true; + } + s.append((const char *) data, len); +} + + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index e5a9df1d05..3d296cf6d8 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -72,6 +72,8 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; + bool warn; + size_t written; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } @@ -95,10 +97,7 @@ struct FdSource : BufferedSource struct StringSink : Sink { string s; - void operator () (const unsigned char * data, size_t len) - { - s.append((const char *) data, len); - } + void operator () (const unsigned char * data, size_t len); }; From b1beed97a0670befbfd5e105a81132e87e58ac37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jun 2014 13:45:50 +0200 Subject: [PATCH 19/41] Report daemon OOM better When copying a large path causes the daemon to run out of memory, you now get: error: Nix daemon out of memory instead of: error: writing to file: Broken pipe --- src/libstore/remote-store.cc | 22 +++++++++++++++++----- src/nix-daemon/nix-daemon.cc | 6 ++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 177b8e7afd..3b021bb2a5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,11 +402,23 @@ Path RemoteStore::addToStore(const Path & _srcPath, writeInt((hashAlgo == htSHA256 && recursive) ? 0 : 1, to); writeInt(recursive ? 1 : 0, to); writeString(printHashType(hashAlgo), to); - to.written = 0; - to.warn = true; - dumpPath(srcPath, to, filter); - to.warn = false; - processStderr(); + + try { + to.written = 0; + to.warn = true; + dumpPath(srcPath, to, filter); + to.warn = false; + processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + processStderr(); + } catch (EndOfFile & e) { } + throw; + } + return readStorePath(from); } diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 8814fe3155..0464ac9653 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -735,12 +735,10 @@ static void processConnection(bool trusted) during addTextToStore() / importPath(). If that happens, just send the error message and exit. */ bool errorAllowed = canSendStderr; - if (!errorAllowed) printMsg(lvlError, format("error processing client input: %1%") % e.msg()); stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); - if (!errorAllowed) break; + if (!errorAllowed) throw; } catch (std::bad_alloc & e) { - if (canSendStderr) - stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); + stopWork(false, "Nix daemon out of memory", GET_PROTOCOL_MINOR(clientVersion) >= 8 ? 1 : 0); throw; } From ee7fe64c0ac00f2be11604a2a6509eb86dc19f0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Jun 2014 14:02:56 +0200 Subject: [PATCH 20/41] == operator: Ignore string context There really is no case I can think of where taking the context into account is useful. Mostly it's just very inconvenient. --- src/libexpr/eval.cc | 13 ++----------- src/libexpr/primops.cc | 2 +- tests/lang/eval-okay-context.nix | 2 +- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 81ce7d9a30..87b6bc9915 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1355,17 +1355,8 @@ bool EvalState::eqValues(Value & v1, Value & v2) case tBool: return v1.boolean == v2.boolean; - case tString: { - /* Compare both the string and its context. */ - if (strcmp(v1.string.s, v2.string.s) != 0) return false; - const char * * p = v1.string.context, * * q = v2.string.context; - if (!p && !q) return true; - if (!p || !q) return false; - for ( ; *p && *q; ++p, ++q) - if (strcmp(*p, *q) != 0) return false; - if (*p || *q) return false; - return true; - } + case tString: + return strcmp(v1.string.s, v2.string.s) == 0; case tPath: return strcmp(v1.path, v2.path) == 0; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 993d290eb3..feb0227acb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1297,7 +1297,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 2); + mkInt(v, 3); addConstant("__langVersion", v); // Miscellaneous diff --git a/tests/lang/eval-okay-context.nix b/tests/lang/eval-okay-context.nix index b3f1748975..8cd8f2e131 100644 --- a/tests/lang/eval-okay-context.nix +++ b/tests/lang/eval-okay-context.nix @@ -1,6 +1,6 @@ let s = "foo ${builtins.substring 33 100 (baseNameOf ./eval-okay-context.nix)} bar"; in - if s == "foo eval-okay-context.nix bar" + if s != "foo eval-okay-context.nix bar" then abort "context not discarded" else builtins.unsafeDiscardStringContext s From 61c464f252271d3d6343e1bfa1e3b39d2c8473cd Mon Sep 17 00:00:00 2001 From: Steve Purcell Date: Thu, 5 Jun 2014 11:04:48 +0100 Subject: [PATCH 21/41] Add autoloads, make code more concise & idiomatic - Use define-derived-mode to declare nix-mode - Use autoloads to ensure nix-mode is usable (and enabled) without needing `require` - Use set + make-local-variable instead of longer 2-step equivalent --- misc/emacs/nix-mode.el | 133 +++++++++++++++++------------------------ 1 file changed, 55 insertions(+), 78 deletions(-) diff --git a/misc/emacs/nix-mode.el b/misc/emacs/nix-mode.el index 2c45d68576..fc64523f1a 100644 --- a/misc/emacs/nix-mode.el +++ b/misc/emacs/nix-mode.el @@ -1,10 +1,51 @@ -;;; nix-mode.el --- Major mode for editing Nix expressions. +;;; nix-mode.el --- Major mode for editing Nix expressions ;; Author: Eelco Dolstra ;; URL: https://github.com/NixOS/nix/tree/master/misc/emacs ;; Version: 1.0 -(defun nix-mode () +;;; Commentary: + +;;; Code: + +(defconst nix-font-lock-keywords + '("\\" "\\" "\\" "\\" "\\" + "\\" "\\" "\\" "\\" "\\" + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("\\" . font-lock-builtin-face) + ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+" + . font-lock-constant-face) + ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\-\.]*\\)[ \t]*=" + (1 font-lock-variable-name-face nil nil)) + ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>" + . font-lock-constant-face) + ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+" + . font-lock-constant-face)) + "Font lock keywords for nix.") + +(defvar nix-mode-syntax-table + (let ((table (make-syntax-table))) + (modify-syntax-entry ?/ ". 14" table) + (modify-syntax-entry ?* ". 23" table) + (modify-syntax-entry ?# "< b" table) + (modify-syntax-entry ?\n "> b" table) + table) + "Syntax table for Nix mode.") + +(defun nix-indent-line () + "Indent current line in a Nix expression." + (interactive) + (indent-relative-maybe)) + + +;;;###autoload +(define-derived-mode nix-mode fundamental-mode "Nix" "Major mode for editing Nix expressions. The following commands may be useful: @@ -25,93 +66,29 @@ The hook `nix-mode-hook' is run when Nix mode is started. \\{nix-mode-map} " - - (interactive) - - (kill-all-local-variables) - - (setq major-mode 'nix-mode) - (setq mode-name "Nix") - - (use-local-map nix-mode-map) - (set-syntax-table nix-mode-syntax-table) ;; Font lock support. - (setq font-lock-defaults '(nix-keywords nil nil nil nil)) + (setq font-lock-defaults '(nix-font-lock-keywords nil nil nil nil)) ;; Automatic indentation [C-j]. - (make-local-variable 'indent-line-function) - (setq indent-line-function 'nix-indent-line) + (set (make-local-variable 'indent-line-function) 'nix-indent-line) ;; Indenting of comments. - (make-local-variable 'comment-start) - (setq comment-start "# ") - (make-local-variable 'comment-end) - (setq comment-end "") - (make-local-variable 'comment-start-skip) - (setq comment-start-skip "\\(^\\|\\s-\\);?#+ *") + (set (make-local-variable 'comment-start) "# ") + (set (make-local-variable 'comment-end) "") + (set (make-local-variable 'comment-start-skip) "\\(^\\|\\s-\\);?#+ *") ;; Filling of comments. - (make-local-variable 'adaptive-fill-mode) - (setq adaptive-fill-mode t) - (make-local-variable 'paragraph-start) - (setq paragraph-start "[ \t]*\\(#+[ \t]*\\)?$") - (make-local-variable 'paragraph-separate) - (setq paragraph-separate paragraph-start) - - (run-hooks 'nix-mode-hook)) + (set (make-local-variable 'adaptive-fill-mode) t) + (set (make-local-variable 'paragraph-start) "[ \t]*\\(#+[ \t]*\\)?$") + (set (make-local-variable 'paragraph-separate) paragraph-start)) -(defvar nix-mode-map nil - "Keymap for Nix mode.") - -(setq nix-mode-map (make-sparse-keymap)) -;(define-key nix-mode-map [tab] 'tab-to-tab-stop) - - -(defvar nix-keywords - '("\\" "\\" "\\" "\\" "\\" - "\\" "\\" "\\" "\\" "\\" - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("\\" . font-lock-builtin-face) - ("[a-zA-Z][a-zA-Z0-9\\+-\\.]*:[a-zA-Z0-9%/\\?:@&=\\+\\$,_\\.!~\\*'-]+" - . font-lock-constant-face) - ("\\<\\([a-zA-Z_][a-zA-Z0-9_'\-\.]*\\)[ \t]*=" - (1 font-lock-variable-name-face nil nil)) - ("<[a-zA-Z0-9._\\+-]+\\(/[a-zA-Z0-9._\\+-]+\\)*>" - . font-lock-constant-face) - ("[a-zA-Z0-9._\\+-]*\\(/[a-zA-Z0-9._\\+-]+\\)+" - . font-lock-constant-face))) - - -(defvar nix-mode-syntax-table nil - "Syntax table for Nix mode.") - -(if nix-mode-syntax-table - nil - (progn - (setq nix-mode-syntax-table (make-syntax-table)) - (modify-syntax-entry ?/ ". 14" nix-mode-syntax-table) - (modify-syntax-entry ?* ". 23" nix-mode-syntax-table) - (modify-syntax-entry ?# "< b" nix-mode-syntax-table) - (modify-syntax-entry ?\n "> b" nix-mode-syntax-table))) - - -(defun nix-indent-line () - "Indent current line in a Nix expression." - (interactive) - (indent-relative-maybe)) - - -(setq auto-mode-alist (cons '("\\.nix\\'" . nix-mode) auto-mode-alist)) -(setq auto-mode-alist (cons '("\\.nix.in\\'" . nix-mode) auto-mode-alist)) +;;;###autoload +(progn + (add-to-list 'auto-mode-alist '("\\.nix\\'" . nix-mode)) + (add-to-list 'auto-mode-alist '("\\.nix.in\\'" . nix-mode))) (provide 'nix-mode) From a8fb575c98726f195d0cf5c7e6b7e51c75a0a9b3 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 30 May 2014 15:04:17 -0400 Subject: [PATCH 22/41] Share code between scopedImport and import In addition to reducing duplication, this fixes both import from derivation and import of derivation for scopedImport --- src/libexpr/primops.cc | 88 +++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index feb0227acb..f270ca302f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,31 +39,34 @@ std::pair decodeContext(const string & s) /* Load and evaluate an expression from path specified by the argument. */ -static void prim_import(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path = state.coerceToPath(pos, *args[1], context); - foreach (PathSet::iterator, i, context) { - Path ctx = decodeContext(*i).first; + PathSet drvs; + for (auto & i : context) { + std::pair decoded = decodeContext(i); + Path ctx = decoded.first; assert(isStorePath(ctx)); if (!store->isValidPath(ctx)) throw EvalError(format("cannot import `%1%', since path `%2%' is not valid, at %3%") % path % ctx % pos); if (isDerivation(ctx)) - try { - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - queryMissing(*store, singleton(ctx), - willBuild, willSubstitute, unknown, downloadSize, narSize); + drvs.insert(decoded.first + "!" + decoded.second); + } + if (!drvs.empty()) { + try { + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + queryMissing(*store, drvs, + willBuild, willSubstitute, unknown, downloadSize, narSize); - /* !!! If using a substitute, we only need to fetch - the selected output of this derivation. */ - store->buildPaths(singleton(ctx)); - } catch (Error & e) { - throw ImportError(e.msg()); - } + store->buildPaths(drvs); + } catch (Error & e) { + throw ImportError(e.msg()); + } } if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { @@ -88,35 +91,30 @@ static void prim_import(EvalState & state, const Pos & pos, Value * * args, Valu mkApp(v, fun, w); state.forceAttrs(v, pos); } else { - state.evalFile(path, v); + state.forceAttrs(*args[0]); + if (args[0]->attrs->empty()) + state.evalFile(path, v); + else { + Env * env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; + + StaticEnv staticEnv(false, &state.staticBaseEnv); + + unsigned int displ = 0; + for (auto & attr : *args[0]->attrs) { + staticEnv.vars[attr.name] = displ; + env->values[displ++] = attr.value; + } + + startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); + + e->eval(state, *env, v); + } } } -static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - state.forceAttrs(*args[0]); - Path path = resolveExprPath(state.coerceToPath(pos, *args[1], context)); - - Env * env = &state.allocEnv(args[0]->attrs->size()); - env->up = &state.baseEnv; - - StaticEnv staticEnv(false, &state.staticBaseEnv); - - unsigned int displ = 0; - for (auto & attr : *args[0]->attrs) { - staticEnv.vars[attr.name] = displ; - env->values[displ++] = attr.value; - } - - startNest(nest, lvlTalkative, format("evaluating file `%1%'") % path); - Expr * e = state.parseExprFromFile(path, staticEnv); - - e->eval(state, *env, v); -} - - /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1301,8 +1299,12 @@ void EvalState::createBaseEnv() addConstant("__langVersion", v); // Miscellaneous - addPrimOp("import", 1, prim_import); addPrimOp("scopedImport", 2, prim_scopedImport); + Value * v2 = allocValue(); + mkAttrs(*v2, 0); + mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2); + forceValue(v); + addConstant("import", v); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); @@ -1388,7 +1390,7 @@ void EvalState::createBaseEnv() mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - Value * v2 = v.list.elems[n++] = allocValue(); + v2 = v.list.elems[n++] = allocValue(); mkAttrs(*v2, 2); mkString(*allocAttr(*v2, symbols.create("path")), i.second); mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); From 718f20da6d79466f91c49849bcf91a688aaa871e Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 30 May 2014 15:43:31 -0400 Subject: [PATCH 23/41] findFile: Realise the context of the path attributes --- src/libexpr/nixexpr.hh | 1 + src/libexpr/primops.cc | 74 ++++++++++++++++++++++++++++-------------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 3df9dd47f0..b8d0929285 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -16,6 +16,7 @@ MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) MakeError(TypeError, EvalError) MakeError(ImportError, EvalError) // error building an imported derivation +MakeError(FindError, EvalError) // error building a nix-path component MakeError(UndefinedVarError, Error) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f270ca302f..fecaf37b6e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -37,6 +37,38 @@ std::pair decodeContext(const string & s) } +struct InvalidPathError : EvalError +{ + Path path; + InvalidPathError(const Path & path) : + EvalError(format("path `%1%' is not valid") % path), path(path) {}; +}; + + +static void realiseContext(const PathSet & context) +{ + PathSet drvs; + for (auto & i : context) { + std::pair decoded = decodeContext(i); + Path ctx = decoded.first; + assert(isStorePath(ctx)); + if (!store->isValidPath(ctx)) + throw InvalidPathError(ctx); + if (isDerivation(ctx)) + drvs.insert(decoded.first + "!" + decoded.second); + } + if (!drvs.empty()) { + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + queryMissing(*store, drvs, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + store->buildPaths(drvs); + } +} + + /* Load and evaluate an expression from path specified by the argument. */ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -44,29 +76,13 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args PathSet context; Path path = state.coerceToPath(pos, *args[1], context); - PathSet drvs; - for (auto & i : context) { - std::pair decoded = decodeContext(i); - Path ctx = decoded.first; - assert(isStorePath(ctx)); - if (!store->isValidPath(ctx)) - throw EvalError(format("cannot import `%1%', since path `%2%' is not valid, at %3%") - % path % ctx % pos); - if (isDerivation(ctx)) - drvs.insert(decoded.first + "!" + decoded.second); - } - if (!drvs.empty()) { - try { - /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - queryMissing(*store, drvs, - willBuild, willSubstitute, unknown, downloadSize, narSize); - - store->buildPaths(drvs); - } catch (Error & e) { - throw ImportError(e.msg()); - } + try { + realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot import `%1%', since path `%2%' is not valid, at %3%") + % path % e.path % pos); + } catch (Error & e) { + throw ImportError(e.msg()); } if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { @@ -660,6 +676,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; + PathSet context; for (unsigned int n = 0; n < args[0]->list.length; ++n) { Value & v2(*args[0]->list.elems[n]); state.forceAttrs(v2, pos); @@ -672,13 +689,22 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va i = v2.attrs->find(state.symbols.create("path")); if (i == v2.attrs->end()) throw EvalError(format("attribute `path' missing, at %1%") % pos); - PathSet context; string path = state.coerceToPath(pos, *i->value, context); searchPath.push_back(std::pair(prefix, path)); } string path = state.forceStringNoCtx(*args[1], pos); + + try { + realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find `%1%', since path `%2%' is not valid, at %3%") + % path % e.path % pos); + } catch (Error & e) { + throw FindError(e.msg()); + } + mkPath(v, state.findFile(searchPath, path).c_str()); } From 0960d674d48808eaaa3475899f45cfd6c7c3e51d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2014 13:00:54 +0200 Subject: [PATCH 24/41] Drop ImportError and FindError We're not catching these anywhere. --- src/libexpr/nixexpr.hh | 2 -- src/libexpr/primops.cc | 4 ---- 2 files changed, 6 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index b8d0929285..9c631d093b 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -15,8 +15,6 @@ MakeError(AssertionError, EvalError) MakeError(ThrownError, AssertionError) MakeError(Abort, EvalError) MakeError(TypeError, EvalError) -MakeError(ImportError, EvalError) // error building an imported derivation -MakeError(FindError, EvalError) // error building a nix-path component MakeError(UndefinedVarError, Error) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fecaf37b6e..366911b54d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -81,8 +81,6 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } catch (InvalidPathError & e) { throw EvalError(format("cannot import `%1%', since path `%2%' is not valid, at %3%") % path % e.path % pos); - } catch (Error & e) { - throw ImportError(e.msg()); } if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { @@ -701,8 +699,6 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va } catch (InvalidPathError & e) { throw EvalError(format("cannot find `%1%', since path `%2%' is not valid, at %3%") % path % e.path % pos); - } catch (Error & e) { - throw FindError(e.msg()); } mkPath(v, state.findFile(searchPath, path).c_str()); From 48495f67ed893c4ee056099ae0ca5a2afacde93c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2014 13:15:35 +0200 Subject: [PATCH 25/41] Fix bogus warnings about dumping large paths Also, yay for C++11 non-static initialisers. --- src/libutil/serialise.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3d296cf6d8..1b3fb744fd 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -72,8 +72,8 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn; - size_t written; + bool warn = false; + size_t written = 0; FdSink() : fd(-1) { } FdSink(int fd) : fd(fd) { } From 9d0709e8c47082cec35d6412053eacfadae23bcd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 12 Jun 2014 17:30:37 +0200 Subject: [PATCH 26/41] Don't use member initialisers They're a little bit too recent (only supported since GCC 4.7). http://hydra.nixos.org/build/11851475 --- src/libutil/serialise.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 1b3fb744fd..6a6f028aa6 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -72,11 +72,11 @@ struct BufferedSource : Source struct FdSink : BufferedSink { int fd; - bool warn = false; - size_t written = 0; + bool warn; + size_t written; - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } + FdSink() : fd(-1), warn(false), written(0) { } + FdSink(int fd) : fd(fd), warn(false), written(0) { } ~FdSink(); void write(const unsigned char * data, size_t len); From 5cd022d6c099c583c0494bdacd06f4eb32661135 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sun, 1 Jun 2014 10:42:56 -0400 Subject: [PATCH 27/41] Add importNative primop This can be used to import a dynamic shared object and return an arbitrary value, including new primops. This can be used both to test new primops without having to recompile nix every time, and to build specialized primops that probably don't belong upstream (e.g. a function that calls out to gpg to decrypt a nixops secret as-needed). The imported function should initialize the Value & as needed. A single import can define multiple values by creating an attrset or list, of course. An example initialization function might look like: extern "C" void initialize(nix::EvalState & state, nix::Value & v) { v.type = nix::tPrimOp; v.primOp = NEW nix::PrimOp(myFun, 1, state.symbols.create("myFun")); } Then `builtins.importNative ./example.so "initialize"` will evaluate to the primop defined in the myFun function. --- src/libexpr/local.mk | 2 ++ src/libexpr/primops.cc | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b3b4086916..75a0e185e3 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,6 +8,8 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc libexpr_LIBS = libutil libstore libformat +libexpr_LDFLAGS = -ldl + # The dependency on libgc must be propagated (i.e. meaning that # programs/libraries that use libexpr must explicitly pass -lgc), # because inline functions in libexpr's header files call libgc. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 366911b54d..d6ac7c9578 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -15,6 +15,7 @@ #include #include +#include namespace nix { @@ -129,6 +130,46 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } +/* Want reasonable symbol names, so extern C */ +/* !!! Should we pass the Pos or the file name too? */ +extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); + +/* Load a ValueInitializer from a dso and return whatever it initializes */ +static void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + + try { + realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot import `%1%', since path `%2%' is not valid, at %3%") + % path % e.path % pos); + } + + string sym = state.forceStringNoCtx(*args[1], pos); + + void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw EvalError(format("could not open `%1%': %2%") % path % dlerror()); + + dlerror(); + ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); + if(!func) { + char *message = dlerror(); + if (message) + throw EvalError(format("could not load symbol `%1%' from `%2%': %3%") % sym % path % message); + else + throw EvalError(format("symbol `%1%' from `%2%' resolved to NULL when a function pointer was expected") + % sym % path); + } + + (func)(state, v); + + /* We don't dlclose because v may be a primop referencing a function in the shared object file */ +} + + /* Return a string representing the type of the expression. */ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1327,6 +1368,7 @@ void EvalState::createBaseEnv() mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2); forceValue(v); addConstant("import", v); + addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); From d62f46e500958bc97ae6837911e27c20a47cc181 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 24 Jun 2014 10:50:03 -0400 Subject: [PATCH 28/41] Only add the importNative primop if the allow-arbitrary-code-during-evaluation option is true (default false) --- src/libexpr/primops.cc | 3 ++- src/libstore/globals.cc | 2 ++ src/libstore/globals.hh | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d6ac7c9578..ff82f36b52 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1368,7 +1368,8 @@ void EvalState::createBaseEnv() mkApp(v, *baseEnv.values[baseEnvDispl - 1], *v2); forceValue(v); addConstant("import", v); - addPrimOp("__importNative", 2, prim_importNative); + if (settings.enableImportNative) + addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__typeOf", 1, prim_typeOf); addPrimOp("isNull", 1, prim_isNull); addPrimOp("__isFunction", 1, prim_isFunction); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 180344e336..5d359e1281 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -61,6 +61,7 @@ Settings::Settings() envKeepDerivations = false; lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; showTrace = false; + enableImportNative = false; } @@ -148,6 +149,7 @@ void Settings::update() get(sshSubstituterHosts, "ssh-substituter-hosts"); get(useSshSubstituter, "use-ssh-substituter"); get(logServers, "log-servers"); + get(enableImportNative, "allow-arbitrary-code-during-evaluation"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 65a6c388b8..8dd59a9c79 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -200,6 +200,9 @@ struct Settings { /* A list of URL prefixes that can return Nix build logs. */ Strings logServers; + /* Whether the importNative primop should be enabled */ + bool enableImportNative; + private: SettingsMap settings, overrides; From 8504e7d60488cb12dd2597734ebd1d3cadf5d153 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jun 2014 11:20:16 +0200 Subject: [PATCH 29/41] allow-arbitrary-code-during-evaluation -> allow-unsafe-native-code-during-evaluation --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5d359e1281..8bffdd73f7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -149,7 +149,7 @@ void Settings::update() get(sshSubstituterHosts, "ssh-substituter-hosts"); get(useSshSubstituter, "use-ssh-substituter"); get(logServers, "log-servers"); - get(enableImportNative, "allow-arbitrary-code-during-evaluation"); + get(enableImportNative, "allow-unsafe-native-code-during-evaluation"); string subs = getEnv("NIX_SUBSTITUTERS", "default"); if (subs == "default") { From 858b8f9760a81540b0a95068d96dc5c1628673c3 Mon Sep 17 00:00:00 2001 From: Paul Colomiets Date: Tue, 24 Jun 2014 00:30:22 +0300 Subject: [PATCH 30/41] Add `--json` argument to `nix-instantiate` --- doc/manual/nix-instantiate.xml | 9 +++++++++ src/nix-instantiate/nix-instantiate.cc | 23 +++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/doc/manual/nix-instantiate.xml b/doc/manual/nix-instantiate.xml index b388560a11..936f154dde 100644 --- a/doc/manual/nix-instantiate.xml +++ b/doc/manual/nix-instantiate.xml @@ -124,6 +124,15 @@ input. + + + When used with and + , print the resulting expression as an + JSON representation of the abstract syntax tree rather than as an + ATerm. + + + When used with , diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index cdd74523ca..7acc9d501c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -5,6 +5,7 @@ #include "get-drvs.hh" #include "attr-path.hh" #include "value-to-xml.hh" +#include "value-to-json.hh" #include "util.hh" #include "store-api.hh" #include "common-opts.hh" @@ -13,6 +14,12 @@ #include #include +enum OutputKind { + OUTPUT_PLAIN, + OUTPUT_XML, + OUTPUT_JSON, +}; + using namespace nix; @@ -37,7 +44,7 @@ static bool indirectRoot = false; void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, - bool evalOnly, bool xmlOutput, bool location, Expr * e) + bool evalOnly, OutputKind output, bool location, Expr * e) { if (parseOnly) { std::cout << format("%1%\n") % *e; @@ -58,8 +65,10 @@ void processExpr(EvalState & state, const Strings & attrPaths, vRes = v; else state.autoCallFunction(autoArgs, v, vRes); - if (xmlOutput) + if (output == OUTPUT_XML) printValueAsXML(state, strict, location, vRes, std::cout, context); + else if (output == OUTPUT_JSON) + printValueAsJSON(state, strict, vRes, std::cout, context); else { if (strict) state.strictForceValue(vRes); std::cout << vRes << std::endl; @@ -108,7 +117,7 @@ void run(Strings args) bool findFile = false; bool evalOnly = false; bool parseOnly = false; - bool xmlOutput = false; + OutputKind outputKind = OUTPUT_PLAIN; bool xmlOutputSourceLocation = true; bool strict = false; Strings attrPaths; @@ -145,7 +154,9 @@ void run(Strings args) else if (arg == "--indirect") indirectRoot = true; else if (arg == "--xml") - xmlOutput = true; + outputKind = OUTPUT_XML; + else if (arg == "--json") + outputKind = OUTPUT_JSON; else if (arg == "--no-location") xmlOutputSourceLocation = false; else if (arg == "--strict") @@ -179,7 +190,7 @@ void run(Strings args) if (readStdin) { Expr * e = parseStdin(state); processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, xmlOutput, xmlOutputSourceLocation, e); + evalOnly, outputKind, xmlOutputSourceLocation, e); } else if (files.empty() && !fromArgs) files.push_back("./default.nix"); @@ -188,7 +199,7 @@ void run(Strings args) ? state.parseExprFromString(*i, absPath(".")) : state.parseExprFromFile(resolveExprPath(lookupFileArg(state, *i))); processExpr(state, attrPaths, parseOnly, strict, autoArgs, - evalOnly, xmlOutput, xmlOutputSourceLocation, e); + evalOnly, outputKind, xmlOutputSourceLocation, e); } state.printStats(); From e477f0e9385d7825f005b7e9a32cd3ad6ee27aab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Jun 2014 11:36:23 +0200 Subject: [PATCH 31/41] Style fix --- src/nix-instantiate/nix-instantiate.cc | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 7acc9d501c..a188b28fe8 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -14,12 +14,6 @@ #include #include -enum OutputKind { - OUTPUT_PLAIN, - OUTPUT_XML, - OUTPUT_JSON, -}; - using namespace nix; @@ -42,6 +36,9 @@ static int rootNr = 0; static bool indirectRoot = false; +enum OutputKind { okPlain, okXML, okJSON }; + + void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, bool evalOnly, OutputKind output, bool location, Expr * e) @@ -65,9 +62,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, vRes = v; else state.autoCallFunction(autoArgs, v, vRes); - if (output == OUTPUT_XML) + if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context); - else if (output == OUTPUT_JSON) + else if (output == okJSON) printValueAsJSON(state, strict, vRes, std::cout, context); else { if (strict) state.strictForceValue(vRes); @@ -117,7 +114,7 @@ void run(Strings args) bool findFile = false; bool evalOnly = false; bool parseOnly = false; - OutputKind outputKind = OUTPUT_PLAIN; + OutputKind outputKind = okPlain; bool xmlOutputSourceLocation = true; bool strict = false; Strings attrPaths; @@ -154,9 +151,9 @@ void run(Strings args) else if (arg == "--indirect") indirectRoot = true; else if (arg == "--xml") - outputKind = OUTPUT_XML; + outputKind = okXML; else if (arg == "--json") - outputKind = OUTPUT_JSON; + outputKind = okJSON; else if (arg == "--no-location") xmlOutputSourceLocation = false; else if (arg == "--strict") From e82951fe23daa961ef18b0c5cc9ba1f5d8906186 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Jul 2014 12:36:58 +0200 Subject: [PATCH 32/41] Manual: html -> xhtml --- doc/manual/local.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 953a4d437a..1c5a3a2fff 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -54,7 +54,7 @@ dist-files += $(man-pages) $(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid $(trace-gen) $(XSLTPROC) --xinclude --stringparam profile.condition manual \ $(docbookxsl)/profiling/profile.xsl $< | \ - $(XSLTPROC) --output $@ $(docbookxsl)/html/docbook.xsl - + $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl - $(foreach file, $(d)/manual.html $(d)/style.css, $(eval $(call install-data-in, $(file), $(docdir)/manual))) @@ -94,12 +94,12 @@ NEWS_OPTS = \ $(d)/release-notes.html: $(d)/release-notes.xml $(trace-gen) $(XSLTPROC) --xinclude --output $@ $(NEWS_OPTS) \ - $(docbookxsl)/html/docbook.xsl $< + $(docbookxsl)/xhtml/docbook.xsl $< NEWS: $(d)/release-notes.xml $(trace-gen) $(XSLTPROC) --xinclude doc/manual/quote-literals.xsl $< | \ $(XSLTPROC) --output $@.tmp.html $(NEWS_OPTS) \ - $(docbookxsl)/html/docbook.xsl - && \ + $(docbookxsl)/xhtml/docbook.xsl - && \ LANG=en_US.UTF-8 $(w3m) -dump $@.tmp.html > $@.tmp && \ sed -e 's/●/*/g' -e 's/○/-/g' -e 's/━/-/g' < $@.tmp > NEWS && \ rm $@.tmp $@.tmp.html From beaf3e90aff14664b98f2c7ab7387c9fa4354fd1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Jul 2014 13:34:15 +0200 Subject: [PATCH 33/41] =?UTF-8?q?Add=20builtin=20function=20=E2=80=98fromJ?= =?UTF-8?q?SON=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #294. --- doc/manual/builtins.xml | 17 ++++ src/libexpr/json-to-value.cc | 144 ++++++++++++++++++++++++++++++ src/libexpr/json-to-value.hh | 13 +++ src/libexpr/primops.cc | 10 +++ tests/lang/eval-okay-fromjson.exp | 1 + tests/lang/eval-okay-fromjson.nix | 32 +++++++ 6 files changed, 217 insertions(+) create mode 100644 src/libexpr/json-to-value.cc create mode 100644 src/libexpr/json-to-value.hh create mode 100644 tests/lang/eval-okay-fromjson.exp create mode 100644 tests/lang/eval-okay-fromjson.nix diff --git a/doc/manual/builtins.xml b/doc/manual/builtins.xml index 6a472291c3..b289c6f0ed 100644 --- a/doc/manual/builtins.xml +++ b/doc/manual/builtins.xml @@ -257,6 +257,22 @@ stdenv.mkDerivation { + builtins.fromJSON e + + Convert a JSON string to a Nix + value. For example, + + +builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' + + + returns the value { x = [ 1 2 3 ]; y = null; + }. Floating point numbers are not + supported. + + + + builtins.getAttr s set @@ -762,6 +778,7 @@ in foo + builtins.toPath s Convert the string value diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc new file mode 100644 index 0000000000..7f60e7cc63 --- /dev/null +++ b/src/libexpr/json-to-value.cc @@ -0,0 +1,144 @@ +#include "config.h" +#include "json-to-value.hh" + +#include + +namespace nix { + + +static void skipWhitespace(const char * & s) +{ + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; +} + + +#if HAVE_BOEHMGC +typedef std::vector > ValueVector; +#else +typedef std::vector ValueVector; +#endif + + +static string parseJSONString(const char * & s) +{ + string res; + if (*s++ != '"') throw JSONParseError("expected JSON string"); + while (*s != '"') { + if (!*s) throw JSONParseError("got end-of-string in JSON string"); + if (*s == '\\') { + s++; + if (*s == '"') res += '"'; + else if (*s == '\\') res += '\\'; + else if (*s == '/') res += '/'; + else if (*s == '/') res += '/'; + else if (*s == 'b') res += '\b'; + else if (*s == 'f') res += '\f'; + else if (*s == 'n') res += '\n'; + else if (*s == 'r') res += '\r'; + else if (*s == 't') res += '\t'; + else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); + else throw JSONParseError("invalid escaped character in JSON string"); + s++; + } else + res += *s++; + } + s++; + return res; +} + + +static void parseJSON(EvalState & state, const char * & s, Value & v) +{ + skipWhitespace(s); + + if (!*s) throw JSONParseError("expected JSON value"); + + if (*s == '[') { + s++; + ValueVector values; + values.reserve(128); + skipWhitespace(s); + while (1) { + if (values.empty() && *s == ']') break; + Value * v2 = state.allocValue(); + parseJSON(state, s, *v2); + values.push_back(v2); + skipWhitespace(s); + if (*s == ']') break; + if (*s != ',') throw JSONParseError("expected `,' or `]' after JSON array element"); + s++; + } + s++; + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) + v.list.elems[n] = values[n]; + } + + else if (*s == '{') { + s++; + state.mkAttrs(v, 1); + while (1) { + skipWhitespace(s); + if (v.attrs->empty() && *s == '}') break; + string name = parseJSONString(s); + skipWhitespace(s); + if (*s != ':') throw JSONParseError("expected `:' in JSON object"); + s++; + Value * v2 = state.allocValue(); + parseJSON(state, s, *v2); + v.attrs->push_back(Attr(state.symbols.create(name), v2)); + skipWhitespace(s); + if (*s == '}') break; + if (*s != ',') throw JSONParseError("expected `,' or `}' after JSON member"); + s++; + } + v.attrs->sort(); + s++; + } + + else if (*s == '"') { + mkString(v, parseJSONString(s)); + } + + else if (isdigit(*s) || *s == '-') { + bool neg = false; + if (*s == '-') { + neg = true; + if (!*++s) throw JSONParseError("unexpected end of JSON number"); + } + NixInt n = 0; + // FIXME: detect overflow + while (isdigit(*s)) n = n * 10 + (*s++ - '0'); + if (*s == '.' || *s == 'e') throw JSONParseError("floating point JSON numbers are not supported"); + mkInt(v, neg ? -n : n); + } + + else if (strncmp(s, "true", 4) == 0) { + s += 4; + mkBool(v, true); + } + + else if (strncmp(s, "false", 5) == 0) { + s += 5; + mkBool(v, false); + } + + else if (strncmp(s, "null", 4) == 0) { + s += 4; + mkNull(v); + } + + else throw JSONParseError("unrecognised JSON value"); +} + + +void parseJSON(EvalState & state, const string & s_, Value & v) +{ + const char * s = s_.c_str(); + parseJSON(state, s, v); + skipWhitespace(s); + if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); +} + + +} diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh new file mode 100644 index 0000000000..33f35b16ce --- /dev/null +++ b/src/libexpr/json-to-value.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "eval.hh" + +#include + +namespace nix { + +MakeError(JSONParseError, EvalError) + +void parseJSON(EvalState & state, const string & s, Value & v); + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ff82f36b52..01c7ca4441 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "value-to-xml.hh" #include "value-to-json.hh" +#include "json-to-value.hh" #include "names.hh" #include "eval-inline.hh" @@ -775,6 +776,14 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu } +/* Parse a JSON string to a value. */ +static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string s = state.forceStringNoCtx(*args[0], pos); + parseJSON(state, s, v); +} + + /* Store a string in the Nix store as a source file that can be used as an input by derivations. */ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1396,6 +1405,7 @@ void EvalState::createBaseEnv() // Creating files addPrimOp("__toXML", 1, prim_toXML); addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__fromJSON", 1, prim_fromJSON); addPrimOp("__toFile", 2, prim_toFile); addPrimOp("__filterSource", 2, prim_filterSource); diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/lang/eval-okay-fromjson.exp new file mode 100644 index 0000000000..27ba77ddaf --- /dev/null +++ b/tests/lang/eval-okay-fromjson.exp @@ -0,0 +1 @@ +true diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix new file mode 100644 index 0000000000..5ed0c1c439 --- /dev/null +++ b/tests/lang/eval-okay-fromjson.nix @@ -0,0 +1,32 @@ +# RFC 7159, section 13. +builtins.fromJSON + '' + { + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100] + } + } + '' +== + { Image = + { Width = 800; + Height = 600; + Title = "View from 15th Floor"; + Thumbnail = + { Url = http://www.example.com/image/481989943; + Height = 125; + Width = 100; + }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + }; + } From beac05c206a801be6f83b4eaaffe75c30eeb7d37 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Jul 2014 20:41:25 +0200 Subject: [PATCH 34/41] Don't build on Ubuntu 10.10 Its C++ compiler is too old. http://hydra.nixos.org/build/12385722 --- release.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/release.nix b/release.nix index 1279ec3c59..e77f65ba04 100644 --- a/release.nix +++ b/release.nix @@ -189,8 +189,6 @@ let deb_debian7i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian7i386) 60; deb_debian7x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian7x86_64) 60; - deb_ubuntu1010i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1010i386) 50; - deb_ubuntu1010x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1010x86_64) 50; deb_ubuntu1110i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1110i386) 60; deb_ubuntu1110x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1110x86_64) 60; deb_ubuntu1204i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1204i386) 60; From 0e5d0c15430cf82861a1ae213cbccff065f71107 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 9 Jul 2014 12:14:40 +0200 Subject: [PATCH 35/41] Fix compilation error on some versions of GCC src/libexpr/primops.cc:42:8: error: looser throw specifier for 'virtual nix::InvalidPathError::~InvalidPathError()' src/libexpr/nixexpr.hh:12:1: error: overriding 'virtual nix::EvalError::~EvalError() noexcept (true)' http://hydra.nixos.org/build/12385750 --- src/libexpr/primops.cc | 1 + src/libutil/types.hh | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 01c7ca4441..543266b22a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -44,6 +44,7 @@ struct InvalidPathError : EvalError Path path; InvalidPathError(const Path & path) : EvalError(format("path `%1%' is not valid") % path), path(path) {}; + ~InvalidPathError() throw () { }; }; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 4b5ce9a78c..906a959e30 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -41,8 +41,8 @@ public: BaseError(const FormatOrString & fs, unsigned int status = 1); ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } - const string & msg() const throw () { return err; } - const string & prefix() const throw () { return prefix_; } + const string & msg() const { return err; } + const string & prefix() const { return prefix_; } BaseError & addPrefix(const FormatOrString & fs); }; From 66dbc0fdeebf509c5d919e9c12b2645136d6deeb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 01:50:29 +0200 Subject: [PATCH 36/41] Add a test for the SSH substituter --- tests/nix-copy-closure.nix | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix index bfd708404d..1418c65897 100644 --- a/tests/nix-copy-closure.nix +++ b/tests/nix-copy-closure.nix @@ -4,7 +4,7 @@ with import { inherit system; }; -makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; in { +makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; pkgC = pkgs.hello; in { nodes = { client = @@ -12,13 +12,14 @@ makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; in { { virtualisation.writableStore = true; virtualisation.pathsInNixDB = [ pkgA ]; nix.package = nix; + nix.binaryCaches = [ ]; }; server = { config, pkgs, ... }: { services.openssh.enable = true; virtualisation.writableStore = true; - virtualisation.pathsInNixDB = [ pkgB ]; + virtualisation.pathsInNixDB = [ pkgB pkgC ]; nix.package = nix; }; }; @@ -49,6 +50,14 @@ makeTest (let pkgA = pkgs.aterm; pkgB = pkgs.wget; in { $client->fail("nix-store --check-validity ${pkgB}"); $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2"); $client->succeed("nix-store --check-validity ${pkgB}"); + + # Copy the closure of package C via the SSH substituter. + $client->fail("nix-store -r ${pkgC}"); + $client->succeed( + "nix-store --option use-ssh-substituter true" + . " --option ssh-substituter-hosts root\@server" + . " -r ${pkgC} >&2"); + $client->succeed("nix-store --check-validity ${pkgC}"); ''; }) From 2c3a8f787ba9da49feafdec4022534184e0a96a3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 11:46:01 +0200 Subject: [PATCH 37/41] =?UTF-8?q?Fix=20security=20hole=20in=20=E2=80=98nix?= =?UTF-8?q?-store=20--serve=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since it didn't check that the path received from the client is a store path, the client could dump any path in the file system. --- src/nix-store/nix-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4fee7258cb..5bcb82f324 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -923,7 +923,7 @@ static void opServe(Strings opFlags, Strings opArgs) } break; case cmdSubstitute: - dumpPath(readString(in), out); + dumpPath(readStorePath(in), out); break; default: throw Error(format("unknown serve command `%1%'") % cmd); From 04170d06bf7d17f882c01d3ab98885e0f3e46d2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 11:51:22 +0200 Subject: [PATCH 38/41] nix-copy-closure: Fix race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is a long-standing race condition when copying a closure to a remote machine, particularly affecting build-remote.pl: the client first asks the remote machine which paths it already has, then copies over the missing paths. If the garbage collector kicks in on the remote machine between the first and second step, the already-present paths may be deleted. The missing paths may then refer to deleted paths, causing nix-copy-closure to fail. The client now performs both steps using a single remote Nix call (using ‘nix-store --serve’), locking all paths in the closure while querying. I changed the --serve protocol a bit (getting rid of QueryCommand), so this breaks the SSH substituter from older versions. But it was marked experimental anyway. Fixes #141. --- perl/lib/Nix/CopyClosure.pm | 77 ++++++++++++++++++- src/download-via-ssh/download-via-ssh.cc | 8 +- src/nix-store/nix-store.cc | 98 +++++++++++++----------- src/nix-store/serve-protocol.hh | 15 ++-- 4 files changed, 137 insertions(+), 61 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index 41ceabd858..cba365aa17 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -4,6 +4,15 @@ use strict; use Nix::Config; use Nix::Store; use List::Util qw(sum); +use IPC::Open2; + + +sub readInt { + my ($from) = @_; + my $resp; + sysread($from, $resp, 8) == 8 or die "did not receive valid reply from remote host\n"; + return unpack("L= 0x300; + }; + if ($@) { + chomp $@; + warn "$@; falling back to old closure copying method\n"; + return oldCopyTo(\@closure, @_); + } + + # Send the "query valid paths" command with the "lock" option + # enabled. This prevens a race where the remote host + # garbage-collect paths that are already there. + my $req = pack("L 0) { - my @ps = splice(@closure, 0, 1500); + while (scalar(@$closure) > 0) { + my @ps = splice(@$closure, 0, 1500); open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|"); while () { chomp; diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index 6361e71e99..6cbcd9891c 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -58,7 +58,7 @@ static std::pair connect(const string & conn) static void substitute(std::pair & pipes, Path storePath, Path destPath) { - writeInt(cmdSubstitute, pipes.first); + writeInt(cmdDumpStorePath, pipes.first); writeString(storePath, pipes.first); pipes.first.flush(); restorePath(destPath, pipes.second); @@ -68,20 +68,20 @@ static void substitute(std::pair & pipes, Path storePath, Path static void query(std::pair & pipes) { - writeInt(cmdQuery, pipes.first); for (string line; getline(std::cin, line);) { Strings tokenized = tokenizeString(line); string cmd = tokenized.front(); tokenized.pop_front(); if (cmd == "have") { - writeInt(qCmdHave, pipes.first); + writeInt(cmdQueryValidPaths, pipes.first); + writeInt(0, pipes.first); // don't lock writeStrings(tokenized, pipes.first); pipes.first.flush(); PathSet paths = readStrings(pipes.second); foreach (PathSet::iterator, i, paths) std::cout << *i << std::endl; } else if (cmd == "info") { - writeInt(qCmdInfo, pipes.first); + writeInt(cmdQueryPathInfos, pipes.first); writeStrings(tokenized, pipes.first); pipes.first.flush(); while (1) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 5bcb82f324..849cb7e8a7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -869,8 +869,12 @@ static void opClearFailedPaths(Strings opFlags, Strings opArgs) /* Serve the nix store in a way usable by a restricted ssh user. */ static void opServe(Strings opFlags, Strings opArgs) { - if (!opArgs.empty() || !opFlags.empty()) - throw UsageError("no arguments or flags expected"); + bool writeAllowed = false; + foreach (Strings::iterator, i, opFlags) + if (*i == "--write") writeAllowed = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource in(STDIN_FILENO); FdSink out(STDOUT_FILENO); @@ -883,50 +887,56 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); readInt(in); // Client version, unused for now - ServeCommand cmd = (ServeCommand) readInt(in); - switch (cmd) { - case cmdQuery: - while (true) { - QueryCommand qCmd; - try { - qCmd = (QueryCommand) readInt(in); - } catch (EndOfFile & e) { - break; - } - switch (qCmd) { - case qCmdHave: { - PathSet paths = readStorePaths(in); - writeStrings(store->queryValidPaths(paths), out); - break; - } - case qCmdInfo: { - PathSet paths = readStorePaths(in); - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!store->isValidPath(*i)) - continue; - ValidPathInfo info = store->queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); - } - writeString("", out); - break; - } - default: - throw Error(format("unknown serve query `%1%'") % cmd); - } + while (true) { + ServeCommand cmd; + try { + cmd = (ServeCommand) readInt(in); + } catch (EndOfFile & e) { + break; + } + + switch (cmd) { + case cmdQueryValidPaths: { + bool lock = readInt(in); + PathSet paths = readStorePaths(in); + if (lock && writeAllowed) + for (auto & path : paths) + store->addTempRoot(path); + writeStrings(store->queryValidPaths(paths), out); out.flush(); + break; } - break; - case cmdSubstitute: - dumpPath(readStorePath(in), out); - break; - default: - throw Error(format("unknown serve command `%1%'") % cmd); + case cmdQueryPathInfos: { + PathSet paths = readStorePaths(in); + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + if (!store->isValidPath(*i)) + continue; + ValidPathInfo info = store->queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); + } + writeString("", out); + out.flush(); + break; + } + case cmdDumpStorePath: + dumpPath(readStorePath(in), out); + out.flush(); + break; + case cmdImportPaths: + if (!writeAllowed) throw Error("importing paths not allowed"); + store->importPaths(false, in); + writeInt(1, out); // indicate success + out.flush(); + break; + default: + throw Error(format("unknown serve command %1%") % cmd); + } } } diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh index 69277bc1b9..07ff4f7a7c 100644 --- a/src/nix-store/serve-protocol.hh +++ b/src/nix-store/serve-protocol.hh @@ -2,23 +2,18 @@ namespace nix { - #define SERVE_MAGIC_1 0x390c9deb #define SERVE_MAGIC_2 0x5452eecb -#define SERVE_PROTOCOL_VERSION 0x101 +#define SERVE_PROTOCOL_VERSION 0x200 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) - typedef enum { - cmdQuery = 0, - cmdSubstitute = 1, + cmdQueryValidPaths = 1, + cmdQueryPathInfos = 2, + cmdDumpStorePath = 3, + cmdImportPaths = 4, } ServeCommand; -typedef enum { - qCmdHave = 0, - qCmdInfo = 1, -} QueryCommand; - } From 7911e4c27a0020a61ace13cfdc44de4af02f315e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jun 2014 09:15:35 -0400 Subject: [PATCH 39/41] Remove maybeVfork --- configure.ac | 4 ---- src/libstore/build.cc | 4 ++-- src/libstore/local-store.cc | 2 +- src/libutil/util.cc | 9 +-------- src/libutil/util.hh | 3 --- 5 files changed, 4 insertions(+), 18 deletions(-) diff --git a/configure.ac b/configure.ac index 55e6191cfa..00c1d495d4 100644 --- a/configure.ac +++ b/configure.ac @@ -89,10 +89,6 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], ]) -# Check for vfork. -#AC_FUNC_FORK() - - # Check for lutimes, optionally used for changing the mtime of # symlinks. AC_CHECK_FUNCS([lutimes]) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f38cd29940..70a3effb23 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -602,7 +602,7 @@ HookInstance::HookInstance() builderOut.create(); /* Fork the hook. */ - pid = maybeVfork(); + pid = fork(); switch (pid) { case -1: @@ -2781,7 +2781,7 @@ void SubstitutionGoal::tryToRun() const char * * argArr = strings2CharPtrs(args); /* Fork the substitute program. */ - pid = maybeVfork(); + pid = fork(); switch (pid) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5d210ae017..08ab269b3a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1083,7 +1083,7 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & setSubstituterEnv(); - run.pid = maybeVfork(); + run.pid = fork(); switch (run.pid) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8fc78b1463..5f6203bc28 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -868,7 +868,7 @@ string runProgram(Path program, bool searchPath, const Strings & args) /* Fork. */ Pid pid; - pid = maybeVfork(); + pid = fork(); switch (pid) { @@ -928,13 +928,6 @@ void closeOnExec(int fd) } -#if HAVE_VFORK -pid_t (*maybeVfork)() = vfork; -#else -pid_t (*maybeVfork)() = fork; -#endif - - ////////////////////////////////////////////////////////////////////// diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 1e9ffcf51b..07c027a1f9 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -266,9 +266,6 @@ void closeMostFDs(const set & exceptions); /* Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(int fd); -/* Call vfork() if available, otherwise fork(). */ -extern pid_t (*maybeVfork)(); - /* User interruption. */ From 1114c7bd57bcab16255d5db5e6f66ae8dece7b1e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 14:15:12 +0200 Subject: [PATCH 40/41] nix-copy-closure: Restore compression and the progress viewer --- perl/lib/Nix/CopyClosure.pm | 52 ++++++++++++++++++++++++++++--------- scripts/nix-copy-closure.in | 4 +-- src/nix-store/nix-store.cc | 49 ++++++++++++++++++++++++++++++++-- 3 files changed, 89 insertions(+), 16 deletions(-) diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm index cba365aa17..5085ec075b 100644 --- a/perl/lib/Nix/CopyClosure.pm +++ b/perl/lib/Nix/CopyClosure.pm @@ -15,6 +15,16 @@ sub readInt { } +sub writeString { + my ($s, $to) = @_; + my $len = length $s; + my $req .= pack("L&" . fileno($to), $to_, + $progressViewer && $compressor ? "$progressViewer | $compressor" : $progressViewer || $compressor); + close $to; + exportPaths(fileno($to_), $sign, @missing); + close $to_; + waitpid $pid2, 0; + + } else { + exportPaths(fileno($to), $sign, @missing); + close $to; + } + + readInt($from) == 1 or die "remote machine \`$sshHost' failed to import closure\n"; # Shut down the server process. - close $to; waitpid $pid, 0; } diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 23d5619519..abd3760fc1 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -42,11 +42,11 @@ while (@ARGV) { } elsif ($arg eq "--gzip") { $compressor = "gzip"; - $decompressor = "gunzip"; + $decompressor = "gzip -d"; } elsif ($arg eq "--bzip2") { $compressor = "bzip2"; - $decompressor = "bunzip2"; + $decompressor = "bzip2 -d"; } elsif ($arg eq "--xz") { $compressor = "xz"; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 849cb7e8a7..bb5a9e2e0b 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -928,12 +928,57 @@ static void opServe(Strings opFlags, Strings opArgs) dumpPath(readStorePath(in), out); out.flush(); break; - case cmdImportPaths: + case cmdImportPaths: { if (!writeAllowed) throw Error("importing paths not allowed"); - store->importPaths(false, in); + string compression = readString(in); + + if (compression != "") { + if (compression != "gzip" && compression != "bzip2" && compression != "xz") + throw Error(format("unsupported compression method `%1%'") % compression); + + Pipe fromDecompressor; + fromDecompressor.create(); + + Pid pid; + pid = fork(); + + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: /* child */ + try { + fromDecompressor.readSide.close(); + if (dup2(fromDecompressor.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + // FIXME: use absolute path. + execlp(compression.c_str(), compression.c_str(), "-d", NULL); + throw SysError(format("executing `%1%'") % compression); + } catch (std::exception & e) { + std::cerr << "error: " << e.what() << std::endl; + } + _exit(1); + } + + fromDecompressor.writeSide.close(); + + FdSource fromDecompressor_(fromDecompressor.readSide); + store->importPaths(false, fromDecompressor_); + + pid.wait(true); + } else + store->importPaths(false, in); + writeInt(1, out); // indicate success out.flush(); + + /* The decompressor will have left stdin in an + undefined state, so we can't continue. */ + if (compression != "") return; + break; + } default: throw Error(format("unknown serve command %1%") % cmd); } From 8e9140cfdef9dbd1eb61e4c75c91d452ab5e4a74 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Jul 2014 16:50:51 +0200 Subject: [PATCH 41/41] Refactoring: Move all fork handling into a higher-order function C++11 lambdas ftw. --- src/download-via-ssh/download-via-ssh.cc | 32 ++----- src/libstore/build.cc | 73 +++++---------- src/libstore/local-store.cc | 35 +++----- src/libutil/util.cc | 109 +++++++++++------------ src/libutil/util.hh | 7 ++ src/nix-daemon/nix-daemon.cc | 49 ++++------ src/nix-store/nix-store.cc | 29 ++---- 7 files changed, 128 insertions(+), 206 deletions(-) diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index 6cbcd9891c..6834634f3d 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -24,30 +24,14 @@ static std::pair connect(const string & conn) Pipe to, from; to.create(); from.create(); - pid_t child = fork(); - switch (child) { - case -1: - throw SysError("unable to fork"); - case 0: - try { - restoreAffinity(); - if (dup2(to.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(from.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - execlp("ssh" - , "ssh" - , "-x" - , "-T" - , conn.c_str() - , "nix-store --serve" - , NULL); - throw SysError("executing ssh"); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } + startProcess([&]() { + if (dup2(to.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(from.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + execlp("ssh", "ssh", "-x", "-T", conn.c_str(), "nix-store --serve", NULL); + throw SysError("executing ssh"); + }); // If child exits unexpectedly, we'll EPIPE or EOF early. // If we exit unexpectedly, child will EPIPE or EOF early. // So no need to keep track of it. diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 70a3effb23..d3184507ce 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -602,42 +602,29 @@ HookInstance::HookInstance() builderOut.create(); /* Fork the hook. */ - pid = fork(); - switch (pid) { + pid = startProcess([&]() { - case -1: - throw SysError("unable to fork"); + commonChildInit(fromHook); - case 0: - try { /* child */ + if (chdir("/") == -1) throw SysError("changing into `/"); - commonChildInit(fromHook); + /* Dup the communication pipes. */ + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); - if (chdir("/") == -1) throw SysError("changing into `/"); + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); - /* Dup the communication pipes. */ - if (dup2(toHook.readSide, STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); + execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), + (format("%1%") % settings.maxSilentTime).str().c_str(), + (format("%1%") % settings.printBuildTrace).str().c_str(), + (format("%1%") % settings.buildTimeout).str().c_str(), + NULL); - /* Use fd 4 for the builder's stdout/stderr. */ - if (dup2(builderOut.writeSide, 4) == -1) - throw SysError("dupping builder's stdout/stderr"); + throw SysError(format("executing `%1%'") % buildHook); + }); - execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(), - (format("%1%") % settings.maxSilentTime).str().c_str(), - (format("%1%") % settings.printBuildTrace).str().c_str(), - (format("%1%") % settings.buildTimeout).str().c_str(), - NULL); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (std::exception & e) { - writeToStderr("build hook error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); fromHook.writeSide.close(); @@ -2781,32 +2768,18 @@ void SubstitutionGoal::tryToRun() const char * * argArr = strings2CharPtrs(args); /* Fork the substitute program. */ - pid = fork(); + pid = startProcess([&]() { - switch (pid) { + commonChildInit(logPipe); - case -1: - throw SysError("unable to fork"); + if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("cannot dup output pipe into stdout"); - case 0: - try { /* child */ + execv(sub.c_str(), (char * *) argArr); - commonChildInit(logPipe); + throw SysError(format("executing `%1%'") % sub); + }); - if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("cannot dup output pipe into stdout"); - - execv(sub.c_str(), (char * *) argArr); - - throw SysError(format("executing `%1%'") % sub); - - } catch (std::exception & e) { - writeToStderr("substitute error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* parent */ pid.setSeparatePG(true); pid.setKillSignal(SIGTERM); outPipe.writeSide.close(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 08ab269b3a..e66042c57f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1083,31 +1083,16 @@ void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & setSubstituterEnv(); - run.pid = fork(); - - switch (run.pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - restoreAffinity(); - if (dup2(toPipe.readSide, STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) - throw SysError("dupping stderr"); - execl(substituter.c_str(), substituter.c_str(), "--query", NULL); - throw SysError(format("executing `%1%'") % substituter); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } - - /* Parent. */ + run.pid = startProcess([&]() { + if (dup2(toPipe.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1) + throw SysError("dupping stderr"); + execl(substituter.c_str(), substituter.c_str(), "--query", NULL); + throw SysError(format("executing `%1%'") % substituter); + }); run.program = baseNameOf(substituter); run.to = toPipe.writeSide.borrow(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5f6203bc28..faa2b83c37 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,8 @@ #include "config.h" +#include "util.hh" +#include "affinity.hh" + #include #include #include @@ -16,8 +19,6 @@ #include #endif -#include "util.hh" - extern char * * environ; @@ -714,6 +715,13 @@ Pid::Pid() } +Pid::Pid(pid_t pid) +{ + Pid(); + *this = pid; +} + + Pid::~Pid() { kill(); @@ -801,43 +809,30 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - Pid pid; - pid = fork(); - switch (pid) { + Pid pid = startProcess([&]() { - case -1: - throw SysError("unable to fork"); + if (setuid(uid) == -1) + throw SysError("setting uid"); - case 0: - try { /* child */ - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { + while (true) { #ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among other - things, determines if kill(-1, signo) affects the calling - process. In the OSX libc, it's set to true, which means - "follow POSIX", which we don't want here + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; #else - if (kill(-1, SIGKILL) == 0) break; + if (kill(-1, SIGKILL) == 0) break; #endif - if (errno == ESRCH) break; /* no more processes */ - if (errno != EINTR) - throw SysError(format("cannot kill processes for uid `%1%'") % uid); - } - - } catch (std::exception & e) { - writeToStderr((format("killing processes belonging to uid `%1%': %2%\n") % uid % e.what()).str()); - _exit(1); + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid `%1%'") % uid); } - _exit(0); - } - /* parent */ + _exit(0); + }); + int status = pid.wait(true); if (status != 0) throw Error(format("cannot kill processes for uid `%1%': %2%") % uid % statusToString(status)); @@ -852,6 +847,25 @@ void killUser(uid_t uid) ////////////////////////////////////////////////////////////////////// +pid_t startProcess(std::function fun, const string & errorPrefix) +{ + pid_t pid = fork(); + if (pid == -1) throw SysError("unable to fork"); + + if (pid == 0) { + try { + restoreAffinity(); + fun(); + } catch (std::exception & e) { + writeToStderr(errorPrefix + string(e.what()) + "\n"); + } + _exit(1); + } + + return pid; +} + + string runProgram(Path program, bool searchPath, const Strings & args) { checkInterrupt(); @@ -867,32 +881,17 @@ string runProgram(Path program, bool searchPath, const Strings & args) pipe.create(); /* Fork. */ - Pid pid; - pid = fork(); + Pid pid = startProcess([&]() { + if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); - switch (pid) { + if (searchPath) + execvp(program.c_str(), (char * *) &cargs[0]); + else + execv(program.c_str(), (char * *) &cargs[0]); - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - if (dup2(pipe.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - - if (searchPath) - execvp(program.c_str(), (char * *) &cargs[0]); - else - execv(program.c_str(), (char * *) &cargs[0]); - throw SysError(format("executing `%1%'") % program); - - } catch (std::exception & e) { - writeToStderr("error: " + string(e.what()) + "\n"); - } - _exit(1); - } - - /* Parent. */ + throw SysError(format("executing `%1%'") % program); + }); pipe.writeSide.close(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 07c027a1f9..ad0d377a4f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -237,6 +238,7 @@ class Pid int killSignal; public: Pid(); + Pid(pid_t pid); ~Pid(); void operator =(pid_t pid); operator pid_t(); @@ -252,6 +254,11 @@ public: void killUser(uid_t uid); +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +pid_t startProcess(std::function fun, const string & errorPrefix = "error: "); + + /* Run a program and return its stdout in a string (i.e., like the shell backtick operator). */ string runProgram(Path program, bool searchPath = false, diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 0464ac9653..265131c613 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -872,40 +872,27 @@ static void daemonLoop() printMsg(lvlInfo, format("accepted connection from pid %1%, uid %2%") % clientPid % clientUid); /* Fork a child to handle the connection. */ - pid_t child; - child = fork(); + startProcess([&]() { + /* Background the daemon. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); - switch (child) { + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - /* Background the daemon. */ - if (setsid() == -1) - throw SysError(format("creating a new session")); - - /* Restore normal handling of SIGCHLD. */ - setSigChldAction(false); - - /* For debugging, stuff the pid into argv[1]. */ - if (clientPid != -1 && argvSaved[1]) { - string processName = int2String(clientPid); - strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); - } - - /* Handle the connection. */ - from.fd = remote; - to.fd = remote; - processConnection(trusted); - - } catch (std::exception & e) { - writeToStderr("unexpected Nix daemon error: " + string(e.what()) + "\n"); + /* For debugging, stuff the pid into argv[1]. */ + if (clientPid != -1 && argvSaved[1]) { + string processName = int2String(clientPid); + strncpy(argvSaved[1], processName.c_str(), strlen(argvSaved[1])); } - exit(0); - } + + /* Handle the connection. */ + from.fd = remote; + to.fd = remote; + processConnection(trusted); + + _exit(0); + }, "unexpected Nix daemon error: "); } catch (Interrupted & e) { throw; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index bb5a9e2e0b..28b205b1fd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -939,27 +939,14 @@ static void opServe(Strings opFlags, Strings opArgs) Pipe fromDecompressor; fromDecompressor.create(); - Pid pid; - pid = fork(); - - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: /* child */ - try { - fromDecompressor.readSide.close(); - if (dup2(fromDecompressor.writeSide, STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - // FIXME: use absolute path. - execlp(compression.c_str(), compression.c_str(), "-d", NULL); - throw SysError(format("executing `%1%'") % compression); - } catch (std::exception & e) { - std::cerr << "error: " << e.what() << std::endl; - } - _exit(1); - } + Pid pid = startProcess([&]() { + fromDecompressor.readSide.close(); + if (dup2(fromDecompressor.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + // FIXME: use absolute path. + execlp(compression.c_str(), compression.c_str(), "-d", NULL); + throw SysError(format("executing `%1%'") % compression); + }); fromDecompressor.writeSide.close();