Delay evaulation of `with` attrs until a variable lookup needs them

Evaluation of attribute sets is strict in the attribute names, which
means immediate evaluation of `with` attribute sets rules out some
potentially interesting use cases (e.g. where the attribute names of one
set depend in some way on another but we want to bring those names into
scope for some values in the second set).

The major example of this is overridable self-referential package sets
(e.g. all-packages.nix). With immediate `with` evaluation, the only
options for such sets are to either make them non-recursive and
explicitly use the name of the overridden set in non-overridden one
every time you want to reference another package, or make the set
recursive and use the `__overrides` hack. As shown in the test case that
comes with this commit, though, delayed `with` evaluation allows a nicer
third alternative.

Signed-off-by: Shea Levy <shea@shealevy.com>

src/libexpr/eval.cc

@@ -310,6 +310,10 @@ inline Value * EvalState::lookupVar(Env * env, const VarRef & var)
     
     if (var.fromWith) {
         while (1) {
+            if (env->values[0] == NULL) {
+                env->values[0] = allocValue();
+                evalAttrs(*env->up, env->withAttrs, *env->values[0]);
+            }
             Bindings::iterator j = env->values[0]->attrs->find(var.name);
             if (j != env->values[0]->attrs->end()) {
                 if (countCalls && j->pos) attrSelects[*j->pos]++;
@@ -337,7 +341,7 @@ Env & EvalState::allocEnv(unsigned int size)
     nrValuesInEnvs += size;
     Env * env = (Env *) GC_MALLOC(sizeof(Env) + size * sizeof(Value *));
 
-    /* Clear the values because maybeThunk() expects this. */
+    /* Clear the values because maybeThunk() and lookupVar fromWith expects this. */
     for (unsigned i = 0; i < size; ++i)
         env->values[i] = 0;
     
@@ -405,10 +409,12 @@ unsigned long nrAvoided = 0;
 
 Value * ExprVar::maybeThunk(EvalState & state, Env & env)
 {
-    Value * v = state.lookupVar(&env, info);
-    /* The value might not be initialised in the environment yet.
-       In that case, ignore it. */
-    if (v) { nrAvoided++; return v; }
+    if (!info.fromWith) {
+        Value * v = state.lookupVar(&env, info);
+        /* The value might not be initialised in the environment yet.
+           In that case, ignore it. */
+        if (v) { nrAvoided++; return v; }
+    }
     return Expr::maybeThunk(state, env);
 }
 
@@ -825,8 +831,7 @@ void ExprWith::eval(EvalState & state, Env & env, Value & v)
     env2.up = &env;
     env2.prevWith = prevWith;
 
-    env2.values[0] = state.allocValue();
-    state.evalAttrs(env, attrs, *env2.values[0]);
+    env2.withAttrs = attrs;
 
     body->eval(state, env2, v);
 }

src/libexpr/eval.hh

@@ -53,6 +53,7 @@ struct Env
 {
     Env * up;
     unsigned int prevWith; // nr of levels up to next `with' environment
+    Expr * withAttrs;
     Value * values[0];
 };
 

tests/lang/eval-okay-delayed-with.exp

@@ -0,0 +1 @@
+"b-overridden"

tests/lang/eval-okay-delayed-with.nix

@@ -0,0 +1,26 @@
+let
+  pkgs_ = with pkgs; {
+    a = derivation {
+      name = "a";
+      system = builtins.currentSystem;
+      builder = "/bin/sh";
+      args = [ "-c" "touch $out" ];
+      inherit b;
+    };
+
+    b = derivation {
+      name = "b";
+      system = builtins.currentSystem;
+      builder = "/bin/sh";
+      args = [ "-c" "touch $out" ];
+    };
+
+    c = b;
+  };
+
+  packageOverrides = p: {
+    b = derivation (p.b.drvAttrs // { name = "b-overridden"; });
+  };
+
+  pkgs = pkgs_ // (packageOverrides pkgs_);
+in pkgs.a.b.name