From 58337e0e6122a97061dcf803954f72469f67afca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 11:51:27 -0400 Subject: [PATCH 01/19] Set release date --- doc/manual/release-notes.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 69ab0874b6..ed06b638ad 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -8,7 +8,7 @@ -
Release 1.1 (TBA) +
Release 1.1 (July 18, 2012) This release has the following improvements: From b7fd2c28224a69476434d69b5d9da3d150c07226 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 14:59:03 -0400 Subject: [PATCH 02/19] Use "#pragma once" to prevent repeated header file inclusion --- .gitignore | 4 ---- src/libexpr/attr-path.hh | 9 +-------- src/libexpr/common-opts.hh | 7 +------ src/libexpr/eval-inline.hh | 7 +------ src/libexpr/eval.hh | 6 +----- src/libexpr/get-drvs.hh | 6 +----- src/libexpr/names.hh | 11 +---------- src/libexpr/nixexpr.hh | 6 +----- src/libexpr/symbol-table.hh | 5 +---- src/libexpr/value-to-xml.hh | 5 +---- src/libexpr/value.hh | 5 +---- src/libmain/shared.hh | 6 +----- src/libstore/derivations.hh | 6 +----- src/libstore/globals.hh | 6 +----- src/libstore/local-store.hh | 6 +----- src/libstore/misc.hh | 6 +----- src/libstore/pathlocks.hh | 6 +----- src/libstore/references.hh | 5 +---- src/libstore/remote-store.hh | 6 +----- src/libstore/store-api.hh | 6 +----- src/libstore/worker-protocol.hh | 7 +------ src/libutil/archive.hh | 6 +----- src/libutil/hash.hh | 6 +----- src/libutil/immutable.hh | 5 +---- src/libutil/serialise.hh | 6 +----- src/libutil/types.hh | 6 +----- src/libutil/util.hh | 6 +----- src/libutil/xml-writer.hh | 6 +----- src/nix-env/profiles.hh | 6 +----- src/nix-env/user-env.hh | 9 +-------- src/nix-store/dotgraph.hh | 5 +---- src/nix-store/xmlgraph.hh | 5 +---- 32 files changed, 31 insertions(+), 166 deletions(-) diff --git a/.gitignore b/.gitignore index d7f151507f..3fb96f1249 100644 --- a/.gitignore +++ b/.gitignore @@ -78,13 +78,9 @@ Makefile.in /src/libexpr/parser-tab.cc /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output -/src/libexpr/nixexpr-ast.hh -/src/libexpr/nixexpr-ast.cc /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/derivations-ast.cc -/src/libstore/derivations-ast.hh /src/libstore/schema.sql.hh # /src/nix-env/ diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index b106da5ef8..d3aad74611 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,20 +1,13 @@ -#ifndef __ATTR_PATH_H -#define __ATTR_PATH_H +#pragma once #include "eval.hh" #include #include - namespace nix { - void findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Expr * e, Value & v); - } - - -#endif /* !__ATTR_PATH_H */ diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index c28641e901..e2e3fe7717 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -1,9 +1,7 @@ -#ifndef __COMMON_OPTS_H -#define __COMMON_OPTS_H +#pragma once #include "eval.hh" - namespace nix { /* Some common option parsing between nix-env and nix-instantiate. */ @@ -17,6 +15,3 @@ bool parseSearchPathArg(const string & arg, Strings::iterator & i, Path lookupFileArg(EvalState & state, string s); } - - -#endif /* !__COMMON_OPTS_H */ diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 6026a7d110..57a9e4c63c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_INLINE_H -#define __EVAL_INLINE_H +#pragma once #include "eval.hh" @@ -8,7 +7,6 @@ namespace nix { - LocalNoInlineNoReturn(void throwEvalError(const char * s)) { throw EvalError(s); @@ -55,7 +53,4 @@ inline void EvalState::forceList(Value & v) throwTypeError("value is %1% while a list was expected", showType(v)); } - } - -#endif /* !__EVAL_INLINE_H */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index bab9303b08..5103ae8cef 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_H -#define __EVAL_H +#pragma once #include "value.hh" #include "nixexpr.hh" @@ -256,6 +255,3 @@ string showType(const Value & v); } - - -#endif /* !__EVAL_H */ diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 2d260c57be..879dc8dbb4 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,5 +1,4 @@ -#ifndef __GET_DRVS_H -#define __GET_DRVS_H +#pragma once #include "eval.hh" @@ -78,6 +77,3 @@ void getDerivations(EvalState & state, Value & v, const string & pathPrefix, } - - -#endif /* !__GET_DRVS_H */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index e189302d6d..ebe113e82a 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -1,12 +1,9 @@ -#ifndef __NAMES_H -#define __NAMES_H +#pragma once #include "types.hh" - namespace nix { - struct DrvName { string fullName; @@ -19,15 +16,9 @@ struct DrvName bool matches(DrvName & n); }; - typedef list DrvNames; - int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); - } - - -#endif /* !__NAMES_H */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 6eb771a726..4c1a0bb2d5 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,5 +1,4 @@ -#ifndef __NIXEXPR_H -#define __NIXEXPR_H +#pragma once #include "value.hh" #include "symbol-table.hh" @@ -290,6 +289,3 @@ struct StaticEnv } - - -#endif /* !__NIXEXPR_H */ diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 976117a20a..143fc495b0 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,5 +1,4 @@ -#ifndef __SYMBOL_TABLE_H -#define __SYMBOL_TABLE_H +#pragma once #include "config.h" @@ -88,5 +87,3 @@ public: }; } - -#endif /* !__SYMBOL_TABLE_H */ diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 3ebc989ffa..97657327ed 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_TO_XML_H -#define __VALUE_TO_XML_H +#pragma once #include "nixexpr.hh" #include "eval.hh" @@ -13,5 +12,3 @@ void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); } - -#endif /* !__VALUE_TO_XML_H */ diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 41512d40b1..c9ec236c47 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_H -#define __VALUE_H +#pragma once #include "symbol-table.hh" @@ -151,5 +150,3 @@ void mkPath(Value & v, const char * s); } - -#endif /* !__VALUE_H */ diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d198df70d1..7849e10e36 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,5 +1,4 @@ -#ifndef __SHARED_H -#define __SHARED_H +#pragma once #include "util.hh" @@ -54,6 +53,3 @@ extern int exitCode; extern char * * argvSaved; } - - -#endif /* !__SHARED_H */ diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 27e471d885..8f22b4afa6 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,5 +1,4 @@ -#ifndef __DERIVATIONS_H -#define __DERIVATIONS_H +#pragma once #include @@ -81,6 +80,3 @@ typedef std::map DrvHashes; extern DrvHashes drvHashes; } - - -#endif /* !__DERIVATIONS_H */ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12a9b9ca15..1c0877a5e1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,5 +1,4 @@ -#ifndef __GLOBALS_H -#define __GLOBALS_H +#pragma once #include "types.hh" @@ -118,6 +117,3 @@ void setDefaultsFromEnvironment(); } - - -#endif /* !__GLOBALS_H */ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 65ee029c26..eb7705219a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,5 +1,4 @@ -#ifndef __LOCAL_STORE_H -#define __LOCAL_STORE_H +#pragma once #include @@ -302,6 +301,3 @@ void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path); } - - -#endif /* !__LOCAL_STORE_H */ diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 850c12af4e..fe0bbdd799 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -1,5 +1,4 @@ -#ifndef __MISC_H -#define __MISC_H +#pragma once #include "derivations.hh" @@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets, } - - -#endif /* !__MISC_H */ diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 57ca1584a6..8a6b1450da 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,5 +1,4 @@ -#ifndef __PATHLOCKS_H -#define __PATHLOCKS_H +#pragma once #include "types.hh" @@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path); } - - -#endif /* !__PATHLOCKS_H */ diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 158c08a776..013809d122 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,5 +1,4 @@ -#ifndef __REFERENCES_H -#define __REFERENCES_H +#pragma once #include "types.hh" #include "hash.hh" @@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); } - -#endif /* !__REFERENCES_H */ diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0e5dbf769..c57b49ce15 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,4 @@ -#ifndef __REMOTE_STORE_H -#define __REMOTE_STORE_H +#pragma once #include @@ -101,6 +100,3 @@ private: } - - -#endif /* !__REMOTE_STORE_H */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0ab15c3806..9ba67852ef 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,4 @@ -#ifndef __STOREAPI_H -#define __STOREAPI_H +#pragma once #include "hash.hh" #include "serialise.hh" @@ -356,6 +355,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */ } - - -#endif /* !__STOREAPI_H */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b08410fa1c..501c0b3db5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,4 @@ -#ifndef __WORKER_PROTOCOL_H -#define __WORKER_PROTOCOL_H - +#pragma once namespace nix { @@ -64,6 +62,3 @@ template T readStorePaths(Source & from); } - - -#endif /* !__WORKER_PROTOCOL_H */ diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fff6203139..ccac92074d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,5 +1,4 @@ -#ifndef __ARCHIVE_H -#define __ARCHIVE_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -74,6 +73,3 @@ void restorePath(const Path & path, Source & source); } - - -#endif /* !__ARCHIVE_H */ diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index e0b6478cc4..781f517428 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,5 +1,4 @@ -#ifndef __HASH_H -#define __HASH_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -109,6 +108,3 @@ public: } - - -#endif /* !__HASH_H */ diff --git a/src/libutil/immutable.hh b/src/libutil/immutable.hh index 5a42a46107..8af4190049 100644 --- a/src/libutil/immutable.hh +++ b/src/libutil/immutable.hh @@ -1,5 +1,4 @@ -#ifndef __IMMUTABLE_H -#define __IMMUTABLE_H +#pragma once #include @@ -15,5 +14,3 @@ void makeImmutable(const Path & path); void makeMutable(const Path & path); } - -#endif /* !__IMMUTABLE_H */ diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index ded4b12a04..42dd271176 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,5 +1,4 @@ -#ifndef __SERIALISE_H -#define __SERIALISE_H +#pragma once #include "types.hh" @@ -130,6 +129,3 @@ MakeError(SerialisationError, Error) } - - -#endif /* !__SERIALISE_H */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 844ad6f76a..165a46fa28 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,5 +1,4 @@ -#ifndef __TYPES_H -#define __TYPES_H +#pragma once #include #include @@ -74,6 +73,3 @@ typedef enum { } - - -#endif /* !__TYPES_H */ diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ee0f3862a8..362d0f65e2 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,5 +1,4 @@ -#ifndef __UTIL_H -#define __UTIL_H +#pragma once #include "types.hh" @@ -333,6 +332,3 @@ void ignoreException(); } - - -#endif /* !__UTIL_H */ diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index e5cc5f8c54..fee2eb495e 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,5 +1,4 @@ -#ifndef __XML_WRITER_H -#define __XML_WRITER_H +#pragma once #include #include @@ -70,6 +69,3 @@ public: } - - -#endif /* !__XML_WRITER_H */ diff --git a/src/nix-env/profiles.hh b/src/nix-env/profiles.hh index a64258dae2..30d2376d99 100644 --- a/src/nix-env/profiles.hh +++ b/src/nix-env/profiles.hh @@ -1,5 +1,4 @@ -#ifndef __PROFILES_H -#define __PROFILES_H +#pragma once #include "types.hh" #include "pathlocks.hh" @@ -54,6 +53,3 @@ void lockProfile(PathLocks & lock, const Path & profile); string optimisticLockProfile(const Path & profile); } - - -#endif /* !__PROFILES_H */ diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 4125d82173..f188efe9b4 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,5 +1,4 @@ -#ifndef __USER_ENV_H -#define __USER_ENV_H +#pragma once #include "get-drvs.hh" @@ -12,9 +11,3 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, const string & lockToken); } - -#endif /* !__USER_ENV_H */ - - - - diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 2318e2fde4..68410d8415 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,5 +1,4 @@ -#ifndef __DOTGRAPH_H -#define __DOTGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printDotGraph(const PathSet & roots); } - -#endif /* !__DOTGRAPH_H */ diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh index 2f9908c436..c2216c5a46 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/xmlgraph.hh @@ -1,5 +1,4 @@ -#ifndef __XMLGRAPH_H -#define __XMLGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printXmlGraph(const PathSet & roots); } - -#endif /* !__XMLGRAPH_H */ From 98193bb440561875d2829f9dd542e38972dbcf63 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 16:50:56 -0400 Subject: [PATCH 03/19] Remove RPM builds that don't evaluate --- release.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/release.nix b/release.nix index 5f20920f3c..47d296c622 100644 --- a/release.nix +++ b/release.nix @@ -153,10 +153,6 @@ let rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50; rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50; rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50; - - rpm_opensuse103i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse103i386) 40; - rpm_opensuse110i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse110i386) 50; - rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImageFuns: diskImageFuns.opensuse110x86_64) 50; deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50; From 1832ab71dbb6b24965eb5a873a56a7231da7af4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Jul 2012 17:17:23 -0400 Subject: [PATCH 04/19] Bump version --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index b123147e2a..ea710abb95 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1 \ No newline at end of file +1.2 \ No newline at end of file From 6852289c46cdfceb07b459cd1028722ffb124ca6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 16:52:25 -0400 Subject: [PATCH 05/19] Use lutimes() if available to canonicalise the timestamp of symlinks Also use utimes() instead of utime() if lutimes() is not available. --- configure.ac | 5 +++++ src/libstore/local-store.cc | 25 ++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/configure.ac b/configure.ac index 7b814dedcb..a2512cfe92 100644 --- a/configure.ac +++ b/configure.ac @@ -115,6 +115,11 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], ]) +# Check for lutimes, optionally used for changing the mtime of +# symlinks. +AC_CHECK_FUNCS([lutimes]) + + # Check for . AC_LANG_PUSH(C++) AC_CHECK_HEADERS([locale]) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 30398a2446..e009191b6c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -444,7 +445,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing owner of `%1%' to %2%") % path % geteuid()); } - + if (!S_ISLNK(st.st_mode)) { /* Mask out all type related bits. */ @@ -458,14 +459,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); } - if (st.st_mtime != mtimeStore) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = mtimeStore; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); } if (recurse && S_ISDIR(st.st_mode)) { From ed59bf7a181bb382dea7dd72da52bf91f60deb8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 17:11:12 -0400 Subject: [PATCH 06/19] =?UTF-8?q?nix-build:=20Support=20the=20=E2=80=98-?= =?UTF-8?q?=E2=80=99=20argument=20to=20build=20an=20expression=20from=20st?= =?UTF-8?q?din?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/nix-build.in | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 35b186bb71..afe0679a47 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -118,6 +118,10 @@ EOF elsif ($arg eq "--show-trace") { push @instArgs, $arg; } + + elsif ($arg eq "-") { + @exprs = ("-"); + } elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") { push @buildArgs, $arg; From 564fb7d9fa80d06397a88d69f26439727cb922c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 12:08:34 -0400 Subject: [PATCH 07/19] optimiseStore(): Use a content-addressed file store in /nix/store/.links optimiseStore() now creates persistent, content-addressed hard links in /nix/store/.links. For instance, if it encounters a file P with hash H, it will create a hard link P' = /nix/store/.link/ to P if P' doesn't already exist; if P' exist, then P is replaced by a hard link to P'. This is better than the previous in-memory map, because it had the tendency to unnecessarily replace hard links with a hard link to whatever happened to be the first file with a given hash it encountered. It also allows on-the-fly, incremental optimisation. --- src/libstore/local-store.hh | 2 +- src/libstore/optimise-store.cc | 211 +++++++++++++++++---------------- src/nix-store/nix-store.cc | 10 +- 3 files changed, 109 insertions(+), 114 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index eb7705219a..1bb47fb3ba 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -167,7 +167,7 @@ public: /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - void optimiseStore(bool dryRun, OptimiseStats & stats); + void optimiseStore(OptimiseStats & stats); /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 2ca98f46dd..0893db9d31 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "local-store.hh" #include "immutable.hh" +#include "globals.hh" #include #include @@ -12,9 +13,6 @@ namespace nix { -typedef std::map > HashToPath; - - static void makeWritable(const Path & path) { struct stat st; @@ -51,132 +49,135 @@ struct MakeImmutable }; -static void hashAndLink(bool dryRun, HashToPath & hashToPath, - OptimiseStats & stats, const Path & path) +const string linksDir = ".links"; + + +static void hashAndLink(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + hashAndLink(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and symlinks. */ + if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) return; + /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under NixOS (example: $fontconfig/var/cache being modified). Skip - those files. */ + those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); return; } - /* We can hard link regular files and symlinks. */ - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). - /* Hash the file. Note that hashPath() returns the hash over - the NAR serialisation, which includes the execute bit on - the file. Thus, executable and non-executable files with - the same contents *won't* be linked (which is good because - otherwise the permissions would be screwed up). + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - Also note that if `path' is a symlink, then we're hashing - the contents of the symlink (i.e. the result of - readlink()), not the contents of the target (which may not - even exist). */ - Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + /* Check if this is a known hash. */ + Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash); - std::pair prevPath = hashToPath[hash]; - - if (prevPath.first == "") { - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - - /* Yes! We've seen a file with the same contents. Replace - the current file with a hard link to that file. */ - stats.sameContents++; - if (prevPath.second == st.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); - return; - } - - if (!dryRun) { - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + makeMutable(path); + MakeImmutable mk1(path); - Path tempLink = (format("%1%.tmp-%2%-%3%") - % path % getpid() % rand()).str(); + if (link(path.c_str(), linkPath.c_str()) == -1) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Make the containing directory writable, but only if - it's not the store itself (we don't want or need to - mess with its permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and - reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - /* If ‘prevPath’ is immutable, we can't create hard links - to it, so make it mutable first (and make it immutable - again when we're done). We also have to make ‘path’ - mutable, otherwise rename() will fail to delete it. */ - makeMutable(prevPath.first); - MakeImmutable mk1(prevPath.first); - - makeMutable(path); - MakeImmutable mk2(path); - - if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on - most file systems). This is likely to happen - with empty files. Just start over, creating - links to the current file. */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - throw SysError(format("cannot link `%1%' to `%2%'") - % tempLink % prevPath.first); - } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (errno == EMLINK) { - /* Some filesystems generate too many links on the - rename, rather than on the original link. - (Probably it temporarily increases the st_nlink - field before decreasing it again.) */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - - /* Unlink the temp link. */ - if (unlink(tempLink.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); - return; - } - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % path); - } - } else - printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + return; } - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; } + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % nixStore % getpid() % rand()).str(); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + /* If ‘linkPath’ is immutable, we can't create hard links to it, + so make it mutable first (and make it immutable again when + we're done). We also have to make ‘path’ mutable, otherwise + rename() will fail to delete it. */ + makeMutable(linkPath); + MakeImmutable mk1(linkPath); + + makeMutable(path); + MakeImmutable mk2(path); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + + /* Unlink the temp link. */ + if (unlink(linkPath.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % linkPath); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; } -void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) +void LocalStore::optimiseStore(OptimiseStats & stats) { - HashToPath hashToPath; + createDirs(nixStore + "/" + linksDir); PathSet paths = queryValidPaths(); @@ -184,7 +185,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(dryRun, hashToPath, stats, *i); + hashAndLink(stats, *i); } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23863525fe..82e08fecf2 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -746,18 +746,12 @@ static void showOptimiseStats(OptimiseStats & stats) files with the same contents. */ static void opOptimise(Strings opFlags, Strings opArgs) { - if (!opArgs.empty()) + if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); - bool dryRun = false; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--dry-run") dryRun = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - OptimiseStats stats; try { - ensureLocalStore().optimiseStore(dryRun, stats); + ensureLocalStore().optimiseStore(stats); } catch (...) { showOptimiseStats(stats); throw; From 619310571002fc74e428824bd603604d1055b61b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 15:02:52 -0400 Subject: [PATCH 08/19] Automatically optimise the Nix store when a new path is added Auto-optimisation is enabled by default. It can be turned off by setting auto-optimise-store to false in nix.conf. --- doc/manual/conf-file.xml | 15 ++++++++++++++- src/libstore/build.cc | 4 ++++ src/libstore/local-store.cc | 7 +++++++ src/libstore/local-store.hh | 7 +++++++ src/libstore/optimise-store.cc | 22 +++++++++++++--------- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 1b19e56b57..c095a001c1 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -337,7 +337,20 @@ build-use-chroot = /dev /proc /bin true. - + + + auto-optimise-store + + If set to true (the default), + Nix automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false, you + can still run nix-store --optimise to get rid + of duplicate files. + + + + diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 26268f6ddb..a3bde34623 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2093,6 +2093,8 @@ void DerivationGoal::computeClosure() if (allowed.find(*i) == allowed.end()) throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() } /* Register each output path as valid, and register the sets of @@ -2546,6 +2548,8 @@ void SubstitutionGoal::finished() HashResult hash = hashPath(htSHA256, storePath); + worker.store.optimisePath(storePath); // FIXME: combine with hashPath() + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e009191b6c..05b2b9c6e5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -209,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace) /* Create missing state directories if they don't already exist. */ createDirs(nixStore); + createDirs(linksDir = nixStore + "/.links"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -1116,6 +1117,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash.second = dump.size(); } else hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; @@ -1170,6 +1173,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, canonicalisePathMetaData(dstPath); HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; @@ -1405,6 +1410,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1bb47fb3ba..7d30a2d408 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -85,6 +85,8 @@ private: typedef std::map RunningSubstituters; RunningSubstituters runningSubstituters; + + Path linksDir; public: @@ -169,6 +171,9 @@ public: files with the same contents. */ void optimiseStore(OptimiseStats & stats); + /* Optimise a single store path. */ + void optimisePath(const Path & path); + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -267,6 +272,8 @@ private: Path importPath(bool requireSignature, Source & source); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); }; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0893db9d31..a7aa14fb49 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -49,10 +49,7 @@ struct MakeImmutable }; -const string linksDir = ".links"; - - -static void hashAndLink(OptimiseStats & stats, const Path & path) +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) @@ -61,7 +58,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) if (S_ISDIR(st.st_mode)) { Strings names = readDirectory(path); foreach (Strings::iterator, i, names) - hashAndLink(stats, path + "/" + *i); + optimisePath_(stats, path + "/" + *i); return; } @@ -91,7 +88,7 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); /* Check if this is a known hash. */ - Path linkPath = nixStore + "/" + linksDir + "/" + printHash32(hash); + Path linkPath = linksDir + "/" + printHash32(hash); if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ @@ -177,15 +174,22 @@ static void hashAndLink(OptimiseStats & stats, const Path & path) void LocalStore::optimiseStore(OptimiseStats & stats) { - createDirs(nixStore + "/" + linksDir); - PathSet paths = queryValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(stats, *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + if (queryBoolSetting("auto-optimise-store", true)) { + OptimiseStats stats; + optimisePath_(stats, path); } } From 680ab6f83def2b636200204542ca352631a46f85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 15:48:30 -0400 Subject: [PATCH 09/19] Garbage collect unused links in /nix/store/.links Incremental optimisation requires creating links in /nix/store/.links to all files in the store. However, this means that if we delete a store path, no files are actually deleted because links in /nix/store/.links still exists. So we need to check /nix/store/.links for files with a link count of 1 and delete them. --- src/libstore/gc.cc | 37 +++++++++++++++++++++++++++++++++++++ src/libstore/local-store.hh | 2 ++ 2 files changed, 39 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f6ed7dd226..874efe4d32 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); + if (path == linksDir) return true; + struct stat st; if (lstat(path.c_str(), &st)) { if (errno == ENOENT) return true; @@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks() +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) continue; + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + } +} + + void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { GCState state(results); @@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) released. */ foreach (PathSet::iterator, i, state.invalidated) deleteGarbage(state, *i); + + /* Clean up the links directory. */ + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7d30a2d408..50910f353a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -264,6 +264,8 @@ private: int openGCLock(LockType lockType); + void removeUnusedLinks(); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); From 0f65793f94bd89c973482ac949be1e96e876762b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 17:40:23 -0400 Subject: [PATCH 10/19] Add a test for Nix store optimisation --- tests/Makefile.am | 2 +- tests/optimise-store.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/optimise-store.sh diff --git a/tests/Makefile.am b/tests/Makefile.am index 517c382b19..a562db52bc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -9,7 +9,7 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ - multiple-outputs.sh import-derivation.sh fetchurl.sh + multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh XFAIL_TESTS = diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh new file mode 100644 index 0000000000..4d8077997d --- /dev/null +++ b/tests/optimise-store.sh @@ -0,0 +1,26 @@ +source common.sh + +clearStore + +outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) +outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) + +inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")" +inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")" +if [ "$inode1" != "$inode2" ]; then + echo "inodes do not match" + exit 1 +fi + +nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")" +if [ "$nlink" != 3 ]; then + echo "link count incorrect" + exit 1 +fi + +nix-store --gc + +if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then + echo ".links directory not empty after GC" + exit 1 +fi From fd63c8bfcd75624e7fbba8899365095400534e01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 18:06:37 -0400 Subject: [PATCH 11/19] Unlink the right file --- src/libstore/optimise-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a7aa14fb49..e5bfa332dc 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -159,8 +159,8 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); /* Unlink the temp link. */ - if (unlink(linkPath.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % linkPath); + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); return; } throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); From e98c029717016dfa3e5c618c9fc46da9b2142dcc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 23 Jul 2012 18:42:18 -0400 Subject: [PATCH 12/19] Handle platforms that don't support linking to a symlink E.g. Darwin doesn't allow this. --- configure.ac | 12 ++++++++++++ src/libstore/optimise-store.cc | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index a2512cfe92..f920b8c10e 100644 --- a/configure.ac +++ b/configure.ac @@ -120,6 +120,18 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], AC_CHECK_FUNCS([lutimes]) +# Check whether the store optimiser can optimise symlinks. +AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) +ln -s bla tmp_link +if ln tmp_link tmp_link2 2> /dev/null; then + AC_MSG_RESULT(yes) + AC_DEFINE(CAN_LINK_SYMLINK, 1, [Whether link() works on symlinks.]) +else + AC_MSG_RESULT(no) +fi +rm -f tmp_link tmp_link2 + + # Check for . AC_LANG_PUSH(C++) AC_CHECK_HEADERS([locale]) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e5bfa332dc..486538bd29 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -62,8 +62,13 @@ void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) return; } - /* We can hard link regular files and symlinks. */ - if (!S_ISREG(st.st_mode) && !S_ISLNK(st.st_mode)) return; + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + x + && !S_ISLNK(st.st_mode) +#endif + ) return; /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under From 566a30c0072690900d4d55679a2981758d6fb888 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 12:51:04 -0400 Subject: [PATCH 13/19] Disable tests temporarily --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 47d296c622..fd9db47932 100644 --- a/release.nix +++ b/release.nix @@ -82,7 +82,7 @@ let enableParallelBuilding = true; - doInstallCheck = true; + doInstallCheck = false; }; binaryTarball = From b1112bbef195bc8397c4e88aa8544537a6d84731 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 13:41:28 -0400 Subject: [PATCH 14/19] import: If the path is a valid .drv file, parse it and generate a derivation attrset. The generated attrset has drvPath and outPath with the right string context, type 'derivation', outputName with the right name, all with a list of outputs, and an attribute for each output. I see three uses for this (though certainly there may be more): * Using derivations generated by something besides nix-instantiate (e.g. guix) * Allowing packages provided by channels to be used in nix expressions. If a channel installed a valid deriver for each package it provides into the store, then those could be imported and used as dependencies or installed in environment.systemPackages, for example. * Enable hydra to be consistent in how it treats inputs that are outputs of another build. Right now, if an input is passed as an argument to the job, it is passed as a derivation, but if it is accessed via NIX_PATH (i.e. through the <> syntax), then it is a path that can be imported. This is problematic because the build being depended upon may have been built with non-obvious arguments passed to its jobset file. With this feature, hydra can just set the name of that input to the path to its drv file in NIX_PATH --- corepkgs/Makefile.am | 3 ++- corepkgs/imported-drv-to-derivation.nix | 21 ++++++++++++++++++++ src/libexpr/primops.cc | 26 ++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 corepkgs/imported-drv-to-derivation.nix diff --git a/corepkgs/Makefile.am b/corepkgs/Makefile.am index 729d15e7b1..4b0b8860be 100644 --- a/corepkgs/Makefile.am +++ b/corepkgs/Makefile.am @@ -1,6 +1,7 @@ all-local: config.nix -files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix +files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \ + imported-drv-to-derivation.nix install-exec-local: $(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs diff --git a/corepkgs/imported-drv-to-derivation.nix b/corepkgs/imported-drv-to-derivation.nix new file mode 100644 index 0000000000..bdb6016986 --- /dev/null +++ b/corepkgs/imported-drv-to-derivation.nix @@ -0,0 +1,21 @@ +attrs @ { drvPath, outputs, ... }: + +let + + commonAttrs = (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvPath; + type = "derivation"; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName attrs; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d5f0bfb3b..2ab3eda53f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -59,7 +59,31 @@ static void prim_import(EvalState & state, Value * * args, Value & v) } } - state.evalFile(path, v); + if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { + Derivation drv = parseDerivation(readFile(path)); + Value w; + state.mkAttrs(w, 1 + drv.outputs.size()); + mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); + state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); + unsigned int outputs_index = 0; + + Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(w, state.symbols.create(i->first)), + i->second.path, singleton("!" + i->first + "!" + path)); + mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), + i->first); + } + w.attrs->sort(); + Value fun; + state.mkThunk_(fun, + state.parseExprFromFile(state.findFile("nix/imported-drv-to-derivation.nix"))); + state.forceFunction(fun); + mkApp(v, fun, w); + state.forceAttrs(v); + } else { + state.evalFile(path, v); + } } From 1ef2d5765be35c3d3c13a2aea8748166f576ec8b Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 23 Jul 2012 13:45:51 -0400 Subject: [PATCH 15/19] Turn tests back on --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index fd9db47932..47d296c622 100644 --- a/release.nix +++ b/release.nix @@ -82,7 +82,7 @@ let enableParallelBuilding = true; - doInstallCheck = false; + doInstallCheck = true; }; binaryTarball = From f5954e2d940c3a41a6ed0cad45660e254eb381a3 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Tue, 24 Jul 2012 12:05:27 -0400 Subject: [PATCH 16/19] prim_import: When importing .drvs, allocate the intermediate attrset on the heap just in case it escapes the stack frame. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2ab3eda53f..0d4efc47e6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -61,7 +61,7 @@ static void prim_import(EvalState & state, Value * * args, Value & v) if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { Derivation drv = parseDerivation(readFile(path)); - Value w; + Value & w = *state.allocValue(); state.mkAttrs(w, 1 + drv.outputs.size()); mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); From 477b0fbeca62bf1957bc0aad26f1a844ebd22231 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jul 2012 16:56:56 -0400 Subject: [PATCH 17/19] Subscribe to the Nixpkgs rather than NixOS channel --- scripts/nix-profile.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index d343385cc1..b18069d940 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -19,9 +19,9 @@ fi export PATH="$HOME/.nix-profile/bin:$PATH" -# Subscribe the root user to the NixOS channel by default. +# Subscribe the root user to the Nixpkgs channel by default. if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "http://nixos.org/releases/nixos/channels/nixos-unstable nixos" > $HOME/.nix-channels + echo "http://nixos.org/releases/nixos/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi # Create the per-user garbage collector roots directory. From 2605f4f4e6a367df67bf8b33b252c350313699c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Jul 2012 17:06:09 -0400 Subject: [PATCH 18/19] nix-profile.sh: Don't set NIX_REMOTE on single user installations Commit 6a214f3e06fa1c5f0a4d40e555f14d87691af297 reused the NixOS environment initialisation for nix-profile.sh, but this is inappropriate on systems that don't have multi-user support enabled. --- scripts/nix-profile.sh.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index b18069d940..bc3dc719eb 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -43,7 +43,7 @@ fi # Set up secure multi-user builds: non-root users build through the # Nix daemon. -if test "$USER" != root; then +if [ "$USER" != root -a -e @localstatedir@/nix/daemon-socket/socket ]; then export NIX_REMOTE=daemon else unset NIX_REMOTE From 3a4623afbbc1bff85bde33167d36e8c5a4a3df0d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Jul 2012 15:04:40 -0400 Subject: [PATCH 19/19] Set permissions on temporary build directories to 0700 Fixes #39. --- src/libstore/build.cc | 3 ++- src/libutil/util.cc | 4 ++-- src/libutil/util.hh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/build.cc b/src/libstore/build.cc index a3bde34623..290635695e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1527,7 +1527,7 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false); + tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700); /* For convenience, set an environment pointing to the top build directory. */ @@ -2178,6 +2178,7 @@ void DerivationGoal::deleteTmpDir(bool force) % drvPath % tmpDir); if (buildUser.enabled() && !amPrivileged()) getOwnership(tmpDir); + chmod(tmpDir.c_str(), 0755); } else deletePathWrapped(tmpDir); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b188a9fc0e..689fc543af 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -380,7 +380,7 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter) + bool includePid, bool useGlobalCounter, mode_t mode) { static int globalCounter = 0; int localCounter = 0; @@ -389,7 +389,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, while (1) { checkInterrupt(); Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), 0777) == 0) { + if (mkdir(tmpDir.c_str(), mode) == 0) { /* Explicitly set the group of the directory. This is to work around around problems caused by BSD's group ownership semantics (directories inherit the group of diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 362d0f65e2..9b8656f704 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -88,7 +88,7 @@ void makePathReadOnly(const Path & path); /* Create a temporary directory. */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true); + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */