diff --git a/.gitignore b/.gitignore index bcc7a79f8a..9160fe43d1 100644 --- a/.gitignore +++ b/.gitignore @@ -79,13 +79,9 @@ Makefile.in /src/libexpr/parser-tab.cc /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output -/src/libexpr/nixexpr-ast.hh -/src/libexpr/nixexpr-ast.cc /src/libexpr/nix.tbl # /src/libstore/ -/src/libstore/derivations-ast.cc -/src/libstore/derivations-ast.hh /src/libstore/schema.sql.hh # /src/nix-env/ diff --git a/configure.ac b/configure.ac index 53d653a0ee..21a87194f4 100644 --- a/configure.ac +++ b/configure.ac @@ -115,6 +115,23 @@ AC_CHECK_HEADERS([sys/mount.h], [], [], ]) +# Check for lutimes, optionally used for changing the mtime of +# symlinks. +AC_CHECK_FUNCS([lutimes]) + + +# Check whether the store optimiser can optimise symlinks. +AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) +ln -s bla tmp_link +if ln tmp_link tmp_link2 2> /dev/null; then + AC_MSG_RESULT(yes) + AC_DEFINE(CAN_LINK_SYMLINK, 1, [Whether link() works on symlinks.]) +else + AC_MSG_RESULT(no) +fi +rm -f tmp_link tmp_link2 + + # Check for . AC_LANG_PUSH(C++) AC_CHECK_HEADERS([locale]) diff --git a/corepkgs/Makefile.am b/corepkgs/Makefile.am index 729d15e7b1..4b0b8860be 100644 --- a/corepkgs/Makefile.am +++ b/corepkgs/Makefile.am @@ -1,6 +1,7 @@ all-local: config.nix -files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix +files = nar.nix buildenv.nix buildenv.pl unpack-channel.nix unpack-channel.sh derivation.nix fetchurl.nix \ + imported-drv-to-derivation.nix install-exec-local: $(INSTALL) -d $(DESTDIR)$(datadir)/nix/corepkgs diff --git a/corepkgs/imported-drv-to-derivation.nix b/corepkgs/imported-drv-to-derivation.nix new file mode 100644 index 0000000000..bdb6016986 --- /dev/null +++ b/corepkgs/imported-drv-to-derivation.nix @@ -0,0 +1,21 @@ +attrs @ { drvPath, outputs, ... }: + +let + + commonAttrs = (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvPath; + type = "derivation"; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName attrs; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/doc/manual/conf-file.xml b/doc/manual/conf-file.xml index 021b42ec63..cccee8d462 100644 --- a/doc/manual/conf-file.xml +++ b/doc/manual/conf-file.xml @@ -357,7 +357,20 @@ build-use-chroot = /dev /proc /bin true. - + + + auto-optimise-store + + If set to true (the default), + Nix automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false, you + can still run nix-store --optimise to get rid + of duplicate files. + + + + diff --git a/doc/manual/release-notes.xml b/doc/manual/release-notes.xml index 4dfcb6b477..7c78122f02 100644 --- a/doc/manual/release-notes.xml +++ b/doc/manual/release-notes.xml @@ -8,7 +8,7 @@ -
Release 1.1 (TBA) +
Release 1.1 (July 18, 2012) This release has the following improvements: diff --git a/release.nix b/release.nix index f2d48691d4..0b382fff31 100644 --- a/release.nix +++ b/release.nix @@ -156,10 +156,6 @@ let rpm_fedora13x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora13x86_64) 50; rpm_fedora16i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.fedora16i386) 50; rpm_fedora16x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora16x86_64) 50; - - rpm_opensuse103i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse103i386) 40; - rpm_opensuse110i386 = makeRPM_i686 (diskImageFuns: diskImageFuns.opensuse110i386) 50; - rpm_opensuse110x86_64 = makeRPM_x86_64 (diskImageFuns: diskImageFuns.opensuse110x86_64) 50; deb_debian60i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian60i386) 50; diff --git a/scripts/nix-build.in b/scripts/nix-build.in index 13404f1dcb..aa3f4661ae 100755 --- a/scripts/nix-build.in +++ b/scripts/nix-build.in @@ -123,6 +123,10 @@ EOF elsif ($arg eq "--show-trace") { push @instArgs, $arg; } + + elsif ($arg eq "-") { + @exprs = ("-"); + } elsif ($arg eq "--verbose" or substr($arg, 0, 2) eq "-v") { push @buildArgs, $arg; diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index d343385cc1..bc3dc719eb 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -19,9 +19,9 @@ fi export PATH="$HOME/.nix-profile/bin:$PATH" -# Subscribe the root user to the NixOS channel by default. +# Subscribe the root user to the Nixpkgs channel by default. if [ "$USER" = root -a ! -e $HOME/.nix-channels ]; then - echo "http://nixos.org/releases/nixos/channels/nixos-unstable nixos" > $HOME/.nix-channels + echo "http://nixos.org/releases/nixos/channels/nixpkgs-unstable nixpkgs" > $HOME/.nix-channels fi # Create the per-user garbage collector roots directory. @@ -43,7 +43,7 @@ fi # Set up secure multi-user builds: non-root users build through the # Nix daemon. -if test "$USER" != root; then +if [ "$USER" != root -a -e @localstatedir@/nix/daemon-socket/socket ]; then export NIX_REMOTE=daemon else unset NIX_REMOTE diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index b106da5ef8..d3aad74611 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -1,20 +1,13 @@ -#ifndef __ATTR_PATH_H -#define __ATTR_PATH_H +#pragma once #include "eval.hh" #include #include - namespace nix { - void findAlongAttrPath(EvalState & state, const string & attrPath, Bindings & autoArgs, Expr * e, Value & v); - } - - -#endif /* !__ATTR_PATH_H */ diff --git a/src/libexpr/common-opts.hh b/src/libexpr/common-opts.hh index c28641e901..e2e3fe7717 100644 --- a/src/libexpr/common-opts.hh +++ b/src/libexpr/common-opts.hh @@ -1,9 +1,7 @@ -#ifndef __COMMON_OPTS_H -#define __COMMON_OPTS_H +#pragma once #include "eval.hh" - namespace nix { /* Some common option parsing between nix-env and nix-instantiate. */ @@ -17,6 +15,3 @@ bool parseSearchPathArg(const string & arg, Strings::iterator & i, Path lookupFileArg(EvalState & state, string s); } - - -#endif /* !__COMMON_OPTS_H */ diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 6026a7d110..57a9e4c63c 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_INLINE_H -#define __EVAL_INLINE_H +#pragma once #include "eval.hh" @@ -8,7 +7,6 @@ namespace nix { - LocalNoInlineNoReturn(void throwEvalError(const char * s)) { throw EvalError(s); @@ -55,7 +53,4 @@ inline void EvalState::forceList(Value & v) throwTypeError("value is %1% while a list was expected", showType(v)); } - } - -#endif /* !__EVAL_INLINE_H */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index bab9303b08..5103ae8cef 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,5 +1,4 @@ -#ifndef __EVAL_H -#define __EVAL_H +#pragma once #include "value.hh" #include "nixexpr.hh" @@ -256,6 +255,3 @@ string showType(const Value & v); } - - -#endif /* !__EVAL_H */ diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 1e5d0a817c..25d8baa559 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,5 +1,4 @@ -#ifndef __GET_DRVS_H -#define __GET_DRVS_H +#pragma once #include "eval.hh" @@ -83,6 +82,3 @@ void getDerivations(EvalState & state, Value & v, const string & pathPrefix, } - - -#endif /* !__GET_DRVS_H */ diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index e189302d6d..ebe113e82a 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -1,12 +1,9 @@ -#ifndef __NAMES_H -#define __NAMES_H +#pragma once #include "types.hh" - namespace nix { - struct DrvName { string fullName; @@ -19,15 +16,9 @@ struct DrvName bool matches(DrvName & n); }; - typedef list DrvNames; - int compareVersions(const string & v1, const string & v2); DrvNames drvNamesFromArgs(const Strings & opArgs); - } - - -#endif /* !__NAMES_H */ diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 6eb771a726..4c1a0bb2d5 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -1,5 +1,4 @@ -#ifndef __NIXEXPR_H -#define __NIXEXPR_H +#pragma once #include "value.hh" #include "symbol-table.hh" @@ -290,6 +289,3 @@ struct StaticEnv } - - -#endif /* !__NIXEXPR_H */ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f20c2f2879..5c011c43e3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -65,7 +65,31 @@ static void prim_import(EvalState & state, Value * * args, Value & v) } } - state.evalFile(path, v); + if (isStorePath(path) && store->isValidPath(path) && isDerivation(path)) { + Derivation drv = parseDerivation(readFile(path)); + Value & w = *state.allocValue(); + state.mkAttrs(w, 1 + drv.outputs.size()); + mkString(*state.allocAttr(w, state.sDrvPath), path, singleton("=" + path)); + state.mkList(*state.allocAttr(w, state.symbols.create("outputs")), drv.outputs.size()); + unsigned int outputs_index = 0; + + Value * outputsVal = w.attrs->find(state.symbols.create("outputs"))->value; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + mkString(*state.allocAttr(w, state.symbols.create(i->first)), + i->second.path, singleton("!" + i->first + "!" + path)); + mkString(*(outputsVal->list.elems[outputs_index++] = state.allocValue()), + i->first); + } + w.attrs->sort(); + Value fun; + state.mkThunk_(fun, + state.parseExprFromFile(state.findFile("nix/imported-drv-to-derivation.nix"))); + state.forceFunction(fun); + mkApp(v, fun, w); + state.forceAttrs(v); + } else { + state.evalFile(path, v); + } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 976117a20a..143fc495b0 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,5 +1,4 @@ -#ifndef __SYMBOL_TABLE_H -#define __SYMBOL_TABLE_H +#pragma once #include "config.h" @@ -88,5 +87,3 @@ public: }; } - -#endif /* !__SYMBOL_TABLE_H */ diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 3ebc989ffa..97657327ed 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_TO_XML_H -#define __VALUE_TO_XML_H +#pragma once #include "nixexpr.hh" #include "eval.hh" @@ -13,5 +12,3 @@ void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); } - -#endif /* !__VALUE_TO_XML_H */ diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 41512d40b1..c9ec236c47 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,4 @@ -#ifndef __VALUE_H -#define __VALUE_H +#pragma once #include "symbol-table.hh" @@ -151,5 +150,3 @@ void mkPath(Value & v, const char * s); } - -#endif /* !__VALUE_H */ diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index d198df70d1..7849e10e36 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,5 +1,4 @@ -#ifndef __SHARED_H -#define __SHARED_H +#pragma once #include "util.hh" @@ -54,6 +53,3 @@ extern int exitCode; extern char * * argvSaved; } - - -#endif /* !__SHARED_H */ diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4bef6f02dc..c57a63db69 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1533,7 +1533,7 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false); + tmpDir = createTempDir("", "nix-build-" + baseNameOf(drvPath), false, false, 0700); /* For convenience, set an environment pointing to the top build directory. */ @@ -2099,6 +2099,8 @@ void DerivationGoal::computeClosure() if (allowed.find(*i) == allowed.end()) throw BuildError(format("output is not allowed to refer to path `%1%'") % *i); } + + worker.store.optimisePath(path); // FIXME: combine with scanForReferences() } /* Register each output path as valid, and register the sets of @@ -2182,6 +2184,7 @@ void DerivationGoal::deleteTmpDir(bool force) % drvPath % tmpDir); if (buildUser.enabled() && !amPrivileged()) getOwnership(tmpDir); + chmod(tmpDir.c_str(), 0755); } else deletePathWrapped(tmpDir); @@ -2562,6 +2565,8 @@ void SubstitutionGoal::finished() HashResult hash = hashPath(htSHA256, storePath); + worker.store.optimisePath(storePath); // FIXME: combine with hashPath() + ValidPathInfo info2; info2.path = storePath; info2.hash = hash.first; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 27e471d885..8f22b4afa6 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,5 +1,4 @@ -#ifndef __DERIVATIONS_H -#define __DERIVATIONS_H +#pragma once #include @@ -81,6 +80,3 @@ typedef std::map DrvHashes; extern DrvHashes drvHashes; } - - -#endif /* !__DERIVATIONS_H */ diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f6ed7dd226..874efe4d32 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -436,6 +436,8 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); + if (path == linksDir) return true; + struct stat st; if (lstat(path.c_str(), &st)) { if (errno == ENOENT) return true; @@ -569,6 +571,37 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks() +{ + AutoCloseDir dir = opendir(linksDir.c_str()); + if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting `%1%'") % path); + + if (st.st_nlink != 1) continue; + + printMsg(lvlTalkative, format("deleting unused link `%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting `%1%'") % path); + } +} + + void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { GCState state(results); @@ -682,6 +715,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) released. */ foreach (PathSet::iterator, i, state.invalidated) deleteGarbage(state, *i); + + /* Clean up the links directory. */ + printMsg(lvlError, format("deleting unused links...")); + removeUnusedLinks(); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12a9b9ca15..1c0877a5e1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,5 +1,4 @@ -#ifndef __GLOBALS_H -#define __GLOBALS_H +#pragma once #include "types.hh" @@ -118,6 +117,3 @@ void setDefaultsFromEnvironment(); } - - -#endif /* !__GLOBALS_H */ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aa21774df4..ebfcc94671 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -208,6 +209,7 @@ LocalStore::LocalStore(bool reserveSpace) /* Create missing state directories if they don't already exist. */ createDirs(nixStore); + createDirs(linksDir = nixStore + "/.links"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -444,7 +446,7 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing owner of `%1%' to %2%") % path % geteuid()); } - + if (!S_ISLNK(st.st_mode)) { /* Mask out all type related bits. */ @@ -458,14 +460,20 @@ void canonicalisePathMetaData(const Path & path, bool recurse) throw SysError(format("changing mode of `%1%' to %2$o") % path % mode); } - if (st.st_mtime != mtimeStore) { - struct utimbuf utimbuf; - utimbuf.actime = st.st_atime; - utimbuf.modtime = mtimeStore; - if (utime(path.c_str(), &utimbuf) == -1) - throw SysError(format("changing modification time of `%1%'") % path); - } - + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of `%1%'") % path); } if (recurse && S_ISDIR(st.st_mode)) { @@ -1134,6 +1142,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, hash.second = dump.size(); } else hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; @@ -1188,6 +1198,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, canonicalisePathMetaData(dstPath); HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); ValidPathInfo info; info.path = dstPath; @@ -1423,6 +1435,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ HashResult hash = hashPath(htSHA256, dstPath); + + optimisePath(dstPath); // FIXME: combine with hashPath() ValidPathInfo info; info.path = dstPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 07d8198eca..15dff1d020 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,5 +1,4 @@ -#ifndef __LOCAL_STORE_H -#define __LOCAL_STORE_H +#pragma once #include @@ -86,6 +85,8 @@ private: typedef std::map RunningSubstituters; RunningSubstituters runningSubstituters; + + Path linksDir; public: @@ -168,8 +169,11 @@ public: /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ - void optimiseStore(bool dryRun, OptimiseStats & stats); + void optimiseStore(OptimiseStats & stats); + /* Optimise a single store path. */ + void optimisePath(const Path & path); + /* Check the integrity of the Nix store. */ void verifyStore(bool checkContents); @@ -260,6 +264,8 @@ private: int openGCLock(LockType lockType); + void removeUnusedLinks(); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); @@ -268,6 +274,8 @@ private: Path importPath(bool requireSignature, Source & source); void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + void optimisePath_(OptimiseStats & stats, const Path & path); }; @@ -302,6 +310,3 @@ void deletePathWrapped(const Path & path, void deletePathWrapped(const Path & path); } - - -#endif /* !__LOCAL_STORE_H */ diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 850c12af4e..fe0bbdd799 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -1,5 +1,4 @@ -#ifndef __MISC_H -#define __MISC_H +#pragma once #include "derivations.hh" @@ -35,6 +34,3 @@ void queryMissing(StoreAPI & store, const PathSet & targets, } - - -#endif /* !__MISC_H */ diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a486e66ef5..84a72604bb 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ #include "util.hh" #include "local-store.hh" #include "immutable.hh" +#include "globals.hh" #include #include @@ -12,9 +13,6 @@ namespace nix { -typedef std::map > HashToPath; - - static void makeWritable(const Path & path) { struct stat st; @@ -51,140 +49,152 @@ struct MakeImmutable }; -static void hashAndLink(bool dryRun, HashToPath & hashToPath, - OptimiseStats & stats, const Path & path) +void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) throw SysError(format("getting attributes of path `%1%'") % path); + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectory(path); + foreach (Strings::iterator, i, names) + optimisePath_(stats, path + "/" + *i); + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + x + && !S_ISLNK(st.st_mode) +#endif + ) return; + /* Sometimes SNAFUs can cause files in the Nix store to be modified, in particular when running programs as root under NixOS (example: $fontconfig/var/cache being modified). Skip - those files. */ + those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); return; } - /* We can hard link regular files and symlinks. */ - if (S_ISREG(st.st_mode) || S_ISLNK(st.st_mode)) { + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). - /* Hash the file. Note that hashPath() returns the hash over - the NAR serialisation, which includes the execute bit on - the file. Thus, executable and non-executable files with - the same contents *won't* be linked (which is good because - otherwise the permissions would be screwed up). + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + stats.totalFiles++; + printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); - Also note that if `path' is a symlink, then we're hashing - the contents of the symlink (i.e. the result of - readlink()), not the contents of the target (which may not - even exist). */ - Hash hash = hashPath(htSHA256, path).first; - stats.totalFiles++; - printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + printHash32(hash); - std::pair prevPath = hashToPath[hash]; - - if (prevPath.first == "") { - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - - /* Yes! We've seen a file with the same contents. Replace - the current file with a hard link to that file. */ - stats.sameContents++; - if (prevPath.second == st.st_ino) { - printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % prevPath.first); - return; - } - - if (!dryRun) { - - printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % prevPath.first); + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + makeMutable(path); + MakeImmutable mk1(path); - Path tempLink = (format("%1%.tmp-%2%-%3%") - % path % getpid() % rand()).str(); + if (link(path.c_str(), linkPath.c_str()) == -1) + throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); - /* Make the containing directory writable, but only if - it's not the store itself (we don't want or need to - mess with its permissions). */ - bool mustToggle = !isStorePath(path); - if (mustToggle) makeWritable(dirOf(path)); - - /* When we're done, make the directory read-only again and - reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); - - /* If ‘prevPath’ is immutable, we can't create hard links - to it, so make it mutable first (and make it immutable - again when we're done). We also have to make ‘path’ - mutable, otherwise rename() will fail to delete it. */ - makeMutable(prevPath.first); - MakeImmutable mk1(prevPath.first); - - makeMutable(path); - MakeImmutable mk2(path); - - if (link(prevPath.first.c_str(), tempLink.c_str()) == -1) { - if (errno == EMLINK) { - /* Too many links to the same file (>= 32000 on - most file systems). This is likely to happen - with empty files. Just start over, creating - links to the current file. */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - return; - } - throw SysError(format("cannot link `%1%' to `%2%'") - % tempLink % prevPath.first); - } - - /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { - if (errno == EMLINK) { - /* Some filesystems generate too many links on the - rename, rather than on the original link. - (Probably it temporarily increases the st_nlink - field before decreasing it again.) */ - printMsg(lvlInfo, format("`%1%' has maximum number of links") % prevPath.first); - hashToPath[hash] = std::pair(path, st.st_ino); - - /* Unlink the temp link. */ - if (unlink(tempLink.c_str()) == -1) - printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); - return; - } - throw SysError(format("cannot rename `%1%' to `%2%'") - % tempLink % path); - } - } else - printMsg(lvlTalkative, format("would link `%1%' to `%2%'") % path % prevPath.first); - - stats.filesLinked++; - stats.bytesFreed += st.st_size; - stats.blocksFreed += st.st_blocks; + return; } - if (S_ISDIR(st.st_mode)) { - Strings names = readDirectory(path); - foreach (Strings::iterator, i, names) - hashAndLink(dryRun, hashToPath, stats, path + "/" + *i); + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path `%1%'") % linkPath); + + stats.sameContents++; + if (st.st_ino == stLink.st_ino) { + printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); + return; } + + printMsg(lvlTalkative, format("linking `%1%' to `%2%'") % path % linkPath); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % nixStore % getpid() % rand()).str(); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = !isStorePath(path); + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + /* If ‘linkPath’ is immutable, we can't create hard links to it, + so make it mutable first (and make it immutable again when + we're done). We also have to make ‘path’ mutable, otherwise + rename() will fail to delete it. */ + makeMutable(linkPath); + MakeImmutable mk1(linkPath); + + makeMutable(path); + MakeImmutable mk2(path); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + return; + } + throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); + + /* Unlink the temp link. */ + if (unlink(tempLink.c_str()) == -1) + printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); + return; + } + throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; } -void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) +void LocalStore::optimiseStore(OptimiseStats & stats) { - HashToPath hashToPath; - PathSet paths = queryAllValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); - hashAndLink(dryRun, hashToPath, stats, *i); + optimisePath_(stats, *i); + } +} + + +void LocalStore::optimisePath(const Path & path) +{ + if (queryBoolSetting("auto-optimise-store", true)) { + OptimiseStats stats; + optimisePath_(stats, path); } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 57ca1584a6..8a6b1450da 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,5 +1,4 @@ -#ifndef __PATHLOCKS_H -#define __PATHLOCKS_H +#pragma once #include "types.hh" @@ -44,6 +43,3 @@ bool pathIsLockedByMe(const Path & path); } - - -#endif /* !__PATHLOCKS_H */ diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 158c08a776..013809d122 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,5 +1,4 @@ -#ifndef __REFERENCES_H -#define __REFERENCES_H +#pragma once #include "types.hh" #include "hash.hh" @@ -10,5 +9,3 @@ PathSet scanForReferences(const Path & path, const PathSet & refs, HashResult & hash); } - -#endif /* !__REFERENCES_H */ diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 68db0640ae..ae4c48dad6 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,5 +1,4 @@ -#ifndef __REMOTE_STORE_H -#define __REMOTE_STORE_H +#pragma once #include @@ -103,6 +102,3 @@ private: } - - -#endif /* !__REMOTE_STORE_H */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5d8c09f5a5..324d802dc4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,4 @@ -#ifndef __STOREAPI_H -#define __STOREAPI_H +#pragma once #include "hash.hh" #include "serialise.hh" @@ -362,6 +361,3 @@ MakeError(BuildError, Error) /* denotes a permanent build failure */ } - - -#endif /* !__STOREAPI_H */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d4ac0ea16f..9677a46c28 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,4 @@ -#ifndef __WORKER_PROTOCOL_H -#define __WORKER_PROTOCOL_H - +#pragma once namespace nix { @@ -67,6 +65,3 @@ template T readStorePaths(Source & from); } - - -#endif /* !__WORKER_PROTOCOL_H */ diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fff6203139..ccac92074d 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -1,5 +1,4 @@ -#ifndef __ARCHIVE_H -#define __ARCHIVE_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -74,6 +73,3 @@ void restorePath(const Path & path, Source & source); } - - -#endif /* !__ARCHIVE_H */ diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index e0b6478cc4..781f517428 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -1,5 +1,4 @@ -#ifndef __HASH_H -#define __HASH_H +#pragma once #include "types.hh" #include "serialise.hh" @@ -109,6 +108,3 @@ public: } - - -#endif /* !__HASH_H */ diff --git a/src/libutil/immutable.hh b/src/libutil/immutable.hh index 5a42a46107..8af4190049 100644 --- a/src/libutil/immutable.hh +++ b/src/libutil/immutable.hh @@ -1,5 +1,4 @@ -#ifndef __IMMUTABLE_H -#define __IMMUTABLE_H +#pragma once #include @@ -15,5 +14,3 @@ void makeImmutable(const Path & path); void makeMutable(const Path & path); } - -#endif /* !__IMMUTABLE_H */ diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index ded4b12a04..42dd271176 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -1,5 +1,4 @@ -#ifndef __SERIALISE_H -#define __SERIALISE_H +#pragma once #include "types.hh" @@ -130,6 +129,3 @@ MakeError(SerialisationError, Error) } - - -#endif /* !__SERIALISE_H */ diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 844ad6f76a..165a46fa28 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,5 +1,4 @@ -#ifndef __TYPES_H -#define __TYPES_H +#pragma once #include #include @@ -74,6 +73,3 @@ typedef enum { } - - -#endif /* !__TYPES_H */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b188a9fc0e..689fc543af 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -380,7 +380,7 @@ static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter) + bool includePid, bool useGlobalCounter, mode_t mode) { static int globalCounter = 0; int localCounter = 0; @@ -389,7 +389,7 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, while (1) { checkInterrupt(); Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), 0777) == 0) { + if (mkdir(tmpDir.c_str(), mode) == 0) { /* Explicitly set the group of the directory. This is to work around around problems caused by BSD's group ownership semantics (directories inherit the group of diff --git a/src/libutil/util.hh b/src/libutil/util.hh index ee0f3862a8..9b8656f704 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -1,5 +1,4 @@ -#ifndef __UTIL_H -#define __UTIL_H +#pragma once #include "types.hh" @@ -89,7 +88,7 @@ void makePathReadOnly(const Path & path); /* Create a temporary directory. */ Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true); + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); /* Create a directory and all its parents, if necessary. Returns the list of created directories, in order of creation. */ @@ -333,6 +332,3 @@ void ignoreException(); } - - -#endif /* !__UTIL_H */ diff --git a/src/libutil/xml-writer.hh b/src/libutil/xml-writer.hh index e5cc5f8c54..fee2eb495e 100644 --- a/src/libutil/xml-writer.hh +++ b/src/libutil/xml-writer.hh @@ -1,5 +1,4 @@ -#ifndef __XML_WRITER_H -#define __XML_WRITER_H +#pragma once #include #include @@ -70,6 +69,3 @@ public: } - - -#endif /* !__XML_WRITER_H */ diff --git a/src/nix-env/profiles.hh b/src/nix-env/profiles.hh index a64258dae2..30d2376d99 100644 --- a/src/nix-env/profiles.hh +++ b/src/nix-env/profiles.hh @@ -1,5 +1,4 @@ -#ifndef __PROFILES_H -#define __PROFILES_H +#pragma once #include "types.hh" #include "pathlocks.hh" @@ -54,6 +53,3 @@ void lockProfile(PathLocks & lock, const Path & profile); string optimisticLockProfile(const Path & profile); } - - -#endif /* !__PROFILES_H */ diff --git a/src/nix-env/user-env.hh b/src/nix-env/user-env.hh index 4125d82173..f188efe9b4 100644 --- a/src/nix-env/user-env.hh +++ b/src/nix-env/user-env.hh @@ -1,5 +1,4 @@ -#ifndef __USER_ENV_H -#define __USER_ENV_H +#pragma once #include "get-drvs.hh" @@ -12,9 +11,3 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, const string & lockToken); } - -#endif /* !__USER_ENV_H */ - - - - diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index 2318e2fde4..68410d8415 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,5 +1,4 @@ -#ifndef __DOTGRAPH_H -#define __DOTGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printDotGraph(const PathSet & roots); } - -#endif /* !__DOTGRAPH_H */ diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 47c76693f1..941301d2e7 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -746,18 +746,12 @@ static void showOptimiseStats(OptimiseStats & stats) files with the same contents. */ static void opOptimise(Strings opFlags, Strings opArgs) { - if (!opArgs.empty()) + if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); - bool dryRun = false; - - foreach (Strings::iterator, i, opFlags) - if (*i == "--dry-run") dryRun = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - OptimiseStats stats; try { - ensureLocalStore().optimiseStore(dryRun, stats); + ensureLocalStore().optimiseStore(stats); } catch (...) { showOptimiseStats(stats); throw; diff --git a/src/nix-store/xmlgraph.hh b/src/nix-store/xmlgraph.hh index 2f9908c436..c2216c5a46 100644 --- a/src/nix-store/xmlgraph.hh +++ b/src/nix-store/xmlgraph.hh @@ -1,5 +1,4 @@ -#ifndef __XMLGRAPH_H -#define __XMLGRAPH_H +#pragma once #include "types.hh" @@ -8,5 +7,3 @@ namespace nix { void printXmlGraph(const PathSet & roots); } - -#endif /* !__XMLGRAPH_H */ diff --git a/tests/Makefile.am b/tests/Makefile.am index 517c382b19..a562db52bc 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -9,7 +9,7 @@ TESTS = init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \ gc-runtime.sh install-package.sh check-refs.sh filter-source.sh \ remote-store.sh export.sh export-graph.sh negative-caching.sh \ binary-patching.sh timeout.sh secure-drv-outputs.sh nix-channel.sh \ - multiple-outputs.sh import-derivation.sh fetchurl.sh + multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh XFAIL_TESTS = diff --git a/tests/optimise-store.sh b/tests/optimise-store.sh new file mode 100644 index 0000000000..4d8077997d --- /dev/null +++ b/tests/optimise-store.sh @@ -0,0 +1,26 @@ +source common.sh + +clearStore + +outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) +outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) + +inode1="$(perl -e "print ((lstat('$outPath1/foo'))[1])")" +inode2="$(perl -e "print ((lstat('$outPath2/foo'))[1])")" +if [ "$inode1" != "$inode2" ]; then + echo "inodes do not match" + exit 1 +fi + +nlink="$(perl -e "print ((lstat('$outPath1/foo'))[3])")" +if [ "$nlink" != 3 ]; then + echo "link count incorrect" + exit 1 +fi + +nix-store --gc + +if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then + echo ".links directory not empty after GC" + exit 1 +fi diff --git a/version b/version index b123147e2a..ea710abb95 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.1 \ No newline at end of file +1.2 \ No newline at end of file