From 77d272623fb4fd57cf27d4b92a7dc1713a2d4098 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 22 Mar 2009 17:36:43 +0000 Subject: [PATCH] * NAR archives: handle files larger than 2^32 bytes. Previously it would just silently store only (fileSize % 2^32) bytes. * Use posix_fallocate if available when unpacking archives. * Provide a better error message when trying to unpack something that isn't a NAR archive. --- configure.ac | 1 + src/libstore/local-store.cc | 2 +- src/libutil/archive.cc | 45 ++++++++++++++++++++++++++++--------- src/libutil/archive.hh | 1 + src/libutil/serialise.cc | 4 ++-- src/libutil/serialise.hh | 3 +++ src/nix-store/nix-store.cc | 9 +++++--- 7 files changed, 48 insertions(+), 17 deletions(-) diff --git a/configure.ac b/configure.ac index 0095e7d5ec..29b7419274 100644 --- a/configure.ac +++ b/configure.ac @@ -266,6 +266,7 @@ AC_CHECK_FUNCS([setresuid setreuid lchown]) # Nice to have, but not essential. AC_CHECK_FUNCS([strsignal]) +AC_CHECK_FUNCS([posix_fallocate]) # This is needed if ATerm, Berkeley DB or bzip2 are static libraries, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1d0c68cb88..fdfc853466 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -865,7 +865,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) store path follows the archive data proper), and besides, we don't know yet whether the signature is valid. */ Path tmpDir = createTempDir(nixStore); - AutoDelete delTmp(tmpDir); + AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */ Path unpacked = tmpDir + "/unpacked"; restorePath(unpacked, hashAndReadSource); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 30b94951bd..3142aa929b 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -2,6 +2,7 @@ #include #include +#define _XOPEN_SOURCE 600 #include #include #include @@ -11,6 +12,8 @@ #include "archive.hh" #include "util.hh" +#include "config.h" + namespace nix { @@ -47,17 +50,17 @@ static void dumpEntries(const Path & path, Sink & sink, PathFilter & filter) } -static void dumpContents(const Path & path, unsigned int size, +static void dumpContents(const Path & path, off_t size, Sink & sink) { writeString("contents", sink); - writeInt(size, sink); + writeLongLong(size, sink); AutoCloseFD fd = open(path.c_str(), O_RDONLY); if (fd == -1) throw SysError(format("opening file `%1%'") % path); unsigned char buf[65536]; - unsigned int left = size; + off_t left = size; while (left > 0) { size_t n = left > sizeof(buf) ? sizeof(buf) : left; @@ -101,7 +104,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) writeString(readLink(path), sink); } - else throw Error("unknown file type: " + path); + else throw Error(format("file `%1%' has an unknown type") % path); writeString(")", sink); } @@ -114,9 +117,9 @@ void dumpPath(const Path & path, Sink & sink, PathFilter & filter) } -static Error badArchive(string s) +static SerialisationError badArchive(string s) { - return Error("bad archive: " + s); + return SerialisationError("bad archive: " + s); } @@ -162,14 +165,17 @@ static void parseEntry(ParseSink & sink, Source & source, const Path & path) static void parseContents(ParseSink & sink, Source & source, const Path & path) { - unsigned int size = readInt(source); - unsigned int left = size; + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; unsigned char buf[65536]; while (left) { checkInterrupt(); unsigned int n = sizeof(buf); - if (n > left) n = left; + if ((unsigned long long) n > left) n = left; source(buf, n); sink.receiveContents(buf, n); left -= n; @@ -248,8 +254,15 @@ static void parse(ParseSink & sink, Source & source, const Path & path) void parseDump(ParseSink & sink, Source & source) { - if (readString(source) != archiveVersion1) - throw badArchive("expected Nix archive"); + string version; + try { + version = readString(source); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != archiveVersion1) + throw badArchive("input doesn't look like a Nix archive"); parse(sink, source, ""); } @@ -282,6 +295,16 @@ struct RestoreSink : ParseSink throw SysError("fchmod"); } + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd, 0, len); + if (errno) throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + void receiveContents(unsigned char * data, unsigned int len) { writeFull(fd, data, len); diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 06f09cd1ec..fff6203139 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -62,6 +62,7 @@ struct ParseSink virtual void createRegularFile(const Path & path) { }; virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { }; virtual void createSymlink(const Path & path, const string & target) { }; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index c13e8c7e38..9b42227132 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -80,7 +80,7 @@ void readPadding(unsigned int len, Source & source) unsigned int n = 8 - (len % 8); source(zero, n); for (unsigned int i = 0; i < n; i++) - if (zero[i]) throw Error("non-zero padding"); + if (zero[i]) throw SerialisationError("non-zero padding"); } } @@ -90,7 +90,7 @@ unsigned int readInt(Source & source) unsigned char buf[8]; source(buf, sizeof(buf)); if (buf[4] || buf[5] || buf[6] || buf[7]) - throw Error("implementation cannot deal with > 32-bit integers"); + throw SerialisationError("implementation cannot deal with > 32-bit integers"); return buf[0] | (buf[1] << 8) | diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6a74c4cf6e..0e797d63bc 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -106,6 +106,9 @@ string readString(Source & source); StringSet readStringSet(Source & source); +MakeError(SerialisationError, Error) + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f6271d56aa..4a70d870a5 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -617,16 +617,19 @@ static void opExport(Strings opFlags, Strings opArgs) static void opImport(Strings opFlags, Strings opArgs) { bool requireSignature = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) + foreach (Strings::iterator, i, opFlags) if (*i == "--require-signature") requireSignature = true; else throw UsageError(format("unknown flag `%1%'") % *i); if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - while (readInt(source) == 1) + while (true) { + int n = readInt(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'"); cout << format("%1%\n") % store->importPath(requireSignature, source) << std::flush; + } }