From a3883cbd28057a3dd2573f77dcda9a26faaac555 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2010 17:11:46 +0000 Subject: [PATCH] * Store the size of a store path in the database (to be precise, the size of the NAR serialisation of the path, i.e., `nix-store --dump PATH'). This is useful for Hydra. --- externals/Makefile.am | 2 +- src/libstore/build.cc | 41 +++++++++++-------- src/libstore/local-store.cc | 72 ++++++++++++++++++++++------------ src/libstore/local-store.hh | 9 ++--- src/libstore/optimise-store.cc | 2 +- src/libstore/references.cc | 2 +- src/libstore/references.hh | 2 +- src/libstore/remote-store.cc | 6 +++ src/libstore/remote-store.hh | 2 + src/libstore/schema.sql | 3 +- src/libstore/store-api.cc | 23 ++++++----- src/libstore/store-api.hh | 40 +++++++++++-------- src/libutil/hash.cc | 10 +++-- src/libutil/hash.hh | 7 ++-- src/nix-hash/nix-hash.cc | 2 +- src/nix-store/nix-store.cc | 12 +++--- 16 files changed, 144 insertions(+), 91 deletions(-) diff --git a/externals/Makefile.am b/externals/Makefile.am index 060ae2c52f..bdc29fcb52 100644 --- a/externals/Makefile.am +++ b/externals/Makefile.am @@ -47,7 +47,7 @@ build-sqlite: else build-sqlite: $(SQLITE) (cd $(SQLITE) && \ - CC="$(CC)" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ + CC="$(CC)" CFLAGS="-DSQLITE_ENABLE_COLUMN_METADATA=1" ./configure --disable-static --prefix=$(pkglibdir)/dummy --libdir=${pkglibdir} $(SUB_CONFIGURE_FLAGS) && \ $(MAKE) ) touch build-sqlite diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 210486fbc2..8b8be3e80d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1547,7 +1547,7 @@ void DerivationGoal::startBuilder() /* Write closure info to `fileName'. */ writeFile(tmpDir + "/" + fileName, - makeValidityRegistration(paths, false, false)); + worker.store.makeValidityRegistration(paths, false, false)); } @@ -1870,7 +1870,7 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) void DerivationGoal::computeClosure() { map allReferences; - map contentHashes; + map contentHashes; /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1927,7 +1927,7 @@ void DerivationGoal::computeClosure() if (ht == htUnknown) throw BuildError(format("unknown hash algorithm `%1%'") % algo); Hash h = parseHash(ht, i->second.hash); - Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path); + Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path); if (h != h2) throw BuildError( format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") @@ -1941,7 +1941,7 @@ void DerivationGoal::computeClosure() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Hash hash; + HashResult hash; PathSet references = scanForReferences(path, allPaths, hash); contentHashes[path] = hash; @@ -1970,14 +1970,18 @@ void DerivationGoal::computeClosure() } /* Register each output path as valid, and register the sets of - paths referenced by each of them. !!! this should be - atomic so that either all paths are registered as valid, or - none are. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerValidPath(i->second.path, - contentHashes[i->second.path], - allReferences[i->second.path], - drvPath); + paths referenced by each of them. */ + ValidPathInfos infos; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + ValidPathInfo info; + info.path = i->second.path; + info.hash = contentHashes[i->second.path].first; + info.narSize = contentHashes[i->second.path].second; + info.references = allReferences[i->second.path]; + info.deriver = drvPath; + infos.push_back(info); + } + worker.store.registerValidPaths(infos); /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not @@ -2385,10 +2389,15 @@ void SubstitutionGoal::finished() canonicalisePathMetaData(storePath); - Hash contentHash = hashPath(htSHA256, storePath); - - worker.store.registerValidPath(storePath, contentHash, - info.references, info.deriver); + HashResult hash = hashPath(htSHA256, storePath); + + ValidPathInfo info2; + info2.path = storePath; + info2.hash = hash.first; + info2.narSize = hash.second; + info2.references = info.references; + info2.deriver = info.deriver; + worker.store.registerValidPath(info2); outputLock->setDeletion(true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0c75e34d5..9595561bcb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,9 +327,16 @@ void LocalStore::openDB(bool create) throw SQLiteError(db, "initialising database schema"); } + /* Backwards compatibility with old (pre-release) databases. Can + remove this eventually. */ + if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { + if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "adding column narSize"); + } + /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -431,19 +438,6 @@ void canonicalisePathMetaData(const Path & path) } -void LocalStore::registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, - const Path & deriver) -{ - ValidPathInfo info; - info.path = path; - info.hash = hash; - info.references = references; - info.deriver = deriver; - registerValidPath(info); -} - - unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { SQLiteStmtUse use(stmtRegisterValidPath); @@ -454,6 +448,10 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtRegisterValidPath.bind(info.deriver); else stmtRegisterValidPath.bind(); // null + if (info.narSize != 0) + stmtRegisterValidPath.bind(info.narSize); + else + stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); unsigned long long id = sqlite3_last_insert_rowid(db); @@ -920,10 +918,18 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - registerValidPath(dstPath, - (recursive && hashAlgo == htSHA256) ? h : - (recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)), - PathSet(), ""); + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + registerValidPath(info); } outputLock.setDeletion(true); @@ -970,9 +976,15 @@ Path LocalStore::addTextToStore(const string & name, const string & s, writeFile(dstPath, s); canonicalisePathMetaData(dstPath); + + HashResult hash = hashPath(htSHA256, dstPath); - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, ""); + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + registerValidPath(info); } outputLock.setDeletion(true); @@ -998,7 +1010,7 @@ struct HashAndWriteSink : Sink Hash currentHash() { HashSink hashSinkClone(hashSink); - return hashSinkClone.finish(); + return hashSinkClone.finish().first; } }; @@ -1136,7 +1148,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); - Hash hash = hashAndReadSource.hashSink.finish(); + Hash hash = hashAndReadSource.hashSink.finish().first; hashAndReadSource.hashing = false; bool haveSignature = readInt(hashAndReadSource) == 1; @@ -1200,9 +1212,15 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ - if (deriver != "" && !isValidPath(deriver)) deriver = ""; - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, deriver); + HashResult hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; + registerValidPath(info); } outputLock.setDeletion(true); @@ -1263,12 +1281,14 @@ void LocalStore::verifyStore(bool checkContents) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - Hash current = hashPath(info.hash.type, *i); + Hash current = hashPath(info.hash.type, *i).first; if (current != info.hash) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") % *i % printHash(info.hash) % printHash(current)); } + + /* !!! Check info.narSize */ } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0d7ec1f495..2fd31c26d5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -97,6 +97,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); @@ -173,8 +175,7 @@ public: execution of the derivation (or something equivalent). Also register the hash of the file system contents of the path. The hash must be a SHA-256 hash. */ - void registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, const Path & deriver); + void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); @@ -224,10 +225,6 @@ private: void addReference(unsigned long long referrer, unsigned long long reference); - void registerValidPath(const ValidPathInfo & info); - - ValidPathInfo queryPathInfo(const Path & path); - void appendReferrer(const Path & from, const Path & to, bool lock); void rewriteReferrers(const Path & path, bool purge, PathSet referrers); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 3ed54e24d7..89be6ac652 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -68,7 +68,7 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath, the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path); + Hash hash = hashPath(htSHA256, path).first; stats.totalFiles++; printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a6f6e85fc8..ade9c9aa20 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -81,7 +81,7 @@ void RefScanSink::operator () (const unsigned char * data, unsigned int len) PathSet scanForReferences(const string & path, - const PathSet & refs, Hash & hash) + const PathSet & refs, HashResult & hash) { RefScanSink sink; std::map backMap; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 7d068eb517..158c08a776 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,7 +7,7 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, - Hash & hash); + HashResult & hash); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 92d517bbb0..517c886b4f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -247,6 +247,12 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, } +ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +{ + throw Error("not implemented"); +} + + Hash RemoteStore::queryPathHash(const Path & path) { openConnection(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 02a1c47525..519f46fd1a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 7438632edb..c1b4a689af 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -3,7 +3,8 @@ create table if not exists ValidPaths ( path text unique not null, hash text not null, registrationTime integer not null, - deriver text + deriver text, + narSize integer ); create table if not exists Refs ( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01dd516216..4b04f5751c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -190,7 +190,7 @@ std::pair computeStorePathForPath(const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { HashType ht(hashAlgo); - Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); + Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); return std::pair(dstPath, h); @@ -216,7 +216,7 @@ Path computeStorePathForText(const string & name, const string & s, /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string makeValidityRegistration(const PathSet & paths, +string StoreAPI::makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash) { string s = ""; @@ -224,18 +224,19 @@ string makeValidityRegistration(const PathSet & paths, foreach (PathSet::iterator, i, paths) { s += *i + "\n"; - if (showHash) - s += printHash(store->queryPathHash(*i)) + "\n"; + ValidPathInfo info = queryPathInfo(*i); - Path deriver = showDerivers ? store->queryDeriver(*i) : ""; + if (showHash) { + s += printHash(info.hash) + "\n"; + s += (format("%1%\n") % info.narSize).str(); + } + + Path deriver = showDerivers ? info.deriver : ""; s += deriver + "\n"; - PathSet references; - store->queryReferences(*i, references); + s += (format("%1%\n") % info.references.size()).str(); - s += (format("%1%\n") % references.size()).str(); - - foreach (PathSet::iterator, j, references) + foreach (PathSet::iterator, j, info.references) s += *j + "\n"; } @@ -252,6 +253,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string s; getline(str, s); info.hash = parseHash(htSHA256, s); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); } getline(str, info.deriver); string s; int n; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 082a9edc44..b0071da83b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -90,6 +90,21 @@ struct SubstitutablePathInfo }; +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + time_t registrationTime; + unsigned long long narSize; // 0 = unknown + unsigned long long id; // internal use only + ValidPathInfo() : registrationTime(0), narSize(0) { } +}; + +typedef list ValidPathInfos; + + class StoreAPI { public: @@ -102,6 +117,9 @@ public: /* Query the set of valid paths. */ virtual PathSet queryValidPaths() = 0; + /* Query information about a valid path. */ + virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -214,6 +232,12 @@ public: /* Clear the "failed" status of the given paths. The special value `*' causes all failed paths to be cleared. */ virtual void clearFailedPaths(const PathSet & paths) = 0; + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); }; @@ -307,22 +331,6 @@ boost::shared_ptr openStore(); string showPaths(const PathSet & paths); -string makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash); - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; - time_t registrationTime; - unsigned long long id; // internal use only - ValidPathInfo() : registrationTime(0) { } -}; - -typedef list ValidPathInfos; - ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index bd7e33a48e..b9e7846992 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -286,12 +286,14 @@ Hash hashFile(HashType ht, const Path & path) HashSink::HashSink(HashType ht) : ht(ht) { ctx = new Ctx; + bytes = 0; start(ht, *ctx); } HashSink::HashSink(const HashSink & h) { ht = h.ht; + bytes = h.bytes; ctx = new Ctx; *ctx = *h.ctx; } @@ -304,18 +306,20 @@ HashSink::~HashSink() void HashSink::operator () (const unsigned char * data, unsigned int len) { + bytes += len; update(ht, *ctx, data, len); } -Hash HashSink::finish() +HashResult HashSink::finish() { Hash hash(ht); nix::finish(ht, *ctx, hash.hash); - return hash; + return HashResult(hash, bytes); } -Hash hashPath(HashType ht, const Path & path, PathFilter & filter) +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) { HashSink sink(ht); dumpPath(path, sink, filter); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 81425b2349..13740954d6 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,7 +40,6 @@ struct Hash /* For sorting. */ bool operator < (const Hash & h) const; - }; @@ -72,7 +71,8 @@ Hash hashFile(HashType ht, const Path & path); (essentially) hashString(ht, dumpPath(path)). */ struct PathFilter; extern PathFilter defaultPathFilter; -Hash hashPath(HashType ht, const Path & path, +typedef std::pair HashResult; +HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); /* Compress a hash to the specified number of bytes by cyclically @@ -93,13 +93,14 @@ class HashSink : public Sink private: HashType ht; Ctx * ctx; + unsigned long long bytes; public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); virtual void operator () (const unsigned char * data, unsigned int len); - Hash finish(); + HashResult finish(); }; diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc index 8a66554033..4867234bff 100644 --- a/src/nix-hash/nix-hash.cc +++ b/src/nix-hash/nix-hash.cc @@ -44,7 +44,7 @@ void run(Strings args) if (op == opHash) { for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i); + Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % (base32 ? printHash32(h) : printHash(h)); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 695eb10dce..120f6ce729 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -393,9 +393,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); PathSet validPaths = store->queryValidPaths(); - foreach (PathSet::iterator, i, validPaths) { - cout << makeValidityRegistration(singleton(*i), true, true); - } + foreach (PathSet::iterator, i, validPaths) + cout << store->makeValidityRegistration(singleton(*i), true, true); } @@ -410,8 +409,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData(info.path); - if (!hashGiven) - info.hash = hashPath(htSHA256, info.path); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.hash = hash.first; + info.narSize = hash.second; + } infos.push_back(info); } }