From 3a38d0f3565a02c034c29b264aceb0eb78dac005 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 6 Feb 2014 11:52:03 -0500 Subject: [PATCH 01/14] Add the nix-store --serve command This is essentially the substituter API operating on the local store, which will be used by the ssh substituter. It runs in a loop rather than just taking one command so that in the future nix will be able to keep one connection open for multiple instances of the substituter. Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 31 +++++++++++++++++++++++++++++++ src/libstore/store-api.hh | 4 ++++ src/nix-store/nix-store.cc | 20 ++++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0f250a3c7c..1a13e7ca32 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -258,6 +258,37 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, } +void StoreAPI::serve(Source & in, Sink & out, bool sign) +{ + for (string cmd = readString(in); !cmd.empty(); cmd = readString(in)) { + if (cmd == "query") { + for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { + PathSet paths = readStrings(in); + if (cmd == "have") { + writeStrings(queryValidPaths(paths), out); + } else if (cmd == "info") { + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + ValidPathInfo info = queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); + } + writeString("", out); + } else + throw Error(format("Unknown serve query `%1%'") % cmd); + } + } else if (cmd == "substitute") + exportPath(readString(in), sign, out); + else + throw Error(format("Unknown serve command `%1%'") % cmd); + } +} + + ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) { ValidPathInfo info; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a82fe32216..57cf517948 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -248,6 +248,10 @@ public: `nix-store --register-validity'. */ string makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash); + + /* Serve the store for ssh substituters by taking commands + * from in and printing results to out */ + void serve(Source & in, Sink & out, bool sign); }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 69a98fe472..fb1d3f541d 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -834,6 +834,24 @@ static void opClearFailedPaths(Strings opFlags, Strings opArgs) } +// Serve the nix store in a way usable by a restricted ssh user +static void opServe(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + // Could eventually take a username argument? + bool sign; + foreach (Strings::iterator, i, opFlags) + if (*i == "--sign") sign = true; + else throw UsageError(format("unknown flag `%1%'") % *i); + + FdSource in(STDIN_FILENO); + FdSink out(STDOUT_FILENO); + + store->serve(in, out, sign); +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -904,6 +922,8 @@ void run(Strings args) indirectRoot = true; else if (arg == "--no-output") noOutput = true; + else if (arg == "--serve") + op = opServe; else if (arg[0] == '-') { opFlags.push_back(arg); if (arg == "--max-freed" || arg == "--max-links" || arg == "--max-atime") { /* !!! hack */ From 94884475947ca8c44dda51d83f3c1fbfeff5ccc0 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 7 Feb 2014 14:07:31 -0500 Subject: [PATCH 02/14] nix-store --serve: Don't loop forever nix-store --export takes a tmproot, which can only release by exiting. Substituters don't currently work in a way that could take advantage of the looping, anyway. Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 51 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1a13e7ca32..d4d53e9daf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -260,32 +260,31 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, void StoreAPI::serve(Source & in, Sink & out, bool sign) { - for (string cmd = readString(in); !cmd.empty(); cmd = readString(in)) { - if (cmd == "query") { - for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { - PathSet paths = readStrings(in); - if (cmd == "have") { - writeStrings(queryValidPaths(paths), out); - } else if (cmd == "info") { - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - ValidPathInfo info = queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); - } - writeString("", out); - } else - throw Error(format("Unknown serve query `%1%'") % cmd); - } - } else if (cmd == "substitute") - exportPath(readString(in), sign, out); - else - throw Error(format("Unknown serve command `%1%'") % cmd); - } + string cmd = readString(in); + if (cmd == "query") { + for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { + PathSet paths = readStrings(in); + if (cmd == "have") { + writeStrings(queryValidPaths(paths), out); + } else if (cmd == "info") { + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + ValidPathInfo info = queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); + } + writeString("", out); + } else + throw Error(format("Unknown serve query `%1%'") % cmd); + } + } else if (cmd == "substitute") + exportPath(readString(in), sign, out); + else + throw Error(format("Unknown serve command `%1%'") % cmd); } From 188f96500bc16891b22c684ad96122635667a8ff Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 7 Feb 2014 15:29:32 -0500 Subject: [PATCH 03/14] nix-store --serve: Don't fail if asked for info about non-valid path Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d4d53e9daf..082c2c4d97 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -269,6 +269,8 @@ void StoreAPI::serve(Source & in, Sink & out, bool sign) } else if (cmd == "info") { // !!! Maybe we want a queryPathInfos? foreach (PathSet::iterator, i, paths) { + if (!isValidPath(*i)) + continue; ValidPathInfo info = queryPathInfo(*i); writeString(info.path, out); writeString(info.deriver, out); From 73874629ef59dc3b237a2c316179e722f971bb5e Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 7 Feb 2014 16:17:52 -0500 Subject: [PATCH 04/14] nix-store --serve: Use dump instead of export Also remove signing support Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 5 +++-- src/libstore/store-api.hh | 2 +- src/nix-store/nix-store.cc | 12 +++--------- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 082c2c4d97..bd95590756 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,7 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" +#include "archive.hh" #include @@ -258,7 +259,7 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, } -void StoreAPI::serve(Source & in, Sink & out, bool sign) +void StoreAPI::serve(Source & in, Sink & out) { string cmd = readString(in); if (cmd == "query") { @@ -284,7 +285,7 @@ void StoreAPI::serve(Source & in, Sink & out, bool sign) throw Error(format("Unknown serve query `%1%'") % cmd); } } else if (cmd == "substitute") - exportPath(readString(in), sign, out); + dumpPath(readString(in), out); else throw Error(format("Unknown serve command `%1%'") % cmd); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 57cf517948..edab6ea5a2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -251,7 +251,7 @@ public: /* Serve the store for ssh substituters by taking commands * from in and printing results to out */ - void serve(Source & in, Sink & out, bool sign); + void serve(Source & in, Sink & out); }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index fb1d3f541d..e3f27820ff 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -837,18 +837,12 @@ static void opClearFailedPaths(Strings opFlags, Strings opArgs) // Serve the nix store in a way usable by a restricted ssh user static void opServe(Strings opFlags, Strings opArgs) { - if (!opArgs.empty()) - throw UsageError("no arguments expected"); - // Could eventually take a username argument? - bool sign; - foreach (Strings::iterator, i, opFlags) - if (*i == "--sign") sign = true; - else throw UsageError(format("unknown flag `%1%'") % *i); - + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments or flags expected"); FdSource in(STDIN_FILENO); FdSink out(STDOUT_FILENO); - store->serve(in, out, sign); + store->serve(in, out); } From 5671188eb2822b7392a6affa5ebe2f1eb8f521a0 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Fri, 7 Feb 2014 16:56:00 -0500 Subject: [PATCH 05/14] nix-store --serve: Flush out after every loop Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 3 ++- src/libstore/store-api.hh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index bd95590756..e4f1802409 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,7 +259,7 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, } -void StoreAPI::serve(Source & in, Sink & out) +void StoreAPI::serve(Source & in, BufferedSink & out) { string cmd = readString(in); if (cmd == "query") { @@ -283,6 +283,7 @@ void StoreAPI::serve(Source & in, Sink & out) writeString("", out); } else throw Error(format("Unknown serve query `%1%'") % cmd); + out.flush(); } } else if (cmd == "substitute") dumpPath(readString(in), out); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index edab6ea5a2..f32824a3bb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -251,7 +251,7 @@ public: /* Serve the store for ssh substituters by taking commands * from in and printing results to out */ - void serve(Source & in, Sink & out); + void serve(Source & in, BufferedSink & out); }; From 64e23d0a38f316a07cef0960d0ed74a450214283 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Sat, 8 Feb 2014 00:05:46 -0500 Subject: [PATCH 06/14] Add download-via-ssh substituter This substituter connects to a remote host, runs nix-store --serve there, and then forwards substituter commands on to the remote host and sends their results to the calling program. The ssh-substituter-hosts option can be specified as a list of hosts to try. This is an initial implementation and, while it works, it has some limitations: * Only the first host is used * There is no caching of query results (all queries are sent to the remote machine) * There is no informative output (such as progress bars) * Some failure modes may cause unhelpful error messages * There is no concept of trusted-ssh-substituter-hosts Signed-off-by: Shea Levy --- Makefile | 1 + src/download-via-ssh/download-via-ssh.cc | 129 +++++++++++++++++++++++ src/download-via-ssh/local.mk | 9 ++ src/libmain/shared.cc | 3 + src/libstore/globals.cc | 20 ++++ src/libstore/globals.hh | 6 ++ 6 files changed, 168 insertions(+) create mode 100644 src/download-via-ssh/download-via-ssh.cc create mode 100644 src/download-via-ssh/local.mk diff --git a/Makefile b/Makefile index 0ad6bb236c..78d85ebc58 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ makefiles = \ src/nix-instantiate/local.mk \ src/nix-env/local.mk \ src/nix-daemon/local.mk \ + src/download-via-ssh/local.mk \ src/nix-log2xml/local.mk \ src/bsdiff-4.3/local.mk \ perl/local.mk \ diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc new file mode 100644 index 0000000000..7adb93e10d --- /dev/null +++ b/src/download-via-ssh/download-via-ssh.cc @@ -0,0 +1,129 @@ +#include "shared.hh" +#include "util.hh" +#include "serialise.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" + +#include +#include + +using namespace nix; +using std::pair; +using std::cout; +using std::endl; + +// !!! TODO: +// * Respect more than the first host +// * use a database +// * show progress + +static pair connect(string conn) { + Pipe to, from; + to.create(); + from.create(); + pid_t child = fork(); + switch (child) { + case -1: + throw SysError("unable to fork"); + case 0: + try { + restoreAffinity(); + if (dup2(to.readSide, STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (dup2(from.writeSide, STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + execlp("ssh" + , "ssh" + , "-x" + , "-T" + , conn.c_str() + , "nix-store --serve" + , NULL); + throw SysError("executing ssh"); + } catch (std::exception & e) { + std::cerr << "error: " << e.what() << std::endl; + } + _exit(1); + } + // If child exits unexpectedly, we'll EPIPE. If we exit unexpectedly, child will + // So no need to keep track of it. + + return pair(to.writeSide.borrow(), from.readSide.borrow()); +} + +static void substitute(pair & pipes, Path storePath, Path destPath) { + writeString("substitute", pipes.first); + writeString(storePath, pipes.first); + pipes.first.flush(); + restorePath(destPath, pipes.second); + cout << endl; +} + +static void query(pair & pipes) { + using std::cin; + writeString("query", pipes.first); + for (string line; getline(cin, line);) { + Strings tokenized = tokenizeString(line); + string cmd = tokenized.front(); + writeString(cmd, pipes.first); + tokenized.pop_front(); + foreach (Strings::iterator, i, tokenized) + writeStrings(tokenized, pipes.first); + pipes.first.flush(); + if (cmd == "have") { + PathSet paths = readStrings(pipes.second); + foreach (PathSet::iterator, i, paths) + cout << *i << endl; + } else if (cmd == "info") { + for (Path path = readString(pipes.second); !path.empty(); path = readString(pipes.second)) { + cout << path << endl; + cout << readString(pipes.second) << endl; + PathSet references = readStrings(pipes.second); + cout << references.size() << endl; + foreach (PathSet::iterator, i, references) + cout << *i << endl; + cout << readLongLong(pipes.second) << endl; + cout << readLongLong(pipes.second) << endl; + } + } else + throw Error(format("Unknown substituter query `%1%'") % cmd); + cout << endl; + } + writeString("", pipes.first); +} + +void run(Strings args) +{ + if (args.empty()) + throw UsageError("download-via-ssh requires an argument"); + + if (settings.sshSubstituterHosts.empty()) + return; + + cout << endl; + + pair pipes = connect(settings.sshSubstituterHosts.front()); + + Strings::iterator i = args.begin(); + if (*i == "--query") + query(pipes); + else if (*i == "--substitute") + if (args.size() != 3) + throw UsageError("download-via-ssh: --substitute takes exactly two arguments"); + else { + Path storePath = *++i; + Path destPath = *++i; + substitute(pipes, storePath, destPath); + } + else + throw UsageError(format("download-via-ssh: unknown command `%1%'") % *i); +} + +void printHelp() +{ + std::cerr << "Usage: download-via-ssh --query|--substitute store-path dest-path" << std::endl; +} + + +string programId = "download-via-ssh"; diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk new file mode 100644 index 0000000000..92bf115946 --- /dev/null +++ b/src/download-via-ssh/local.mk @@ -0,0 +1,9 @@ +programs += download-via-ssh + +download-via-ssh_DIR := $(d) + +download-via-ssh_SOURCES := $(d)/download-via-ssh.cc + +download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters + +download-via-ssh_LIBS = libmain libstore libutil libformat diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index fb70cb0767..30238c7fbe 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -223,6 +223,9 @@ static void initAndRun(int argc, char * * argv) else remaining.push_back(arg); } + if (char *pack = getenv("_NIX_OPTIONS")) + settings.unpack(pack); + settings.update(); run(remaining); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 68add1982f..c1ffc26fa3 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -79,6 +79,7 @@ void Settings::processEnvironment() #endif substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-via-ssh"); } else substituters = tokenizeString(subs, ":"); } @@ -151,6 +152,7 @@ void Settings::update() get(gcKeepDerivations, "gc-keep-derivations"); get(autoOptimiseStore, "auto-optimise-store"); get(envKeepDerivations, "env-keep-derivations"); + get(sshSubstituterHosts, "ssh-substituter-hosts"); } @@ -182,6 +184,13 @@ void Settings::get(StringSet & res, const string & name) res.insert(ss.begin(), ss.end()); } +void Settings::get(Strings & res, const string & name) +{ + SettingsMap::iterator i = settings.find(name); + if (i == settings.end()) return; + res = tokenizeString(i->second); +} + template void Settings::get(N & res, const string & name) { @@ -206,6 +215,17 @@ string Settings::pack() } +void Settings::unpack(string pack) { + Strings lines = tokenizeString(pack, "\n"); + foreach (Strings::iterator, i, lines) { + string::size_type eq = i->find('='); + if (eq == string::npos) + throw Error("illegal option name/value"); + set(i->substr(0, eq), i->substr(eq + 1)); + } +} + + Settings::SettingsMap Settings::getOverrides() { return overrides; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9300edbe96..5e7cfda9a1 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -25,6 +25,8 @@ struct Settings { string pack(); + void unpack(string pack); + SettingsMap getOverrides(); /* The directory where we store sources and derived files. */ @@ -144,6 +146,9 @@ struct Settings { chroot. */ StringSet dirsInChroot; + /* Set of ssh connection strings for the ssh substituter */ + Strings sshSubstituterHosts; + /* Whether to impersonate a Linux 2.6 machine on newer kernels. */ bool impersonateLinux26; @@ -195,6 +200,7 @@ private: void get(string & res, const string & name); void get(bool & res, const string & name); void get(StringSet & res, const string & name); + void get(Strings & res, const string & name); template void get(N & res, const string & name); }; From 78d979567fa304fa4445fe7403932d9d97183ebd Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 10 Feb 2014 06:43:29 -0500 Subject: [PATCH 07/14] Clarify comment Signed-off-by: Shea Levy --- src/download-via-ssh/download-via-ssh.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index 7adb93e10d..003d7de2bf 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -46,7 +46,8 @@ static pair connect(string conn) { } _exit(1); } - // If child exits unexpectedly, we'll EPIPE. If we exit unexpectedly, child will + // If child exits unexpectedly, we'll EPIPE or EOF early. + // If we exit unexpectedly, child will EPIPE or EOF early. // So no need to keep track of it. return pair(to.writeSide.borrow(), from.readSide.borrow()); From 16146031659eae475cd5933b8553b13d725ca436 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 10 Feb 2014 06:49:37 -0500 Subject: [PATCH 08/14] Pass in params by const ref Signed-off-by: Shea Levy --- src/libstore/globals.cc | 2 +- src/libstore/globals.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index c1ffc26fa3..1d4bcd94f7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -215,7 +215,7 @@ string Settings::pack() } -void Settings::unpack(string pack) { +void Settings::unpack(const string &pack) { Strings lines = tokenizeString(pack, "\n"); foreach (Strings::iterator, i, lines) { string::size_type eq = i->find('='); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 5e7cfda9a1..13772c65cb 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -25,7 +25,7 @@ struct Settings { string pack(); - void unpack(string pack); + void unpack(const string &pack); SettingsMap getOverrides(); From 38c3beac1a8ac9ddf4fdbbcafd400dabcf195076 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 10 Feb 2014 06:52:48 -0500 Subject: [PATCH 09/14] Move StoreApi::serve into opServe Signed-off-by: Shea Levy --- src/libstore/store-api.cc | 34 ---------------------------------- src/libstore/store-api.hh | 4 ---- src/nix-store/nix-store.cc | 30 +++++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4f1802409..0f250a3c7c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,7 +1,6 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" -#include "archive.hh" #include @@ -259,39 +258,6 @@ string StoreAPI::makeValidityRegistration(const PathSet & paths, } -void StoreAPI::serve(Source & in, BufferedSink & out) -{ - string cmd = readString(in); - if (cmd == "query") { - for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { - PathSet paths = readStrings(in); - if (cmd == "have") { - writeStrings(queryValidPaths(paths), out); - } else if (cmd == "info") { - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!isValidPath(*i)) - continue; - ValidPathInfo info = queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); - } - writeString("", out); - } else - throw Error(format("Unknown serve query `%1%'") % cmd); - out.flush(); - } - } else if (cmd == "substitute") - dumpPath(readString(in), out); - else - throw Error(format("Unknown serve command `%1%'") % cmd); -} - - ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) { ValidPathInfo info; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f32824a3bb..a82fe32216 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -248,10 +248,6 @@ public: `nix-store --register-validity'. */ string makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash); - - /* Serve the store for ssh substituters by taking commands - * from in and printing results to out */ - void serve(Source & in, BufferedSink & out); }; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e3f27820ff..68ad902674 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -839,10 +839,38 @@ static void opServe(Strings opFlags, Strings opArgs) { if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments or flags expected"); + FdSource in(STDIN_FILENO); FdSink out(STDOUT_FILENO); - store->serve(in, out); + string cmd = readString(in); + if (cmd == "query") { + for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { + PathSet paths = readStrings(in); + if (cmd == "have") { + writeStrings(store->queryValidPaths(paths), out); + } else if (cmd == "info") { + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + if (!store->isValidPath(*i)) + continue; + ValidPathInfo info = store->queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); + } + writeString("", out); + } else + throw Error(format("Unknown serve query `%1%'") % cmd); + out.flush(); + } + } else if (cmd == "substitute") + dumpPath(readString(in), out); + else + throw Error(format("Unknown serve command `%1%'") % cmd); } From c89d6b9b63b629ff936a56855be5689523910c58 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 10 Feb 2014 07:43:13 -0500 Subject: [PATCH 10/14] nix-store --serve: Use a versioned protocol Signed-off-by: Shea Levy --- src/download-via-ssh/download-via-ssh.cc | 28 ++++++-- src/download-via-ssh/local.mk | 2 + src/nix-store/nix-store.cc | 82 ++++++++++++++++-------- src/nix-store/serve-protocol.hh | 24 +++++++ 4 files changed, 102 insertions(+), 34 deletions(-) create mode 100644 src/nix-store/serve-protocol.hh diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index 003d7de2bf..be70a374fe 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -4,6 +4,7 @@ #include "archive.hh" #include "affinity.hh" #include "globals.hh" +#include "serve-protocol.hh" #include #include @@ -54,7 +55,7 @@ static pair connect(string conn) { } static void substitute(pair & pipes, Path storePath, Path destPath) { - writeString("substitute", pipes.first); + writeInt(cmdSubstitute, pipes.first); writeString(storePath, pipes.first); pipes.first.flush(); restorePath(destPath, pipes.second); @@ -63,20 +64,24 @@ static void substitute(pair & pipes, Path storePath, Path dest static void query(pair & pipes) { using std::cin; - writeString("query", pipes.first); + writeInt(cmdQuery, pipes.first); for (string line; getline(cin, line);) { Strings tokenized = tokenizeString(line); string cmd = tokenized.front(); - writeString(cmd, pipes.first); tokenized.pop_front(); - foreach (Strings::iterator, i, tokenized) - writeStrings(tokenized, pipes.first); - pipes.first.flush(); if (cmd == "have") { + writeInt(qCmdHave, pipes.first); + foreach (Strings::iterator, i, tokenized) + writeStrings(tokenized, pipes.first); + pipes.first.flush(); PathSet paths = readStrings(pipes.second); foreach (PathSet::iterator, i, paths) cout << *i << endl; } else if (cmd == "info") { + writeInt(qCmdInfo, pipes.first); + foreach (Strings::iterator, i, tokenized) + writeStrings(tokenized, pipes.first); + pipes.first.flush(); for (Path path = readString(pipes.second); !path.empty(); path = readString(pipes.second)) { cout << path << endl; cout << readString(pipes.second) << endl; @@ -91,7 +96,6 @@ static void query(pair & pipes) { throw Error(format("Unknown substituter query `%1%'") % cmd); cout << endl; } - writeString("", pipes.first); } void run(Strings args) @@ -106,6 +110,16 @@ void run(Strings args) pair pipes = connect(settings.sshSubstituterHosts.front()); + /* Exchange the greeting */ + writeInt(SERVE_MAGIC_1, pipes.first); + pipes.first.flush(); + unsigned int magic = readInt(pipes.second); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch"); + readInt(pipes.second); // Server version, unused for now + writeInt(SERVE_PROTOCOL_VERSION, pipes.first); + pipes.first.flush(); + Strings::iterator i = args.begin(); if (*i == "--query") query(pipes); diff --git a/src/download-via-ssh/local.mk b/src/download-via-ssh/local.mk index 92bf115946..80f4c385ac 100644 --- a/src/download-via-ssh/local.mk +++ b/src/download-via-ssh/local.mk @@ -6,4 +6,6 @@ download-via-ssh_SOURCES := $(d)/download-via-ssh.cc download-via-ssh_INSTALL_DIR := $(libexecdir)/nix/substituters +download-via-ssh_CXXFLAGS = -Isrc/nix-store + download-via-ssh_LIBS = libmain libstore libutil libformat diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 68ad902674..638d244988 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -6,6 +6,7 @@ #include "xmlgraph.hh" #include "local-store.hh" #include "util.hh" +#include "serve-protocol.hh" #include #include @@ -843,34 +844,61 @@ static void opServe(Strings opFlags, Strings opArgs) FdSource in(STDIN_FILENO); FdSink out(STDOUT_FILENO); - string cmd = readString(in); - if (cmd == "query") { - for (cmd = readString(in); !cmd.empty(); cmd = readString(in)) { - PathSet paths = readStrings(in); - if (cmd == "have") { - writeStrings(store->queryValidPaths(paths), out); - } else if (cmd == "info") { - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!store->isValidPath(*i)) - continue; - ValidPathInfo info = store->queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); + /* Exchange the greeting. */ + unsigned int magic = readInt(in); + if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); + writeInt(SERVE_MAGIC_2, out); + writeInt(SERVE_PROTOCOL_VERSION, out); + out.flush(); + readInt(in); // Client version, unused for now + + ServeCommand cmd = (ServeCommand) readInt(in); + switch (cmd) { + case cmdQuery: + while (true) { + QueryCommand qCmd; + try { + qCmd = (QueryCommand) readInt(in); + } catch (EndOfFile & e) { + break; } - writeString("", out); - } else - throw Error(format("Unknown serve query `%1%'") % cmd); - out.flush(); - } - } else if (cmd == "substitute") - dumpPath(readString(in), out); - else - throw Error(format("Unknown serve command `%1%'") % cmd); + switch (qCmd) { + case qCmdHave: + { + PathSet paths = readStrings(in); + writeStrings(store->queryValidPaths(paths), out); + } + break; + case qCmdInfo: + { + PathSet paths = readStrings(in); + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + if (!store->isValidPath(*i)) + continue; + ValidPathInfo info = store->queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); + } + writeString("", out); + } + break; + default: + throw Error(format("Unknown serve query `%1%'") % cmd); + } + out.flush(); + } + break; + case cmdSubstitute: + dumpPath(readString(in), out); + break; + default: + throw Error(format("Unknown serve command `%1%'") % cmd); + } } diff --git a/src/nix-store/serve-protocol.hh b/src/nix-store/serve-protocol.hh new file mode 100644 index 0000000000..69277bc1b9 --- /dev/null +++ b/src/nix-store/serve-protocol.hh @@ -0,0 +1,24 @@ +#pragma once + +namespace nix { + + +#define SERVE_MAGIC_1 0x390c9deb +#define SERVE_MAGIC_2 0x5452eecb + +#define SERVE_PROTOCOL_VERSION 0x101 +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + + +typedef enum { + cmdQuery = 0, + cmdSubstitute = 1, +} ServeCommand; + +typedef enum { + qCmdHave = 0, + qCmdInfo = 1, +} QueryCommand; + +} From 2246aa77d291e07141f6a508e46730e2c28e1d84 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 12 Feb 2014 07:22:36 -0500 Subject: [PATCH 11/14] Remove using declarations from download-via-ssh Signed-off-by: Shea Levy --- src/download-via-ssh/download-via-ssh.cc | 36 +++++++++++------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index be70a374fe..5aa1e00ee3 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -10,16 +10,13 @@ #include using namespace nix; -using std::pair; -using std::cout; -using std::endl; // !!! TODO: // * Respect more than the first host // * use a database // * show progress -static pair connect(string conn) { +static std::pair connect(string conn) { Pipe to, from; to.create(); from.create(); @@ -51,21 +48,20 @@ static pair connect(string conn) { // If we exit unexpectedly, child will EPIPE or EOF early. // So no need to keep track of it. - return pair(to.writeSide.borrow(), from.readSide.borrow()); + return std::pair(to.writeSide.borrow(), from.readSide.borrow()); } -static void substitute(pair & pipes, Path storePath, Path destPath) { +static void substitute(std::pair & pipes, Path storePath, Path destPath) { writeInt(cmdSubstitute, pipes.first); writeString(storePath, pipes.first); pipes.first.flush(); restorePath(destPath, pipes.second); - cout << endl; + std::cout << std::endl; } -static void query(pair & pipes) { - using std::cin; +static void query(std::pair & pipes) { writeInt(cmdQuery, pipes.first); - for (string line; getline(cin, line);) { + for (string line; getline(std::cin, line);) { Strings tokenized = tokenizeString(line); string cmd = tokenized.front(); tokenized.pop_front(); @@ -76,25 +72,25 @@ static void query(pair & pipes) { pipes.first.flush(); PathSet paths = readStrings(pipes.second); foreach (PathSet::iterator, i, paths) - cout << *i << endl; + std::cout << *i << std::endl; } else if (cmd == "info") { writeInt(qCmdInfo, pipes.first); foreach (Strings::iterator, i, tokenized) writeStrings(tokenized, pipes.first); pipes.first.flush(); for (Path path = readString(pipes.second); !path.empty(); path = readString(pipes.second)) { - cout << path << endl; - cout << readString(pipes.second) << endl; + std::cout << path << std::endl; + std::cout << readString(pipes.second) << std::endl; PathSet references = readStrings(pipes.second); - cout << references.size() << endl; + std::cout << references.size() << std::endl; foreach (PathSet::iterator, i, references) - cout << *i << endl; - cout << readLongLong(pipes.second) << endl; - cout << readLongLong(pipes.second) << endl; + std::cout << *i << std::endl; + std::cout << readLongLong(pipes.second) << std::endl; + std::cout << readLongLong(pipes.second) << std::endl; } } else throw Error(format("Unknown substituter query `%1%'") % cmd); - cout << endl; + std::cout << std::endl; } } @@ -106,9 +102,9 @@ void run(Strings args) if (settings.sshSubstituterHosts.empty()) return; - cout << endl; + std::cout << std::endl; - pair pipes = connect(settings.sshSubstituterHosts.front()); + std::pair pipes = connect(settings.sshSubstituterHosts.front()); /* Exchange the greeting */ writeInt(SERVE_MAGIC_1, pipes.first); From 7438f0bc2bc4b92bddf7159744ab2923e34b7457 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 12 Feb 2014 07:26:35 -0500 Subject: [PATCH 12/14] error messages start in lowercase Signed-off-by: Shea Levy --- src/download-via-ssh/download-via-ssh.cc | 2 +- src/nix-store/nix-store.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index 5aa1e00ee3..dee4fd78c9 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -89,7 +89,7 @@ static void query(std::pair & pipes) { std::cout << readLongLong(pipes.second) << std::endl; } } else - throw Error(format("Unknown substituter query `%1%'") % cmd); + throw Error(format("unknown substituter query `%1%'") % cmd); std::cout << std::endl; } } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 638d244988..038f099649 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -888,7 +888,7 @@ static void opServe(Strings opFlags, Strings opArgs) } break; default: - throw Error(format("Unknown serve query `%1%'") % cmd); + throw Error(format("unknown serve query `%1%'") % cmd); } out.flush(); } @@ -897,7 +897,7 @@ static void opServe(Strings opFlags, Strings opArgs) dumpPath(readString(in), out); break; default: - throw Error(format("Unknown serve command `%1%'") % cmd); + throw Error(format("unknown serve command `%1%'") % cmd); } } From 62eb9eb76ddacc1aa97400bef9f25b6ca4c50c8c Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 12 Feb 2014 07:27:45 -0500 Subject: [PATCH 13/14] Remove relic of old code Signed-off-by: Shea Levy --- src/download-via-ssh/download-via-ssh.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/download-via-ssh/download-via-ssh.cc b/src/download-via-ssh/download-via-ssh.cc index dee4fd78c9..d85f1572f3 100644 --- a/src/download-via-ssh/download-via-ssh.cc +++ b/src/download-via-ssh/download-via-ssh.cc @@ -67,7 +67,6 @@ static void query(std::pair & pipes) { tokenized.pop_front(); if (cmd == "have") { writeInt(qCmdHave, pipes.first); - foreach (Strings::iterator, i, tokenized) writeStrings(tokenized, pipes.first); pipes.first.flush(); PathSet paths = readStrings(pipes.second); @@ -75,7 +74,6 @@ static void query(std::pair & pipes) { std::cout << *i << std::endl; } else if (cmd == "info") { writeInt(qCmdInfo, pipes.first); - foreach (Strings::iterator, i, tokenized) writeStrings(tokenized, pipes.first); pipes.first.flush(); for (Path path = readString(pipes.second); !path.empty(); path = readString(pipes.second)) { From f67f52751f21b2fe70b5a7352053f130eb6f0f59 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Wed, 12 Feb 2014 07:33:07 -0500 Subject: [PATCH 14/14] Indendation fix Signed-off-by: Shea Levy --- src/nix-store/nix-store.cc | 40 ++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 038f099649..350a4ce0fa 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -863,30 +863,28 @@ static void opServe(Strings opFlags, Strings opArgs) break; } switch (qCmd) { - case qCmdHave: - { - PathSet paths = readStrings(in); - writeStrings(store->queryValidPaths(paths), out); - } + case qCmdHave: { + PathSet paths = readStrings(in); + writeStrings(store->queryValidPaths(paths), out); break; - case qCmdInfo: - { - PathSet paths = readStrings(in); - // !!! Maybe we want a queryPathInfos? - foreach (PathSet::iterator, i, paths) { - if (!store->isValidPath(*i)) - continue; - ValidPathInfo info = store->queryPathInfo(*i); - writeString(info.path, out); - writeString(info.deriver, out); - writeStrings(info.references, out); - // !!! Maybe we want compression? - writeLongLong(info.narSize, out); // downloadSize - writeLongLong(info.narSize, out); - } - writeString("", out); + } + case qCmdInfo: { + PathSet paths = readStrings(in); + // !!! Maybe we want a queryPathInfos? + foreach (PathSet::iterator, i, paths) { + if (!store->isValidPath(*i)) + continue; + ValidPathInfo info = store->queryPathInfo(*i); + writeString(info.path, out); + writeString(info.deriver, out); + writeStrings(info.references, out); + // !!! Maybe we want compression? + writeLongLong(info.narSize, out); // downloadSize + writeLongLong(info.narSize, out); } + writeString("", out); break; + } default: throw Error(format("unknown serve query `%1%'") % cmd); }