From 5bbd693caedd5d50994938555b3a4b535875347e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Nov 2011 15:13:37 +0000 Subject: [PATCH] =?UTF-8?q?*=20Add=20an=20API=20function=20exportPaths()?= =?UTF-8?q?=20that=20provides=20the=20functionality=20of=20=20=20=E2=80=98?= =?UTF-8?q?nix-store=20--export=E2=80=99.=20*=20Add=20a=20Perl=20module=20?= =?UTF-8?q?that=20provides=20the=20functionality=20of=20=20=20=E2=80=98nix?= =?UTF-8?q?-copy-closure=20--to=E2=80=99.=20=20This=20is=20used=20by=20bui?= =?UTF-8?q?ld-remote.pl=20so=20it=20no=20=20=20longer=20needs=20to=20start?= =?UTF-8?q?=20a=20separate=20nix-copy-closure=20process.=20=20Also,=20it?= =?UTF-8?q?=20=20=20uses=20the=20Perl=20API=20to=20do=20the=20export,=20so?= =?UTF-8?q?=20it=20doesn't=20need=20to=20start=20a=20=20=20separate=20nix-?= =?UTF-8?q?store=20process=20either.=20=20As=20a=20result,=20nix-copy-clos?= =?UTF-8?q?ure=20=20=20and=20build-remote.pl=20should=20no=20longer=20fail?= =?UTF-8?q?=20on=20very=20large=20closures=20due=20=20=20to=20an=20"Argume?= =?UTF-8?q?nt=20list=20too=20long"=20error.=20=20(Note=20that=20having=20v?= =?UTF-8?q?ery=20many=20=20=20dependencies=20in=20a=20single=20derivation?= =?UTF-8?q?=20can=20still=20fail=20because=20the=20=20=20environment=20can?= =?UTF-8?q?=20become=20too=20large.=20=20Can't=20be=20helped=20though.)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perl/Makefile.am | 2 +- perl/lib/Nix/CopyClosure.pm | 46 +++++++++++++++++++++++++++++++++++++ perl/lib/Nix/Store.pm | 2 +- perl/lib/Nix/Store.xs | 13 +++++++++++ scripts/build-remote.pl.in | 4 ++-- scripts/nix-copy-closure.in | 36 +++++++---------------------- src/libstore/local-store.cc | 2 +- src/libstore/store-api.cc | 11 +++++++++ src/libstore/store-api.hh | 6 +++++ src/nix-store/nix-store.cc | 6 +---- 10 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 perl/lib/Nix/CopyClosure.pm diff --git a/perl/Makefile.am b/perl/Makefile.am index eded469f92..93f5415c8b 100644 --- a/perl/Makefile.am +++ b/perl/Makefile.am @@ -2,7 +2,7 @@ perlversion := $(shell perl -e 'use Config; print $$Config{version};') perlarchname := $(shell perl -e 'use Config; print $$Config{archname};') perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname) -PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/Config.pm.in +PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in all: $(PERL_MODULES:.in=) ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so diff --git a/perl/lib/Nix/CopyClosure.pm b/perl/lib/Nix/CopyClosure.pm new file mode 100644 index 0000000000..045f6bfaf1 --- /dev/null +++ b/perl/lib/Nix/CopyClosure.pm @@ -0,0 +1,46 @@ +package Nix::CopyClosure; + +use strict; +use Nix::Config; +use Nix::Store; + + +sub copyTo { + my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor, $includeOutputs, $dryRun, $sign) = @_; + + $compressor = "$compressor |" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; + + # Get the closure of this path. + my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, + map { followLinksToStorePath $_ } @{$storePaths}))); + + # Ask the remote host which paths are invalid. Because of limits + # to the command line length, do this in chunks. Eventually, + # we'll want to use ‘--from-stdin’, but we can't rely on the + # target having this option yet. + my @missing = (); + while (scalar(@closure) > 0) { + my @ps = splice(@closure, 0, 1500); + open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store --check-validity --print-invalid @ps|"); + while () { + chomp; + push @missing, $_; + } + close READ or die; + } + + # Export the store paths and import them on the remote machine. + if (scalar @missing > 0) { + print STDERR "copying ", scalar @missing, " missing paths to ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + unless ($dryRun) { + open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor nix-store --import'" or die; + exportPaths(fileno(SSH), $sign, @missing); + close SSH or die "copying store paths to remote machine `$sshHost' failed: $?"; + } + } +} + + +1; diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index bef6e7460c..d96f8e9ab6 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -12,7 +12,7 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); -our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath); +our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure followLinksToStorePath exportPaths); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 9e51ea337c..b50451f45f 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -146,3 +146,16 @@ SV * followLinksToStorePath(char * path) } OUTPUT: RETVAL + + +void exportPaths(int fd, int sign, ...) + PPCODE: + try { + doInit(); + Paths paths; + for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + FdSink sink(fd); + exportPaths(*store, paths, sign, sink); + } catch (Error & e) { + croak(e.what()); + } diff --git a/scripts/build-remote.pl.in b/scripts/build-remote.pl.in index 110c95f223..72d09565bc 100755 --- a/scripts/build-remote.pl.in +++ b/scripts/build-remote.pl.in @@ -5,6 +5,7 @@ use English '-no_match_vars'; use IO::Handle; use Nix::Config; use Nix::SSH qw/sshOpts openSSHConnection/; +use Nix::CopyClosure; no warnings('once'); @@ -225,8 +226,7 @@ sub removeRoots { # Copy the derivation and its dependencies to the build machine. -system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName $maybeSign $drvPath @inputs") == 0 - or die "cannot copy inputs to $hostName: $?"; +Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "", "", 0, 0, $maybeSign ne ""); # Perform the build. diff --git a/scripts/nix-copy-closure.in b/scripts/nix-copy-closure.in index 172acd9e7d..2eac56e3f2 100755 --- a/scripts/nix-copy-closure.in +++ b/scripts/nix-copy-closure.in @@ -3,6 +3,7 @@ use Nix::SSH; use Nix::Config; use Nix::Store; +use Nix::CopyClosure; if (scalar @ARGV < 1) { @@ -39,8 +40,8 @@ while (@ARGV) { $sign = 1; } elsif ($arg eq "--gzip") { - $compressor = "| gzip"; - $decompressor = "gunzip |"; + $compressor = "gzip"; + $decompressor = "gunzip"; } elsif ($arg eq "--from") { $toMode = 0; @@ -67,30 +68,7 @@ openSSHConnection $sshHost or die "$0: unable to start SSH\n"; if ($toMode) { # Copy TO the remote machine. - - # Get the closure of this path. - my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, map { followLinksToStorePath $_ } @storePaths))); - - # Ask the remote host which paths are invalid. - open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity --print-invalid @allStorePaths|"); - my @missing = (); - while () { - chomp; - push @missing, $_; - } - close READ or die; - - # Export the store paths and import them on the remote machine. - if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; - unless ($dryRun) { - my $extraOpts = $sign ? "--sign" : ""; - system("set -f; nix-store --export $extraOpts @missing $compressor | ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0 - or die "copying store paths to remote machine `$sshHost' failed: $?"; - } - } - + Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ], $compressor, $decompressor, $includeOutputs, $dryRun, $sign); } else { # Copy FROM the remote machine. @@ -112,8 +90,10 @@ else { # Copy FROM the remote machine. # Export the store paths on the remote machine and import them on locally. if (scalar @missing > 0) { - print STDERR "copying these missing paths:\n"; - print STDERR " $_\n" foreach @missing; + print STDERR "copying ", scalar @missing, " missing paths from ‘$sshHost’...\n"; + #print STDERR " $_\n" foreach @missing; + $compressor = "| $compressor" if $compressor ne ""; + $decompressor = "$decompressor |" if $decompressor ne ""; unless ($dryRun) { my $extraOpts = $sign ? "--sign" : ""; system("set -f; ssh $sshHost @sshOpts 'nix-store --export $extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store --import") == 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 06cadcb0ff..3c1f2ecacd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1255,7 +1255,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) bool haveSignature = readInt(hashAndReadSource) == 1; if (requireSignature && !haveSignature) - throw Error("imported archive lacks a signature"); + throw Error(format("imported archive of `%1%' lacks a signature") % dstPath); if (haveSignature) { string signature = readString(hashAndReadSource); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index d67ff2c772..36ade21708 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -298,6 +298,17 @@ string showPaths(const PathSet & paths) } +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink) +{ + foreach (Paths::const_iterator, i, paths) { + writeInt(1, sink); + store.exportPath(*i, sign, sink); + } + writeInt(0, sink); +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b3e67436c6..8bfb09880e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -342,6 +342,12 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); +/* Export multiple paths in the format expected by ‘nix-store + --import’. */ +void exportPaths(StoreAPI & store, const Paths & paths, + bool sign, Sink & sink); + + } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 371ca54af0..84d3da032c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -594,11 +594,7 @@ static void opExport(Strings opFlags, Strings opArgs) else throw UsageError(format("unknown flag `%1%'") % *i); FdSink sink(STDOUT_FILENO); - for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) { - writeInt(1, sink); - store->exportPath(*i, sign, sink); - } - writeInt(0, sink); + exportPaths(*store, opArgs, sign, sink); }