diff --git a/scripts/download-using-manifests.pl.in b/scripts/download-using-manifests.pl.in index e02d82c140..65ab36084a 100644 --- a/scripts/download-using-manifests.pl.in +++ b/scripts/download-using-manifests.pl.in @@ -34,17 +34,31 @@ print "\n*** Trying to download/patch `$targetPath'\n"; # Load all manifests. my %narFiles; +my %localPaths; my %patches; for my $manifest (glob "$manifestDir/*.nixmanifest") { # print STDERR "reading $manifest\n"; - if (readManifest($manifest, \%narFiles, \%patches) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) { print STDERR "you have an old-style manifest `$manifest'; please delete it\n"; exit 1; } } +# If we can copy from a local path, do that. +my $localPathList = $localPaths{$targetPath}; +foreach my $localPath (@{$localPathList}) { + my $sourcePath = $localPath->{copyFrom}; + if (-e $sourcePath) { + print "\n*** Step 1/1: copying from $sourcePath\n"; + system("@bindir@/nix-store --dump $sourcePath | @bindir@/nix-store --restore $targetPath") == 0 + or die "cannot copy `$sourcePath' to `$targetPath'"; + exit 0; + } +} + + # Build a graph of all store paths that might contribute to the # construction of $targetPath, and the special node "start". The # edges are either patch operations, or downloads of full NAR files. diff --git a/scripts/generate-patches.pl.in b/scripts/generate-patches.pl.in index 9e355f6aee..9a5c3423f2 100755 --- a/scripts/generate-patches.pl.in +++ b/scripts/generate-patches.pl.in @@ -28,16 +28,18 @@ print "TEMP = $tmpDir\n"; #END { rmdir $tmpDir; } my %srcNarFiles; +my %srcLocalPaths; my %srcPatches; my %dstNarFiles; +my %dstLocalPaths; my %dstPatches; readManifest "$srcDir/MANIFEST", - \%srcNarFiles, \%srcPatches; + \%srcNarFiles, \%srcLocalPaths, \%srcPatches; readManifest "$dstDir/MANIFEST", - \%dstNarFiles, \%dstPatches; + \%dstNarFiles, \%dstLocalPaths, \%dstPatches; sub findOutputPaths { diff --git a/scripts/maintenance/gc-releases.pl b/scripts/maintenance/gc-releases.pl index 80b2726c5a..b8abb03477 100755 --- a/scripts/maintenance/gc-releases.pl +++ b/scripts/maintenance/gc-releases.pl @@ -7,11 +7,12 @@ use readcache; # Read the manifests. my %narFiles; +my %localPaths; my %patches; foreach my $manifest (@ARGV) { print STDERR "loading $manifest\n"; - if (readManifest($manifest, \%narFiles, \%patches, 1) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches, 1) < 3) { # die "manifest `$manifest' is too old (i.e., for Nix <= 0.7)\n"; } } diff --git a/scripts/maintenance/shrink-manifest.pl b/scripts/maintenance/shrink-manifest.pl index 7be5814488..60cf9c5b46 100755 --- a/scripts/maintenance/shrink-manifest.pl +++ b/scripts/maintenance/shrink-manifest.pl @@ -6,11 +6,12 @@ use readcache; my %allNarFiles; +my %allLocalPaths; my %allPatches; foreach my $manifest (glob "/data/webserver/dist/*/*/MANIFEST") { print STDERR "loading $manifest\n"; - readManifest($manifest, \%allNarFiles, \%allPatches, 1); + readManifest($manifest, \%allNarFiles, \%allLocalPaths, \%allPatches, 1); } diff --git a/scripts/nix-pull.in b/scripts/nix-pull.in index 21fa30c61e..46f9f147c9 100644 --- a/scripts/nix-pull.in +++ b/scripts/nix-pull.in @@ -28,6 +28,7 @@ umask 0022; # Process the URLs specified on the command line. my %narFiles; +my %localPaths; my %patches; my $skipWrongStore = 0; @@ -42,7 +43,7 @@ sub processURL { "'$url' > '$manifest'") == 0 or die "curl failed: $?"; - if (readManifest($manifest, \%narFiles, \%patches) < 3) { + if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) { die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n"; } @@ -80,7 +81,7 @@ while (@ARGV) { } -my $size = scalar (keys %narFiles); +my $size = scalar (keys %narFiles) + scalar (keys %localPaths); print "$size store paths in manifest\n"; @@ -90,19 +91,32 @@ print STDERR "registering substitutes...\n"; my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes") or die "cannot run nix-store"; +sub writeRegistration { + my $storePath = shift; + my $object = shift; + print WRITE "$storePath\n"; + print WRITE "$object->{deriver}\n"; + print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; + print WRITE "0\n"; + my @references = split " ", $object->{references}; + my $count = scalar @references; + print WRITE "$count\n"; + foreach my $reference (@references) { + print WRITE "$reference\n"; + } +} + foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath}; foreach my $narFile (@{$narFileList}) { - print WRITE "$storePath\n"; - print WRITE "$narFile->{deriver}\n"; - print WRITE "$libexecDir/nix/download-using-manifests.pl\n"; - print WRITE "0\n"; - my @references = split " ", $narFile->{references}; - my $count = scalar @references; - print WRITE "$count\n"; - foreach my $reference (@references) { - print WRITE "$reference\n"; - } + writeRegistration $storePath, $narFile; + } +} + +foreach my $storePath (keys %localPaths) { + my $localPathList = $localPaths{$storePath}; + foreach my $localPath (@{$localPathList}) { + writeRegistration $storePath, $localPath; } } diff --git a/scripts/readmanifest.pm.in b/scripts/readmanifest.pm.in index 0e15133cf1..1d44444700 100644 --- a/scripts/readmanifest.pm.in +++ b/scripts/readmanifest.pm.in @@ -34,6 +34,7 @@ sub addPatch { sub readManifest { my $manifest = shift; my $narFiles = shift; + my $localPaths = shift; my $patches = shift; my $allowConflicts = shift; $allowConflicts = 0 unless defined $allowConflicts; @@ -57,6 +58,7 @@ sub readManifest { my $references; my $deriver; my $hashAlgo; + my $copyFrom; while () { chomp; @@ -125,9 +127,25 @@ sub readManifest { }, $allowConflicts; } + elsif ($type eq "localPath") { + + $$localPaths{$storePath} = [] + unless defined $$localPaths{$storePath}; + + my $localPathsList = $$localPaths{$storePath}; + + # !!! remove duplicates + + push @{$localPathsList}, + { copyFrom => $copyFrom, references => $references + , deriver => "" + }; + } + } elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } + elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } @@ -158,6 +176,7 @@ sub writeManifest my $manifest = shift; my $narFiles = shift; my $patches = shift; + my $copySources = shift; open MANIFEST, ">$manifest.tmp"; # !!! check exclusive diff --git a/scripts/remove-patches.pl b/scripts/remove-patches.pl index 75eb0ce778..1a94dfa164 100755 --- a/scripts/remove-patches.pl +++ b/scripts/remove-patches.pl @@ -6,13 +6,12 @@ use readmanifest; for my $p (@ARGV) { my %narFiles; + my %localPaths; my %patches; - readManifest $p, - \%narFiles, \%patches; + readManifest $p, \%narFiles, \%localPaths, \%patches; %patches = (); - writeManifest $p, - \%narFiles, \%patches; + writeManifest $p, \%narFiles, \%patches; } diff --git a/scripts/update-manifest.pl b/scripts/update-manifest.pl index 566f646732..ac0ce749d9 100755 --- a/scripts/update-manifest.pl +++ b/scripts/update-manifest.pl @@ -8,9 +8,10 @@ die unless scalar @ARGV == 2; my $cache = $ARGV[0]; my $manifest = $ARGV[1]; my %narFiles; +my %localPaths; my %patches; -readManifest $manifest, \%narFiles, \%patches; +readManifest $manifest, \%narFiles, \%localPaths, \%patches; foreach my $storePath (keys %narFiles) { my $narFileList = $narFiles{$storePath};