download-from-binary-cache: cache binary cache info in a SQLite DB

This commit is contained in:
Eelco Dolstra 2012-07-03 17:29:33 -04:00
parent 8319b1ab9f
commit d694c599e2

View file

@ -4,15 +4,66 @@ use strict;
use File::Basename; use File::Basename;
use Nix::Config; use Nix::Config;
use Nix::Store; use Nix::Store;
use DBI;
my @binaryCacheUrls = split / /, ($ENV{"NIX_BINARY_CACHES"} || ""); my @binaryCacheUrls = split / /, ($ENV{"NIX_BINARY_CACHES"} || "");
my ($dbh, $insertNAR, $queryNAR);
my %cacheIds;
sub initCache {
my $dbPath = "$Nix::Config::stateDir/binary-cache-v1.sqlite";
# Open/create the database.
$dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
or die "cannot open database `$dbPath'";
$dbh->{RaiseError} = 1;
$dbh->{PrintError} = 0;
$dbh->do("pragma synchronous = off"); # we can always reproduce the cache
$dbh->do("pragma journal_mode = truncate");
# Initialise the database schema, if necessary.
$dbh->do(<<EOF);
create table if not exists BinaryCaches (
id integer primary key autoincrement not null,
url text unique not null
);
EOF
$dbh->do(<<EOF);
create table if not exists NARs (
cache integer not null,
storePath text not null,
url text not null,
compression text not null,
fileHash text,
fileSize integer,
narHash text,
narSize integer,
refs text,
deriver text,
system text,
timestamp integer not null,
primary key (cache, storePath),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
EOF
$insertNAR = $dbh->prepare(
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, " .
"narSize, refs, deriver, system, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") or die;
$queryNAR = $dbh->prepare("select * from NARs where cache = ? and storePath = ?") or die;
}
sub getInfoFrom { sub getInfoFrom {
my ($storePath, $pathHash, $binaryCacheUrl) = @_; my ($storePath, $pathHash, $binaryCacheUrl) = @_;
my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo"; my $infoUrl = "$binaryCacheUrl/$pathHash.narinfo";
#print STDERR "checking $infoUrl...\n"; print STDERR "checking $infoUrl...\n";
my $s = `$Nix::Config::curl --fail --silent --location $infoUrl`; my $s = `$Nix::Config::curl --fail --silent --location $infoUrl`;
if ($? != 0) { if ($? != 0) {
my $status = $? >> 8; my $status = $? >> 8;
@ -34,6 +85,7 @@ sub getInfoFrom {
elsif ($1 eq "References") { @refs = split / /, $2; } elsif ($1 eq "References") { @refs = split / /, $2; }
elsif ($1 eq "Deriver") { $deriver = $2; } elsif ($1 eq "Deriver") { $deriver = $2; }
} }
return undef if $storePath ne $storePath2;
if ($storePath ne $storePath2 || !defined $url || !defined $narHash) { if ($storePath ne $storePath2 || !defined $url || !defined $narHash) {
print STDERR "bad NAR info file $infoUrl\n"; print STDERR "bad NAR info file $infoUrl\n";
return undef; return undef;
@ -45,9 +97,63 @@ sub getInfoFrom {
, fileSize => $fileSize , fileSize => $fileSize
, narHash => $narHash , narHash => $narHash
, narSize => $narSize , narSize => $narSize
, refs => [ map { "$Nix::Config::storeDir/$_" } @refs ] , refs => [ @refs ]
, deriver => defined $deriver ? "$Nix::Config::storeDir/$deriver" : undef , deriver => $deriver
};
} }
sub getCacheId {
my ($binaryCacheUrl) = @_;
my $cacheId = $cacheIds{$binaryCacheUrl};
return $cacheId if defined $cacheId;
# FIXME: not atomic.
my @res = @{$dbh->selectcol_arrayref("select id from BinaryCaches where url = ?", {}, $binaryCacheUrl)};
if (scalar @res == 1) {
$cacheId = $res[0];
} else {
$dbh->do("insert into BinaryCaches(url) values (?)",
{}, $binaryCacheUrl);
$cacheId = $dbh->last_insert_id("", "", "", "");
}
$cacheIds{$binaryCacheUrl} = $cacheId;
return $cacheId;
}
sub cachedGetInfoFrom {
my ($storePath, $pathHash, $binaryCacheUrl) = @_;
my $cacheId = getCacheId($binaryCacheUrl);
# Look up $storePath in the SQLite cache.
$queryNAR->execute($cacheId, basename($storePath));
my $res = $queryNAR->fetchrow_hashref();
return
{ url => $res->{url}
, compression => $res->{compression}
, fileHash => $res->{fileHash}
, fileSize => $res->{fileSize}
, narHash => $res->{narHash}
, narSize => $res->{narSize}
, refs => [ split " ", $res->{refs} ]
, deriver => $res->{deriver}
} if defined $res;
# Not found, so do an HTTP request to get the info.
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl);
# Cache the result.
$insertNAR->execute(
$cacheId, basename($storePath), $info->{url}, $info->{compression}, $info->{fileHash}, $info->{fileSize},
$info->{narHash}, $info->{narSize}, join(" ", @{$info->{refs}}),
$info->{deriver}, $info->{system}, time())
if defined $info;
return $info;
} }
@ -57,7 +163,7 @@ sub getInfo {
my $pathHash = substr(basename($storePath), 0, 32); my $pathHash = substr(basename($storePath), 0, 32);
cache: foreach my $binaryCacheUrl (@binaryCacheUrls) { cache: foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl); my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl);
return $info if defined $info; return $info if defined $info;
} }
@ -71,7 +177,7 @@ sub downloadBinary {
my $pathHash = substr(basename($storePath), 0, 32); my $pathHash = substr(basename($storePath), 0, 32);
cache: foreach my $binaryCacheUrl (@binaryCacheUrls) { cache: foreach my $binaryCacheUrl (@binaryCacheUrls) {
my $info = getInfoFrom($storePath, $pathHash, $binaryCacheUrl); my $info = cachedGetInfoFrom($storePath, $pathHash, $binaryCacheUrl);
if (defined $info) { if (defined $info) {
my $decompressor; my $decompressor;
if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; } if ($info->{compression} eq "bzip2") { $decompressor = "$Nix::Config::bzip2 -d"; }
@ -99,6 +205,9 @@ sub downloadBinary {
} }
initCache();
if ($ARGV[0] eq "--query") { if ($ARGV[0] eq "--query") {
while (<STDIN>) { while (<STDIN>) {
@ -117,9 +226,9 @@ if ($ARGV[0] eq "--query") {
my $info = getInfo($storePath); my $info = getInfo($storePath);
if (defined $info) { if (defined $info) {
print "1\n"; print "1\n";
print $info->{deriver} || "", "\n"; print $info->{deriver} ? "$Nix::Config::storeDir/$info->{deriver}" : "", "\n";
print scalar @{$info->{refs}}, "\n"; print scalar @{$info->{refs}}, "\n";
print "$_\n" foreach @{$info->{refs}}; print "$Nix::Config::storeDir/$_\n" foreach @{$info->{refs}};
print $info->{fileSize} || 0, "\n"; print $info->{fileSize} || 0, "\n";
print $info->{narSize} || 0, "\n"; print $info->{narSize} || 0, "\n";
} else { } else {