projects
/
mir.git
/ commitdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
| commitdiff |
tree
raw
|
patch
|
inline
| side by side (parent:
76425ee
)
Stream fetched files to disk rather than reading them
author
zak
<zak>
Sun, 12 Nov 2006 23:35:23 +0000
(23:35 +0000)
committer
zak
<zak>
Sun, 12 Nov 2006 23:35:23 +0000
(23:35 +0000)
into memory.
scripts/mirror-scripts/update.pl
patch
|
blob
|
history
diff --git
a/scripts/mirror-scripts/update.pl
b/scripts/mirror-scripts/update.pl
index
b6690bb
..
47eece6
100755
(executable)
--- a/
scripts/mirror-scripts/update.pl
+++ b/
scripts/mirror-scripts/update.pl
@@
-164,7
+164,14
@@
sub ensureDir($) {
}
}
}
}
-# get a file, optionally saving it locally
+# get a file, optionally saving it locally.
+# if a local filename is given, return true if the file was
+# found on the server and undef otherwise
+#
+# if no local filename is given, return the content of the
+# file, or undef if it was not found on the server
+#
+# on all other errors, die
sub fetchFile($;$) {
my ($remotefile, $localfile) = @_;
sub fetchFile($;$) {
my ($remotefile, $localfile) = @_;
@@
-175,40
+182,35
@@
sub fetchFile($;$) {
my $req = new HTTP::Request(GET => "$remotefile");
my $req = new HTTP::Request(GET => "$remotefile");
- if ($localfile and -e $localfile) {
- # Don't fetch unless more recent than local copy
- my $stat = stat($localfile);
- $req->header("If-Modified-Since" => time2str($stat->mtime));
+ if ($localfile) {
+ if (-e $localfile) {
+ # Don't fetch unless more recent than local copy
+ my $stat = stat($localfile);
+ $req->header("If-Modified-Since" => time2str($stat->mtime));
+ }
+ else {
+ ensureDir(dirPart($localfile));
+ }
}
}
- my $resp = $ua->request($req);
+ my $resp = $ua->request($req
, $localfile
);
if ($resp->is_success) { # 2xx codes
my $mtime = str2time($resp->header("Last-Modified"));
if ($resp->is_success) { # 2xx codes
my $mtime = str2time($resp->header("Last-Modified"));
- if ($localfile) {
- if ($verbose) {
- print STDERR " -> success";
- print STDERR "; mtime ".time2str($mtime) if $mtime;
- print STDERR "\n";
- }
- ensureDir(dirPart($localfile));
-
- open (LOCAL, ">", "$localfile") or die "Can't open $localfile for writing ($!)";
- print LOCAL $resp->content or die "Error writing $localfile ($!)";
- close LOCAL or die "Error writing $localfile ($!)";
+ if ($verbose) {
+ print STDERR " -> success";
+ print STDERR "; mtime ".time2str($mtime) if $mtime;
+ print STDERR "\n";
+ }
- if ($mtime) {
- utime $mtime, $mtime, $localfile;
- }
+ if ($localfile and $mtime) {
+ utime $mtime, $mtime, $localfile;
}
}
- return $resp->content;
+ return $
localfile ? 1 : $
resp->content;
}
elsif ($resp->is_redirect) { # 3xx codes
if ($resp->code == RC_NOT_MODIFIED) { # 304
print STDERR " -> not modified\n" if $verbose;
}
elsif ($resp->is_redirect) { # 3xx codes
if ($resp->code == RC_NOT_MODIFIED) { # 304
print STDERR " -> not modified\n" if $verbose;
- open (LOCAL, "<", "$localfile") or die "Can't open $localfile ($!)";
- local $/; # slurp whole file
- my $content = <LOCAL>;
- close LOCAL;
- return $content;
+ die "Got 304 with no local file" if not $localfile;
+ return 1;
}
print STDERR " -> redirect (".$resp->code.")\n" if $verbose;
die "Can't fetch $remotefile (got redirect, not yet handled)";
}
print STDERR " -> redirect (".$resp->code.")\n" if $verbose;
die "Can't fetch $remotefile (got redirect, not yet handled)";
@@
-256,18
+258,16
@@
if ($listchangefiles) {
}
# get the changes files
}
# get the changes files
-my %changesfilecontent;
-foreach my $file (@changesfiles) { $changesfilecontent{$file} = getChangesFile($file); }
+foreach my $file (@changesfiles) { getChangesFile($file); }
-# if the file has not changed (response code 304) then ignore it
+#
TODO:
if the file has not changed (response code 304) then ignore it
# iterate over all the fetched files, building up a list of files
# to fetch/delete
my %files;
foreach my $changesfile (@changesfiles) {
# iterate over all the fetched files, building up a list of files
# to fetch/delete
my %files;
foreach my $changesfile (@changesfiles) {
- my $changesfilecontent = $changesfilecontent{$changesfile};
- if (not defined $changesfilecontent) {
- print STDERR "Skipping changes file $changesfile; not present at remote end\n" if $verbose;
+ if (! -e "$changesdir/$changesfile") {
+ print STDERR "Skipping changes file $changesfile; not present\n" if $verbose;
next;
}
next;
}
@@
-276,8
+276,8
@@
foreach my $changesfile (@changesfiles) {
print STDERR "Processing changes file $changesfile\n" if $verbose;
print STDERR "Processing changes file $changesfile\n" if $verbose;
-
my @changes = split /[\r\n]+/, $changesfilecontent
;
-
foreach my $change (@changes
) {
+
open (CHANGES, "<", "$changesdir/$changesfile") or die "Can't open $changesfile"
;
+
while (my $change = <CHANGES>
) {
my ($time, $op, $path) = split ' ', $change;
# Ignore malformed lines, especially wacky paths that could be malicious
my ($time, $op, $path) = split ' ', $change;
# Ignore malformed lines, especially wacky paths that could be malicious
@@
-321,9
+321,7
@@
while (my ($file, $op) = each %files) {
}
elsif ($op eq "add" or $op eq "change" or $op eq "Modification") {
# add/change: re-fetch the file
}
elsif ($op eq "add" or $op eq "change" or $op eq "Modification") {
# add/change: re-fetch the file
- # FIXME: don't insist on reading entire file into memory
- my $content = fetchFile("$remoteroot/$file","$workingdir/$file");
- die "File $remoteroot/$file not found" if not defined $content;
+ fetchFile("$remoteroot/$file","$workingdir/$file") or die "File $remoteroot/$file not found";
}
else {
die "Unknown operation '$op'";
}
else {
die "Unknown operation '$op'";