Stream fetched files to disk rather than reading them

into memory.

diff --git a/scripts/mirror-scripts/update.pl b/scripts/mirror-scripts/update.pl
index b6690bb..47eece6 100755 (executable)
--- a/scripts/mirror-scripts/update.pl
+++ b/scripts/mirror-scripts/update.pl
@@ -164,7 +164,14 @@ sub ensureDir($) {
   }
 }
 
-# get a file, optionally saving it locally
+# get a file, optionally saving it locally.
+#   if a local filename is given, return true if the file was
+#   found on the server and undef otherwise
+#
+#   if no local filename is given, return the content of the
+#   file, or undef if it was not found on the server
+#
+#   on all other errors, die
 sub fetchFile($;$) {
   my ($remotefile, $localfile) = @_;
 
@@ -175,40 +182,35 @@ sub fetchFile($;$) {
 
   my $req = new HTTP::Request(GET => "$remotefile");
 
-  if ($localfile and -e $localfile) {
-    # Don't fetch unless more recent than local copy
-    my $stat = stat($localfile);
-    $req->header("If-Modified-Since" => time2str($stat->mtime));
+  if ($localfile) {
+    if (-e $localfile) {
+      # Don't fetch unless more recent than local copy
+      my $stat = stat($localfile);
+      $req->header("If-Modified-Since" => time2str($stat->mtime));
+    }
+    else {
+      ensureDir(dirPart($localfile));
+    }
   }
-  my $resp = $ua->request($req);
+  my $resp = $ua->request($req, $localfile);
   if ($resp->is_success) { # 2xx codes
     my $mtime = str2time($resp->header("Last-Modified"));
-    if ($localfile) {
-      if ($verbose) {
-        print STDERR " -> success";
-        print STDERR "; mtime ".time2str($mtime) if $mtime;
-       print STDERR "\n";
-      }
-      ensureDir(dirPart($localfile));
-
-      open (LOCAL, ">", "$localfile") or die "Can't open $localfile for writing ($!)";
-      print LOCAL $resp->content or die "Error writing $localfile ($!)";
-      close LOCAL or die "Error writing $localfile ($!)";
+    if ($verbose) {
+      print STDERR " -> success";
+      print STDERR "; mtime ".time2str($mtime) if $mtime;
+      print STDERR "\n";
+    }
 
-      if ($mtime) {
-        utime $mtime, $mtime, $localfile;
-      }
+    if ($localfile and $mtime) {
+      utime $mtime, $mtime, $localfile;
     }
-    return $resp->content;
+    return $localfile ? 1 : $resp->content;
   }
   elsif ($resp->is_redirect) { # 3xx codes
     if ($resp->code == RC_NOT_MODIFIED) { # 304
       print STDERR " -> not modified\n" if $verbose;
-      open (LOCAL, "<", "$localfile") or die "Can't open $localfile ($!)";
-      local $/; # slurp whole file
-      my $content = <LOCAL>;
-      close LOCAL;
-      return $content;
+      die "Got 304 with no local file" if not $localfile;
+      return 1;
     }
     print STDERR " -> redirect (".$resp->code.")\n" if $verbose;
     die "Can't fetch $remotefile (got redirect, not yet handled)";
@@ -256,18 +258,16 @@ if ($listchangefiles) {
 }
 
 # get the changes files
-my %changesfilecontent;
-foreach my $file (@changesfiles) { $changesfilecontent{$file} = getChangesFile($file); }
+foreach my $file (@changesfiles) { getChangesFile($file); }
 
-# if the file has not changed (response code 304) then ignore it
+# TODO: if the file has not changed (response code 304) then ignore it
 
 # iterate over all the fetched files, building up a list of files
 # to fetch/delete
 my %files;
 foreach my $changesfile (@changesfiles) {
-  my $changesfilecontent = $changesfilecontent{$changesfile};
-  if (not defined $changesfilecontent) {
-    print STDERR "Skipping changes file $changesfile; not present at remote end\n" if $verbose;
+  if (! -e "$changesdir/$changesfile") {
+    print STDERR "Skipping changes file $changesfile; not present\n" if $verbose;
     next;
   }
 
@@ -276,8 +276,8 @@ foreach my $changesfile (@changesfiles) {
 
   print STDERR "Processing changes file $changesfile\n" if $verbose;
 
-  my @changes = split /[\r\n]+/, $changesfilecontent;
-  foreach my $change (@changes) {
+  open (CHANGES, "<", "$changesdir/$changesfile") or die "Can't open $changesfile";
+  while (my $change = <CHANGES>) {
     my ($time, $op, $path) = split ' ', $change;
 
     # Ignore malformed lines, especially wacky paths that could be malicious
@@ -321,9 +321,7 @@ while (my ($file, $op) = each %files) {
   }
   elsif ($op eq "add" or $op eq "change" or $op eq "Modification") {
     # add/change: re-fetch the file
-    # FIXME: don't insist on reading entire file into memory
-    my $content = fetchFile("$remoteroot/$file","$workingdir/$file");
-    die "File $remoteroot/$file not found" if not defined $content;
+    fetchFile("$remoteroot/$file","$workingdir/$file") or die "File $remoteroot/$file not found";
   }
   else {
     die "Unknown operation '$op'";