From 57e7d97a94d4cd271c2e412f8d542b3a351a0625 Mon Sep 17 00:00:00 2001 From: zak Date: Sat, 18 Nov 2006 18:41:52 +0000 Subject: [PATCH] More time-handling tweaks in update.pl --- scripts/mirror-scripts/update.pl | 112 +++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 33 deletions(-) diff --git a/scripts/mirror-scripts/update.pl b/scripts/mirror-scripts/update.pl index 47eece63..710a5166 100755 --- a/scripts/mirror-scripts/update.pl +++ b/scripts/mirror-scripts/update.pl @@ -30,6 +30,7 @@ use strict; use Date::Calc::Object; # For date/time math +use File::Copy; # For moving files use File::stat; # For getting mtimes use Getopt::Long; # For parsing command-line options use HTTP::Status; # For HTTP status codes @@ -57,6 +58,14 @@ my $listchangefiles = 0; # The root URL to fetch files from my $remoteroot; +# The time we believe we were last up to date +my $timeoflastupdate; + +# The most recent update we've just processed, for updating the above. +# We use this rather than $timenow, in case of clock discrepancies +# between the local and remote systems. +my $mostrecentupdateprocessed; + ################################ # process command line arguments ################################ @@ -65,24 +74,26 @@ sub usage() { print STDERR < \$verbose, - "workingdir=s" => \$workingdir, - "now=s" => sub($) { $timenow = str2time($_[0]); }, - "remoteroot=s" => \$remoteroot, +GetOptions( "verbose!" => \$verbose, + "workingdir=s" => \$workingdir, + "now=s" => sub($$) { $timenow = str2time($_[1]) or die "Can't parse argument to --".join('=',@_); }, + "lastupdate=s" => sub($$) { $timeoflastupdate = str2time($_[1]) or die "Can't parse argument to --".join('=',@_); }, + "remoteroot=s" => \$remoteroot, "list-change-files!" => \$listchangefiles, - "help" => sub() { usage(); } ); + "help" => sub() { usage(); } ); usage() if not defined $remoteroot; @@ -126,10 +137,14 @@ sub findLastUpdateTime() { return str2time ($lastupdatetimestr); } -# write the time now into the last update file -sub saveLastUpdateTime() { +# write the given time into the last update file +sub saveLastUpdateTime($) { + my $time = $_[0]; + $time = $timeoflastupdate if !defined $time; + + print STDERR "Updating timestamp to ".time2str($time)."\n" if $verbose; open (UPDATETIME, ">", $lastupdatefile) or die "Can't open $lastupdatefile for writing ($!)"; - print UPDATETIME time2str($timenow); + print UPDATETIME time2str($time); close (UPDATETIME); } @@ -164,9 +179,19 @@ sub ensureDir($) { } } +# get the mtime of a file +sub getmtime($) { + my $file = $_[0]; + my $stat = stat($file); + die "Can't stat $file ($!)" if !$stat; + return $stat->mtime; +} + # get a file, optionally saving it locally. -# if a local filename is given, return true if the file was -# found on the server and undef otherwise +# if a local filename is given, return: +# undef if not found on the server +# 1 if found but not updated since local version +# 2 if found and more recent than local version # # if no local filename is given, return the content of the # file, or undef if it was not found on the server @@ -185,14 +210,13 @@ sub fetchFile($;$) { if ($localfile) { if (-e $localfile) { # Don't fetch unless more recent than local copy - my $stat = stat($localfile); - $req->header("If-Modified-Since" => time2str($stat->mtime)); + $req->header("If-Modified-Since" => time2str(getmtime($localfile))); } else { ensureDir(dirPart($localfile)); } } - my $resp = $ua->request($req, $localfile); + my $resp = $ua->request($req, $localfile.".part"); if ($resp->is_success) { # 2xx codes my $mtime = str2time($resp->header("Last-Modified")); if ($verbose) { @@ -201,10 +225,13 @@ sub fetchFile($;$) { print STDERR "\n"; } - if ($localfile and $mtime) { - utime $mtime, $mtime, $localfile; + if ($localfile) { + if ($mtime) { + utime $mtime, $mtime, $localfile.".part"; + } + move($localfile.".part", $localfile) or die "Can't move $localfile into place"; } - return $localfile ? 1 : $resp->content; + return $localfile ? 2 : $resp->content; } elsif ($resp->is_redirect) { # 3xx codes if ($resp->code == RC_NOT_MODIFIED) { # 304 @@ -232,14 +259,20 @@ sub getChangesFile($) { } +# update the timestamp if the new one is more recent +sub updatestamp(\$$) { + my ($stampref, $newtime) = @_; + $$stampref = $newtime if (!defined $$stampref or $newtime > $$stampref); +} + ################## # the program flow ################## -# first work out when the last time we were up to date is and -# find present time. -my $timeoflastupdate = findLastUpdateTime(); +# first work out when the last time we were up to date is, if +# it wasn't overridden on the command line +$timeoflastupdate = findLastUpdateTime() if !defined $timeoflastupdate; if ($verbose) { print STDERR "timenow is ".time2str($timenow)." \n"; @@ -257,15 +290,14 @@ if ($listchangefiles) { exit 0; } -# get the changes files -foreach my $file (@changesfiles) { getChangesFile($file); } - -# TODO: if the file has not changed (response code 304) then ignore it - -# iterate over all the fetched files, building up a list of files +# fetch each changes file in turn, building up a list of files # to fetch/delete my %files; -foreach my $changesfile (@changesfiles) { +foreach my $changesfile (@changesfiles) +{ + my $rv = getChangesFile($changesfile); + + # If the file isn't there, ignore it if (! -e "$changesdir/$changesfile") { print STDERR "Skipping changes file $changesfile; not present\n" if $verbose; next; @@ -274,6 +306,17 @@ foreach my $changesfile (@changesfiles) { my $date = $changesfile; $date =~ s{^(?:.*/)?changes([0-9]+)\.txt$}{$1} or die "Can't extract date from changes filename $changesfile"; + # The file exists, set most recent update to at lease "YYYY-MM-DD 00:00:00" + updatestamp($mostrecentupdateprocessed, str2time("$date 00:00:00")); + + # If the file has not changed (response code 304) then ignore it + # Also check mtime against "last update" time so that we won't ignore + # it if previous runs have been failing + if ((!$rv or $rv < 2) and getmtime("$changesdir/$changesfile") < $timeoflastupdate) { + print STDERR "Skipping changes file $changesfile; not changed since last run\n" if $verbose; + next; + } + print STDERR "Processing changes file $changesfile\n" if $verbose; open (CHANGES, "<", "$changesdir/$changesfile") or die "Can't open $changesfile"; @@ -301,6 +344,9 @@ foreach my $changesfile (@changesfiles) { # Ignore changes prior to $timeoflastupdate next if $time < $timeoflastupdate; + # Update timestamp + updatestamp($mostrecentupdateprocessed, $time); + $files{$path} = $op; print STDERR "Marked $path as '$op'\n" if $verbose; } @@ -330,7 +376,7 @@ while (my ($file, $op) = each %files) { # update the last "up-to-date" time -saveLastUpdateTime(); +saveLastUpdateTime($mostrecentupdateprocessed); # finish exit 0; -- 2.11.0