crypto: fix bug in large buffer handling

* lib/sha512.c (sha512_process_block):
Work even if size_t is wider than 64 bits.

diff --git a/ChangeLog b/ChangeLog
index c1d7fdf..7dfccd7 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -11,7 +11,8 @@
        Don't assume the buffer length is less than 2**32.
        * lib/sha512.c (sha512_process_block): Likewise.
        Here, the bug is present only in the rare case where the host does
-       not support uint64_t; use u64size to work around the problem.
+       not support uint64_t or where size_t is wider than 64 bits.
+       Use u64size to work around the problems.
        * lib/u64.h (u64size): New macro.
 
 2012-05-15  Pádraig Brady  <P@draigBrady.com>

diff --git a/lib/sha512.c b/lib/sha512.c
index 1934507..cf62f20 100644 (file)
--- a/lib/sha512.c
+++ b/lib/sha512.c
@@ -485,14 +485,15 @@ sha512_process_block (const void *buffer, size_t len, struct sha512_ctx *ctx)
   u64 f = ctx->state[5];
   u64 g = ctx->state[6];
   u64 h = ctx->state[7];
-  u64 len64 = u64size (len);
+  u64 lolen = u64size (len);
 
   /* First increment the byte count.  FIPS PUB 180-2 specifies the possible
      length of the file up to 2^128 bits.  Here we only compute the
      number of bytes.  Do a double word increment.  */
-  ctx->total[0] = u64plus (ctx->total[0], len64);
-  if (u64lt (ctx->total[0], len64))
-    ctx->total[1] = u64plus (ctx->total[1], u64lo (1));
+  ctx->total[0] = u64plus (ctx->total[0], lolen);
+  ctx->total[1] = u64plus (ctx->total[1],
+                           u64plus (u64size (len >> 31 >> 31 >> 2),
+                                    u64lo (u64lt (ctx->total[0], lolen))));
 
 #define S0(x) u64xor (u64rol(x, 63), u64xor (u64rol (x, 56), u64shr (x, 7)))
 #define S1(x) u64xor (u64rol (x, 45), u64xor (u64rol (x, 3), u64shr (x, 6)))