From: Eric Blake <ebb9@byu.net>
Date: Fri, 13 Nov 2009 19:53:17 +0000 (-0700)
Subject: getgroups: work around FreeBSD bug
X-Git-Tag: stable/20091120~32
X-Git-Url: http://erislabs.net/gitweb/?a=commitdiff_plain;h=867cf3bc81870ba2896b5c2098e76b458c057a14;p=gnulib.git

getgroups: work around FreeBSD bug

FreeBSD 7.2 mistakenly succeeds on getgroups(-1,ptr) (POSIX
requires EINVAL failure since -1 is less than the proper result).

* lib/getgroups.c (rpl_getgroups): Work around the bug.
* m4/getgroups.m4 (gl_FUNC_GETGROUPS): Detect the bug.
* doc/posix-functions/getgroups.texi (getgroups): Document it.
* tests/test-getgroups.c (main): Fix buffer overrun.

Signed-off-by: Eric Blake <ebb9@byu.net>
---

diff --git a/ChangeLog b/ChangeLog
index 7d02121b7..a5b88a454 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,11 @@
 2009-11-13  Eric Blake  <ebb9@byu.net>
 
+	getgroups: work around FreeBSD bug
+	* lib/getgroups.c (rpl_getgroups): Work around the bug.
+	* m4/getgroups.m4 (gl_FUNC_GETGROUPS): Detect the bug.
+	* doc/posix-functions/getgroups.texi (getgroups): Document it.
+	* tests/test-getgroups.c (main): Fix buffer overrun.
+
 	getgroups: avoid compilation failure
 	* lib/getgroups.c (includes): Include <stdint.h> for SIZE_MAX.
 	* modules/getgroups (Depends-on): Add stdint.
diff --git a/doc/posix-functions/getgroups.texi b/doc/posix-functions/getgroups.texi
index 951705e8a..e503b2225 100644
--- a/doc/posix-functions/getgroups.texi
+++ b/doc/posix-functions/getgroups.texi
@@ -12,7 +12,11 @@ Portability problems fixed by Gnulib:
 This function is missing on some platforms:
 mingw.
 @item
-On Ultrix 4.3, @code{getgroups (0, 0)} always fails.  See macro
+On some platforms, this function fails to reject a negative count,
+even though that is less than the size that would be returned:
+FreeBSD 7.2.
+@item
+On Ultrix 4.3, @code{getgroups (0, NULL)} always fails.  See macro
 @samp{AC_FUNC_GETGROUPS}.
 @end itemize
 
diff --git a/lib/getgroups.c b/lib/getgroups.c
index 9c0d6f7e4..24d7816ba 100644
--- a/lib/getgroups.c
+++ b/lib/getgroups.c
@@ -40,6 +40,9 @@ getgroups (int n _UNUSED_PARAMETER_, GETGROUPS_T *groups _UNUSED_PARAMETER_)
 #else /* HAVE_GETGROUPS */
 
 # undef getgroups
+# ifndef GETGROUPS_ZERO_BUG
+#  define GETGROUPS_ZERO_BUG 0
+# endif
 
 /* On at least Ultrix 4.3 and NextStep 3.2, getgroups (0, NULL) always
    fails.  On other systems, it returns the number of supplemental
@@ -55,8 +58,39 @@ rpl_getgroups (int n, GETGROUPS_T *group)
   GETGROUPS_T *gbuf;
   int saved_errno;
 
-  if (n != 0)
-    return getgroups (n, group);
+  if (n < 0)
+    {
+      errno = EINVAL;
+      return -1;
+    }
+
+  if (n != 0 || !GETGROUPS_ZERO_BUG)
+    {
+      int result;
+      int saved_errno;
+      if (sizeof *group == sizeof *gbuf)
+        return getgroups (n, (GETGROUPS_T *) group);
+
+      if (SIZE_MAX / sizeof *gbuf <= n)
+        {
+          errno = ENOMEM;
+          return -1;
+        }
+      gbuf = malloc (n * sizeof *gbuf);
+      if (!gbuf)
+        return -1;
+      result = getgroups (n, gbuf);
+      if (0 <= result)
+        {
+          n = result;
+          while (n--)
+            group[n] = gbuf[n];
+        }
+      saved_errno = errno;
+      free (gbuf);
+      errno == saved_errno;
+      return result;
+    }
 
   n = 20;
   while (1)
diff --git a/m4/getgroups.m4 b/m4/getgroups.m4
index 1dd39eacc..96260e336 100644
--- a/m4/getgroups.m4
+++ b/m4/getgroups.m4
@@ -1,4 +1,4 @@
-# serial 13
+# serial 15
 
 dnl From Jim Meyering.
 dnl A wrapper around AC_FUNC_GETGROUPS.
@@ -20,6 +20,23 @@ AC_DEFUN([gl_FUNC_GETGROUPS],
   elif test "$ac_cv_func_getgroups_works" != yes; then
     AC_LIBOBJ([getgroups])
     REPLACE_GETGROUPS=1
+    AC_DEFINE([GETGROUPS_ZERO_BUG], [1], [Define this to 1 if
+      getgroups(0,NULL) does not return the number of groups.])
+  else
+    dnl Detect FreeBSD bug; POSIX requires getgroups(-1,ptr) to fail.
+    AC_CACHE_CHECK([whether getgroups handles negative values],
+      [gl_cv_func_getgroups_works],
+      [AC_RUN_IFELSE([AC_LANG_PROGRAM([AC_INCLUDES_DEFAULT],
+        [[int size = getgroups (0, 0);
+          gid_t *list = malloc (size * sizeof *list);
+          return getgroups (-1, list) != -1;]])],
+        [gl_cv_func_getgroups_works=yes],
+        [gl_cv_func_getgroups_works=no],
+        [gl_cv_func_getgroups_works="guessing no"])])
+    if test "$gl_cv_func_getgroups_works" != yes; then
+      AC_LIBOBJ([getgroups])
+      REPLACE_GETGROUPS=1
+    fi
   fi
   test -n "$GETGROUPS_LIB" && LIBS="$GETGROUPS_LIB $LIBS"
 ])
diff --git a/tests/test-getgroups.c b/tests/test-getgroups.c
index 8ecfb963a..8d8d2aba5 100644
--- a/tests/test-getgroups.c
+++ b/tests/test-getgroups.c
@@ -52,7 +52,7 @@ main (int argc, char **argv _UNUSED_PARAMETER_)
     }
   ASSERT (0 <= result);
   ASSERT (result + 1 < SIZE_MAX / sizeof *groups);
-  groups = malloc (result + 1 * sizeof *groups);
+  groups = malloc ((result + 1) * sizeof *groups);
   ASSERT (groups);
   groups[result] = -1;
   /* Check for EINVAL handling.  Not all processes have supplemental