ptsname_r: Fix test failures on IRIX, Solaris.

* m4/ptsname_r.m4 (gl_PREREQ_PTSNAME_R): Test whether isatty sets
errno when it fails. Define ISATTY_FAILS_WITHOUT_SETTING_ERRNO
accordingly.
* lib/ptsname_r.c: Include <fcntl.h>.
(__ptsname_r): When isatty returned false, then on IRIX, Solaris
set errno if fd is invalid.
* tests/test-isatty.c (main): Update comments.

diff --git a/ChangeLog b/ChangeLog
index 45173a4..d81b727 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,16 @@
 2012-06-24  Bruno Haible  <bruno@clisp.org>
 
+       ptsname_r: Fix test failures on IRIX, Solaris.
+       * m4/ptsname_r.m4 (gl_PREREQ_PTSNAME_R): Test whether isatty sets
+       errno when it fails. Define ISATTY_FAILS_WITHOUT_SETTING_ERRNO
+       accordingly.
+       * lib/ptsname_r.c: Include <fcntl.h>.
+       (__ptsname_r): When isatty returned false, then on IRIX, Solaris
+       set errno if fd is invalid.
+       * tests/test-isatty.c (main): Update comments.
+
+2012-06-24  Bruno Haible  <bruno@clisp.org>
+
        ptsname test: Extend test.
        * tests/test-ptsname.c: Include <errno.h>.
        (main): Test behaviour with invalid file descriptor.

diff --git a/lib/ptsname_r.c b/lib/ptsname_r.c
index f2e3410..aa3ba38 100644 (file)
--- a/lib/ptsname_r.c
+++ b/lib/ptsname_r.c
@@ -19,6 +19,7 @@
 #include <stdlib.h>
 
 #include <errno.h>
+#include <fcntl.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <unistd.h>
@@ -59,8 +60,16 @@ __ptsname_r (int fd, char *buf, size_t buflen)
     }
 
   if (!__isatty (fd))
-    /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
-    return errno;
+    {
+#if ISATTY_FAILS_WITHOUT_SETTING_ERRNO && defined F_GETFL /* IRIX, Solaris */
+      /* Set errno.  */
+      if (fcntl (fd, F_GETFL) != -1)
+        errno = ENOTTY;
+#else
+      /* We rely on isatty to set errno properly (i.e. EBADF or ENOTTY).  */
+#endif
+      return errno;
+    }
 
   if (buflen < strlen (_PATH_TTY) + 3)
     {

diff --git a/m4/ptsname_r.m4 b/m4/ptsname_r.m4
index 8dd07b1..bc5adb7 100644 (file)
--- a/m4/ptsname_r.m4
+++ b/m4/ptsname_r.m4
@@ -1,4 +1,4 @@
-# ptsname_r.m4 serial 2
+# ptsname_r.m4 serial 3
 dnl Copyright (C) 2010-2012 Free Software Foundation, Inc.
 dnl This file is free software; the Free Software Foundation
 dnl gives unlimited permission to copy and/or distribute it,
@@ -46,5 +46,38 @@ AC_DEFUN([gl_FUNC_PTSNAME_R],
 
 # Prerequisites of lib/ptsname.c.
 AC_DEFUN([gl_PREREQ_PTSNAME_R], [
-  :
+  AC_REQUIRE([AC_CANONICAL_HOST]) dnl for cross-compiles
+  AC_CACHE_CHECK([whether isatty sets errno when it fails],
+    [gl_cv_func_isatty_sets_errno],
+    [AC_RUN_IFELSE(
+       [AC_LANG_PROGRAM(
+          [[#include <errno.h>
+            #if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__
+            # include <io.h>
+            #else
+            # include <unistd.h>
+            #endif
+          ]],
+          [[errno = 0;
+            isatty (-1);
+            return errno == 0;
+          ]])
+       ],
+       [gl_cv_func_isatty_sets_errno=yes],
+       [gl_cv_func_isatty_sets_errno=no],
+       [case "$host_os" in
+          irix* | solaris* | mingw*)
+            gl_cv_func_isatty_sets_errno="guessing no";;
+          *)
+            gl_cv_func_isatty_sets_errno="guessing yes";;
+        esac
+       ])
+    ])
+  case "$gl_cv_func_isatty_sets_errno" in
+    *yes) ;;
+    *)
+      AC_DEFINE([ISATTY_FAILS_WITHOUT_SETTING_ERRNO], [1],
+        [Define to 1 if isatty() may fail without setting errno.])
+      ;;
+  esac
 ])

diff --git a/tests/test-isatty.c b/tests/test-isatty.c
index 263e05e..195a906 100644 (file)
--- a/tests/test-isatty.c
+++ b/tests/test-isatty.c
@@ -45,14 +45,14 @@ main (void)
     errno = 0;
     ASSERT (isatty (-1) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }
   {
     errno = 0;
     ASSERT (isatty (99) == 0);
     ASSERT (errno == EBADF
-            || errno == 0 /* seen on Solaris 10 */
+            || errno == 0 /* seen on IRIX 6.5, Solaris 10 */
            );
   }