X-Git-Url: http://erislabs.net/gitweb/?a=blobdiff_plain;f=lib%2Fstriconveh.c;h=05dbb73fb9bf3c562bd9e78ca8d20846c5dfa236;hb=d1d75d5540bf75df8bc41c85b5a9fd111ca1f2dd;hp=1324da5a0de1b2d9657b2109980bfdf365e6b0a5;hpb=00065b7cd8ec54f7d94a60b2fcf8b3ce87bea953;p=gnulib.git diff --git a/lib/striconveh.c b/lib/striconveh.c index 1324da5a0..05dbb73fb 100644 --- a/lib/striconveh.c +++ b/lib/striconveh.c @@ -1,11 +1,11 @@ /* Character set conversion with error handling. - Copyright (C) 2001-2007 Free Software Foundation, Inc. + Copyright (C) 2001-2009 Free Software Foundation, Inc. Written by Bruno Haible and Simon Josefsson. - This program is free software; you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2, or (at your option) - any later version. + the Free Software Foundation; either version 3 of the License, or + (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -13,8 +13,7 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software Foundation, - Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ + along with this program. If not, see . */ #include @@ -28,8 +27,6 @@ #if HAVE_ICONV # include -# include "utf8-ucs4-safe.h" -# include "ucs4-utf8.h" # include "unistr.h" #endif @@ -43,9 +40,98 @@ #if HAVE_ICONV -/* The caller must provide CD, CD1, CD2, not just CD, because when a conversion - error occurs, we may have to determine the Unicode representation of the - inconvertible character. */ +/* The caller must provide an iconveh_t, not just an iconv_t, because when a + conversion error occurs, we may have to determine the Unicode representation + of the inconvertible character. */ + +int +iconveh_open (const char *to_codeset, const char *from_codeset, iconveh_t *cdp) +{ + iconv_t cd; + iconv_t cd1; + iconv_t cd2; + + /* Avoid glibc-2.1 bug with EUC-KR. */ +# if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION + if (c_strcasecmp (from_codeset, "EUC-KR") == 0 + || c_strcasecmp (to_codeset, "EUC-KR") == 0) + { + errno = EINVAL; + return -1; + } +# endif + + cd = iconv_open (to_codeset, from_codeset); + + if (STRCASEEQ (from_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)) + cd1 = (iconv_t)(-1); + else + { + cd1 = iconv_open ("UTF-8", from_codeset); + if (cd1 == (iconv_t)(-1)) + { + int saved_errno = errno; + if (cd != (iconv_t)(-1)) + iconv_close (cdp->cd); + errno = saved_errno; + return -1; + } + } + + if (STRCASEEQ (to_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0) +# if (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 2) || __GLIBC__ > 2 || _LIBICONV_VERSION >= 0x0105 + || c_strcasecmp (to_codeset, "UTF-8//TRANSLIT") == 0 +# endif + ) + cd2 = (iconv_t)(-1); + else + { + cd2 = iconv_open (to_codeset, "UTF-8"); + if (cd2 == (iconv_t)(-1)) + { + int saved_errno = errno; + if (cd1 != (iconv_t)(-1)) + iconv_close (cd1); + if (cd != (iconv_t)(-1)) + iconv_close (cd); + errno = saved_errno; + return -1; + } + } + + cdp->cd = cd; + cdp->cd1 = cd1; + cdp->cd2 = cd2; + return 0; +} + +int +iconveh_close (const iconveh_t *cd) +{ + if (cd->cd2 != (iconv_t)(-1) && iconv_close (cd->cd2) < 0) + { + /* Return -1, but preserve the errno from iconv_close. */ + int saved_errno = errno; + if (cd->cd1 != (iconv_t)(-1)) + iconv_close (cd->cd1); + if (cd->cd != (iconv_t)(-1)) + iconv_close (cd->cd); + errno = saved_errno; + return -1; + } + if (cd->cd1 != (iconv_t)(-1) && iconv_close (cd->cd1) < 0) + { + /* Return -1, but preserve the errno from iconv_close. */ + int saved_errno = errno; + if (cd->cd != (iconv_t)(-1)) + iconv_close (cd->cd); + errno = saved_errno; + return -1; + } + if (cd->cd != (iconv_t)(-1) && iconv_close (cd->cd) < 0) + return -1; + return 0; +} /* iconv_carefully is like iconv, except that it stops as soon as it encounters a conversion error, and it returns in *INCREMENTED a boolean telling whether @@ -82,9 +168,13 @@ iconv_carefully (iconv_t cd, &outptr, &outsize); if (!(res == (size_t)(-1) && errno == EINVAL)) break; - /* We expect that no input bytes have been consumed so far. */ - if (inptr != inptr_before) - abort (); + /* iconv can eat up a shift sequence but give EINVAL while attempting + to convert the first character. E.g. libiconv does this. */ + if (inptr > inptr_before) + { + res = 0; + break; + } } if (res == 0) @@ -120,31 +210,36 @@ iconv_carefully (iconv_t cd, # endif /* iconv_carefully_1 is like iconv_carefully, except that it stops after - converting one character. */ + converting one character or one shift sequence. */ static size_t iconv_carefully_1 (iconv_t cd, const char **inbuf, size_t *inbytesleft, char **outbuf, size_t *outbytesleft, bool *incremented) { - const char *inptr = *inbuf; - const char *inptr_end = inptr + *inbytesleft; + const char *inptr_before = *inbuf; + const char *inptr = inptr_before; + const char *inptr_end = inptr_before + *inbytesleft; char *outptr = *outbuf; size_t outsize = *outbytesleft; - const char *inptr_before = inptr; size_t res = (size_t)(-1); size_t insize; - for (insize = 1; inptr + insize <= inptr_end; insize++) + for (insize = 1; inptr_before + insize <= inptr_end; insize++) { + inptr = inptr_before; res = iconv (cd, (ICONV_CONST char **) &inptr, &insize, &outptr, &outsize); if (!(res == (size_t)(-1) && errno == EINVAL)) break; - /* We expect that no input bytes have been consumed so far. */ - if (inptr != inptr_before) - abort (); + /* iconv can eat up a shift sequence but give EINVAL while attempting + to convert the first character. E.g. libiconv does this. */ + if (inptr > inptr_before) + { + res = 0; + break; + } } *inbuf = inptr; @@ -175,6 +270,79 @@ iconv_carefully_1 (iconv_t cd, return res; } +/* utf8conv_carefully is like iconv, except that + - it converts from UTF-8 to UTF-8, + - it stops as soon as it encounters a conversion error, and it returns + in *INCREMENTED a boolean telling whether it has incremented the input + pointers past the error location, + - if one_character_only is true, it stops after converting one + character. */ +static size_t +utf8conv_carefully (bool one_character_only, + const char **inbuf, size_t *inbytesleft, + char **outbuf, size_t *outbytesleft, + bool *incremented) +{ + const char *inptr = *inbuf; + size_t insize = *inbytesleft; + char *outptr = *outbuf; + size_t outsize = *outbytesleft; + size_t res; + + res = 0; + do + { + ucs4_t uc; + int n; + int m; + + n = u8_mbtoucr (&uc, (const uint8_t *) inptr, insize); + if (n < 0) + { + errno = (n == -2 ? EINVAL : EILSEQ); + n = u8_mbtouc (&uc, (const uint8_t *) inptr, insize); + inptr += n; + insize -= n; + res = (size_t)(-1); + *incremented = true; + break; + } + if (outsize == 0) + { + errno = E2BIG; + res = (size_t)(-1); + *incremented = false; + break; + } + m = u8_uctomb ((uint8_t *) outptr, uc, outsize); + if (m == -2) + { + errno = E2BIG; + res = (size_t)(-1); + *incremented = false; + break; + } + inptr += n; + insize -= n; + if (m == -1) + { + errno = EILSEQ; + res = (size_t)(-1); + *incremented = true; + break; + } + outptr += m; + outsize -= m; + } + while (!one_character_only && insize > 0); + + *inbuf = inptr; + *inbytesleft = insize; + *outbuf = outptr; + *outbytesleft = outsize; + return res; +} + static int mem_cd_iconveh_internal (const char *src, size_t srclen, iconv_t cd, iconv_t cd1, iconv_t cd2, @@ -212,6 +380,10 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, } result = initial_result; + /* Test whether a direct conversion is possible at all. */ + if (cd == (iconv_t)(-1)) + goto indirectly; + if (offsets != NULL) { size_t i; @@ -410,8 +582,7 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, goto done; indirectly: - /* The direct conversion failed, handler != iconveh_error, - and cd2 != (iconv_t)(-1). + /* The direct conversion failed. Use a conversion through UTF-8. */ if (offsets != NULL) { @@ -424,6 +595,7 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, } length = 0; { + const bool slowly = (offsets != NULL || handler == iconveh_error); # define utf8bufsize 4096 /* may also be smaller or larger than tmpbufsize */ char utf8buf[utf8bufsize + 1]; size_t utf8len = 0; @@ -438,7 +610,8 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, /* Set to the initial state. */ if (cd1 != (iconv_t)(-1)) iconv (cd1, NULL, NULL, NULL, NULL); - iconv (cd2, NULL, NULL, NULL, NULL); + if (cd2 != (iconv_t)(-1)) + iconv (cd2, NULL, NULL, NULL, NULL); # endif while (in1size > 0 || do_final_flush1 || utf8len > 0 || do_final_flush2) @@ -460,7 +633,7 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, } if (cd1 != (iconv_t)(-1)) { - if (offsets != NULL) + if (slowly) res1 = iconv_carefully_1 (cd1, &in1ptr, &in1size, &out1ptr, &out1size, @@ -474,55 +647,10 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, else { /* FROM_CODESET is UTF-8. */ - res1 = 0; - do - { - ucs4_t uc; - int n; - int m; - - n = u8_mbtouc_safe (&uc, (const uint8_t *) in1ptr, in1size); - if (uc == 0xfffd - && !(n >= 3 - && (uint8_t)in1ptr[0] == 0xEF - && (uint8_t)in1ptr[1] == 0xBF - && (uint8_t)in1ptr[2] == 0xBD)) - { - in1ptr += n; - in1size -= n; - errno = EILSEQ; - res1 = (size_t)(-1); - incremented1 = true; - break; - } - if (out1size == 0) - { - errno = E2BIG; - res1 = (size_t)(-1); - incremented1 = false; - break; - } - m = u8_uctomb ((uint8_t *) out1ptr, uc, out1size); - if (m == -2) - { - errno = E2BIG; - res1 = (size_t)(-1); - incremented1 = false; - break; - } - in1ptr += n; - in1size -= n; - if (m == -1) - { - errno = EILSEQ; - res1 = (size_t)(-1); - incremented1 = true; - break; - } - out1ptr += m; - out1size -= m; - } - while (offsets == NULL && in1size > 0); + res1 = utf8conv_carefully (slowly, + &in1ptr, &in1size, + &out1ptr, &out1size, + &incremented1); } } else if (do_final_flush1) @@ -568,7 +696,8 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, in1ptr++; in1size--; } - utf8buf[utf8len++] = '?'; + *out1ptr++ = '?'; + res1 = 0; } errno1 = errno; utf8len = out1ptr - utf8buf; @@ -592,10 +721,19 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, bool grow; if (in2size > 0) - res2 = iconv_carefully (cd2, - &in2ptr, &in2size, - &out2ptr, &out2size, - &incremented2); + { + if (cd2 != (iconv_t)(-1)) + res2 = iconv_carefully (cd2, + &in2ptr, &in2size, + &out2ptr, &out2size, + &incremented2); + else + /* TO_CODESET is UTF-8. */ + res2 = utf8conv_carefully (false, + &in2ptr, &in2size, + &out2ptr, &out2size, + &incremented2); + } else /* in1size == 0 && !do_final_flush1 && in2size == 0 && do_final_flush2 */ { @@ -603,10 +741,11 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, state. But avoid glibc-2.1 bug and Solaris 2.7 bug. */ # if defined _LIBICONV_VERSION \ || !((__GLIBC__ == 2 && __GLIBC_MINOR__ <= 1) || defined __sun) - res2 = iconv (cd2, NULL, NULL, &out2ptr, &out2size); -# else - res2 = 0; + if (cd2 != (iconv_t)(-1)) + res2 = iconv (cd2, NULL, NULL, &out2ptr, &out2size); + else # endif + res2 = 0; do_final_flush2 = false; incremented2 = true; } @@ -677,9 +816,28 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, inptr = scratchbuf; insize = scratchlen; - res = iconv (cd2, - (ICONV_CONST char **) &inptr, &insize, - &out2ptr, &out2size); + if (cd2 != (iconv_t)(-1)) + res = iconv (cd2, + (ICONV_CONST char **) &inptr, &insize, + &out2ptr, &out2size); + else + { + /* TO_CODESET is UTF-8. */ + if (out2size >= insize) + { + memcpy (out2ptr, inptr, insize); + out2ptr += insize; + out2size -= insize; + inptr += insize; + insize = 0; + res = 0; + } + else + { + errno = E2BIG; + res = (size_t)(-1); + } + } length = out2ptr - result; if (res == (size_t)(-1) && errno == E2BIG) { @@ -706,9 +864,23 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, out2ptr = result + length; out2size = allocated - extra_alloc - length; - res = iconv (cd2, - (ICONV_CONST char **) &inptr, &insize, - &out2ptr, &out2size); + if (cd2 != (iconv_t)(-1)) + res = iconv (cd2, + (ICONV_CONST char **) &inptr, + &insize, + &out2ptr, &out2size); + else + { + /* TO_CODESET is UTF-8. */ + if (!(out2size >= insize)) + abort (); + memcpy (out2ptr, inptr, insize); + out2ptr += insize; + out2size -= insize; + inptr += insize; + insize = 0; + res = 0; + } length = out2ptr - result; } # if !defined _LIBICONV_VERSION && !defined __GLIBC__ @@ -797,9 +969,10 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, /* Now the final memory allocation. */ if (result == tmpbuf) { + size_t memsize = length + extra_alloc; char *memory; - memory = (char *) malloc (length + extra_alloc); + memory = (char *) malloc (memsize > 0 ? memsize : 1); if (memory != NULL) { memcpy (memory, tmpbuf, length); @@ -814,9 +987,10 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, else if (result != *resultp && length + extra_alloc < allocated) { /* Shrink the allocated memory if possible. */ + size_t memsize = length + extra_alloc; char *memory; - memory = (char *) realloc (result, length + extra_alloc); + memory = (char *) realloc (result, memsize > 0 ? memsize : 1); if (memory != NULL) result = memory; } @@ -829,18 +1003,18 @@ mem_cd_iconveh_internal (const char *src, size_t srclen, int mem_cd_iconveh (const char *src, size_t srclen, - iconv_t cd, iconv_t cd1, iconv_t cd2, + const iconveh_t *cd, enum iconv_ilseq_handler handler, size_t *offsets, char **resultp, size_t *lengthp) { - return mem_cd_iconveh_internal (src, srclen, cd, cd1, cd2, handler, 0, - offsets, resultp, lengthp); + return mem_cd_iconveh_internal (src, srclen, cd->cd, cd->cd1, cd->cd2, + handler, 0, offsets, resultp, lengthp); } char * str_cd_iconveh (const char *src, - iconv_t cd, iconv_t cd1, iconv_t cd2, + const iconveh_t *cd, enum iconv_ilseq_handler handler) { /* For most encodings, a trailing NUL byte in the input will be converted @@ -850,8 +1024,8 @@ str_cd_iconveh (const char *src, char *result = NULL; size_t length = 0; int retval = mem_cd_iconveh_internal (src, strlen (src), - cd, cd1, cd2, handler, 1, NULL, - &result, &length); + cd->cd, cd->cd1, cd->cd2, handler, 1, + NULL, &result, &length); if (retval < 0) { @@ -908,103 +1082,32 @@ mem_iconveh (const char *src, size_t srclen, else { #if HAVE_ICONV - iconv_t cd; - iconv_t cd1; - iconv_t cd2; + iconveh_t cd; char *result; size_t length; int retval; - /* Avoid glibc-2.1 bug with EUC-KR. */ -# if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION - if (c_strcasecmp (from_codeset, "EUC-KR") == 0 - || c_strcasecmp (to_codeset, "EUC-KR") == 0) - { - errno = EINVAL; - return -1; - } -# endif - - cd = iconv_open (to_codeset, from_codeset); - if (cd == (iconv_t)(-1)) + if (iconveh_open (to_codeset, from_codeset, &cd) < 0) return -1; - if (STRCASEEQ (from_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)) - cd1 = (iconv_t)(-1); - else - { - cd1 = iconv_open ("UTF-8", from_codeset); - if (cd1 == (iconv_t)(-1)) - { - int saved_errno = errno; - iconv_close (cd); - errno = saved_errno; - return -1; - } - } - - if (STRCASEEQ (to_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)) - cd2 = (iconv_t)(-1); - else - { - cd2 = iconv_open (to_codeset, "UTF-8"); - if (cd2 == (iconv_t)(-1)) - { - int saved_errno = errno; - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); - errno = saved_errno; - return -1; - } - } - result = *resultp; length = *lengthp; - retval = mem_cd_iconveh (src, srclen, cd, cd1, cd2, handler, offsets, + retval = mem_cd_iconveh (src, srclen, &cd, handler, offsets, &result, &length); if (retval < 0) { - /* Close cd, cd1, cd2, but preserve the errno from str_cd_iconv. */ + /* Close cd, but preserve the errno from str_cd_iconv. */ int saved_errno = errno; - if (cd2 != (iconv_t)(-1)) - iconv_close (cd2); - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); + iconveh_close (&cd); errno = saved_errno; } else { - if (cd2 != (iconv_t)(-1) && iconv_close (cd2) < 0) - { - /* Return -1, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ - int saved_errno = errno; - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); - if (result != *resultp && result != NULL) - free (result); - errno = saved_errno; - return -1; - } - if (cd1 != (iconv_t)(-1) && iconv_close (cd1) < 0) + if (iconveh_close (&cd) < 0) { /* Return -1, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ - int saved_errno = errno; - iconv_close (cd); - if (result != *resultp && result != NULL) - free (result); - errno = saved_errno; - return -1; - } - if (iconv_close (cd) < 0) - { - /* Return -1, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ + that, preserve the errno from iconveh_close. */ int saved_errno = errno; if (result != *resultp && result != NULL) free (result); @@ -1043,96 +1146,27 @@ str_iconveh (const char *src, else { #if HAVE_ICONV - iconv_t cd; - iconv_t cd1; - iconv_t cd2; + iconveh_t cd; char *result; - /* Avoid glibc-2.1 bug with EUC-KR. */ -# if (__GLIBC__ - 0 == 2 && __GLIBC_MINOR__ - 0 <= 1) && !defined _LIBICONV_VERSION - if (c_strcasecmp (from_codeset, "EUC-KR") == 0 - || c_strcasecmp (to_codeset, "EUC-KR") == 0) - { - errno = EINVAL; - return NULL; - } -# endif - - cd = iconv_open (to_codeset, from_codeset); - if (cd == (iconv_t)(-1)) + if (iconveh_open (to_codeset, from_codeset, &cd) < 0) return NULL; - if (STRCASEEQ (from_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)) - cd1 = (iconv_t)(-1); - else - { - cd1 = iconv_open ("UTF-8", from_codeset); - if (cd1 == (iconv_t)(-1)) - { - int saved_errno = errno; - iconv_close (cd); - errno = saved_errno; - return NULL; - } - } - - if (STRCASEEQ (to_codeset, "UTF-8", 'U','T','F','-','8',0,0,0,0)) - cd2 = (iconv_t)(-1); - else - { - cd2 = iconv_open (to_codeset, "UTF-8"); - if (cd2 == (iconv_t)(-1)) - { - int saved_errno = errno; - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); - errno = saved_errno; - return NULL; - } - } - - result = str_cd_iconveh (src, cd, cd1, cd2, handler); + result = str_cd_iconveh (src, &cd, handler); if (result == NULL) { - /* Close cd, cd1, cd2, but preserve the errno from str_cd_iconv. */ + /* Close cd, but preserve the errno from str_cd_iconv. */ int saved_errno = errno; - if (cd2 != (iconv_t)(-1)) - iconv_close (cd2); - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); + iconveh_close (&cd); errno = saved_errno; } else { - if (cd2 != (iconv_t)(-1) && iconv_close (cd2) < 0) - { - /* Return NULL, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ - int saved_errno = errno; - if (cd1 != (iconv_t)(-1)) - iconv_close (cd1); - iconv_close (cd); - free (result); - errno = saved_errno; - return NULL; - } - if (cd1 != (iconv_t)(-1) && iconv_close (cd1) < 0) - { - /* Return NULL, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ - int saved_errno = errno; - iconv_close (cd); - free (result); - errno = saved_errno; - return NULL; - } - if (iconv_close (cd) < 0) + if (iconveh_close (&cd) < 0) { /* Return NULL, but free the allocated memory, and while doing - that, preserve the errno from iconv_close. */ + that, preserve the errno from iconveh_close. */ int saved_errno = errno; free (result); errno = saved_errno;