wimlib.net Git - wimlib/blob - src/encoding.c

   1 /*
   2  * encoding.c:  Convert "multibyte" strings (the locale-default encoding---
   3  * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice
   4  * versa.  Also, convert UTF-8 strings to multibyte strings.
   5  */
   6
   7 /*
   8  * Copyright (C) 2012, 2013 Eric Biggers
   9  *
  10  * This file is part of wimlib, a library for working with WIM files.
  11  *
  12  * wimlib is free software; you can redistribute it and/or modify it under the
  13  * terms of the GNU General Public License as published by the Free
  14  * Software Foundation; either version 3 of the License, or (at your option)
  15  * any later version.
  16  *
  17  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
  18  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  19  * A PARTICULAR PURPOSE. See the GNU General Public License for more
  20  * details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with wimlib; if not, see http://www.gnu.org/licenses/.
  24  */
  25
  26 #include "wimlib_internal.h"
  27
  28 #include <errno.h>
  29 #include <iconv.h>
  30 #include <pthread.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33
  34 bool wimlib_mbs_is_utf8 = !TCHAR_IS_UTF16LE;
  35
  36 /* List of iconv_t conversion descriptors for a specific character conversion.
  37  * The idea is that it is not thread-safe to have just one conversion
  38  * descriptor, but it also is inefficient to open a new conversion descriptor to
  39  * convert every string.  Both these problems can be solved by maintaining a
  40  * list of conversion descriptors; then, a thread can use an existing conversion
  41  * descriptor if available. */
  42 struct iconv_list_head {
  43         const char *from_encoding;
  44         const char *to_encoding;
  45         struct list_head list;
  46         pthread_mutex_t mutex;
  47 };
  48
  49 struct iconv_node {
  50         iconv_t cd;
  51         struct list_head list;
  52         struct iconv_list_head *head;
  53 };
  54
  55 #define ICONV_LIST(name, from, to)                      \
  56 struct iconv_list_head name = {                         \
  57         .from_encoding = from,                          \
  58         .to_encoding = to,                              \
  59         .list = LIST_HEAD_INIT(name.list),              \
  60         .mutex = PTHREAD_MUTEX_INITIALIZER,             \
  61 }
  62
  63 static iconv_t *
  64 get_iconv(struct iconv_list_head *head)
  65 {
  66         iconv_t cd;
  67         iconv_t *cd_p;
  68         struct iconv_node *i;
  69
  70         pthread_mutex_lock(&head->mutex);
  71         if (list_empty(&head->list)) {
  72                 cd = iconv_open(head->to_encoding, head->from_encoding);
  73                 if (cd == (iconv_t)-1) {
  74                         ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
  75                                          head->from_encoding, head->to_encoding);
  76                         cd_p = NULL;
  77                 } else {
  78                         i = MALLOC(sizeof(struct iconv_node));
  79                         if (i) {
  80                                 i->head = head;
  81                                 i->cd = cd;
  82                                 cd_p = &i->cd;
  83                         } else {
  84                                 iconv_close(cd);
  85                                 cd_p = NULL;
  86                         }
  87                 }
  88         } else {
  89                 i = container_of(head->list.next, struct iconv_node, list);
  90                 list_del(head->list.next);
  91                 cd_p = &i->cd;
  92         }
  93         pthread_mutex_unlock(&head->mutex);
  94         return cd_p;
  95 }
  96
  97 static void
  98 put_iconv(iconv_t *cd)
  99 {
 100         int errno_save = errno;
 101         struct iconv_node *i = container_of(cd, struct iconv_node, cd);
 102         struct iconv_list_head *head = i->head;
 103
 104         pthread_mutex_lock(&head->mutex);
 105         list_add(&i->list, &head->list);
 106         pthread_mutex_unlock(&head->mutex);
 107         errno = errno_save;
 108 }
 109
 110 /* Prevent printing an error message if a character conversion error occurs
 111  * while printing an error message.  (This variable is not per-thread but it
 112  * doesn't matter too much since it's just the error messages.) */
 113 static bool error_message_being_printed = false;
 114
 115 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
 116                                          varname2, longname2, chartype2,\
 117                                          earlyreturn,                   \
 118                                          worst_case_len_expr,           \
 119                                          err_return,                    \
 120                                          err_msg)                       \
 121 static ICONV_LIST(iconv_##varname1##_to_##varname2,                     \
 122                   longname1, longname2);                                \
 123                                                                         \
 124 int                                                                     \
 125 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
 126                                   size_t *out_nbytes_ret)               \
 127 {                                                                       \
 128         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
 129         if (cd == NULL)                                                 \
 130                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
 131                                                                         \
 132         /* Worst case length */                                         \
 133         chartype2 buf[worst_case_len_expr];                             \
 134         char *inbuf = (char*)in;                                        \
 135         size_t inbytesleft = in_nbytes;                                 \
 136         char *outbuf = (char*)buf;                                      \
 137         size_t outbytesleft = sizeof(buf);                              \
 138         size_t len;                                                     \
 139         int ret;                                                        \
 140                                                                         \
 141         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
 142         if (len == (size_t)-1) {                                        \
 143                 if (!error_message_being_printed) {                     \
 144                         error_message_being_printed = true;             \
 145                         err_msg;                                        \
 146                         error_message_being_printed = false;            \
 147                 }                                                       \
 148                 ret = err_return;                                       \
 149         } else {                                                        \
 150                 *out_nbytes_ret = sizeof(buf) - outbytesleft;           \
 151                 ret = 0;                                                \
 152         }                                                               \
 153         put_iconv(cd);                                                  \
 154         return ret;                                                     \
 155 }                                                                       \
 156                                                                         \
 157 int                                                                     \
 158 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes,   \
 159                                chartype2 *out)                          \
 160 {                                                                       \
 161         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
 162         if (cd == NULL)                                                 \
 163                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
 164                                                                         \
 165         char *inbuf = (char*)in;                                        \
 166         size_t inbytesleft = in_nbytes;                                 \
 167         char *outbuf = (char*)out;                                      \
 168         const size_t LARGE_NUMBER = 1000000000;                         \
 169         size_t outbytesleft = LARGE_NUMBER;                             \
 170         size_t len;                                                     \
 171         int ret;                                                        \
 172                                                                         \
 173         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
 174         if (len == (size_t)-1) {                                        \
 175                 if (!error_message_being_printed) {                     \
 176                         error_message_being_printed = true;             \
 177                         err_msg;                                        \
 178                         error_message_being_printed = false;            \
 179                 }                                                       \
 180                 ret = err_return;                                       \
 181         } else {                                                        \
 182                 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
 183                 ret = 0;                                                \
 184         }                                                               \
 185         put_iconv(cd);                                                  \
 186         return ret;                                                     \
 187 }                                                                       \
 188                                                                         \
 189 int                                                                     \
 190 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes,         \
 191                          chartype2 **out_ret,                           \
 192                          size_t *out_nbytes_ret)                        \
 193 {                                                                       \
 194         int ret;                                                        \
 195         chartype2 *out;                                                 \
 196         size_t out_nbytes;                                              \
 197                                                                         \
 198         if (earlyreturn) {                                              \
 199                 /* Out same as in */                                    \
 200                 out = MALLOC(in_nbytes + sizeof(chartype2));            \
 201                 if (!out)                                               \
 202                         return WIMLIB_ERR_NOMEM;                        \
 203                 memcpy(out, in, in_nbytes);                             \
 204                 out[in_nbytes / sizeof(chartype2)] = 0;                 \
 205                 *out_ret = out;                                         \
 206                 *out_nbytes_ret = in_nbytes;                            \
 207                 return 0;                                               \
 208         }                                                               \
 209                                                                         \
 210         ret = varname1##_to_##varname2##_nbytes(in, in_nbytes,          \
 211                                                 &out_nbytes);           \
 212         if (ret)                                                        \
 213                 return ret;                                             \
 214                                                                         \
 215         out = MALLOC(out_nbytes + sizeof(chartype2));                   \
 216         if (!out)                                                       \
 217                 return WIMLIB_ERR_NOMEM;                                \
 218                                                                         \
 219         ret = varname1##_to_##varname2##_buf(in, in_nbytes, out);       \
 220         if (ret) {                                                      \
 221                 int errno_save = errno;                                 \
 222                 FREE(out);                                              \
 223                 errno = errno_save;                                     \
 224         } else {                                                        \
 225                 *out_ret = out;                                         \
 226                 *out_nbytes_ret = out_nbytes;                           \
 227         }                                                               \
 228         return ret;                                                     \
 229 }
 230
 231 #if 0
 232 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
 233                                  mbs, "", mbchar,
 234                                  false,
 235                                  in_nbytes / 2 * MB_CUR_MAX,
 236                                  WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
 237                                  ERROR("Failed to convert UTF-16LE string "
 238                                        "to multibyte string!");
 239                                  ERROR("This may be because the UTF-16LE data "
 240                                        "could not be represented in your "
 241                                        "locale's character encoding."))
 242 #endif
 243
 244 #if !TCHAR_IS_UTF16LE
 245 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
 246                                  utf16le, "UTF-16LE", utf16lechar,
 247                                  false,
 248                                  in_nbytes * 2,
 249                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
 250                                  ERROR_WITH_ERRNO("Failed to convert multibyte "
 251                                                   "string \"%"TS"\" to UTF-16LE string!", in);
 252                                  ERROR("If the data you provided was UTF-8, please make sure "
 253                                        "the character encoding of your current locale is UTF-8."))
 254 #endif
 255
 256 /* tchar to UTF-8 and back */
 257 #if TCHAR_IS_UTF16LE
 258 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "UTF16-LE", tchar,
 259                                  utf8, "UTF-8", utf8char,
 260                                  false,
 261                                  in_nbytes * 2,
 262                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
 263                                  ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
 264                                                   "string \"%"TS"\" to UTF-8 string!", in);
 265                                 )
 266
 267 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
 268                                  tstr, "UTF16-LE", tchar,
 269                                  false,
 270                                  in_nbytes * 2,
 271                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
 272                                  ERROR_WITH_ERRNO("Failed to convert UTF-8 string "
 273                                                   "to UTF-16LE string!");
 274                                 )
 275 #else
 276 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
 277                                  utf8, "UTF-8", utf8char,
 278                                  wimlib_mbs_is_utf8,
 279                                  in_nbytes * 4,
 280                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
 281                                  ERROR_WITH_ERRNO("Failed to convert multibyte "
 282                                                   "string \"%"TS"\" to UTF-8 string!", in);
 283                                  ERROR("If the data you provided was UTF-8, please make sure "
 284                                        "the character encoding of your current locale is UTF-8.");)
 285
 286 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
 287                                  tstr, "", tchar,
 288                                  wimlib_mbs_is_utf8,
 289                                  in_nbytes * 4,
 290                                  WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
 291                                  ERROR("Failed to convert UTF-8 string to "
 292                                        "multibyte string!");
 293                                  ERROR("This may be because the UTF-8 data "
 294                                        "could not be represented in your "
 295                                        "locale's character encoding.");)
 296 #endif
 297
 298 int
 299 tstr_to_utf8_simple(const tchar *tstr, utf8char **out)
 300 {
 301         size_t out_nbytes;
 302         return tstr_to_utf8(tstr, tstrlen(tstr), out, &out_nbytes);
 303 }
 304
 305 int
 306 utf8_to_tstr_simple(const utf8char *utf8str, tchar **out)
 307 {
 308         size_t out_nbytes;
 309         return utf8_to_tstr(utf8str, strlen(utf8str), out, &out_nbytes);
 310 }
 311
 312 static void
 313 iconv_cleanup(struct iconv_list_head *head)
 314 {
 315         pthread_mutex_destroy(&head->mutex);
 316         while (!list_empty(&head->list)) {
 317                 struct iconv_node *i;
 318
 319                 i = container_of(head->list.next, struct iconv_node, list);
 320                 list_del(&i->list);
 321                 iconv_close(i->cd);
 322                 FREE(i);
 323         }
 324 }
 325
 326 void
 327 iconv_global_cleanup()
 328 {
 329         /*iconv_cleanup(&iconv_utf16le_to_mbs);*/
 330 #if !TCHAR_IS_UTF16LE
 331         iconv_cleanup(&iconv_mbs_to_utf16le);
 332 #endif
 333         /*iconv_cleanup(&iconv_utf8_to_mbs);*/
 334 }
 335
 336 #if 0
 337 bool
 338 utf8_str_contains_nonascii_chars(const utf8char *utf8_str)
 339 {
 340         do {
 341                 if ((unsigned char)*utf8_str > 127)
 342                         return true;
 343         } while (*++utf8_str);
 344         return false;
 345 }
 346 #endif