wimlib.net Git - wimlib/blob - src/encoding.c

   1 /*
   2  * encoding.c:  Convert "multibyte" strings (the locale-default encoding---
   3  * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice
   4  * versa.
   5  */
   6
   7 /*
   8  * Copyright (C) 2012, 2013 Eric Biggers
   9  *
  10  * This file is part of wimlib, a library for working with WIM files.
  11  *
  12  * wimlib is free software; you can redistribute it and/or modify it under the
  13  * terms of the GNU General Public License as published by the Free
  14  * Software Foundation; either version 3 of the License, or (at your option)
  15  * any later version.
  16  *
  17  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
  18  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  19  * A PARTICULAR PURPOSE. See the GNU General Public License for more
  20  * details.
  21  *
  22  * You should have received a copy of the GNU General Public License
  23  * along with wimlib; if not, see http://www.gnu.org/licenses/.
  24  */
  25
  26 #include "config.h"
  27 #include "wimlib_internal.h"
  28 #include <pthread.h>
  29 #include "list.h"
  30
  31 #include <iconv.h>
  32 #include <stdlib.h>
  33
  34 bool wimlib_mbs_is_utf8 = false;
  35
  36 struct iconv_list_head {
  37         const char *from_encoding;
  38         const char *to_encoding;
  39         struct list_head list;
  40         pthread_mutex_t mutex;
  41 };
  42
  43 struct iconv_node {
  44         iconv_t cd;
  45         struct list_head list;
  46         struct iconv_list_head *head;
  47 };
  48
  49 #define ICONV_LIST(name, from, to)                      \
  50 struct iconv_list_head name = {                         \
  51         .from_encoding = from,                          \
  52         .to_encoding = to,                              \
  53         .list = LIST_HEAD_INIT(name.list),              \
  54         .mutex = PTHREAD_MUTEX_INITIALIZER,             \
  55 }
  56
  57 static iconv_t *
  58 get_iconv(struct iconv_list_head *head)
  59 {
  60         iconv_t cd;
  61         iconv_t *cd_p;
  62         struct iconv_node *i;
  63
  64         pthread_mutex_lock(&head->mutex);
  65         if (list_empty(&head->list)) {
  66                 cd = iconv_open(head->to_encoding, head->from_encoding);
  67                 if (cd == (iconv_t)-1) {
  68                         ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
  69                                          head->from_encoding, head->to_encoding);
  70                         cd_p = NULL;
  71                 } else {
  72                         i = MALLOC(sizeof(struct iconv_node));
  73                         if (i) {
  74                                 i->head = head;
  75                                 i->cd = cd;
  76                                 cd_p = &i->cd;
  77                         } else {
  78                                 iconv_close(cd);
  79                                 cd_p = NULL;
  80                         }
  81                 }
  82         } else {
  83                 i = container_of(head->list.next, struct iconv_node, list);
  84                 list_del(head->list.next);
  85                 cd_p = &i->cd;
  86         }
  87         pthread_mutex_unlock(&head->mutex);
  88         return cd_p;
  89 }
  90
  91 static void
  92 put_iconv(iconv_t *cd)
  93 {
  94         int errno_save = errno;
  95         struct iconv_node *i = container_of(cd, struct iconv_node, cd);
  96         struct iconv_list_head *head = i->head;
  97
  98         pthread_mutex_lock(&head->mutex);
  99         list_add(&i->list, &head->list);
 100         pthread_mutex_unlock(&head->mutex);
 101         errno = errno_save;
 102 }
 103
 104 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
 105                                          varname2, longname2, chartype2,\
 106                                          worst_case_len_expr,           \
 107                                          err_return,                    \
 108                                          err_msg)                       \
 109 static ICONV_LIST(iconv_##varname1##_to_##varname2,                     \
 110                   longname1, longname2);                                \
 111                                                                         \
 112 int                                                                     \
 113 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
 114                                   size_t *out_nbytes_ret)               \
 115 {                                                                       \
 116         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
 117         if (cd == NULL)                                                 \
 118                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
 119                                                                         \
 120         /* Worst case length */                                         \
 121         chartype2 buf[worst_case_len_expr];                             \
 122         char *inbuf = (char*)in;                                        \
 123         size_t inbytesleft = in_nbytes;                                 \
 124         char *outbuf = (char*)buf;                                      \
 125         size_t outbytesleft = sizeof(buf);                              \
 126         size_t len;                                                     \
 127         int ret;                                                        \
 128                                                                         \
 129         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
 130         if (len == (size_t)-1) {                                        \
 131                 err_msg;                                                \
 132                 ret = err_return;                                       \
 133         } else {                                                        \
 134                 *out_nbytes_ret = sizeof(buf) - outbytesleft;           \
 135                 ret = 0;                                                \
 136         }                                                               \
 137         put_iconv(cd);                                                  \
 138         return ret;                                                     \
 139 }                                                                       \
 140                                                                         \
 141 int                                                                     \
 142 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes,   \
 143                                chartype2 *out)                          \
 144 {                                                                       \
 145         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
 146         if (cd == NULL)                                                 \
 147                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
 148                                                                         \
 149         char *inbuf = (char*)in;                                        \
 150         size_t inbytesleft = in_nbytes;                                 \
 151         char *outbuf = (char*)out;                                      \
 152         const size_t LARGE_NUMBER = 1000000000;                         \
 153         size_t outbytesleft = LARGE_NUMBER;                             \
 154         size_t len;                                                     \
 155         int ret;                                                        \
 156                                                                         \
 157         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
 158         if (len == (size_t)-1) {                                        \
 159                 err_msg;                                                \
 160                 ret = err_return;                                       \
 161         } else {                                                        \
 162                 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
 163                 ret = 0;                                                \
 164         }                                                               \
 165         put_iconv(cd);                                                  \
 166         return ret;                                                     \
 167 }                                                                       \
 168                                                                         \
 169 int                                                                     \
 170 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes,         \
 171                          chartype2 **out_ret,                           \
 172                          size_t *out_nbytes_ret)                        \
 173 {                                                                       \
 174         int ret;                                                        \
 175         chartype2 *out;                                                 \
 176         size_t out_nbytes;                                              \
 177                                                                         \
 178         ret = varname1##_to_##varname2##_nbytes(in, in_nbytes,          \
 179                                                 &out_nbytes);           \
 180         if (ret)                                                        \
 181                 return ret;                                             \
 182                                                                         \
 183         out = MALLOC(out_nbytes + sizeof(chartype2));                   \
 184         if (!out)                                                       \
 185                 return WIMLIB_ERR_NOMEM;                                \
 186                                                                         \
 187         ret = varname1##_to_##varname2##_buf(in, in_nbytes, out);       \
 188         if (ret) {                                                      \
 189                 int errno_save = errno;                                 \
 190                 FREE(out);                                              \
 191                 errno = errno_save;                                     \
 192         } else {                                                        \
 193                 *out_ret = out;                                         \
 194                 *out_nbytes_ret = out_nbytes;                           \
 195         }                                                               \
 196         return ret;                                                     \
 197 }
 198
 199 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
 200                                  mbs, "", mbchar,
 201                                  in_nbytes / 2 * MB_CUR_MAX,
 202                                  WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
 203                                  ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
 204                                                   "string %U to multibyte string", in))
 205
 206 DEFINE_CHAR_CONVERSION_FUNCTIONS(mbs, "", mbchar,
 207                                  utf16le, "UTF-16LE", utf16lechar,
 208                                  in_nbytes * 2,
 209                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
 210                                  ERROR_WITH_ERRNO("Failed to convert multibyte "
 211                                                   "string %s to UTF-16LE string", in))
 212
 213 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
 214                                  mbs, "", mbchar,
 215                                  in_nbytes,
 216                                  WIMLIB_ERR_INVALID_UTF8_STRING,
 217                                  ERROR_WITH_ERRNO("Failed to convert UTF-8 "
 218                                                   "string %U to multibyte string", in))
 219
 220
 221 static void
 222 iconv_cleanup(struct iconv_list_head *head)
 223 {
 224         pthread_mutex_destroy(&head->mutex);
 225         while (!list_empty(&head->list)) {
 226                 struct iconv_node *i;
 227
 228                 i = container_of(head->list.next, struct iconv_node, list);
 229                 list_del(&i->list);
 230                 iconv_close(i->cd);
 231                 FREE(i);
 232         }
 233 }
 234
 235 void
 236 iconv_global_cleanup()
 237 {
 238         iconv_cleanup(&iconv_utf16le_to_mbs);
 239         iconv_cleanup(&iconv_mbs_to_utf16le);
 240         iconv_cleanup(&iconv_utf8_to_mbs);
 241 }
 242
 243
 244
 245 bool
 246 utf8_str_contains_nonascii_chars(const utf8char *utf8_str)
 247 {
 248         do {
 249                 if ((unsigned char)*utf8_str > 127)
 250                         return false;
 251         } while (*++utf8_str);
 252         return true;
 253 }