2 * encoding.c: Convert "multibyte" strings (the locale-default encoding---
3 * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice
8 * Copyright (C) 2012, 2013 Eric Biggers
10 * This file is part of wimlib, a library for working with WIM files.
12 * wimlib is free software; you can redistribute it and/or modify it under the
13 * terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option)
17 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19 * A PARTICULAR PURPOSE. See the GNU General Public License for more
22 * You should have received a copy of the GNU General Public License
23 * along with wimlib; if not, see http://www.gnu.org/licenses/.
27 #include "wimlib_internal.h"
35 bool wimlib_mbs_is_utf8 = false;
37 struct iconv_list_head {
38 const char *from_encoding;
39 const char *to_encoding;
40 struct list_head list;
41 pthread_mutex_t mutex;
46 struct list_head list;
47 struct iconv_list_head *head;
50 #define ICONV_LIST(name, from, to) \
51 struct iconv_list_head name = { \
52 .from_encoding = from, \
54 .list = LIST_HEAD_INIT(name.list), \
55 .mutex = PTHREAD_MUTEX_INITIALIZER, \
59 get_iconv(struct iconv_list_head *head)
65 pthread_mutex_lock(&head->mutex);
66 if (list_empty(&head->list)) {
67 cd = iconv_open(head->to_encoding, head->from_encoding);
68 if (cd == (iconv_t)-1) {
69 ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
70 head->from_encoding, head->to_encoding);
73 i = MALLOC(sizeof(struct iconv_node));
84 i = container_of(head->list.next, struct iconv_node, list);
85 list_del(head->list.next);
88 pthread_mutex_unlock(&head->mutex);
93 put_iconv(iconv_t *cd)
95 int errno_save = errno;
96 struct iconv_node *i = container_of(cd, struct iconv_node, cd);
97 struct iconv_list_head *head = i->head;
99 pthread_mutex_lock(&head->mutex);
100 list_add(&i->list, &head->list);
101 pthread_mutex_unlock(&head->mutex);
105 /* Prevent printing an error message if a character conversion error occurs
106 * while printing an error message. (This variable is not per-thread but it
107 * doesn't matter too much since it's just the error messages.) */
108 static bool error_message_being_printed = false;
110 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
111 varname2, longname2, chartype2,\
112 worst_case_len_expr, \
115 static ICONV_LIST(iconv_##varname1##_to_##varname2, \
116 longname1, longname2); \
119 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
120 size_t *out_nbytes_ret) \
122 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
124 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
126 /* Worst case length */ \
127 chartype2 buf[worst_case_len_expr]; \
128 char *inbuf = (char*)in; \
129 size_t inbytesleft = in_nbytes; \
130 char *outbuf = (char*)buf; \
131 size_t outbytesleft = sizeof(buf); \
135 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
136 if (len == (size_t)-1) { \
137 if (!error_message_being_printed) { \
138 error_message_being_printed = true; \
140 error_message_being_printed = false; \
144 *out_nbytes_ret = sizeof(buf) - outbytesleft; \
152 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes, \
155 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
157 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
159 char *inbuf = (char*)in; \
160 size_t inbytesleft = in_nbytes; \
161 char *outbuf = (char*)out; \
162 const size_t LARGE_NUMBER = 1000000000; \
163 size_t outbytesleft = LARGE_NUMBER; \
167 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
168 if (len == (size_t)-1) { \
172 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
180 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes, \
181 chartype2 **out_ret, \
182 size_t *out_nbytes_ret) \
188 ret = varname1##_to_##varname2##_nbytes(in, in_nbytes, \
193 out = MALLOC(out_nbytes + sizeof(chartype2)); \
195 return WIMLIB_ERR_NOMEM; \
197 ret = varname1##_to_##varname2##_buf(in, in_nbytes, out); \
199 int errno_save = errno; \
201 errno = errno_save; \
204 *out_nbytes_ret = out_nbytes; \
209 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
211 in_nbytes / 2 * MB_CUR_MAX,
212 WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
213 ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
214 "string %U to multibyte string", in))
216 DEFINE_CHAR_CONVERSION_FUNCTIONS(mbs, "", mbchar,
217 utf16le, "UTF-16LE", utf16lechar,
219 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
220 ERROR_WITH_ERRNO("Failed to convert multibyte "
221 "string %s to UTF-16LE string", in))
223 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
226 WIMLIB_ERR_INVALID_UTF8_STRING,
227 ERROR_WITH_ERRNO("Failed to convert UTF-8 "
228 "string %U to multibyte string", in))
232 iconv_cleanup(struct iconv_list_head *head)
234 pthread_mutex_destroy(&head->mutex);
235 while (!list_empty(&head->list)) {
236 struct iconv_node *i;
238 i = container_of(head->list.next, struct iconv_node, list);
246 iconv_global_cleanup()
248 iconv_cleanup(&iconv_utf16le_to_mbs);
249 iconv_cleanup(&iconv_mbs_to_utf16le);
250 iconv_cleanup(&iconv_utf8_to_mbs);
256 utf8_str_contains_nonascii_chars(const utf8char *utf8_str)
259 if ((unsigned char)*utf8_str > 127)
261 } while (*++utf8_str);