6 * Copyright (C) 2012, 2013 Eric Biggers
8 * This file is part of wimlib, a library for working with WIM files.
10 * wimlib is free software; you can redistribute it and/or modify it under the
11 * terms of the GNU General Public License as published by the Free
12 * Software Foundation; either version 3 of the License, or (at your option)
15 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
16 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
17 * A PARTICULAR PURPOSE. See the GNU General Public License for more
20 * You should have received a copy of the GNU General Public License
21 * along with wimlib; if not, see http://www.gnu.org/licenses/.
24 #include "wimlib_internal.h"
32 bool wimlib_mbs_is_utf8 = !TCHAR_IS_UTF16LE;
34 /* List of iconv_t conversion descriptors for a specific character conversion.
35 * The idea is that it is not thread-safe to have just one conversion
36 * descriptor, but it also is inefficient to open a new conversion descriptor to
37 * convert every string. Both these problems can be solved by maintaining a
38 * list of conversion descriptors; then, a thread can use an existing conversion
39 * descriptor if available. */
40 struct iconv_list_head {
41 const char *from_encoding;
42 const char *to_encoding;
43 struct list_head list;
44 pthread_mutex_t mutex;
49 struct list_head list;
50 struct iconv_list_head *head;
53 #define ICONV_LIST(name, from, to) \
54 struct iconv_list_head name = { \
55 .from_encoding = from, \
57 .list = LIST_HEAD_INIT(name.list), \
58 .mutex = PTHREAD_MUTEX_INITIALIZER, \
62 get_iconv(struct iconv_list_head *head)
68 pthread_mutex_lock(&head->mutex);
69 if (list_empty(&head->list)) {
70 cd = iconv_open(head->to_encoding, head->from_encoding);
71 if (cd == (iconv_t)-1) {
72 ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
73 head->from_encoding, head->to_encoding);
76 i = MALLOC(sizeof(struct iconv_node));
87 i = container_of(head->list.next, struct iconv_node, list);
88 list_del(head->list.next);
91 pthread_mutex_unlock(&head->mutex);
96 put_iconv(iconv_t *cd)
98 int errno_save = errno;
99 struct iconv_node *i = container_of(cd, struct iconv_node, cd);
100 struct iconv_list_head *head = i->head;
102 pthread_mutex_lock(&head->mutex);
103 list_add(&i->list, &head->list);
104 pthread_mutex_unlock(&head->mutex);
108 /* Prevent printing an error message if a character conversion error occurs
109 * while printing an error message. (This variable is not per-thread but it
110 * doesn't matter too much since it's just the error messages.) */
111 static bool error_message_being_printed = false;
113 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
114 varname2, longname2, chartype2,\
116 worst_case_len_expr, \
120 static ICONV_LIST(iconv_##varname1##_to_##varname2, \
121 longname1, longname2); \
124 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
125 size_t *out_nbytes_ret) \
127 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
129 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
131 /* Worst case length */ \
132 chartype2 buf[worst_case_len_expr]; \
133 char *inbuf = (char*)in; \
134 size_t inbytesleft = in_nbytes; \
135 char *outbuf = (char*)buf; \
136 size_t outbytesleft = sizeof(buf); \
140 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
141 if (len == (size_t)-1) { \
142 if (!error_message_being_printed) { \
143 error_message_being_printed = true; \
145 error_message_being_printed = false; \
149 *out_nbytes_ret = sizeof(buf) - outbytesleft; \
157 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes, \
160 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
162 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
164 char *inbuf = (char*)in; \
165 size_t inbytesleft = in_nbytes; \
166 char *outbuf = (char*)out; \
167 const size_t LARGE_NUMBER = 1000000000; \
168 size_t outbytesleft = LARGE_NUMBER; \
172 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
173 if (len == (size_t)-1) { \
174 if (!error_message_being_printed) { \
175 error_message_being_printed = true; \
177 error_message_being_printed = false; \
181 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
189 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes, \
190 chartype2 **out_ret, \
191 size_t *out_nbytes_ret) \
198 /* Out same as in */ \
199 out = MALLOC(in_nbytes + sizeof(chartype2)); \
201 return WIMLIB_ERR_NOMEM; \
202 memcpy(out, in, in_nbytes); \
203 out[in_nbytes / sizeof(chartype2)] = 0; \
205 *out_nbytes_ret = in_nbytes; \
209 ret = varname1##_to_##varname2##_nbytes(in, in_nbytes, \
214 out = MALLOC(out_nbytes + sizeof(chartype2)); \
216 return WIMLIB_ERR_NOMEM; \
218 ret = varname1##_to_##varname2##_buf(in, in_nbytes, out); \
220 int errno_save = errno; \
222 errno = errno_save; \
225 *out_nbytes_ret = out_nbytes; \
230 #if !TCHAR_IS_UTF16LE
231 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
232 utf16le, "UTF-16LE", utf16lechar,
235 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
236 ERROR_WITH_ERRNO("Failed to convert multibyte "
237 "string \"%"TS"\" to UTF-16LE string!", in);
238 ERROR("If the data you provided was UTF-8, please make sure "
239 "the character encoding of your current locale is UTF-8."),
242 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
246 WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
247 ERROR("Failed to convert UTF-16LE string to "
248 "multibyte string!");
249 ERROR("This may be because the UTF-16LE string "
250 "could not be represented in your "
251 "locale's character encoding."),
255 /* tchar to UTF-8 and back */
257 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "UTF-16LE", tchar,
261 WIMLIB_ERR_INVALID_UTF16_STRING,
262 ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
263 "string \"%"TS"\" to UTF-8 string!", in),
266 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", char,
267 tstr, "UTF-16LE", tchar,
270 WIMLIB_ERR_INVALID_UTF8_STRING,
271 ERROR_WITH_ERRNO("Failed to convert UTF-8 string "
272 "to UTF-16LE string!"),
275 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
279 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
280 ERROR_WITH_ERRNO("Failed to convert multibyte "
281 "string \"%"TS"\" to UTF-8 string!", in);
282 ERROR("If the data you provided was UTF-8, please make sure "
283 "the character encoding of your current locale is UTF-8."),
286 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", char,
290 WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
291 ERROR("Failed to convert UTF-8 string to "
292 "multibyte string!");
293 ERROR("This may be because the UTF-8 data "
294 "could not be represented in your "
295 "locale's character encoding."),
300 tstr_to_utf8_simple(const tchar *tstr, char **out)
303 return tstr_to_utf8(tstr, tstrlen(tstr) * sizeof(tchar),
308 utf8_to_tstr_simple(const char *utf8str, tchar **out)
311 return utf8_to_tstr(utf8str, strlen(utf8str), out, &out_nbytes);
315 iconv_cleanup(struct iconv_list_head *head)
317 pthread_mutex_destroy(&head->mutex);
318 while (!list_empty(&head->list)) {
319 struct iconv_node *i;
321 i = container_of(head->list.next, struct iconv_node, list);
329 iconv_global_cleanup()
331 iconv_cleanup(&iconv_utf8_to_tstr);
332 iconv_cleanup(&iconv_tstr_to_utf8);
333 #if !TCHAR_IS_UTF16LE
334 iconv_cleanup(&iconv_utf16le_to_tstr);
335 iconv_cleanup(&iconv_tstr_to_utf16le);