2 * encoding.c: Convert "multibyte" strings (the locale-default encoding---
3 * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice
4 * versa. Also, convert UTF-8 strings to multibyte strings.
8 * Copyright (C) 2012, 2013 Eric Biggers
10 * This file is part of wimlib, a library for working with WIM files.
12 * wimlib is free software; you can redistribute it and/or modify it under the
13 * terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 3 of the License, or (at your option)
17 * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19 * A PARTICULAR PURPOSE. See the GNU General Public License for more
22 * You should have received a copy of the GNU General Public License
23 * along with wimlib; if not, see http://www.gnu.org/licenses/.
26 #include "wimlib_internal.h"
34 bool wimlib_mbs_is_utf8 = !TCHAR_IS_UTF16LE;
36 /* List of iconv_t conversion descriptors for a specific character conversion.
37 * The idea is that it is not thread-safe to have just one conversion
38 * descriptor, but it also is inefficient to open a new conversion descriptor to
39 * convert every string. Both these problems can be solved by maintaining a
40 * list of conversion descriptors; then, a thread can use an existing conversion
41 * descriptor if available. */
42 struct iconv_list_head {
43 const char *from_encoding;
44 const char *to_encoding;
45 struct list_head list;
46 pthread_mutex_t mutex;
51 struct list_head list;
52 struct iconv_list_head *head;
55 #define ICONV_LIST(name, from, to) \
56 struct iconv_list_head name = { \
57 .from_encoding = from, \
59 .list = LIST_HEAD_INIT(name.list), \
60 .mutex = PTHREAD_MUTEX_INITIALIZER, \
64 get_iconv(struct iconv_list_head *head)
70 pthread_mutex_lock(&head->mutex);
71 if (list_empty(&head->list)) {
72 cd = iconv_open(head->to_encoding, head->from_encoding);
73 if (cd == (iconv_t)-1) {
74 ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
75 head->from_encoding, head->to_encoding);
78 i = MALLOC(sizeof(struct iconv_node));
89 i = container_of(head->list.next, struct iconv_node, list);
90 list_del(head->list.next);
93 pthread_mutex_unlock(&head->mutex);
98 put_iconv(iconv_t *cd)
100 int errno_save = errno;
101 struct iconv_node *i = container_of(cd, struct iconv_node, cd);
102 struct iconv_list_head *head = i->head;
104 pthread_mutex_lock(&head->mutex);
105 list_add(&i->list, &head->list);
106 pthread_mutex_unlock(&head->mutex);
110 /* Prevent printing an error message if a character conversion error occurs
111 * while printing an error message. (This variable is not per-thread but it
112 * doesn't matter too much since it's just the error messages.) */
113 static bool error_message_being_printed = false;
115 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
116 varname2, longname2, chartype2,\
118 worst_case_len_expr, \
121 static ICONV_LIST(iconv_##varname1##_to_##varname2, \
122 longname1, longname2); \
125 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
126 size_t *out_nbytes_ret) \
128 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
130 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
132 /* Worst case length */ \
133 chartype2 buf[worst_case_len_expr]; \
134 char *inbuf = (char*)in; \
135 size_t inbytesleft = in_nbytes; \
136 char *outbuf = (char*)buf; \
137 size_t outbytesleft = sizeof(buf); \
141 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
142 if (len == (size_t)-1) { \
143 if (!error_message_being_printed) { \
144 error_message_being_printed = true; \
146 error_message_being_printed = false; \
150 *out_nbytes_ret = sizeof(buf) - outbytesleft; \
158 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes, \
161 iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2); \
163 return WIMLIB_ERR_ICONV_NOT_AVAILABLE; \
165 char *inbuf = (char*)in; \
166 size_t inbytesleft = in_nbytes; \
167 char *outbuf = (char*)out; \
168 const size_t LARGE_NUMBER = 1000000000; \
169 size_t outbytesleft = LARGE_NUMBER; \
173 len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
174 if (len == (size_t)-1) { \
175 if (!error_message_being_printed) { \
176 error_message_being_printed = true; \
178 error_message_being_printed = false; \
182 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
190 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes, \
191 chartype2 **out_ret, \
192 size_t *out_nbytes_ret) \
199 /* Out same as in */ \
200 out = MALLOC(in_nbytes + sizeof(chartype2)); \
202 return WIMLIB_ERR_NOMEM; \
203 memcpy(out, in, in_nbytes); \
204 out[in_nbytes / sizeof(chartype2)] = 0; \
206 *out_nbytes_ret = in_nbytes; \
210 ret = varname1##_to_##varname2##_nbytes(in, in_nbytes, \
215 out = MALLOC(out_nbytes + sizeof(chartype2)); \
217 return WIMLIB_ERR_NOMEM; \
219 ret = varname1##_to_##varname2##_buf(in, in_nbytes, out); \
221 int errno_save = errno; \
223 errno = errno_save; \
226 *out_nbytes_ret = out_nbytes; \
232 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
235 in_nbytes / 2 * MB_CUR_MAX,
236 WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
237 ERROR("Failed to convert UTF-16LE string "
238 "to multibyte string!");
239 ERROR("This may be because the UTF-16LE data "
240 "could not be represented in your "
241 "locale's character encoding."))
244 #if !TCHAR_IS_UTF16LE
245 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
246 utf16le, "UTF-16LE", utf16lechar,
249 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
250 ERROR_WITH_ERRNO("Failed to convert multibyte "
251 "string \"%"TS"\" to UTF-16LE string!", in);
252 ERROR("If the data you provided was UTF-8, please make sure "
253 "the character encoding of your current locale is UTF-8."))
256 /* tchar to UTF-8 and back */
258 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "UTF16-LE", tchar,
259 utf8, "UTF-8", utf8char,
262 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
263 ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
264 "string \"%"TS"\" to UTF-8 string!", in);
267 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
268 tstr, "UTF16-LE", tchar,
271 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
272 ERROR_WITH_ERRNO("Failed to convert UTF-8 string "
273 "to UTF-16LE string!");
276 DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar,
277 utf8, "UTF-8", utf8char,
280 WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
281 ERROR_WITH_ERRNO("Failed to convert multibyte "
282 "string \"%"TS"\" to UTF-8 string!", in);
283 ERROR("If the data you provided was UTF-8, please make sure "
284 "the character encoding of your current locale is UTF-8.");)
286 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
290 WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
291 ERROR("Failed to convert UTF-8 string to "
292 "multibyte string!");
293 ERROR("This may be because the UTF-8 data "
294 "could not be represented in your "
295 "locale's character encoding.");)
299 tstr_to_utf8_simple(const tchar *tstr, utf8char **out)
302 return tstr_to_utf8(tstr, tstrlen(tstr), out, &out_nbytes);
306 utf8_to_tstr_simple(const utf8char *utf8str, tchar **out)
309 return utf8_to_tstr(utf8str, strlen(utf8str), out, &out_nbytes);
313 iconv_cleanup(struct iconv_list_head *head)
315 pthread_mutex_destroy(&head->mutex);
316 while (!list_empty(&head->list)) {
317 struct iconv_node *i;
319 i = container_of(head->list.next, struct iconv_node, list);
327 iconv_global_cleanup()
329 /*iconv_cleanup(&iconv_utf16le_to_mbs);*/
330 #if !TCHAR_IS_UTF16LE
331 iconv_cleanup(&iconv_mbs_to_utf16le);
333 /*iconv_cleanup(&iconv_utf8_to_mbs);*/
338 utf8_str_contains_nonascii_chars(const utf8char *utf8_str)
341 if ((unsigned char)*utf8_str > 127)
343 } while (*++utf8_str);