aa8f9281d0e3bea3e46cb83d1d4ebfb00cb4d9d2
[wimlib] / src / encoding.c
1 /*
2  * encoding.c:  Convert "multibyte" strings (the locale-default encoding---
3  * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice
4  * versa.
5  */
6
7 /*
8  * Copyright (C) 2012, 2013 Eric Biggers
9  *
10  * This file is part of wimlib, a library for working with WIM files.
11  *
12  * wimlib is free software; you can redistribute it and/or modify it under the
13  * terms of the GNU General Public License as published by the Free
14  * Software Foundation; either version 3 of the License, or (at your option)
15  * any later version.
16  *
17  * wimlib is distributed in the hope that it will be useful, but WITHOUT ANY
18  * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
19  * A PARTICULAR PURPOSE. See the GNU General Public License for more
20  * details.
21  *
22  * You should have received a copy of the GNU General Public License
23  * along with wimlib; if not, see http://www.gnu.org/licenses/.
24  */
25
26 #include "config.h"
27 #include "wimlib_internal.h"
28 #include <pthread.h>
29 #include "list.h"
30
31 #include <iconv.h>
32 #include <stdlib.h>
33 #include <errno.h>
34
35 bool wimlib_mbs_is_utf8 = true;
36
37 struct iconv_list_head {
38         const char *from_encoding;
39         const char *to_encoding;
40         struct list_head list;
41         pthread_mutex_t mutex;
42 };
43
44 struct iconv_node {
45         iconv_t cd;
46         struct list_head list;
47         struct iconv_list_head *head;
48 };
49
50 #define ICONV_LIST(name, from, to)                      \
51 struct iconv_list_head name = {                         \
52         .from_encoding = from,                          \
53         .to_encoding = to,                              \
54         .list = LIST_HEAD_INIT(name.list),              \
55         .mutex = PTHREAD_MUTEX_INITIALIZER,             \
56 }
57
58 static iconv_t *
59 get_iconv(struct iconv_list_head *head)
60 {
61         iconv_t cd;
62         iconv_t *cd_p;
63         struct iconv_node *i;
64
65         pthread_mutex_lock(&head->mutex);
66         if (list_empty(&head->list)) {
67                 cd = iconv_open(head->to_encoding, head->from_encoding);
68                 if (cd == (iconv_t)-1) {
69                         ERROR_WITH_ERRNO("Failed to open iconv from %s to %s",
70                                          head->from_encoding, head->to_encoding);
71                         cd_p = NULL;
72                 } else {
73                         i = MALLOC(sizeof(struct iconv_node));
74                         if (i) {
75                                 i->head = head;
76                                 i->cd = cd;
77                                 cd_p = &i->cd;
78                         } else {
79                                 iconv_close(cd);
80                                 cd_p = NULL;
81                         }
82                 }
83         } else {
84                 i = container_of(head->list.next, struct iconv_node, list);
85                 list_del(head->list.next);
86                 cd_p = &i->cd;
87         }
88         pthread_mutex_unlock(&head->mutex);
89         return cd_p;
90 }
91
92 static void
93 put_iconv(iconv_t *cd)
94 {
95         int errno_save = errno;
96         struct iconv_node *i = container_of(cd, struct iconv_node, cd);
97         struct iconv_list_head *head = i->head;
98         
99         pthread_mutex_lock(&head->mutex);
100         list_add(&i->list, &head->list);
101         pthread_mutex_unlock(&head->mutex);
102         errno = errno_save;
103 }
104
105 /* Prevent printing an error message if a character conversion error occurs
106  * while printing an error message.  (This variable is not per-thread but it
107  * doesn't matter too much since it's just the error messages.) */
108 static bool error_message_being_printed = false;
109
110 #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\
111                                          varname2, longname2, chartype2,\
112                                          worst_case_len_expr,           \
113                                          err_return,                    \
114                                          err_msg)                       \
115 static ICONV_LIST(iconv_##varname1##_to_##varname2,                     \
116                   longname1, longname2);                                \
117                                                                         \
118 int                                                                     \
119 varname1##_to_##varname2##_nbytes(const chartype1 *in, size_t in_nbytes,\
120                                   size_t *out_nbytes_ret)               \
121 {                                                                       \
122         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
123         if (cd == NULL)                                                 \
124                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
125                                                                         \
126         /* Worst case length */                                         \
127         chartype2 buf[worst_case_len_expr];                             \
128         char *inbuf = (char*)in;                                        \
129         size_t inbytesleft = in_nbytes;                                 \
130         char *outbuf = (char*)buf;                                      \
131         size_t outbytesleft = sizeof(buf);                              \
132         size_t len;                                                     \
133         int ret;                                                        \
134                                                                         \
135         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
136         if (len == (size_t)-1) {                                        \
137                 if (!error_message_being_printed) {                     \
138                         error_message_being_printed = true;             \
139                         err_msg;                                        \
140                         error_message_being_printed = false;            \
141                 }                                                       \
142                 ret = err_return;                                       \
143         } else {                                                        \
144                 *out_nbytes_ret = sizeof(buf) - outbytesleft;           \
145                 ret = 0;                                                \
146         }                                                               \
147         put_iconv(cd);                                                  \
148         return ret;                                                     \
149 }                                                                       \
150                                                                         \
151 int                                                                     \
152 varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes,   \
153                                chartype2 *out)                          \
154 {                                                                       \
155         iconv_t *cd = get_iconv(&iconv_##varname1##_to_##varname2);     \
156         if (cd == NULL)                                                 \
157                 return WIMLIB_ERR_ICONV_NOT_AVAILABLE;                  \
158                                                                         \
159         char *inbuf = (char*)in;                                        \
160         size_t inbytesleft = in_nbytes;                                 \
161         char *outbuf = (char*)out;                                      \
162         const size_t LARGE_NUMBER = 1000000000;                         \
163         size_t outbytesleft = LARGE_NUMBER;                             \
164         size_t len;                                                     \
165         int ret;                                                        \
166                                                                         \
167         len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \
168         if (len == (size_t)-1) {                                        \
169                 err_msg;                                                \
170                 ret = err_return;                                       \
171         } else {                                                        \
172                 out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \
173                 ret = 0;                                                \
174         }                                                               \
175         put_iconv(cd);                                                  \
176         return ret;                                                     \
177 }                                                                       \
178                                                                         \
179 int                                                                     \
180 varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes,         \
181                          chartype2 **out_ret,                           \
182                          size_t *out_nbytes_ret)                        \
183 {                                                                       \
184         int ret;                                                        \
185         chartype2 *out;                                                 \
186         size_t out_nbytes;                                              \
187                                                                         \
188         ret = varname1##_to_##varname2##_nbytes(in, in_nbytes,          \
189                                                 &out_nbytes);           \
190         if (ret)                                                        \
191                 return ret;                                             \
192                                                                         \
193         out = MALLOC(out_nbytes + sizeof(chartype2));                   \
194         if (!out)                                                       \
195                 return WIMLIB_ERR_NOMEM;                                \
196                                                                         \
197         ret = varname1##_to_##varname2##_buf(in, in_nbytes, out);       \
198         if (ret) {                                                      \
199                 int errno_save = errno;                                 \
200                 FREE(out);                                              \
201                 errno = errno_save;                                     \
202         } else {                                                        \
203                 *out_ret = out;                                         \
204                 *out_nbytes_ret = out_nbytes;                           \
205         }                                                               \
206         return ret;                                                     \
207 }
208
209 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar,
210                                  mbs, "", mbchar,
211                                  in_nbytes / 2 * MB_CUR_MAX,
212                                  WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE,
213                                  ERROR_WITH_ERRNO("Failed to convert UTF-16LE "
214                                                   "string %U to multibyte string", in))
215
216 DEFINE_CHAR_CONVERSION_FUNCTIONS(mbs, "", mbchar,
217                                  utf16le, "UTF-16LE", utf16lechar,
218                                  in_nbytes * 2,
219                                  WIMLIB_ERR_INVALID_MULTIBYTE_STRING,
220                                  ERROR_WITH_ERRNO("Failed to convert multibyte "
221                                                   "string %s to UTF-16LE string", in))
222
223 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char,
224                                  mbs, "", mbchar,
225                                  in_nbytes,
226                                  WIMLIB_ERR_INVALID_UTF8_STRING,
227                                  ERROR_WITH_ERRNO("Failed to convert UTF-8 "
228                                                   "string %U to multibyte string", in))
229
230
231 static void
232 iconv_cleanup(struct iconv_list_head *head)
233 {
234         pthread_mutex_destroy(&head->mutex);
235         while (!list_empty(&head->list)) {
236                 struct iconv_node *i;
237                 
238                 i = container_of(head->list.next, struct iconv_node, list);
239                 list_del(&i->list);
240                 iconv_close(i->cd);
241                 FREE(i);
242         }
243 }
244
245 void
246 iconv_global_cleanup()
247 {
248         iconv_cleanup(&iconv_utf16le_to_mbs);
249         iconv_cleanup(&iconv_mbs_to_utf16le);
250         iconv_cleanup(&iconv_utf8_to_mbs);
251 }
252
253
254
255 bool
256 utf8_str_contains_nonascii_chars(const utf8char *utf8_str)
257 {
258         do {
259                 if ((unsigned char)*utf8_str > 127)
260                         return true;
261         } while (*++utf8_str);
262         return false;
263 }