X-Git-Url: https://wimlib.net/git/?a=blobdiff_plain;f=src%2Fencoding.c;h=256103a4971634c0e280a12906dabf1154f42605;hb=670a54f5de37bff02c49f37d690b26654cf7fa1d;hp=e8bc40273f4180ab19f6a83cc0e178a5b84ce756;hpb=b5d427e440402aff987a060ff6ceefcb97117fa4;p=wimlib diff --git a/src/encoding.c b/src/encoding.c index e8bc4027..256103a4 100644 --- a/src/encoding.c +++ b/src/encoding.c @@ -1,7 +1,7 @@ /* * encoding.c: Convert "multibyte" strings (the locale-default encoding--- * generally, UTF-8 or something like ISO-8859-1) to UTF-16LE strings, and vice - * versa. + * versa. Also, convert UTF-8 strings to multibyte strings. */ /* @@ -23,17 +23,22 @@ * along with wimlib; if not, see http://www.gnu.org/licenses/. */ -#include "config.h" #include "wimlib_internal.h" -#include -#include "list.h" +#include #include +#include #include -#include +#include -bool wimlib_mbs_is_utf8 = false; +bool wimlib_mbs_is_utf8 = !TCHAR_IS_UTF16LE; +/* List of iconv_t conversion descriptors for a specific character conversion. + * The idea is that it is not thread-safe to have just one conversion + * descriptor, but it also is inefficient to open a new conversion descriptor to + * convert every string. Both these problems can be solved by maintaining a + * list of conversion descriptors; then, a thread can use an existing conversion + * descriptor if available. */ struct iconv_list_head { const char *from_encoding; const char *to_encoding; @@ -95,7 +100,7 @@ put_iconv(iconv_t *cd) int errno_save = errno; struct iconv_node *i = container_of(cd, struct iconv_node, cd); struct iconv_list_head *head = i->head; - + pthread_mutex_lock(&head->mutex); list_add(&i->list, &head->list); pthread_mutex_unlock(&head->mutex); @@ -109,6 +114,7 @@ static bool error_message_being_printed = false; #define DEFINE_CHAR_CONVERSION_FUNCTIONS(varname1, longname1, chartype1,\ varname2, longname2, chartype2,\ + earlyreturn, \ worst_case_len_expr, \ err_return, \ err_msg) \ @@ -166,7 +172,11 @@ varname1##_to_##varname2##_buf(const chartype1 *in, size_t in_nbytes, \ \ len = iconv(*cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft); \ if (len == (size_t)-1) { \ - err_msg; \ + if (!error_message_being_printed) { \ + error_message_being_printed = true; \ + err_msg; \ + error_message_being_printed = false; \ + } \ ret = err_return; \ } else { \ out[(LARGE_NUMBER-outbytesleft)/sizeof(chartype2)] = 0; \ @@ -185,6 +195,18 @@ varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes, \ chartype2 *out; \ size_t out_nbytes; \ \ + if (earlyreturn) { \ + /* Out same as in */ \ + out = MALLOC(in_nbytes + sizeof(chartype2)); \ + if (!out) \ + return WIMLIB_ERR_NOMEM; \ + memcpy(out, in, in_nbytes); \ + out[in_nbytes / sizeof(chartype2)] = 0; \ + *out_ret = out; \ + *out_nbytes_ret = in_nbytes; \ + return 0; \ + } \ + \ ret = varname1##_to_##varname2##_nbytes(in, in_nbytes, \ &out_nbytes); \ if (ret) \ @@ -206,27 +228,86 @@ varname1##_to_##varname2(const chartype1 *in, size_t in_nbytes, \ return ret; \ } +#if 0 DEFINE_CHAR_CONVERSION_FUNCTIONS(utf16le, "UTF-16LE", utf16lechar, mbs, "", mbchar, + false, in_nbytes / 2 * MB_CUR_MAX, WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE, - ERROR_WITH_ERRNO("Failed to convert UTF-16LE " - "string %U to multibyte string", in)) + ERROR("Failed to convert UTF-16LE string " + "to multibyte string!"); + ERROR("This may be because the UTF-16LE data " + "could not be represented in your " + "locale's character encoding.")) +#endif -DEFINE_CHAR_CONVERSION_FUNCTIONS(mbs, "", mbchar, +#if !TCHAR_IS_UTF16LE +DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar, utf16le, "UTF-16LE", utf16lechar, + false, + in_nbytes * 2, + WIMLIB_ERR_INVALID_MULTIBYTE_STRING, + ERROR_WITH_ERRNO("Failed to convert multibyte " + "string \"%"TS"\" to UTF-16LE string!", in); + ERROR("If the data you provided was UTF-8, please make sure " + "the character encoding of your current locale is UTF-8.")) +#endif + +/* tchar to UTF-8 and back */ +#if TCHAR_IS_UTF16LE +DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "UTF16-LE", tchar, + utf8, "UTF-8", utf8char, + false, in_nbytes * 2, WIMLIB_ERR_INVALID_MULTIBYTE_STRING, + ERROR_WITH_ERRNO("Failed to convert UTF-16LE " + "string \"%"TS"\" to UTF-8 string!", in); + ) + +DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char, + tstr, "UTF16-LE", tchar, + false, + in_nbytes * 2, + WIMLIB_ERR_INVALID_MULTIBYTE_STRING, + ERROR_WITH_ERRNO("Failed to convert UTF-8 string " + "to UTF-16LE string!"); + ) +#else +DEFINE_CHAR_CONVERSION_FUNCTIONS(tstr, "", tchar, + utf8, "UTF-8", utf8char, + wimlib_mbs_is_utf8, + in_nbytes * 4, + WIMLIB_ERR_INVALID_MULTIBYTE_STRING, ERROR_WITH_ERRNO("Failed to convert multibyte " - "string %s to UTF-16LE string", in)) + "string \"%"TS"\" to UTF-8 string!", in); + ERROR("If the data you provided was UTF-8, please make sure " + "the character encoding of your current locale is UTF-8.");) DEFINE_CHAR_CONVERSION_FUNCTIONS(utf8, "UTF-8", utf8char, - mbs, "", mbchar, - in_nbytes, - WIMLIB_ERR_INVALID_UTF8_STRING, - ERROR_WITH_ERRNO("Failed to convert UTF-8 " - "string %U to multibyte string", in)) + tstr, "", tchar, + wimlib_mbs_is_utf8, + in_nbytes * 4, + WIMLIB_ERR_UNICODE_STRING_NOT_REPRESENTABLE, + ERROR("Failed to convert UTF-8 string to " + "multibyte string!"); + ERROR("This may be because the UTF-8 data " + "could not be represented in your " + "locale's character encoding.");) +#endif +int +tstr_to_utf8_simple(const tchar *tstr, utf8char **out) +{ + size_t out_nbytes; + return tstr_to_utf8(tstr, tstrlen(tstr), out, &out_nbytes); +} + +int +utf8_to_tstr_simple(const utf8char *utf8str, tchar **out) +{ + size_t out_nbytes; + return utf8_to_tstr(utf8str, strlen(utf8str), out, &out_nbytes); +} static void iconv_cleanup(struct iconv_list_head *head) @@ -234,7 +315,7 @@ iconv_cleanup(struct iconv_list_head *head) pthread_mutex_destroy(&head->mutex); while (!list_empty(&head->list)) { struct iconv_node *i; - + i = container_of(head->list.next, struct iconv_node, list); list_del(&i->list); iconv_close(i->cd); @@ -245,13 +326,14 @@ iconv_cleanup(struct iconv_list_head *head) void iconv_global_cleanup() { - iconv_cleanup(&iconv_utf16le_to_mbs); + /*iconv_cleanup(&iconv_utf16le_to_mbs);*/ +#if !TCHAR_IS_UTF16LE iconv_cleanup(&iconv_mbs_to_utf16le); - iconv_cleanup(&iconv_utf8_to_mbs); +#endif + /*iconv_cleanup(&iconv_utf8_to_mbs);*/ } - - +#if 0 bool utf8_str_contains_nonascii_chars(const utf8char *utf8_str) { @@ -261,3 +343,4 @@ utf8_str_contains_nonascii_chars(const utf8char *utf8_str) } while (*++utf8_str); return false; } +#endif