procyberian/gcc
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

re PR libstdc++/64797 (22_locale/conversions/string/2.cc FAILs)

Browse source 
PR libstdc++/64797
	* include/bits/locale_conv.h (wstring_convert::_M_conv): Handle
	incomplete multibyte sequences correctly.
	* include/std/codecvt (codecvt_utf8, codecvt_utf16,
	codecvt_utf8_utf16): Limit _Maxcode to maximum Unicode code point.
	* src/c++11/codecvt.cc (invalid_mb_sequence, incomplete_mb_character):
	Define constants.
	(is_high_surrogate, is_low_surrogate, surrogate_pair_to_code_point):
	Define convenience functions.
	(read_utf8_code_point): Return relevant constant to distinguish
	incomplete characters from invalid sequences.
	(read_utf16_code_point): Likewise. Check for invalid sequences.
	(ucs4_in, utf16_in): Use incomplete_mb_character constant.
	(utf16_out): Check for invalid sequences.
	(utf16_span): Fix condition.
	(ucs2_out): Use is_high_surrogate.
	(ucs2_in): Use incomplete_mb_character constant and fix condition.
	* testsuite/22_locale/codecvt/char16_t.cc: Fix whitespace.
	* testsuite/22_locale/conversions/buffer/1.cc: New.
	* testsuite/22_locale/conversions/string/2.cc: Use char16_t and
	char32_t instead of wchar_t.
	* testsuite/22_locale/conversions/string/3.cc: New.

From-SVN: r221189
This commit is contained in:
Jonathan Wakely 2015-03-04 17:19:55 +00:00 • committed by Jonathan Wakely
parent d50a26f2ba
commit b6584a72ac
8 changed files with 273 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
libstdc++-v3/ChangeLog
View file

@ -1,3 +1,28 @@
2015-03-04 Jonathan Wakely <jwakely@redhat.com>
PR libstdc++/64797
* include/bits/locale_conv.h (wstring_convert::_M_conv): Handle
incomplete multibyte sequences correctly.
* include/std/codecvt (codecvt_utf8, codecvt_utf16,
codecvt_utf8_utf16): Limit _Maxcode to maximum Unicode code point.
* src/c++11/codecvt.cc (invalid_mb_sequence, incomplete_mb_character):
Define constants.
(is_high_surrogate, is_low_surrogate, surrogate_pair_to_code_point):
Define convenience functions.
(read_utf8_code_point): Return relevant constant to distinguish
incomplete characters from invalid sequences.
(read_utf16_code_point): Likewise. Check for invalid sequences.
(ucs4_in, utf16_in): Use incomplete_mb_character constant.
(utf16_out): Check for invalid sequences.
(utf16_span): Fix condition.
(ucs2_out): Use is_high_surrogate.
(ucs2_in): Use incomplete_mb_character constant and fix condition.
* testsuite/22_locale/codecvt/char16_t.cc: Fix whitespace.
* testsuite/22_locale/conversions/buffer/1.cc: New.
* testsuite/22_locale/conversions/string/2.cc: Use char16_t and
char32_t instead of wchar_t.
* testsuite/22_locale/conversions/string/3.cc: New.
2015-03-03 Iain Sandoe <iain@codesourcery.com>
PR libstdc++/64883

8
libstdc++-v3/include/bits/locale_conv.h
View file

@ -198,18 +198,20 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
auto __outstr = __err ? _OutStr(__err->get_allocator()) : _OutStr();
size_t __outchars = 0;
auto __next = __first;
const auto __maxlen = _M_cvt->max_length();
codecvt_base::result __result;
do
{
__outstr.resize(__outstr.size() + (__last - __next));
__outstr.resize(__outstr.size() + (__last - __next) + __maxlen);
auto __outnext = &__outstr.front() + __outchars;
auto const __outlast = &__outstr.back() + 1;
__result = ((*_M_cvt).*__memfn)(_M_state, __next, __last, __next,
__outnext, __outlast, __outnext);
__outchars = __outnext - &__outstr.front();
}
while (__result == codecvt_base::partial && __next != __last);
while (__result == codecvt_base::partial && __next != __last
&& (__outstr.size() - __outchars) < __maxlen);
__outstr.resize(__outchars);
_M_count = __next - __first;
@ -428,7 +430,7 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
return _M_put(__next, __pending);
if (!_M_put(__outbuf, __outnext - __outbuf))
return false;
return false;
}
while (__next != __last && __next != __start);

4
libstdc++-v3/include/std/codecvt
View file

@ -148,7 +148,9 @@ _GLIBCXX_BEGIN_NAMESPACE_VERSION
public: \
explicit \
_NAME(size_t __refs = 0) \
: __ ## _NAME ## _base<_ELEM>(_Maxcode, _Mode, __refs) { } \
: __ ## _NAME ## _base<_ELEM>(std::min(_Maxcode, 0x10fffful), \
_Mode, __refs) \
{ } \
}
template<typename _Elem> class __codecvt_utf8_base;

113
libstdc++-v3/src/c++11/codecvt.cc
View file

@ -35,8 +35,14 @@ namespace
{
// Largest code point that fits in a single UTF-16 code unit.
const char32_t max_single_utf16_unit = 0xFFFF;
const char32_t max_code_point = 0x10FFFF;
// The functions below rely on maxcode < incomplete_mb_character
// (which is enforced by the codecvt_utf* classes on construction).
const char32_t incomplete_mb_character = char32_t(-2);
const char32_t invalid_mb_sequence = char32_t(-1);
template<typename Elem>
struct range
{
@ -131,13 +137,13 @@ namespace
// Read a codepoint from a UTF-8 multibyte sequence.
// Updates from.next if the codepoint is not greater than maxcode.
// Returns -1 if there is an invalid or incomplete multibyte character.
// Returns invalid_mb_sequence, incomplete_mb_character or the code point.
char32_t
read_utf8_code_point(range<const char>& from, unsigned long maxcode)
{
size_t avail = from.size();
const size_t avail = from.size();
if (avail == 0)
return -1;
return incomplete_mb_character;
unsigned char c1 = from.next[0];
// https://en.wikipedia.org/wiki/UTF-8#Sample_code
if (c1 < 0x80)
@ -146,14 +152,14 @@ namespace
return c1;
}
else if (c1 < 0xC2) // continuation or overlong 2-byte sequence
return -1;
return invalid_mb_sequence;
else if (c1 < 0xE0) // 2-byte sequence
{
if (avail < 2)
return -1;
return incomplete_mb_character;
unsigned char c2 = from.next[1];
if ((c2 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
char32_t c = (c1 << 6) + c2 - 0x3080;
if (c <= maxcode)
from.next += 2;
@ -162,15 +168,15 @@ namespace
else if (c1 < 0xF0) // 3-byte sequence
{
if (avail < 3)
return -1;
return incomplete_mb_character;
unsigned char c2 = from.next[1];
if ((c2 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
if (c1 == 0xE0 && c2 < 0xA0) // overlong
return -1;
return invalid_mb_sequence;
unsigned char c3 = from.next[2];
if ((c3 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
char32_t c = (c1 << 12) + (c2 << 6) + c3 - 0xE2080;
if (c <= maxcode)
from.next += 3;
@ -179,27 +185,27 @@ namespace
else if (c1 < 0xF5) // 4-byte sequence
{
if (avail < 4)
return -1;
return incomplete_mb_character;
unsigned char c2 = from.next[1];
if ((c2 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
if (c1 == 0xF0 && c2 < 0x90) // overlong
return -1;
return invalid_mb_sequence;
if (c1 == 0xF4 && c2 >= 0x90) // > U+10FFFF
return -1;
return invalid_mb_sequence;
unsigned char c3 = from.next[2];
if ((c3 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
unsigned char c4 = from.next[3];
if ((c4 & 0xC0) != 0x80)
return -1;
return invalid_mb_sequence;
char32_t c = (c1 << 18) + (c2 << 12) + (c3 << 6) + c4 - 0x3C82080;
if (c <= maxcode)
from.next += 4;
return c;
}
else // > U+10FFFF
return -1;
return invalid_mb_sequence;
}
bool
@ -250,27 +256,54 @@ namespace
#endif
}
// Return true if c is a high-surrogate (aka leading) code point.
inline bool
is_high_surrogate(char32_t c)
{
return c >= 0xD800 && c <= 0xDBFF;
}
// Return true if c is a low-surrogate (aka trailing) code point.
inline bool
is_low_surrogate(char32_t c)
{
return c >= 0xDC00 && c <= 0xDFFF;
}
inline char32_t
surrogate_pair_to_code_point(char32_t high, char32_t low)
{
return (high << 10) + low - 0x35FDC00;
}
// Read a codepoint from a UTF-16 multibyte sequence.
// The sequence's endianness is indicated by (mode & little_endian).
// Updates from.next if the codepoint is not greater than maxcode.
// Returns -1 if there is an incomplete multibyte character.
// Returns invalid_mb_sequence, incomplete_mb_character or the code point.
char32_t
read_utf16_code_point(range<const char16_t>& from, unsigned long maxcode,
codecvt_mode mode)
{
const size_t avail = from.size();
if (avail == 0)
return incomplete_mb_character;
int inc = 1;
char32_t c = adjust_byte_order(from.next[0], mode);
if (c >= 0xD800 && c <= 0xDBFF)
if (is_high_surrogate(c))
{
if (from.size() < 2)
return -1;
if (avail < 2)
return incomplete_mb_character;
const char16_t c2 = adjust_byte_order(from.next[1], mode);
if (c2 >= 0xDC00 && c2 <= 0xDFFF)
if (is_low_surrogate(c2))
{
c = (c << 10) + c2 - 0x35FDC00;
c = surrogate_pair_to_code_point(c, c2);
inc = 2;
}
else
return invalid_mb_sequence;
}
else if (is_low_surrogate(c))
return invalid_mb_sequence;
if (c <= maxcode)
from.next += inc;
return c;
@ -314,8 +347,8 @@ namespace
while (from.size() && to.size())
{
const char32_t codepoint = read_utf8_code_point(from, maxcode);
if (codepoint == char32_t(-1))
break;
if (codepoint == incomplete_mb_character)
return codecvt_base::partial;
if (codepoint > maxcode)
return codecvt_base::error;
*to.next++ = codepoint;
@ -352,8 +385,8 @@ namespace
while (from.size() && to.size())
{
const char32_t codepoint = read_utf16_code_point(from, maxcode, mode);
if (codepoint == char32_t(-1))
break;
if (codepoint == incomplete_mb_character)
return codecvt_base::partial;
if (codepoint > maxcode)
return codecvt_base::error;
*to.next++ = codepoint;
@ -389,11 +422,9 @@ namespace
read_utf8_bom(from, mode);
while (from.size() && to.size())
{
const char* first = from.next;
if ((unsigned char)*first >= 0xF0 && to.size() < 2)
return codecvt_base::partial;
const char* const first = from.next;
const char32_t codepoint = read_utf8_code_point(from, maxcode);
if (codepoint == char32_t(-1))
if (codepoint == incomplete_mb_character)
return codecvt_base::partial;
if (codepoint > maxcode)
return codecvt_base::error;
@ -418,20 +449,22 @@ namespace
{
char32_t c = from.next[0];
int inc = 1;
if (c >= 0xD800 && c <= 0xDBFF) // start of surrogate pair
if (is_high_surrogate(c))
{
if (from.size() < 2)
return codecvt_base::ok; // stop converting at this point
const char32_t c2 = from.next[1];
if (c2 >= 0xDC00 && c2 <= 0xDFFF)
if (is_low_surrogate(c2))
{
c = surrogate_pair_to_code_point(c, c2);
inc = 2;
c = (c << 10) + c2 - 0x35FDC00;
}
else
return codecvt_base::error;
}
else if (is_low_surrogate(c))
return codecvt_base::error;
if (c > maxcode)
return codecvt_base::error;
if (!write_utf8_code_point(to, c))
@ -452,8 +485,8 @@ namespace
while (count+1 < max)
{
char32_t c = read_utf8_code_point(from, maxcode);
if (c == char32_t(-1))
break;
if (c > maxcode)
return from.next;
else if (c > max_single_utf16_unit)
++count;
++count;
@ -489,7 +522,7 @@ namespace
while (from.size() && to.size())
{
char16_t c = from.next[0];
if (c >= 0xD800 && c <= 0xDBFF) // start of surrogate pair
if (is_high_surrogate(c))
return codecvt_base::error;
if (c > maxcode)
return codecvt_base::error;
@ -510,9 +543,9 @@ namespace
while (from.size() && to.size())
{
const char32_t c = read_utf16_code_point(from, maxcode, mode);
if (c == char32_t(-1))
break;
if (c >= maxcode)
if (c == incomplete_mb_character)
return codecvt_base::partial;
if (c > maxcode)
return codecvt_base::error;
*to.next++ = c;
}

3
libstdc++-v3/testsuite/22_locale/codecvt/char16_t.cc
View file

@ -79,8 +79,7 @@ test01()
codecvt_c16::state_type state01;
state01 = {};
codecvt_base::result res = cvt->out(state01, u16dat, u16dat_end,
from_next,
codecvt_base::result res = cvt->out(state01, u16dat, u16dat_end, from_next,
buffer, buffer_end, to_next);
VERIFY(res == codecvt_base::ok);

78
libstdc++-v3/testsuite/22_locale/conversions/buffer/1.cc Normal file
View file

@ -0,0 +1,78 @@
// { dg-options "-std=gnu++11" }
// Copyright (C) 2012 Free Software Foundation
//
// This file is part of the GNU ISO C++ Library. This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along
// with this library; see the file COPYING3. If not see
// <http://www.gnu.org/licenses/>.
// 22.3.3.2.3 Buffer conversions
#include <locale>
#include <sstream>
#include <testsuite_hooks.h>
template<typename Elem>
struct cvt : std::codecvt<Elem, char, std::mbstate_t> { };
template<typename Elem>
using buf_conv = std::wbuffer_convert<cvt<Elem>, Elem>;
using std::string;
using std::stringstream;
using std::wstring;
using std::wstringstream;
void test01()
{
buf_conv<wchar_t> buf;
std::stringbuf sbuf;
VERIFY( buf.rdbuf() == nullptr );
VERIFY( buf.rdbuf(&sbuf) == nullptr );
VERIFY( buf.rdbuf() == &sbuf );
VERIFY( buf.rdbuf(nullptr) == &sbuf );
}
void test02()
{
std::stringbuf sbuf;
buf_conv<char> buf(&sbuf); // noconv
stringstream ss;
ss.std::ios::rdbuf(&buf);
string input = "King for a day...";
ss << input << std::flush;
string output = sbuf.str();
VERIFY( input == output );
}
void test03()
{
std::stringbuf sbuf;
buf_conv<wchar_t> buf(&sbuf);
wstringstream ss;
ss.std::wios::rdbuf(&buf);
wstring input = L"Fool for a lifetime";
ss << input << std::flush;
string output = sbuf.str();
VERIFY( output == "Fool for a lifetime" );
}
int main()
{
test01();
test02();
test03();
}

36
libstdc++-v3/testsuite/22_locale/conversions/string/2.cc
View file

@ -30,26 +30,43 @@ template<typename Elem>
using str_conv = std::wstring_convert<cvt<Elem>, Elem>;
using std::string;
using std::wstring;
using std::u16string;
using std::u32string;
// test conversion errors, with and without error strings
void test01()
{
typedef str_conv<wchar_t> sc;
typedef str_conv<char16_t> sc;
const sc::byte_string berr = "invalid wide string";
const sc::wide_string werr = L"invalid byte string";
const sc::wide_string werr = u"invalid byte string";
sc c(berr, werr);
string input = "Stop";
input += char(0xff);
input += char(0xff);
wstring woutput = c.from_bytes(input);
input += char(0xFF);
u16string woutput = c.from_bytes(input);
VERIFY( werr == woutput );
wstring winput = L"Stop";
winput += wchar_t(0xff);
winput += wchar_t(0xff);
u16string winput = u"Stop";
winput += char16_t(0xDC00);
string output = c.to_bytes(winput);
VERIFY( berr == output );
}
void test02()
{
typedef str_conv<char32_t> sc;
const sc::byte_string berr = "invalid wide string";
const sc::wide_string werr = U"invalid byte string";
sc c(berr, werr);
string input = "Halt";
input += char(0xff);
u32string woutput = c.from_bytes(input);
VERIFY( werr == woutput );
u32string winput = U"Halt";
winput += char32_t(-1);
string output = c.to_bytes(winput);
VERIFY( berr == output );
}
@ -57,4 +74,5 @@ void test01()
int main()
{
test01();
test02();
}

61
libstdc++-v3/testsuite/22_locale/conversions/string/3.cc Normal file
View file

@ -0,0 +1,61 @@
// { dg-options "-std=gnu++11" }
// Copyright (C) 2012 Free Software Foundation
//
// This file is part of the GNU ISO C++ Library. This library is free
// software; you can redistribute it and/or modify it under the
// terms of the GNU General Public License as published by the
// Free Software Foundation; either version 3, or (at your option)
// any later version.
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
// You should have received a copy of the GNU General Public License along
// with this library; see the file COPYING3. If not see
// <http://www.gnu.org/licenses/>.
// 22.3.3.2.2 String conversions
#include <locale>
#include <string>
#include <testsuite_hooks.h>
template<typename Elem>
struct cvt : std::codecvt<Elem, char, std::mbstate_t> { };
template<typename Elem>
using str_conv = std::wstring_convert<cvt<Elem>, Elem>;
using std::string;
using std::u32string;
// test construction with state, for partial conversions
void test01()
{
typedef str_conv<char32_t> wsc;
wsc c;
string input = u8"\u00a3 shillings pence";
u32string woutput = c.from_bytes(input.substr(0, 1));
auto partial_state = c.state();
auto partial_count = c.converted();
auto woutput2 = c.from_bytes("state reset on next conversion");
VERIFY( woutput2 == U"state reset on next conversion" );
wsc c2(new cvt<char32_t>, partial_state);
woutput += c2.from_bytes(input.substr(partial_count));
VERIFY( U"\u00a3 shillings pence" == woutput );
string roundtrip = c2.to_bytes(woutput);
VERIFY( input == roundtrip );
}
int main()
{
test01();
}