Skip to content

Commit

Permalink
update utf8-cpp library.
Browse files Browse the repository at this point in the history
  • Loading branch information
@scamille
scamille committed Nov 10, 2019
1 parent e7933df commit ae19c05
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 197 deletions.
18 changes: 9 additions & 9 deletions engine/util/utf8/checked.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,29 +42,29 @@ namespace utf8
uint32_t cp;
public:
invalid_code_point(uint32_t cp) : cp(cp) {}
const char* what() const throw() override { return "Invalid code point"; }
virtual const char* what() const throw() { return "Invalid code point"; }
uint32_t code_point() const {return cp;}
};

class invalid_utf8 : public exception {
uint8_t u8;
public:
invalid_utf8 (uint8_t u) : u8(u) {}
const char* what() const throw() override { return "Invalid UTF-8"; }
virtual const char* what() const throw() { return "Invalid UTF-8"; }
uint8_t utf8_octet() const {return u8;}
};

class invalid_utf16 : public exception {
uint16_t u16;
public:
invalid_utf16 (uint16_t u) : u16(u) {}
const char* what() const throw() override { return "Invalid UTF-16"; }
virtual const char* what() const throw() { return "Invalid UTF-16"; }
uint16_t utf16_word() const {return u16;}
};

class not_enough_room : public exception {
public:
const char* what() const throw() override { return "Not enough space"; }
virtual const char* what() const throw() { return "Not enough space"; }
};

/// The library API - functions intended to be called by the users
Expand Down Expand Up @@ -109,16 +109,16 @@ namespace utf8
case internal::NOT_ENOUGH_ROOM:
throw not_enough_room();
case internal::INVALID_LEAD:
utf8::append (replacement, out);
out = utf8::append (replacement, out);
++start;
break;
case internal::INCOMPLETE_SEQUENCE:
case internal::OVERLONG_SEQUENCE:
case internal::INVALID_CODE_POINT:
utf8::append (replacement, out);
out = utf8::append (replacement, out);
++start;
// just one replacement mark for the sequence
while (utf8::internal::is_trail(*start) && start != end)
while (start != end && utf8::internal::is_trail(*start))
++start;
break;
}
Expand All @@ -137,7 +137,7 @@ namespace utf8
uint32_t next(octet_iterator& it, octet_iterator end)
{
uint32_t cp = 0;
internal::utf_error err_code = utf8::internal::validate_next(it, end, &cp);
internal::utf_error err_code = utf8::internal::validate_next(it, end, cp);
switch (err_code) {
case internal::UTF8_OK :
break;
Expand Down Expand Up @@ -270,7 +270,7 @@ namespace utf8
octet_iterator range_start;
octet_iterator range_end;
public:
iterator () {};
iterator () {}
explicit iterator (const octet_iterator& octet_it,
const octet_iterator& range_start,
const octet_iterator& range_end) :
Expand Down
207 changes: 86 additions & 121 deletions engine/util/utf8/core.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,11 @@ namespace internal
return (cp <= CODE_POINT_MAX && !utf8::internal::is_surrogate(cp));
}

inline unsigned int
sequence_length_cp( uint8_t lead )
template <typename octet_iterator>
inline typename std::iterator_traits<octet_iterator>::difference_type
sequence_length(octet_iterator lead_it)
{
uint8_t lead = utf8::internal::mask8(*lead_it);
if (lead < 0x80)
return 1;
else if ((lead >> 5) == 0x6)
Expand All @@ -110,27 +112,19 @@ namespace internal
return 0;
}

template <typename octet_iterator>
inline typename std::iterator_traits<octet_iterator>::difference_type
sequence_length(octet_iterator lead_it)
{
uint8_t lead = utf8::internal::mask8(*lead_it);
return sequence_length_cp( lead );
}

template <typename octet_difference_type>
inline bool is_overlong_sequence(uint32_t cp, octet_difference_type length)
{
if (cp < 0x80) {
if (length != 1)
if (length != 1)
return true;
}
else if (cp < 0x800) {
if (length != 2)
if (length != 2)
return true;
}
else if (cp < 0x10000) {
if (length != 3)
if (length != 3)
return true;
}

Expand All @@ -139,123 +133,94 @@ namespace internal

enum utf_error {UTF8_OK, NOT_ENOUGH_ROOM, INVALID_LEAD, INCOMPLETE_SEQUENCE, OVERLONG_SEQUENCE, INVALID_CODE_POINT};

/// get_sequence_x functions decode utf-8 sequences of the length x
/// Helper for get_sequence_x
template <typename octet_iterator>
utf_error increase_safely(octet_iterator& it, octet_iterator end)
{
if (++it == end)
return NOT_ENOUGH_ROOM;

if (!utf8::internal::is_trail(*it))
return INCOMPLETE_SEQUENCE;

return UTF8_OK;
}

#define UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(IT, END) {utf_error ret = increase_safely(IT, END); if (ret != UTF8_OK) return ret;}

/// get_sequence_x functions decode utf-8 sequences of the length x
template <typename octet_iterator>
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t* code_point)
utf_error get_sequence_1(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
if (it != end) {
if (code_point)
*code_point = utf8::internal::mask8(*it);
return UTF8_OK;
}
return NOT_ENOUGH_ROOM;
if (it == end)
return NOT_ENOUGH_ROOM;

code_point = utf8::internal::mask8(*it);

return UTF8_OK;
}

template <typename octet_iterator>
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t* code_point)
utf_error get_sequence_2(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;
if (it == end)
return NOT_ENOUGH_ROOM;

code_point = utf8::internal::mask8(*it);

if (it != end) {
uint32_t cp = utf8::internal::mask8(*it);
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
code_point = ((code_point << 6) & 0x7ff) + ((*it) & 0x3f);

return ret_code;
return UTF8_OK;
}

template <typename octet_iterator>
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t* code_point)
utf_error get_sequence_3(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;

if (it != end) {
uint32_t cp = utf8::internal::mask8(*it);
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp = ((cp << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp += (*it) & 0x3f;

if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
if (it == end)
return NOT_ENOUGH_ROOM;

code_point = utf8::internal::mask8(*it);

return ret_code;
UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

code_point = ((code_point << 12) & 0xffff) + ((utf8::internal::mask8(*it) << 6) & 0xfff);

UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

code_point += (*it) & 0x3f;

return UTF8_OK;
}

template <typename octet_iterator>
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t* code_point)
utf_error get_sequence_4(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
utf_error ret_code = NOT_ENOUGH_ROOM;

if (it != end) {
uint32_t cp = utf8::internal::mask8(*it);
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp = ((cp << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp += (utf8::internal::mask8(*it) << 6) & 0xfff;
if (++it != end) {
if (utf8::internal::is_trail(*it)) {
cp += (*it) & 0x3f;

if (code_point)
*code_point = cp;
ret_code = UTF8_OK;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
else
ret_code = INCOMPLETE_SEQUENCE;
}
else
ret_code = NOT_ENOUGH_ROOM;
}
if (it == end)
return NOT_ENOUGH_ROOM;

code_point = utf8::internal::mask8(*it);

UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

code_point = ((code_point << 18) & 0x1fffff) + ((utf8::internal::mask8(*it) << 12) & 0x3ffff);

UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

return ret_code;
code_point += (utf8::internal::mask8(*it) << 6) & 0xfff;

UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR(it, end)

code_point += (*it) & 0x3f;

return UTF8_OK;
}

#undef UTF8_CPP_INCREASE_AND_RETURN_ON_ERROR

template <typename octet_iterator>
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t* code_point)
utf_error validate_next(octet_iterator& it, octet_iterator end, uint32_t& code_point)
{
// Save the original value of it so we can go back in case of failure
// Of course, it does not make much sense with i.e. stream iterators
Expand All @@ -264,24 +229,24 @@ namespace internal
uint32_t cp = 0;
// Determine the sequence length based on the lead octet
typedef typename std::iterator_traits<octet_iterator>::difference_type octet_difference_type;
octet_difference_type length = utf8::internal::sequence_length(it);
if (length == 0)
return INVALID_LEAD;
const octet_difference_type length = utf8::internal::sequence_length(it);

// Now that we have a valid sequence length, get trail octets and calculate the code point
// Get trail octets and calculate the code point
utf_error err = UTF8_OK;
switch (length) {
case 0:
return INVALID_LEAD;
case 1:
err = utf8::internal::get_sequence_1(it, end, &cp);
err = utf8::internal::get_sequence_1(it, end, cp);
break;
case 2:
err = utf8::internal::get_sequence_2(it, end, &cp);
err = utf8::internal::get_sequence_2(it, end, cp);
break;
case 3:
err = utf8::internal::get_sequence_3(it, end, &cp);
err = utf8::internal::get_sequence_3(it, end, cp);
break;
case 4:
err = utf8::internal::get_sequence_4(it, end, &cp);
err = utf8::internal::get_sequence_4(it, end, cp);
break;
}

Expand All @@ -290,15 +255,14 @@ namespace internal
if (utf8::internal::is_code_point_valid(cp)) {
if (!utf8::internal::is_overlong_sequence(cp, length)){
// Passed! Return here.
if (code_point)
*code_point = cp;
code_point = cp;
++it;
return UTF8_OK;
}
else
err = OVERLONG_SEQUENCE;
}
else
else
err = INVALID_CODE_POINT;
}

Expand All @@ -309,7 +273,8 @@ namespace internal

template <typename octet_iterator>
inline utf_error validate_next(octet_iterator& it, octet_iterator end) {
return utf8::internal::validate_next(it, end, 0);
uint32_t ignored;
return utf8::internal::validate_next(it, end, ignored);
}

} // namespace internal
Expand Down Expand Up @@ -346,8 +311,8 @@ namespace internal
((it != end) && (utf8::internal::mask8(*it)) == bom[2])
);
}

//Deprecated in release 2.3
//Deprecated in release 2.3
template <typename octet_iterator>
inline bool is_bom (octet_iterator it)
{
Expand Down
Loading

0 comments on commit ae19c05

Please sign in to comment.