Skip to content

Commit

Permalink
Fix a bug in the generation of the Unicode joining property data
Browse files Browse the repository at this point in the history 
The code that was supposed to initialize the joining property of
characters to the correct defaults was actually applied after reading
in the data from the Unicode file, and was in a couple of cases
overwriting explicitly specified data in ArabicShaping.txt

Task-number: QTBUG-63191
Change-Id: Ie35261039b2211a827322ca11afacd9555ccefc7
Reviewed-by: Konstantin Ritt <[email protected]>
  • Loading branch information
@laknoll
laknoll committed Dec 13, 2017
1 parent 457fe3e commit 59febb4
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 26 deletions.
20 changes: 10 additions & 10 deletions src/corelib/tools/qunicodetables.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6083,9 +6083,9 @@ static const Properties uc_properties[] = {
{ 18, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 8, 13, 7 },
{ 25, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 8, 12, 7 },
{ 25, 1, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 12, 0, 12, 7 },
{ 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 4, 4, 12, 8 },
{ 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 4, 4, 12, 8 },
{ 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 4, 4, 12, 2 },
{ 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 3, 4, 4, 12, 8 },
{ 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 3, 4, 4, 12, 8 },
{ 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3, 4, 4, 12, 2 },
{ 26, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 12, 8 },
{ 26, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 12, 8 },
{ 25, 4, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 10, 8 },
Expand Down Expand Up @@ -6146,7 +6146,7 @@ static const Properties uc_properties[] = {
{ 18, 13, 0, 3, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 8, 8, 12, 8 },
{ 25, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 12, 6, 8 },
{ 0, 17, 230, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 8 },
{ 10, 5, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 12, 2 },
{ 10, 5, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 12, 2 },
{ 29, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 12, 8 },
{ 0, 17, 220, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 8 },
{ 17, 13, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 8, 12, 8 },
Expand Down Expand Up @@ -6749,7 +6749,7 @@ static const Properties uc_properties[] = {
{ 25, 10, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 12, 6, 33 },
{ 25, 10, 0, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 12, 33 },
{ 0, 17, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 4, 4, 4, 21, 33 },
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4, 4, 4, 33 },
{ 10, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 3, 4, 4, 4, 33 },
{ 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
{ 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
{ 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 14, 9, 11, 33 },
Expand Down Expand Up @@ -7058,7 +7058,7 @@ static const Properties uc_properties[] = {
{ 6, 9, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 5, 17, 2 },
{ 6, 9, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 5, 4, 2 },
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 4, 20, 2 },
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
{ 10, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
{ 10, 18, 0, 1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 4, 4, 21, 1 },
{ 10, 0, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
{ 10, 1, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
Expand Down Expand Up @@ -7108,10 +7108,10 @@ static const Properties uc_properties[] = {
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 4, 4, 12, 2 },
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 3, 4, 4, 12, 2 },
{ 13, 18, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 12, 0 },
{ 10, 19, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 20, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 21, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 22, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 19, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 20, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 21, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 22, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 3, 4, 4, 21, 2 },
{ 10, 18, 0, 5, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 4, 4, 21, 2 },
{ 5, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 80, 0, 0, 0, 12, 2 },
{ 17, 0, 0, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 80, 0, 8, 6, 12, 3 },
Expand Down
4 changes: 2 additions & 2 deletions src/corelib/tools/qunicodetables_p.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,11 @@
// We mean it.
//

#include <QtCore/private/qglobal_p.h>

#ifndef QUNICODETABLES_P_H
#define QUNICODETABLES_P_H

#include <QtCore/private/qglobal_p.h>

#include <QtCore/qchar.h>

QT_BEGIN_NAMESPACE
Expand Down
30 changes: 16 additions & 14 deletions util/unicode/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1293,6 +1293,18 @@ static void readArabicShaping()
{
qDebug("Reading ArabicShaping.txt");

// Initialize defaults:
// Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
// - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
// - All others not explicitly listed have joining type U.
for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
UnicodeData &d = UnicodeData::valueRef(codepoint);
if (d.p.joining == QChar::Joining_None) {
if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
d.p.joining = QChar::Joining_Transparent;
}
}

QFile f("data/ArabicShaping.txt");
if (!f.exists())
qFatal("Couldn't find ArabicShaping.txt");
Expand Down Expand Up @@ -1338,17 +1350,6 @@ static void readArabicShaping()
break;
}
}

// Code points that are not explicitly listed in ArabicShaping.txt are either of joining type T or U:
// - Those that not explicitly listed that are of General Category Mn, Me, or Cf have joining type T.
// - All others not explicitly listed have joining type U.
for (int codepoint = 0; codepoint <= QChar::LastValidCodePoint; ++codepoint) {
UnicodeData &d = UnicodeData::valueRef(codepoint);
if (d.p.joining == QChar::Joining_None) {
if (d.p.category == QChar::Mark_NonSpacing || d.p.category == QChar::Mark_Enclosing || d.p.category == QChar::Other_Format)
d.p.joining = QChar::Joining_Transparent;
}
}
}

static void readDerivedAge()
Expand Down Expand Up @@ -2525,7 +2526,7 @@ static QByteArray createSpecialCaseMap()
out.chop(1);
out += "\n};\n\n";

qDebug(" memory usage: %d bytes", specialCaseMap.size()*sizeof(unsigned short));
qDebug(" memory usage: %ld bytes", specialCaseMap.size()*sizeof(unsigned short));

return out;
}
Expand Down Expand Up @@ -3021,7 +3022,7 @@ int main(int, char **)
"****************************************************************************/\n\n";

QByteArray note =
"/* This file is autogenerated from the Unicode "DATA_VERSION_S" database. Do not edit */\n\n";
"/* This file is autogenerated from the Unicode " DATA_VERSION_S " database. Do not edit */\n\n";

QByteArray warning =
"//\n"
Expand Down Expand Up @@ -3062,9 +3063,10 @@ int main(int, char **)
f.write(warning);
f.write("#ifndef QUNICODETABLES_P_H\n"
"#define QUNICODETABLES_P_H\n\n"
"#include <QtCore/private/qglobal_p.h>\n\n"
"#include <QtCore/qchar.h>\n\n"
"QT_BEGIN_NAMESPACE\n\n");
f.write("#define UNICODE_DATA_VERSION "DATA_VERSION_STR"\n\n");
f.write("#define UNICODE_DATA_VERSION " DATA_VERSION_STR "\n\n");
f.write("namespace QUnicodeTables {\n\n");
f.write(property_string);
f.write(grapheme_break_class_string);
Expand Down

0 comments on commit 59febb4

Please sign in to comment.