Skip to content

Commit

Permalink
ICU-21330 Use =0 and =1 plural forms in compact notation
Browse files Browse the repository at this point in the history 
See unicode-org#1632
  • Loading branch information
@sffc
sffc committed Mar 11, 2021
1 parent 6c26ea2 commit c26d99b
Show file tree
Hide file tree
Showing 16 changed files with 216 additions and 68 deletions.
31 changes: 20 additions & 11 deletions icu4c/source/i18n/number_compact.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ int32_t countZeros(const UChar *patternString, int32_t patternLength) {
} // namespace

// NOTE: patterns and multipliers both get zero-initialized.
CompactData::CompactData() : patterns(), multipliers(), largestMagnitude(0), isEmpty(TRUE) {
CompactData::CompactData() : patterns(), multipliers(), largestMagnitude(0), isEmpty(true) {
}

void CompactData::populate(const Locale &locale, const char *nsName, CompactStyle compactStyle,
Expand Down Expand Up @@ -104,14 +104,30 @@ int32_t CompactData::getMultiplier(int32_t magnitude) const {
return multipliers[magnitude];
}

const UChar *CompactData::getPattern(int32_t magnitude, StandardPlural::Form plural) const {
const UChar *CompactData::getPattern(
int32_t magnitude,
const PluralRules *rules,
const DecimalQuantity &dq) const {
if (magnitude < 0) {
return nullptr;
}
if (magnitude > largestMagnitude) {
magnitude = largestMagnitude;
}
const UChar *patternString = patterns[getIndex(magnitude, plural)];
const UChar *patternString = nullptr;
if (dq.hasIntegerValue()) {
int64_t i = dq.toLong(true);
if (i == 0) {
patternString = patterns[getIndex(magnitude, StandardPlural::Form::EQ_0)];
} else if (i == 1) {
patternString = patterns[getIndex(magnitude, StandardPlural::Form::EQ_1)];
}
if (patternString != nullptr) {
return patternString;
}
}
StandardPlural::Form plural = utils::getStandardPlural(rules, dq);
patternString = patterns[getIndex(magnitude, plural)];
if (patternString == nullptr && plural != StandardPlural::OTHER) {
// Fall back to "other" plural variant
patternString = patterns[getIndex(magnitude, StandardPlural::OTHER)];
Expand Down Expand Up @@ -166,12 +182,6 @@ void CompactData::CompactDataSink::put(const char *key, ResourceValue &value, UB
ResourceTable pluralVariantsTable = value.getTable(status);
if (U_FAILURE(status)) { return; }
for (int i4 = 0; pluralVariantsTable.getKeyAndValue(i4, key, value); ++i4) {

if (uprv_strcmp(key, "0") == 0 || uprv_strcmp(key, "1") == 0) {
// TODO(ICU-21258): Handle this case. For now, skip.
continue;
}

// Skip this magnitude/plural if we already have it from a child locale.
// Note: This also skips USE_FALLBACK entries.
StandardPlural::Form plural = StandardPlural::fromString(key, status);
Expand Down Expand Up @@ -296,8 +306,7 @@ void CompactHandler::processQuantity(DecimalQuantity &quantity, MicroProps &micr
magnitude -= multiplier;
}

StandardPlural::Form plural = utils::getStandardPlural(rules, quantity);
const UChar *patternString = data.getPattern(magnitude, plural);
const UChar *patternString = data.getPattern(magnitude, rules, quantity);
if (patternString == nullptr) {
// Use the default (non-compact) modifier.
// No need to take any action.
Expand Down
5 changes: 4 additions & 1 deletion icu4c/source/i18n/number_compact.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ class CompactData : public MultiplierProducer {

int32_t getMultiplier(int32_t magnitude) const U_OVERRIDE;

const UChar *getPattern(int32_t magnitude, StandardPlural::Form plural) const;
const UChar *getPattern(
int32_t magnitude,
const PluralRules *rules,
const DecimalQuantity &dq) const;

void getUniquePatterns(UVector &output, UErrorCode &status) const;

Expand Down
48 changes: 41 additions & 7 deletions icu4c/source/i18n/standardplural.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
U_NAMESPACE_BEGIN

static const char *gKeywords[StandardPlural::COUNT] = {
"zero", "one", "two", "few", "many", "other"
"zero", "one", "two", "few", "many", "other", "=0", "=1"
};

const char *StandardPlural::getKeyword(Form p) {
Expand Down Expand Up @@ -60,21 +60,55 @@ int32_t StandardPlural::indexOrNegativeFromString(const char *keyword) {
return ZERO;
}
break;
case '=':
if (uprv_strcmp(keyword, "0") == 0) {
return EQ_0;
} else if (uprv_strcmp(keyword, "1") == 0) {
return EQ_1;
}
break;
// Also allow "0" and "1"
case '0':
if (*keyword == 0) {
return EQ_0;
}
break;
case '1':
if (*keyword == 0) {
return EQ_1;
}
break;
default:
break;
}
return -1;
}

static const UChar gZero[] = { 0x7A, 0x65, 0x72, 0x6F };
static const UChar gOne[] = { 0x6F, 0x6E, 0x65 };
static const UChar gTwo[] = { 0x74, 0x77, 0x6F };
static const UChar gFew[] = { 0x66, 0x65, 0x77 };
static const UChar gMany[] = { 0x6D, 0x61, 0x6E, 0x79 };
static const UChar gOther[] = { 0x6F, 0x74, 0x68, 0x65, 0x72 };
static const UChar gZero[] = u"zero";
static const UChar gOne[] = u"one";
static const UChar gTwo[] = u"two";
static const UChar gFew[] = u"few";
static const UChar gMany[] = u"many";
static const UChar gOther[] = u"other";
static const UChar gEq0[] = u"=0";
static const UChar gEq1[] = u"=1";

int32_t StandardPlural::indexOrNegativeFromString(const UnicodeString &keyword) {
switch (keyword.length()) {
case 1:
if (keyword.charAt(0) == '0') {
return EQ_0;
} else if (keyword.charAt(0) == '1') {
return EQ_1;
}
break;
case 2:
if (keyword.compare(gEq0, 2) == 0) {
return EQ_0;
} else if (keyword.compare(gEq1, 2) == 0) {
return EQ_1;
}
break;
case 3:
if (keyword.compare(gOne, 3) == 0) {
return ONE;
Expand Down
2 changes: 2 additions & 0 deletions icu4c/source/i18n/standardplural.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ class U_I18N_API StandardPlural {
FEW,
MANY,
OTHER,
EQ_0,
EQ_1,
COUNT
};

Expand Down
1 change: 1 addition & 0 deletions icu4c/source/test/intltest/numbertest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ class DecimalQuantityTest : public IntlTest {
void testConvertToAccurateDouble();
void testUseApproximateDoubleWhenAble();
void testHardDoubleConversion();
void testFitsInLong();
void testToDouble();
void testMaxDigits();
void testNickelRounding();
Expand Down
18 changes: 8 additions & 10 deletions icu4c/source/test/intltest/numbertest_api.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -498,16 +498,14 @@ void NumberFormatterApiTest::notationCompact() {
1e7,
u"1000\u842C");

if (!logKnownIssue("21258", "StandardPlural cannot handle keywords 1, 0")) {
assertFormatSingle(
u"Compact with plural form =1 (ICU-21258)",
u"compact-long",
u"K",
NumberFormatter::with().notation(Notation::compactLong()),
Locale("fr-FR"),
1e3,
u"mille");
}
assertFormatSingle(
u"Compact with plural form =1 (ICU-21258)",
u"compact-long",
u"KK",
NumberFormatter::with().notation(Notation::compactLong()),
Locale("fr-FR"),
1e3,
u"mille");

assertFormatSingle(
u"Compact Infinity",
Expand Down
43 changes: 41 additions & 2 deletions icu4c/source/test/intltest/numbertest_decimalquantity.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ void DecimalQuantityTest::runIndexedTest(int32_t index, UBool exec, const char *
}
TESTCASE_AUTO(testUseApproximateDoubleWhenAble);
TESTCASE_AUTO(testHardDoubleConversion);
TESTCASE_AUTO(testFitsInLong);
TESTCASE_AUTO(testToDouble);
TESTCASE_AUTO(testMaxDigits);
TESTCASE_AUTO(testNickelRounding);
Expand Down Expand Up @@ -357,6 +358,44 @@ void DecimalQuantityTest::testHardDoubleConversion() {
}
}

void DecimalQuantityTest::testFitsInLong() {
IcuTestErrorCode status(*this, "testFitsInLong");
DecimalQuantity quantity;
quantity.setToInt(0);
assertTrue("Zero should fit", quantity.fitsInLong());
quantity.setToInt(42);
assertTrue("Small int should fit", quantity.fitsInLong());
quantity.setToDouble(0.1);
assertFalse("Fraction should not fit", quantity.fitsInLong());
quantity.setToDouble(42.1);
assertFalse("Fraction should not fit", quantity.fitsInLong());
quantity.setToLong(1000000);
assertTrue("Large low-precision int should fit", quantity.fitsInLong());
quantity.setToLong(1000000000000000000L);
assertTrue("10^19 should fit", quantity.fitsInLong());
quantity.setToLong(1234567890123456789L);
assertTrue("A number between 10^19 and max long should fit", quantity.fitsInLong());
quantity.setToLong(1234567890000000000L);
assertTrue("A number with trailing zeros less than max long should fit", quantity.fitsInLong());
quantity.setToLong(9223372026854775808L);
assertTrue("A number less than max long but with similar digits should fit",
quantity.fitsInLong());
quantity.setToLong(9223372036854775806L);
assertTrue("One less than max long should fit", quantity.fitsInLong());
quantity.setToLong(9223372036854775807L);
assertTrue("Max long should fit", quantity.fitsInLong());
assertEquals("Max long should equal toLong", 9223372036854775807L, quantity.toLong(false));
quantity.setToDecNumber("9223372036854775808", status);
assertFalse("One greater than max long should not fit", quantity.fitsInLong());
assertEquals("toLong(true) should truncate", 223372036854775808L, quantity.toLong(true));
quantity.setToDecNumber("9223372046854775806", status);
assertFalse("A number between max long and 10^20 should not fit", quantity.fitsInLong());
quantity.setToDecNumber("9223372046800000000", status);
assertFalse("A large 10^19 number with trailing zeros should not fit", quantity.fitsInLong());
quantity.setToDecNumber("10000000000000000000", status);
assertFalse("10^20 should not fit", quantity.fitsInLong());
}

void DecimalQuantityTest::testToDouble() {
IcuTestErrorCode status(*this, "testToDouble");
static const struct TestCase {
Expand Down Expand Up @@ -531,12 +570,12 @@ void DecimalQuantityTest::testScientificAndCompactSuppressedExponent() {
{u"scientific", 0.012, u"1,2E-2", 0L, 0.012, u"0.012", -2, -2},

{u"", 999.9, u"999,9", 999L, 999.9, u"999.9", 0, 0},
{u"compact-long", 999.9, u"1 millier", 1000L, 1000.0, u"1000", 3, 3},
{u"compact-long", 999.9, u"mille", 1000L, 1000.0, u"1000", 3, 3},
{u"compact-short", 999.9, u"1 k", 1000L, 1000.0, u"1000", 3, 3},
{u"scientific", 999.9, u"9,999E2", 999L, 999.9, u"999.9", 2, 2},

{u"", 1000.0, u"1 000", 1000L, 1000.0, u"1000", 0, 0},
{u"compact-long", 1000.0, u"1 millier", 1000L, 1000.0, u"1000", 3, 3},
{u"compact-long", 1000.0, u"mille", 1000L, 1000.0, u"1000", 3, 3},
{u"compact-short", 1000.0, u"1 k", 1000L, 1000.0, u"1000", 3, 3},
{u"scientific", 1000.0, u"1E3", 1000L, 1000.0, u"1000", 3, 3},
};
Expand Down
18 changes: 17 additions & 1 deletion icu4j/main/classes/core/src/com/ibm/icu/impl/StandardPlural.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ public enum StandardPlural {
TWO("two"),
FEW("few"),
MANY("many"),
OTHER("other");
OTHER("other"),
EQ_0("=0"),
EQ_1("=1");

/**
* Numeric index of OTHER, same as OTHER.ordinal().
Expand Down Expand Up @@ -60,6 +62,20 @@ public final String getKeyword() {
*/
public static final StandardPlural orNullFromString(CharSequence keyword) {
switch (keyword.length()) {
case 1:
if (keyword.charAt(0) == '0') {
return EQ_0;
} else if (keyword.charAt(0) == '1') {
return EQ_1;
}
break;
case 2:
if ("=0".contentEquals(keyword)) {
return EQ_0;
} else if ("=1".contentEquals(keyword)) {
return EQ_1;
}
break;
case 3:
if ("one".contentEquals(keyword)) {
return ONE;
Expand Down
28 changes: 16 additions & 12 deletions icu4j/main/classes/core/src/com/ibm/icu/impl/number/CompactData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.ibm.icu.impl.ICUResourceBundle;
import com.ibm.icu.impl.StandardPlural;
import com.ibm.icu.impl.UResource;
import com.ibm.icu.text.PluralRules;
import com.ibm.icu.text.CompactDecimalFormat.CompactStyle;
import com.ibm.icu.util.ICUException;
import com.ibm.icu.util.ULocale;
Expand Down Expand Up @@ -99,10 +100,6 @@ public void populate(Map<String, Map<String, String>> powersToPluralsToPatterns)
byte magnitude = (byte) (magnitudeEntry.getKey().length() - 1);
for (Map.Entry<String, String> pluralEntry : magnitudeEntry.getValue().entrySet()) {
String pluralString = pluralEntry.getKey().toString();
if ("0".equals(pluralString) || "1".equals(pluralString)) {
// TODO(ICU-21258): Handle this case. For now, skip.
continue;
}
StandardPlural plural = StandardPlural.fromString(pluralString);
String patternString = pluralEntry.getValue().toString();
patterns[getIndex(magnitude, plural)] = patternString;
Expand Down Expand Up @@ -130,14 +127,27 @@ public int getMultiplier(int magnitude) {
return multipliers[magnitude];
}

public String getPattern(int magnitude, StandardPlural plural) {
public String getPattern(int magnitude, PluralRules rules, DecimalQuantity dq) {
if (magnitude < 0) {
return null;
}
if (magnitude > largestMagnitude) {
magnitude = largestMagnitude;
}
String patternString = patterns[getIndex(magnitude, plural)];
String patternString = null;
if (dq.isHasIntegerValue()) {
long i = dq.toLong(true);
if (i == 0) {
patternString = patterns[getIndex(magnitude, StandardPlural.EQ_0)];
} else if (i == 1) {
patternString = patterns[getIndex(magnitude, StandardPlural.EQ_1)];
}
if (patternString != null) {
return patternString;
}
}
StandardPlural plural = dq.getStandardPlural(rules);
patternString = patterns[getIndex(magnitude, plural)];
if (patternString == null && plural != StandardPlural.OTHER) {
// Fall back to "other" plural variant
patternString = patterns[getIndex(magnitude, StandardPlural.OTHER)];
Expand Down Expand Up @@ -181,12 +191,6 @@ public void put(UResource.Key key, UResource.Value value, boolean isRoot) {
// Iterate over the plural variants ("one", "other", etc)
UResource.Table pluralVariantsTable = value.getTable();
for (int i4 = 0; pluralVariantsTable.getKeyAndValue(i4, key, value); ++i4) {

if ("0".equals(key.toString()) || "1".equals(key.toString())) {
// TODO(ICU-21258): Handle this case. For now, skip.
continue;
}

// Skip this magnitude/plural if we already have it from a child locale.
// Note: This also skips USE_FALLBACK entries.
StandardPlural plural = StandardPlural.fromString(key.toString());
Expand Down
12 changes: 12 additions & 0 deletions icu4j/main/classes/core/src/com/ibm/icu/impl/number/DecimalQuantity.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,18 @@ public interface DecimalQuantity extends PluralRules.IFixedDecimal {

public BigDecimal toBigDecimal();

/**
* Returns a long approximating the decimal quantity. A long can only represent the
* integral part of the number. Note: this method incorporates the value of
* {@code getExponent} (for cases such as compact notation) to return the proper long
* value represented by the result.
*
* @param truncateIfOverflow if false and the number does NOT fit, fails with an error.
* See comment about call site guards in DecimalQuantity_AbstractBCD.java
* @return A 64-bit integer representation of the internal number.
*/
public long toLong(boolean truncateIfOverflow);

public void setToBigDecimal(BigDecimal input);

public int maxRepresentableDigits();
Expand Down
15 changes: 6 additions & 9 deletions icu4j/main/classes/core/src/com/ibm/icu/impl/number/DecimalQuantity_AbstractBCD.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,11 @@ public void adjustExponent(int delta) {
exponent = exponent + delta;
}

@Override
public boolean isHasIntegerValue() {
return scale >= 0;
}

@Override
public StandardPlural getStandardPlural(PluralRules rules) {
if (rules == null) {
Expand Down Expand Up @@ -603,15 +608,7 @@ private void _setToBigDecimal(BigDecimal n) {
scale -= fracLength;
}

/**
* Returns a long approximating the internal BCD. A long can only represent the integral part of the
* number. Note: this method incorporates the value of {@code exponent}
* (for cases such as compact notation) to return the proper long value
* represented by the result.
*
* @param truncateIfOverflow if false and the number does NOT fit, fails with an assertion error.
* @return A 64-bit integer representation of the internal BCD.
*/
@Override
public long toLong(boolean truncateIfOverflow) {
// NOTE: Call sites should be guarded by fitsInLong(), like this:
// if (dq.fitsInLong()) { /* use dq.toLong() */ } else { /* use some fallback */ }
Expand Down
Loading

0 comments on commit c26d99b

Please sign in to comment.