From 66f918db45dbf0c388fbc58bfdd1c6f99e063118 Mon Sep 17 00:00:00 2001 From: Mihai Nita Date: Thu, 13 Feb 2025 17:36:01 +0000 Subject: [PATCH] ICU-23044 MF2, ICU4J, bring the implementation close to the spec at LDML 47 --- .../dev/test/message2/Mf2FeaturesTest.java | 14 +- .../message2/DateTimeFormatterFactory.java | 15 +- .../com/ibm/icu/message2/Directionality.java | 78 +++++ .../icu/message2/FormattedPlaceholder.java | 47 ++- .../message2/IdentityFormatterFactory.java | 11 +- .../com/ibm/icu/message2/MFDataModel.java | 60 ++-- .../icu/message2/MFDataModelFormatter.java | 208 ++++++++++---- .../icu/message2/MFDataModelValidator.java | 24 +- .../ibm/icu/message2/MFFunctionRegistry.java | 16 +- .../java/com/ibm/icu/message2/MFParser.java | 267 +++++++++++------- .../com/ibm/icu/message2/MFSerializer.java | 31 +- .../ibm/icu/message2/MessageFormatter.java | 71 ++++- .../icu/message2/NumberFormatterFactory.java | 213 +++++++++++--- .../java/com/ibm/icu/message2/OptUtils.java | 120 +++++++- .../com/ibm/icu/message2/StringUtils.java | 38 ++- .../ibm/icu/message2/TextSelectorFactory.java | 7 +- .../ibm/icu/dev/test/message2/CoreTest.java | 72 +++-- .../CustomFormatterMessageRefTest.java | 6 +- .../message2/CustomFormatterPersonTest.java | 4 +- .../test/message2/DefaultTestProperties.java | 25 +- .../ibm/icu/dev/test/message2/ExpErrors.java | 44 +++ .../test/message2/ExpectedErrorAdapter.java | 68 +++++ .../dev/test/message2/MessageFormat2Test.java | 29 +- .../ibm/icu/dev/test/message2/Mf2IcuTest.java | 3 +- .../com/ibm/icu/dev/test/message2/Param.java | 7 + .../dev/test/message2/SerializationTest.java | 21 +- .../test/message2/StringToListAdapter.java | 1 + .../test/message2/TestFunctionFactory.java | 252 +++++++++++++++++ .../ibm/icu/dev/test/message2/TestUtils.java | 50 +++- .../com/ibm/icu/dev/test/message2/Unit.java | 12 +- .../message2/alias-selector-annotations.json | 4 +- .../dev/test/message2/icu-test-functions.json | 4 - .../message2/icu-test-previous-release.json | 24 +- .../dev/test/message2/icu-test-selectors.json | 78 +++-- .../dev/test/message2/matches-whitespace.json | 25 +- .../test/message2/more-data-model-errors.json | 26 +- .../icu/dev/test/message2/normalization.json | 8 +- .../dev/test/message2/resolution-errors.json | 4 +- .../icu/dev/test/message2/runtime-errors.json | 6 +- .../ibm/icu/dev/test/message2/spec/bidi.json | 146 ++++++++++ .../test/message2/spec/data-model-errors.json | 24 +- .../icu/dev/test/message2/spec/fallback.json | 52 ++++ .../message2/spec/functions/currency.json | 61 ++++ .../test/message2/spec/functions/date.json | 15 +- .../message2/spec/functions/datetime.json | 13 +- .../test/message2/spec/functions/integer.json | 21 +- .../test/message2/spec/functions/math.json | 77 +++++ .../test/message2/spec/functions/number.json | 240 ++-------------- .../test/message2/spec/functions/string.json | 48 +++- .../test/message2/spec/functions/time.json | 15 +- .../test/message2/spec/pattern-selection.json | 120 ++++++++ .../dev/test/message2/spec/syntax-errors.json | 109 +++++-- .../icu/dev/test/message2/spec/syntax.json | 161 +++++++---- .../icu/dev/test/message2/spec/u-options.json | 148 ++++++++++ .../icu/dev/test/message2/valid-tests.json | 2 +- 55 files changed, 2441 insertions(+), 804 deletions(-) create mode 100644 icu4j/main/core/src/main/java/com/ibm/icu/message2/Directionality.java create mode 100644 icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpErrors.java create mode 100644 icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpectedErrorAdapter.java create mode 100644 icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestFunctionFactory.java create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/bidi.json create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/fallback.json create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/currency.json create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/math.json create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/pattern-selection.json create mode 100644 icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/u-options.json diff --git a/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java b/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java index 77c0fa3604d7..bf53948aae6f 100644 --- a/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java +++ b/icu4j/main/common_tests/src/test/java/com/ibm/icu/dev/test/message2/Mf2FeaturesTest.java @@ -328,7 +328,7 @@ public void testAllKindOfNumbers() { public void testSpecialPluralWithDecimals() { String message; message = ".local $amount = {$count :number}\n" - + ".match {$amount :number}\n" + + ".match $amount\n" + " 1 {{I have {$amount} dollar.}}\n" + " * {{I have {$amount} dollars.}}"; TestUtils.runTestCase(new TestCase.Builder() @@ -338,7 +338,7 @@ public void testSpecialPluralWithDecimals() { .expected("I have 1 dollar.") .build()); message = ".local $amount = {$count :number minimumFractionDigits=2}\n" - + ".match {$amount :number minimumFractionDigits=2}\n" + + ".match $amount\n" + " one {{I have {$amount} dollar.}}\n" + " * {{I have {$amount} dollars.}}"; TestUtils.runTestCase(new TestCase.Builder() @@ -367,7 +367,8 @@ public void testDefaultFunctionAndOptions() { @Test public void testSimpleSelection() { - String message = ".match {$count :number}\n" + String message = ".input {$count :number}\n" + + ".match $count\n" + " 1 {{You have one notification.}}\n" + " * {{You have {$count} notifications.}}"; @@ -386,7 +387,9 @@ public void testSimpleSelection() { @Test public void testComplexSelection() { String message = "" - + ".match {$photoCount :number} {$userGender :string}\n" + + ".input {$photoCount :number}\n" + + ".input {$userGender :string}\n" + + ".match $photoCount $userGender\n" + " 1 masculine {{{$userName} added a new photo to his album.}}\n" + " 1 feminine {{{$userName} added a new photo to her album.}}\n" + " 1 * {{{$userName} added a new photo to their album.}}\n" @@ -437,8 +440,9 @@ public void testSimpleLocaleVariable() { @Test public void testLocaleVariableWithSelect() { String message = "" + + ".input {$count :number}\n" + ".local $exp = {$expDate :date year=numeric month=short day=numeric weekday=short}\n" - + ".match {$count :number}\n" + + ".match $count\n" + " 1 {{Your ticket expires on {$exp}.}}\n" + " * {{Your {$count} tickets expire on {$exp}.}}"; diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/DateTimeFormatterFactory.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/DateTimeFormatterFactory.java index 573c4ba3b5bd..d38c049be27e 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/DateTimeFormatterFactory.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/DateTimeFormatterFactory.java @@ -59,6 +59,10 @@ private static int stringToStyle(String option) { */ @Override public Formatter createFormatter(Locale locale, Map fixedOptions) { + locale = OptUtils.getBestLocale(fixedOptions, locale); + Directionality dir = OptUtils.getBestDirectionality(fixedOptions, locale); + + boolean reportErrors = OptUtils.reportErrors(fixedOptions); int dateStyle = DateFormat.NONE; int timeStyle = DateFormat.NONE; switch (kind) { @@ -98,7 +102,7 @@ public Formatter createFormatter(Locale locale, Map fixedOptions } if (!skeleton.isEmpty()) { DateFormat df = DateFormat.getInstanceForSkeleton(skeleton, locale); - return new DateTimeFormatter(locale, df); + return new DateTimeFormatter(locale, df, reportErrors); } // No skeletons, custom or otherwise, match fallback to short / short as per spec. @@ -119,7 +123,7 @@ public Formatter createFormatter(Locale locale, Map fixedOptions } DateFormat df = DateFormat.getDateTimeInstance(dateStyle, timeStyle, locale); - return new DateTimeFormatter(locale, df); + return new DateTimeFormatter(locale, df, reportErrors); } private static int getDateTimeStyle(Map options, String key) { @@ -329,10 +333,12 @@ private static String getTimeFieldOptions(Map options) { private static class DateTimeFormatter implements Formatter { private final DateFormat icuFormatter; private final Locale locale; + private final boolean reportErrors; - private DateTimeFormatter(Locale locale, DateFormat df) { + private DateTimeFormatter(Locale locale, DateFormat df, boolean reportErrors) { this.locale = locale; this.icuFormatter = df; + this.reportErrors = reportErrors; } /** @@ -348,6 +354,9 @@ public FormattedPlaceholder format(Object toFormat, Map variable toFormat = parseIso8601(toFormat.toString()); // We were unable to parse the input as iso date if (toFormat instanceof CharSequence) { + if (reportErrors) { + throw new IllegalArgumentException("bad-operand: argument must be ISO 8601"); + } return new FormattedPlaceholder( toFormat, new PlainStringFormattedValue("{|" + toFormat + "|}")); } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/Directionality.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/Directionality.java new file mode 100644 index 000000000000..e74f724d7e2e --- /dev/null +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/Directionality.java @@ -0,0 +1,78 @@ +// © 2025 and later: Unicode, Inc. and others. +// License & terms of use: https://www.unicode.org/copyright.html + +package com.ibm.icu.message2; + +import com.ibm.icu.util.ULocale; + +/** + * Encodes info about the direction of the message. + * + *

It is used to implement the @code u:dir} functionality.

+ * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ +@Deprecated +public enum Directionality { + /** + * Not initialized or unknown. + * + *

No special processing will be used. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + UNKNOWN, + /** + * Left-to-right directionality. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + LTR, + /** + * Right-to-left directionality. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + RTL, + /** + * Directionality determined from expression contents. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + AUTO, + /** + * Directionality inherited from the message or from the resolved value + * of the operand without requiring isolation of the expression value. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + INHERIT; + + /** + * Determines the directionality appropriate for a given locale. + * + * @param ulocale the locale to determine the directionality from. + * @return the appropriate directionality for the locale given. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + public static Directionality of(ULocale ulocale) { + if (ulocale == null ) { + return Directionality.INHERIT; + } + return ulocale.isRightToLeft() ? Directionality.RTL : Directionality.LTR; + } +} diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/FormattedPlaceholder.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/FormattedPlaceholder.java index 9112c3acef48..4c4b17693e06 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/FormattedPlaceholder.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/FormattedPlaceholder.java @@ -17,23 +17,42 @@ public class FormattedPlaceholder { private final FormattedValue formattedValue; private final Object inputValue; + private final Directionality directionality; + private final boolean isolate; /** * Constructor creating the {@code FormattedPlaceholder}. * * @param inputValue the original value to be formatted. * @param formattedValue the result of formatting the placeholder. + * @param directionality the directionality of the formatted placeholder. * * @internal ICU 72 technology preview * @deprecated This API is for ICU internal use only. */ @Deprecated - public FormattedPlaceholder(Object inputValue, FormattedValue formattedValue) { + public FormattedPlaceholder(Object inputValue, FormattedValue formattedValue, Directionality directionality, boolean isolate) { if (formattedValue == null) { throw new IllegalAccessError("Should not try to wrap a null formatted value"); } this.inputValue = inputValue; this.formattedValue = formattedValue; + this.directionality = directionality; + this.isolate = isolate; + } + + /** + * Constructor creating the {@code FormattedPlaceholder}. + * + * @param inputValue the original value to be formatted. + * @param formattedValue the result of formatting the placeholder. + * + * @internal ICU 72 technology preview + * @deprecated This API is for ICU internal use only. + */ + @Deprecated + public FormattedPlaceholder(Object inputValue, FormattedValue formattedValue) { + this(inputValue, formattedValue, Directionality.LTR, false); } /** @@ -62,6 +81,32 @@ public FormattedValue getFormattedValue() { return formattedValue; } + /** + * Retrieve the directionality of the formatted the placeholder. + * + * @return the directionality. + * + * @internal ICU 77 technology preview + * @deprecated This API is for ICU internal use only. + */ + @Deprecated + public Directionality getDirectionality() { + return directionality; + } + + /** + * Retrieve the BiDi isolate setting of the formatted the placeholder. + * + * @return the BiDi isolate setting. + * + * @internal ICU 77 technology preview + * @deprecated This API is for ICU internal use only. + */ + @Deprecated + public boolean getIsolate() { + return isolate; + } + /** * Returns a string representation of the object. * It can be null, which is unusual, and we plan to change that. diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/IdentityFormatterFactory.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/IdentityFormatterFactory.java index 7e570df7dff2..953d4a55a2bf 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/IdentityFormatterFactory.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/IdentityFormatterFactory.java @@ -16,17 +16,24 @@ class IdentityFormatterFactory implements FormatterFactory { */ @Override public Formatter createFormatter(Locale locale, Map fixedOptions) { - return new IdentityFormatterImpl(); + return new IdentityFormatterImpl(OptUtils.getDirectionality(fixedOptions)); } private static class IdentityFormatterImpl implements Formatter { + private final Directionality directionality; + + public IdentityFormatterImpl(Directionality directionality) { + this.directionality = directionality == null ? Directionality.INHERIT : directionality; + } + /** * {@inheritDoc} */ @Override public FormattedPlaceholder format(Object toFormat, Map variableOptions) { return new FormattedPlaceholder( - toFormat, new PlainStringFormattedValue(Objects.toString(toFormat))); + toFormat, new PlainStringFormattedValue(Objects.toString(toFormat)), + directionality, true); } /** diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModel.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModel.java index 7cbf42dbfaf0..8e6345c15207 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModel.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModel.java @@ -165,7 +165,12 @@ public Variant(List keys, Pattern value) { */ @Deprecated public static class CatchallKey implements LiteralOrCatchallKey { + final static String AS_KEY_STRING = "<<::CatchallKey::>>"; // String value; // Always '*' in MF2 + + public static boolean isCatchAll(String key) { + return AS_KEY_STRING.equals(key); + } } // Patterns @@ -213,7 +218,7 @@ public static class StringPart implements PatternPart { @Deprecated public interface Expression extends PatternPart { // Provides a common type for all kind of expressions: - // LiteralExpression, VariableExpression, FunctionExpression, UnsupportedExpression, Markup + // LiteralExpression, VariableExpression, FunctionExpression, Markup } /** @@ -223,7 +228,7 @@ public interface Expression extends PatternPart { @Deprecated public static class LiteralExpression implements Expression { public final Literal arg; - public final Annotation annotation; + public final Function function; public final List attributes; /** @@ -231,9 +236,9 @@ public static class LiteralExpression implements Expression { * @deprecated This API is for technology preview only. */ @Deprecated - public LiteralExpression(Literal arg, Annotation annotation, List attributes) { + public LiteralExpression(Literal arg, Function function, List attributes) { this.arg = arg; - this.annotation = annotation; + this.function = function; this.attributes = attributes; } } @@ -245,7 +250,7 @@ public LiteralExpression(Literal arg, Annotation annotation, List att @Deprecated public static class VariableExpression implements Expression { public final VariableRef arg; - public final Annotation annotation; + public final Function function; public final List attributes; /** @@ -254,9 +259,9 @@ public static class VariableExpression implements Expression { */ @Deprecated public VariableExpression( - VariableRef arg, Annotation annotation, List attributes) { + VariableRef arg, Function function, List attributes) { this.arg = arg; - this.annotation = annotation; + this.function = function; this.attributes = attributes; } } @@ -266,8 +271,19 @@ public VariableExpression( * @deprecated This API is for technology preview only. */ @Deprecated - public interface Annotation { - // Provides a common type for FunctionAnnotation, UnsupportedAnnotation + public static class Function { + public final String name; + public final Map options; + + /** + * @internal ICU 72 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + public Function(String name, Map options) { + this.name = name; + this.options = options; + } } /** @@ -276,7 +292,7 @@ public interface Annotation { */ @Deprecated public static class FunctionExpression implements Expression { - public final FunctionAnnotation annotation; + public final Function function; public final List attributes; /** @@ -284,8 +300,8 @@ public static class FunctionExpression implements Expression { * @deprecated This API is for technology preview only. */ @Deprecated - public FunctionExpression(FunctionAnnotation annotation, List attributes) { - this.annotation = annotation; + public FunctionExpression(Function function, List attributes) { + this.function = function; this.attributes = attributes; } } @@ -359,26 +375,6 @@ public VariableRef(String name) { } } - /** - * @internal ICU 72 technology preview - * @deprecated This API is for technology preview only. - */ - @Deprecated - public static class FunctionAnnotation implements Annotation { - public final String name; - public final Map options; - - /** - * @internal ICU 72 technology preview - * @deprecated This API is for technology preview only. - */ - @Deprecated - public FunctionAnnotation(String name, Map options) { - this.name = name; - this.options = options; - } - } - /** * @internal ICU 72 technology preview * @deprecated This API is for technology preview only. diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelFormatter.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelFormatter.java index 4146a08478a3..6e15dc281326 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelFormatter.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelFormatter.java @@ -11,11 +11,10 @@ import java.util.Locale; import java.util.Map; -import com.ibm.icu.message2.MFDataModel.Annotation; import com.ibm.icu.message2.MFDataModel.CatchallKey; import com.ibm.icu.message2.MFDataModel.Declaration; import com.ibm.icu.message2.MFDataModel.Expression; -import com.ibm.icu.message2.MFDataModel.FunctionAnnotation; +import com.ibm.icu.message2.MFDataModel.Function; import com.ibm.icu.message2.MFDataModel.FunctionExpression; import com.ibm.icu.message2.MFDataModel.InputDeclaration; import com.ibm.icu.message2.MFDataModel.Literal; @@ -29,6 +28,7 @@ import com.ibm.icu.message2.MFDataModel.StringPart; import com.ibm.icu.message2.MFDataModel.VariableRef; import com.ibm.icu.message2.MFDataModel.Variant; +import com.ibm.icu.message2.MessageFormatter.BidiIsolation; import com.ibm.icu.message2.MessageFormatter.ErrorHandlingBehavior; import com.ibm.icu.util.Calendar; import com.ibm.icu.util.CurrencyAmount; @@ -39,8 +39,15 @@ */ // TODO: move this in the MessageFormatter? class MFDataModelFormatter { + // Bidi controls. For code readability only. + private static final char LRI = '\u2066'; // LEFT-TO-RIGHT ISOLATE (LRI) + private static final char RLI = '\u2067'; // RIGHT-TO-LEFT ISOLATE (RLI) + private static final char FSI = '\u2068'; // FIRST STRONG ISOLATE (FSI) + private static final char PDI = '\u2069'; // POP DIRECTIONAL ISOLATE (PDI) + private final Locale locale; private final ErrorHandlingBehavior errorHandlingBehavior; + private final BidiIsolation bidiIsolation; private final MFDataModel.Message dm; private final MFFunctionRegistry standardFunctions; @@ -51,10 +58,13 @@ class MFDataModelFormatter { MFDataModel.Message dm, Locale locale, ErrorHandlingBehavior errorHandlingBehavior, + BidiIsolation bidiIsolation, MFFunctionRegistry customFunctionRegistry) { this.locale = locale; this.errorHandlingBehavior = errorHandlingBehavior == null ? ErrorHandlingBehavior.BEST_EFFORT : errorHandlingBehavior; + this.bidiIsolation = bidiIsolation == null + ? BidiIsolation.NONE : bidiIsolation; this.dm = dm; this.customFunctions = customFunctionRegistry == null ? EMPTY_REGISTY : customFunctionRegistry; @@ -73,10 +83,12 @@ class MFDataModelFormatter { // Number formatting .setFormatter("number", new NumberFormatterFactory("number")) .setFormatter("integer", new NumberFormatterFactory("integer")) + .setFormatter("currency", new NumberFormatterFactory("currency")) + .setFormatter("math", new NumberFormatterFactory("math")) .setDefaultFormatterNameForType(Integer.class, "number") .setDefaultFormatterNameForType(Double.class, "number") .setDefaultFormatterNameForType(Number.class, "number") - .setDefaultFormatterNameForType(CurrencyAmount.class, "number") + .setDefaultFormatterNameForType(CurrencyAmount.class, "currency") // Format that returns "to string" .setFormatter("string", new IdentityFormatterFactory()) @@ -86,6 +98,7 @@ class MFDataModelFormatter { // Register the standard selectors .setSelector("number", new NumberFormatterFactory("number")) .setSelector("integer", new NumberFormatterFactory("integer")) + .setSelector("math", new NumberFormatterFactory("math")) .setSelector("string", new TextSelectorFactory()) .setSelector("icu:gender", new TextSelectorFactory()) .build(); @@ -93,22 +106,20 @@ class MFDataModelFormatter { String format(Map arguments) { MFDataModel.Pattern patternToRender = null; - if (arguments == null) { - arguments = new HashMap<>(); - } + MapWithNfcKeys nfcArguments = new MapWithNfcKeys(arguments); - Map variables; + MapWithNfcKeys variables; if (dm instanceof MFDataModel.PatternMessage) { MFDataModel.PatternMessage pm = (MFDataModel.PatternMessage) dm; - variables = resolveDeclarations(pm.declarations, arguments); + variables = resolveDeclarations(pm.declarations, nfcArguments); if (pm.pattern == null) { fatalFormattingError("The PatternMessage is null."); } patternToRender = pm.pattern; } else if (dm instanceof MFDataModel.SelectMessage) { MFDataModel.SelectMessage sm = (MFDataModel.SelectMessage) dm; - variables = resolveDeclarations(sm.declarations, arguments); - patternToRender = findBestMatchingPattern(sm, variables, arguments); + variables = resolveDeclarations(sm.declarations, nfcArguments); + patternToRender = findBestMatchingPattern(sm, variables, nfcArguments); if (patternToRender == null) { fatalFormattingError("Cannor find a match for the selector."); } @@ -118,6 +129,7 @@ String format(Map arguments) { return "ERROR!"; } + Directionality msgdir = Directionality.LTR; StringBuilder result = new StringBuilder(); for (MFDataModel.PatternPart part : patternToRender.parts) { if (part instanceof MFDataModel.StringPart) { @@ -125,10 +137,14 @@ String format(Map arguments) { result.append(strPart.value); } else if (part instanceof MFDataModel.Expression) { FormattedPlaceholder formattedExpression = - formatExpression((Expression) part, variables, arguments); - result.append(formattedExpression.getFormattedValue().toString()); + formatExpression((Expression) part, variables, nfcArguments); + if (this.bidiIsolation == BidiIsolation.DEFAULT) { + implementBiDiDefault(result, msgdir, formattedExpression); + } else { + result.append(formattedExpression.getFormattedValue().toString()); + } } else if (part instanceof MFDataModel.Markup) { - // Ignore + // Ignore, we don't output markup to string } else { fatalFormattingError("Unknown part type: " + part); } @@ -136,8 +152,29 @@ String format(Map arguments) { return result.toString(); } + private void implementBiDiDefault(StringBuilder result, Directionality msgdir, FormattedPlaceholder formattedExpression) { + String fmt = formattedExpression.getFormattedValue().toString(); + Directionality dir = formattedExpression.getDirectionality(); + boolean isolate = formattedExpression.getIsolate(); + switch (dir) { + case LTR: + if (msgdir == Directionality.LTR && !isolate) { + result.append(fmt); + } else { + result.append(LRI).append(fmt).append(PDI); + } + break; + case RTL: + result.append(RLI).append(fmt).append(PDI); + break; + default: + result.append(FSI).append(fmt).append(PDI); + break; + } + } + private Pattern findBestMatchingPattern( - SelectMessage sm, Map variables, Map arguments) { + SelectMessage sm, MapWithNfcKeys variables, MapWithNfcKeys arguments) { Pattern patternToRender = null; // ==================================== @@ -154,7 +191,7 @@ private Pattern findBestMatchingPattern( FormattedPlaceholder fph = formatExpression(sel, variables, arguments); String functionName = null; Object argument = null; - Map options = new HashMap<>(); + MapWithNfcKeys options = new MapWithNfcKeys(); if (fph.getInput() instanceof ResolvedExpression) { ResolvedExpression re = (ResolvedExpression) fph.getInput(); argument = re.argument; @@ -163,14 +200,14 @@ private Pattern findBestMatchingPattern( } else if (fph.getInput() instanceof MFDataModel.VariableExpression) { MFDataModel.VariableExpression ve = (MFDataModel.VariableExpression) fph.getInput(); argument = resolveLiteralOrVariable(ve.arg, variables, arguments); - if (ve.annotation instanceof FunctionAnnotation) { - functionName = ((FunctionAnnotation) ve.annotation).name; + if (ve.function instanceof Function) { + functionName = ((Function) ve.function).name; } } else if (fph.getInput() instanceof LiteralExpression) { LiteralExpression le = (LiteralExpression) fph.getInput(); argument = le.arg; - if (le.annotation instanceof FunctionAnnotation) { - functionName = ((FunctionAnnotation) le.annotation).name; + if (le.function instanceof Function) { + functionName = ((Function) le.function).name; } } SelectorFactory funcFactory = standardFunctions.getSelector(functionName); @@ -179,7 +216,7 @@ private Pattern findBestMatchingPattern( } // spec: If selection is supported for `rv`: if (funcFactory != null) { - Selector selectorFunction = funcFactory.createSelector(locale, options); + Selector selectorFunction = funcFactory.createSelector(locale, options.getMap()); ResolvedSelector rs = new ResolvedSelector(argument, options, selectorFunction); // spec: Append `rv` as the last element of the list `res`. res.add(rs); @@ -213,7 +250,7 @@ private Pattern findBestMatchingPattern( LiteralOrCatchallKey key = var.keys.get(i); // spec: If `key` is not the catch-all key `'*'`: if (key instanceof CatchallKey) { - keys.add("*"); + keys.add(CatchallKey.AS_KEY_STRING); } else if (key instanceof Literal) { // spec: Assert that `key` is a _literal_. // spec: Let `ks` be the resolved value of `key`. @@ -364,8 +401,8 @@ private static int sortVariants(IntVarTuple o1, IntVarTuple o2) { for (int i = 0; i < v1.size(); i++) { LiteralOrCatchallKey k1 = v1.get(i); LiteralOrCatchallKey k2 = v2.get(i); - String s1 = k1 instanceof Literal ? ((Literal) k1).value : "*"; - String s2 = k2 instanceof Literal ? ((Literal) k2).value : "*"; + String s1 = k1 instanceof Literal ? ((Literal) k1).value : CatchallKey.AS_KEY_STRING; + String s2 = k2 instanceof Literal ? ((Literal) k2).value : CatchallKey.AS_KEY_STRING; int cmp = s1.compareTo(s2); if (cmp != 0) { return cmp; @@ -386,18 +423,18 @@ private static int sortVariants(IntVarTuple o1, IntVarTuple o2) { */ @SuppressWarnings("static-method") private List matchSelectorKeys(ResolvedSelector rv, List keys) { - return rv.selectorFunction.matches(rv.argument, keys, rv.options); + return rv.selectorFunction.matches(rv.argument, keys, rv.options.getMap()); } private static class ResolvedSelector { final Object argument; - final Map options; + final MapWithNfcKeys options; final Selector selectorFunction; public ResolvedSelector( - Object argument, Map options, Selector selectorFunction) { + Object argument, MapWithNfcKeys options, Selector selectorFunction) { this.argument = argument; - this.options = new HashMap<>(options); + this.options = new MapWithNfcKeys(options); this.selectorFunction = selectorFunction; } } @@ -435,8 +472,8 @@ private FormatterFactory getFormattingFunctionFactoryByName( private static Object resolveLiteralOrVariable( LiteralOrVariableRef value, - Map localVars, - Map arguments) { + MapWithNfcKeys localVars, + MapWithNfcKeys arguments) { if (value instanceof Literal) { String val = ((Literal) value).value; // "The resolution of a text or literal MUST resolve to a string." @@ -449,18 +486,18 @@ private static Object resolveLiteralOrVariable( val = localVars.get(varName); } if (val == null) { - val = arguments.get(varName); + val = arguments.get(StringUtils.toNfc(varName)); } return val; } return value; } - private static Map convertOptions( + private static MapWithNfcKeys convertOptions( Map options, - Map localVars, - Map arguments) { - Map result = new HashMap<>(); + MapWithNfcKeys localVars, + MapWithNfcKeys arguments) { + MapWithNfcKeys result = new MapWithNfcKeys(); for (Option option : options.values()) { result.put(option.name, resolveLiteralOrVariable(option.value, localVars, arguments)); } @@ -475,9 +512,9 @@ private static Map convertOptions( * @param arguments the arguments passed at runtime to be formatted (`mf.format(arguments)`) */ private FormattedPlaceholder formatExpression( - Expression expression, Map variables, Map arguments) { + Expression expression, MapWithNfcKeys variables, MapWithNfcKeys arguments) { - Annotation annotation = null; // function name + Function function = null; // function name String functionName = null; Object toFormat = null; Map options = new HashMap<>(); @@ -486,7 +523,7 @@ private FormattedPlaceholder formatExpression( if (expression instanceof MFDataModel.VariableExpression) { MFDataModel.VariableExpression varPart = (MFDataModel.VariableExpression) expression; fallbackString = "{$" + varPart.arg.name + "}"; - annotation = varPart.annotation; // function name & options + function = varPart.function; // function name & options Object resolved = resolveLiteralOrVariable(varPart.arg, variables, arguments); if (resolved instanceof FormattedPlaceholder) { Object input = ((FormattedPlaceholder) resolved).getInput(); @@ -504,11 +541,11 @@ private FormattedPlaceholder formatExpression( } else if (expression instanceof MFDataModel.FunctionExpression) { // Function without arguments MFDataModel.FunctionExpression fe = (FunctionExpression) expression; - fallbackString = "{:" + fe.annotation.name + "}"; - annotation = fe.annotation; + fallbackString = "{:" + fe.function.name + "}"; + function = fe.function; } else if (expression instanceof MFDataModel.LiteralExpression) { MFDataModel.LiteralExpression le = (LiteralExpression) expression; - annotation = le.annotation; + function = le.function; fallbackString = "{|" + le.arg.value + "|}"; toFormat = resolveLiteralOrVariable(le.arg, variables, arguments); } else if (expression instanceof MFDataModel.Markup) { @@ -523,15 +560,11 @@ private FormattedPlaceholder formatExpression( } } - if (annotation instanceof FunctionAnnotation) { - FunctionAnnotation fa = (FunctionAnnotation) annotation; - if (functionName != null && !functionName.equals(fa.name)) { - fatalFormattingError( - "invalid function overrides, '" + functionName + "' <> '" + fa.name + "'"); - } + if (function instanceof Function) { + Function fa = (Function) function; functionName = fa.name; - Map newOptions = convertOptions(fa.options, variables, arguments); - options.putAll(newOptions); + MapWithNfcKeys newOptions = convertOptions(fa.options, variables, arguments); + options.putAll(newOptions.getMap()); } FormatterFactory funcFactory = getFormattingFunctionFactoryByName(toFormat, functionName); @@ -541,8 +574,14 @@ private FormattedPlaceholder formatExpression( } return new FormattedPlaceholder(expression, new PlainStringFormattedValue(fallbackString)); } + // TODO 78: hack. + // How do we pass the error handling policy to formatters? + // I am afraid a clean solution for this would require some changes in the public APIs + // And it is too late for that. + options.put("icu:impl:errorPolicy", this.errorHandlingBehavior.name()); Formatter ff = funcFactory.createFormatter(locale, options); - String res = ff.formatToString(toFormat, arguments); + FormattedPlaceholder resultToWrap = ff.format(toFormat, arguments.getMap()); + String res = resultToWrap == null ? null : resultToWrap.toString(); if (res == null) { if (errorHandlingBehavior == ErrorHandlingBehavior.STRICT) { fatalFormattingError("unable to format string at " + fallbackString); @@ -550,8 +589,16 @@ private FormattedPlaceholder formatExpression( res = fallbackString; } + if (resultToWrap != null) { + toFormat = resultToWrap.getInput(); + } ResolvedExpression resExpression = new ResolvedExpression(toFormat, functionName, options); - return new FormattedPlaceholder(resExpression, new PlainStringFormattedValue(res)); + if (resultToWrap == null) { + return new FormattedPlaceholder(resExpression, new PlainStringFormattedValue(res)); + } + // We wrap the result in a ResolvedExpression, but also propagate the direction info + return new FormattedPlaceholder(resExpression, new PlainStringFormattedValue(res), + resultToWrap.getDirectionality(), resultToWrap.getIsolate()); } static class ResolvedExpression implements Expression { @@ -562,14 +609,14 @@ static class ResolvedExpression implements Expression { public ResolvedExpression( Object argument, String functionName, Map options) { this.argument = argument; - this.functionName = functionName; + this.functionName = StringUtils.toNfc(functionName); this.options = options; } } - private Map resolveDeclarations( - List declarations, Map arguments) { - Map variables = new HashMap<>(); + private MapWithNfcKeys resolveDeclarations( + List declarations, MapWithNfcKeys arguments) { + MapWithNfcKeys variables = new MapWithNfcKeys(); String name; Expression value; if (declarations != null) { @@ -590,7 +637,11 @@ private Map resolveDeclarations( // {{ Hello {$user}! }} FormattedPlaceholder fmt = formatExpression(value, variables, arguments); // If it works, all good - variables.put(name, fmt); + variables.put(StringUtils.toNfc(name), fmt); + } catch (IllegalArgumentException e) { + if (this.errorHandlingBehavior == ErrorHandlingBehavior.STRICT) { + throw(e); + } } catch (Exception e) { // It's OK to ignore the failure in this context, see comment above. } @@ -608,4 +659,51 @@ public IntVarTuple(int integer, Variant variant) { this.variant = variant; } } + + /* + * I considered extending a HashMap. + * But then we would need to override all the methods that use keys: + * `compute`, `computeIfAbsent`, `computeIfPresent`, `containsKey`, `getOrDefault`, + * `merge`, `put`, `putIfAbsent`, `remove`, `replace`, and so on. + * If we don't and some refactoring in the code above starts using one of + * the methods that was not overridden then it will bypass the normalization + * and will create a map with mixed keys (some not normalized). + */ + private static class MapWithNfcKeys { + private final Map theMap = new HashMap<>(); + + Map getMap() { + return theMap; + } + + MapWithNfcKeys() { + super(); + } + + MapWithNfcKeys(MapWithNfcKeys org) { + super(); + theMap.putAll(org.getMap()); + } + + MapWithNfcKeys(Map orgMap) { + super(); + if (orgMap != null) { + for (Map.Entry e : orgMap.entrySet()) { + this.put(StringUtils.toNfc(e.getKey()), e.getValue()); + } + } + } + + public Object put(String key, Object value) { + return theMap.put(StringUtils.toNfc(key), value); + } + + public void putAll(Map m) { + theMap.putAll(m); + } + + public Object get(String key) { + return theMap.get(key); + } + } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java index a17728434b34..e04d738df83c 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFDataModelValidator.java @@ -8,11 +8,10 @@ import java.util.Set; import java.util.StringJoiner; -import com.ibm.icu.message2.MFDataModel.Annotation; import com.ibm.icu.message2.MFDataModel.CatchallKey; import com.ibm.icu.message2.MFDataModel.Declaration; import com.ibm.icu.message2.MFDataModel.Expression; -import com.ibm.icu.message2.MFDataModel.FunctionAnnotation; +import com.ibm.icu.message2.MFDataModel.Function; import com.ibm.icu.message2.MFDataModel.FunctionExpression; import com.ibm.icu.message2.MFDataModel.InputDeclaration; import com.ibm.icu.message2.MFDataModel.Literal; @@ -70,7 +69,7 @@ private boolean validateVariants(List variants, int selectorCount) for (LiteralOrCatchallKey key : variant.keys) { if (key instanceof CatchallKey) { catchAllCount++; - fakeKey.add("*"); + fakeKey.add(CatchallKey.AS_KEY_STRING); } else if (key instanceof Literal) { fakeKey.add(((Literal) key).value); } @@ -140,29 +139,31 @@ private boolean validateDeclarations(List declarations) throws MFPa private void validateExpression(Expression expression, boolean fromInput) throws MFParseException { String argName = null; - Annotation annotation = null; + boolean wasLiteral = false; + Function function = null; if (expression instanceof Literal) { // ...{foo}... or ...{|foo|}... or ...{123}... // does not declare anything } else if (expression instanceof LiteralExpression) { LiteralExpression le = (LiteralExpression) expression; argName = le.arg.value; - annotation = le.annotation; + function = le.function; + wasLiteral = true; } else if (expression instanceof VariableExpression) { VariableExpression ve = (VariableExpression) expression; // ...{$foo :bar opt1=|str| opt2=$x opt3=$y}... // .input {$foo :number} => declares `foo`, if already declared is an error // .local $a={$foo} => declares `a`, but only used `foo`, does not declare it argName = ve.arg.name; - annotation = ve.annotation; + function = ve.function; } else if (expression instanceof FunctionExpression) { // ...{$foo :bar opt1=|str| opt2=$x opt3=$y}... FunctionExpression fe = (FunctionExpression) expression; - annotation = fe.annotation; + function = fe.function; } - if (annotation instanceof FunctionAnnotation) { - FunctionAnnotation fa = (FunctionAnnotation) annotation; + if (function instanceof Function) { + Function fa = (Function) function; if (fa.options != null) { for (Option opt : fa.options.values()) { LiteralOrVariableRef val = opt.value; @@ -184,7 +185,10 @@ private void validateExpression(Expression expression, boolean fromInput) addVariableDeclaration(argName); } else { // Remember that we've seen it, to complain if there is a declaration later - declaredVars.add(argName); + if (!wasLiteral) { + // We don't consider {|bar| :func} to be a declaration of a "bar" variable + declaredVars.add(argName); + } } } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFFunctionRegistry.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFFunctionRegistry.java index 3804b6acb4e8..2b1c46a6be67 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFFunctionRegistry.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFFunctionRegistry.java @@ -26,6 +26,8 @@ * * * + *

NOTE: all function and selector names are normalized to NFC. + * * @internal ICU 72 technology preview * @deprecated This API is for technology preview only. */ @@ -75,7 +77,7 @@ public static Builder builder() { */ @Deprecated public FormatterFactory getFormatter(String formatterName) { - return formattersMap.get(formatterName); + return formattersMap.get(StringUtils.toNfc(formatterName)); } /** @@ -147,7 +149,7 @@ public Set> getDefaultFormatterTypes() { */ @Deprecated public SelectorFactory getSelector(String selectorName) { - return selectorsMap.get(selectorName); + return selectorsMap.get(StringUtils.toNfc(selectorName)); } /** @@ -207,7 +209,7 @@ public Builder addAll(MFFunctionRegistry functionRegistry) { */ @Deprecated public Builder setFormatter(String formatterName, FormatterFactory formatterFactory) { - formattersMap.put(formatterName, formatterFactory); + formattersMap.put(StringUtils.toNfc(formatterName), formatterFactory); return this; } @@ -222,7 +224,7 @@ public Builder setFormatter(String formatterName, FormatterFactory formatterFact */ @Deprecated public Builder removeFormatter(String formatterName) { - formattersMap.remove(formatterName); + formattersMap.remove(StringUtils.toNfc(formatterName)); return this; } @@ -252,7 +254,7 @@ public Builder clearFormatters() { */ @Deprecated public Builder setDefaultFormatterNameForType(Class clazz, String formatterName) { - classToFormatter.put(clazz, formatterName); + classToFormatter.put(clazz, StringUtils.toNfc(formatterName)); return this; } @@ -297,7 +299,7 @@ public Builder clearDefaultFormatterNames() { */ @Deprecated public Builder setSelector(String selectorName, SelectorFactory selectorFactory) { - selectorsMap.put(selectorName, selectorFactory); + selectorsMap.put(StringUtils.toNfc(selectorName), selectorFactory); return this; } @@ -312,7 +314,7 @@ public Builder setSelector(String selectorName, SelectorFactory selectorFactory) */ @Deprecated public Builder removeSelector(String selectorName) { - selectorsMap.remove(selectorName); + selectorsMap.remove(StringUtils.toNfc(selectorName)); return this; } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java index 61d18ffde8a4..39e3019484b6 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFParser.java @@ -51,6 +51,7 @@ private MFDataModel.Message parseImpl() throws MFParseException { int savedPosition = input.getPosition(); skipOptionalWhitespaces(); int cp = input.peekChar(); + // abnf: message = simple-message / complex-message if (cp == '.') { // declarations or .match // No need to restore whitespace result = getComplexMessage(); @@ -59,7 +60,7 @@ private MFDataModel.Message parseImpl() throws MFParseException { cp = input.peekChar(); if (cp == '{') { // `{{`, complex body without declarations input.backup(1); // let complexBody deal with the wrapping {{ and }} - // abnf: complex-message = [s] *(declaration [s]) complex-body [s] + // abnf: complex-message = o *(declaration o) complex-body o MFDataModel.Pattern pattern = getQuotedPattern(); skipOptionalWhitespaces(); result = new MFDataModel.PatternMessage(new ArrayList<>(), pattern); @@ -80,9 +81,8 @@ private MFDataModel.Message parseImpl() throws MFParseException { return result; } - // abnf: simple-message = [simple-start pattern] - // abnf: simple-start = simple-start-char / text-escape / placeholder - // abnf: pattern = *(text-char / text-escape / placeholder) + // abnf: simple-message = o [simple-start pattern] + // abnf: simple-start = simple-start-char / escaped-char / placeholder private MFDataModel.Pattern getPattern() throws MFParseException { MFDataModel.Pattern pattern = new MFDataModel.Pattern(); while (true) { @@ -96,6 +96,7 @@ private MFDataModel.Pattern getPattern() throws MFParseException { return pattern; } + // abnf: pattern = *(text-char / escaped-char / placeholder) private MFDataModel.PatternPart getPatternPart() throws MFParseException { int cp = input.peekChar(); switch (cp) { @@ -113,6 +114,7 @@ private MFDataModel.PatternPart getPatternPart() throws MFParseException { } } + // abnf: text-char = content-char / ws / "." / "@" / "|" private String getText() { StringBuilder result = new StringBuilder(); while (true) { @@ -121,6 +123,7 @@ private String getText() { case EOF: return result.toString(); case '\\': + // abnf: escaped-char = backslash ( backslash / "{" / "|" / "}" ) cp = input.readCodePoint(); if (cp == '\\' || cp == '{' || cp == '|' | cp == '}') { result.appendCodePoint(cp); @@ -148,19 +151,19 @@ private String getText() { // abnf: placeholder = expression / markup // abnf: expression = literal-expression // abnf: / variable-expression - // abnf: / annotation-expression - // abnf: literal-expression = "{" [s] literal [s annotation] *(s attribute) [s] "}" - // abnf: variable-expression = "{" [s] variable [s annotation] *(s attribute) [s] "}" - // abnf: annotation-expression = "{" [s] annotation *(s attribute) [s] "}" - // abnf: markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}" ; open and standalone - // abnf: / "{" [s] "/" identifier *(s option) *(s attribute) [s] "}" ; close + // abnf: / function-expression + // abnf: literal-expression = "{" o literal [s function] *(s attribute) o "}" + // abnf: variable-expression = "{" o variable [s function] *(s attribute) o "}" + // abnf: function-expression = "{" o function *(s attribute) o "}" + // abnf: markup = "{" o "#" identifier *(s option) *(s attribute) o ["/"] "}" ; open and standalone + // abnf: / "{" o "/" identifier *(s option) *(s attribute) o "}" ; close private MFDataModel.Expression getPlaceholder() throws MFParseException { int cp = input.peekChar(); if (cp != '{') { return null; } input.readCodePoint(); // consume the '{' - skipOptionalWhitespaces(); + skipOptionalWhitespaces(); // the o after '{' cp = input.peekChar(); MFDataModel.Expression result; @@ -169,7 +172,7 @@ private MFDataModel.Expression getPlaceholder() throws MFParseException { } else if (cp == '$') { result = getVariableExpression(); } else if (StringUtils.isFunctionSigil(cp)) { - result = getAnnotationExpression(); + result = getFunctionExpression(); } else { result = getLiteralExpression(); } @@ -181,10 +184,10 @@ private MFDataModel.Expression getPlaceholder() throws MFParseException { return result; } - private MFDataModel.Annotation getAnnotation(boolean whitespaceRequired) throws MFParseException { + private MFDataModel.Function getFunction(boolean whitespaceRequired) throws MFParseException { int position = input.getPosition(); - // Handle absent annotation first (before parsing mandatory whitespace) + // Handle absent function first (before parsing mandatory whitespace) int cp = input.peekChar(); if (cp == '}') { return null; @@ -192,7 +195,7 @@ private MFDataModel.Annotation getAnnotation(boolean whitespaceRequired) throws int whitespaceCount = 0; if (whitespaceRequired) { - whitespaceCount = skipMandatoryWhitespaces(); + whitespaceCount = skipRequiredWhitespaces(); } else { whitespaceCount = skipOptionalWhitespaces(); } @@ -200,18 +203,18 @@ private MFDataModel.Annotation getAnnotation(boolean whitespaceRequired) throws cp = input.peekChar(); switch (cp) { case '}': { - // No annotation -- push the whitespace back, + // No function -- push the whitespace back, // in case it's the required whitespace before an attribute input.backup(whitespaceCount); return null; } - case ':': // annotation, function + case ':': // function // abnf: function = ":" identifier *(s option) input.readCodePoint(); // Consume the sigil String identifier = getIdentifier(); - checkCondition(identifier != null, "Annotation / function name missing"); + checkCondition(identifier != null, "Function name missing"); Map options = getOptions(); - return new MFDataModel.FunctionAnnotation(identifier, options); + return new MFDataModel.Function(identifier, options); default: // OK to continue and return null, it is an error. } @@ -219,7 +222,7 @@ private MFDataModel.Annotation getAnnotation(boolean whitespaceRequired) throws return null; } - private MFDataModel.Annotation getMarkupAnnotation() throws MFParseException { + private MFDataModel.Function getMarkupFunction() throws MFParseException { skipOptionalWhitespaces(); int cp = input.peekChar(); @@ -228,66 +231,70 @@ private MFDataModel.Annotation getMarkupAnnotation() throws MFParseException { return null; case '#': case '/': - // abnf: markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}" ; open and standalone - // abnf: / "{" [s] "/" identifier *(s option) *(s attribute) [s] "}" ; close + // abnf: markup = "{" o "#" identifier *(s option) *(s attribute) o ["/"] "}" ; open and standalone + // abnf: / "{" o "/" identifier *(s option) *(s attribute) o "}" ; close input.readCodePoint(); // Consume the sigil String identifier = getIdentifier(); - checkCondition(identifier != null, "Annotation / function name missing"); + checkCondition(identifier != null, "Function name missing"); Map options = getOptions(); - return new MFDataModel.FunctionAnnotation(identifier, options); + return new MFDataModel.Function(identifier, options); default: // function or something else, return null; } } - // abnf: literal-expression = "{" [s] literal [s annotation] *(s attribute) [s] "}" + // abnf: literal-expression = "{" o literal [s function] *(s attribute) o "}" private MFDataModel.Expression getLiteralExpression() throws MFParseException { - MFDataModel.Literal literal = getLiteral(); + MFDataModel.Literal literal = getLiteral(false); checkCondition(literal != null, "Literal expression expected."); - MFDataModel.Annotation annotation = null; + MFDataModel.Function function = null; boolean hasWhitespace = StringUtils.isWhitespace(input.peekChar()); - if (hasWhitespace) { // we might have an annotation - annotation = getAnnotation(true); - if (annotation == null) { - // We had some spaces, but no annotation. + if (hasWhitespace) { // we might have an function + function = getFunction(true); + if (function == null) { + // We had some spaces, but no function. // So we put (some) back for the possible attributes. // input.backup(1); } } + hasWhitespace = StringUtils.isWhitespace(input.peekChar()); List attributes = getAttributes(); + if (!hasWhitespace && !attributes.isEmpty()) { + error("syntax-error: missing space before attributes"); + } // Literal without a function, for example {|hello|} or {123} - return new MFDataModel.LiteralExpression(literal, annotation, attributes); + return new MFDataModel.LiteralExpression(literal, function, attributes); } - // abnf: variable-expression = "{" [s] variable [s annotation] *(s attribute) [s] "}" + // abnf: variable-expression = "{" o variable [s function] *(s attribute) o "}" private MFDataModel.VariableExpression getVariableExpression() throws MFParseException { MFDataModel.VariableRef variableRef = getVariableRef(); - MFDataModel.Annotation annotation = getAnnotation(true); + MFDataModel.Function function = getFunction(true); List attributes = getAttributes(); // Variable without a function, for example {$foo} - return new MFDataModel.VariableExpression(variableRef, annotation, attributes); + return new MFDataModel.VariableExpression(variableRef, function, attributes); } - // abnf: annotation-expression = "{" [s] annotation *(s attribute) [s] "}" - private MFDataModel.Expression getAnnotationExpression() throws MFParseException { - MFDataModel.Annotation annotation = getAnnotation(false); + // abnf: function-expression = "{" o function *(s attribute) o "}" + private MFDataModel.Expression getFunctionExpression() throws MFParseException { + MFDataModel.Function function = getFunction(false); List attributes = getAttributes(); - if (annotation instanceof MFDataModel.FunctionAnnotation) { + if (function instanceof MFDataModel.Function) { return new MFDataModel.FunctionExpression( - (MFDataModel.FunctionAnnotation) annotation, attributes); + (MFDataModel.Function) function, attributes); } else { - error("Unexpected annotation : " + annotation); + error("Unexpected function : " + function); } return null; } - // abnf: markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}" ; open and standalone - // abnf: / "{" [s] "/" identifier *(s option) *(s attribute) [s] "}" ; close + // abnf: markup = "{" o "#" identifier *(s option) *(s attribute) o ["/"] "}" ; open and standalone + // abnf: / "{" o "/" identifier *(s option) *(s attribute) o "}" ; close private MFDataModel.Markup getMarkup() throws MFParseException { int cp = input.peekChar(); // consume the '{' checkCondition(cp == '#' || cp == '/', "Should not happen. Expecting a markup."); @@ -295,11 +302,11 @@ private MFDataModel.Markup getMarkup() throws MFParseException { MFDataModel.Markup.Kind kind = cp == '/' ? MFDataModel.Markup.Kind.CLOSE : MFDataModel.Markup.Kind.OPEN; - MFDataModel.Annotation annotation = getMarkupAnnotation(); + MFDataModel.Function function = getMarkupFunction(); List attributes = getAttributes(); // Parse optional whitespace after attribute list - skipOptionalWhitespaces(); + skipOptionalWhitespaces(); // the o before '/}' or '}' cp = input.peekChar(); if (cp == '/') { @@ -307,8 +314,8 @@ private MFDataModel.Markup getMarkup() throws MFParseException { input.readCodePoint(); } - if (annotation instanceof MFDataModel.FunctionAnnotation) { - MFDataModel.FunctionAnnotation fa = (MFDataModel.FunctionAnnotation) annotation; + if (function instanceof MFDataModel.Function) { + MFDataModel.Function fa = (MFDataModel.Function) function; return new MFDataModel.Markup(kind, fa.name, fa.options, attributes); } @@ -327,24 +334,21 @@ private List getAttributes() throws MFParseException { return result; } - // abnf: attribute = "@" identifier [[s] "=" [s] (literal / variable)] + // abnf: attribute = "@" identifier [o "=" o literal] private MFDataModel.Attribute getAttribute() throws MFParseException { int position = input.getPosition(); - if (skipWhitespaces() == 0) { - input.gotoPosition(position); - return null; - } + skipOptionalWhitespaces(); int cp = input.peekChar(); if (cp == '@') { input.readCodePoint(); // consume the '@' String id = getIdentifier(); - int wsCount = skipWhitespaces(); + int wsCount = skipOptionalWhitespaces(); cp = input.peekChar(); MFDataModel.LiteralOrVariableRef literalOrVariable = null; if (cp == '=') { input.readCodePoint(); skipOptionalWhitespaces(); - literalOrVariable = getLiteralOrVariableRef(); + literalOrVariable = getLiteral(false); checkCondition(literalOrVariable != null, "Attributes must have a value after `=`"); } else { // was not equal, attribute without a value, put the "spaces" back. @@ -359,7 +363,6 @@ private MFDataModel.Attribute getAttribute() throws MFParseException { // abnf: identifier = [namespace ":"] name // abnf: namespace = name - // abnf: name = name-start *name-char private String getIdentifier() throws MFParseException { String namespace = getName(); if (namespace == null) { @@ -386,7 +389,6 @@ private Map getOptions() throws MFParseException { if (option == null) { break; } - // function = ":" identifier *(s option) checkCondition(first || skipCount != 0, "Expected whitespace before option " + option.name); first = false; @@ -404,7 +406,7 @@ private Map getOptions() throws MFParseException { return options; } - // abnf: option = identifier [s] "=" [s] (literal / variable) + // abnf: option = identifier o "=" o (literal / variable) private MFDataModel.Option getOption() throws MFParseException { int position = input.getPosition(); skipOptionalWhitespaces(); @@ -429,21 +431,22 @@ private MFDataModel.LiteralOrVariableRef getLiteralOrVariableRef() throws MFPars if (cp == '$') { return getVariableRef(); } - return getLiteral(); + return getLiteral(false); } - // abnf: literal = quoted / unquoted - private MFDataModel.Literal getLiteral() throws MFParseException { + // abnf: literal = quoted-literal / unquoted-literal + private MFDataModel.Literal getLiteral(boolean normalize) throws MFParseException { int cp = input.readCodePoint(); switch (cp) { - case '|': // quoted - // abnf: quoted = "|" *(quoted-char / quoted-escape) "|" + case '|': // quoted-literal + // abnf: quoted-literal = "|" *(quoted-char / escaped-char) "|" input.backup(1); - MFDataModel.Literal ql = getQuotedLiteral(); + MFDataModel.Literal ql = getQuotedLiteral(normalize); return ql; - default: // unquoted + default: // unquoted-literal + // abnf: unquoted-literal = name / number-literal input.backup(1); - MFDataModel.Literal unql = getUnQuotedLiteral(); + MFDataModel.Literal unql = getUnQuotedLiteral(normalize); return unql; } } @@ -460,7 +463,7 @@ private MFDataModel.VariableRef getVariableRef() throws MFParseException { return new MFDataModel.VariableRef(name); } - private MFDataModel.Literal getQuotedLiteral() throws MFParseException { + private MFDataModel.Literal getQuotedLiteral(boolean normalize) throws MFParseException { StringBuilder result = new StringBuilder(); int cp = input.readCodePoint(); checkCondition(cp == '|', "expected starting '|'"); @@ -471,6 +474,7 @@ private MFDataModel.Literal getQuotedLiteral() throws MFParseException { } else if (StringUtils.isQuotedChar(cp)) { result.appendCodePoint(cp); } else if (cp == '\\') { + // abnf: escaped-char = backslash ( backslash / "{" / "|" / "}" ) cp = input.readCodePoint(); boolean isValidEscape = cp == '|' || cp == '\\' || cp == '{' || cp == '}'; checkCondition(isValidEscape, "Invalid escape sequence inside quoted literal"); @@ -482,13 +486,13 @@ private MFDataModel.Literal getQuotedLiteral() throws MFParseException { checkCondition(cp == '|', "expected ending '|'"); - return new MFDataModel.Literal(result.toString()); + return new MFDataModel.Literal(normalize ? StringUtils.toNfc(result) : result.toString()); } - private MFDataModel.Literal getUnQuotedLiteral() throws MFParseException { + private MFDataModel.Literal getUnQuotedLiteral(boolean normalize) throws MFParseException { String name = getName(); if (name != null) { - return new MFDataModel.Literal(name); + return new MFDataModel.Literal(normalize ? StringUtils.toNfc(name) : name); } return getNumberLiteral(); } @@ -506,31 +510,74 @@ private MFDataModel.Literal getNumberLiteral() { return null; } - private int skipMandatoryWhitespaces() throws MFParseException { + /* + * ; Required whitespace + * abnf: s = *bidi ws o + */ + private int skipRequiredWhitespaces() throws MFParseException { + int position = input.getPosition(); + skipOptionalBidi(); int count = skipWhitespaces(); checkCondition(count > 0, "Space expected"); + skipOptionalWhitespaces(); return count; } - private int skipOptionalWhitespaces() { - return skipWhitespaces(); + private int skipOptionalBidi() { + int skipCount = 0; + while (true) { + int cp = input.peekChar(); + if (StringUtils.isBidi(cp)) { + skipCount++; + input.readCodePoint(); + } else { + return skipCount; + } + } } - private int skipWhitespaces() { + /* + * ; Optional whitespace + * abnf: o = *(ws / bidi) + */ + private int skipOptionalWhitespaces() { int skipCount = 0; while (true) { - int cp = input.readCodePoint(); - if (cp == EOF) { + int cp = input.peekChar(); + if (StringUtils.isWhitespace(cp) || StringUtils.isBidi(cp)) { + input.readCodePoint(); + skipCount++; + } else { return skipCount; } - if (!StringUtils.isWhitespace(cp)) { - input.backup(1); + } + } + + // abnf: ws = SP / HTAB / CR / LF / %x3000 + private int skipWhitespaces() { + int skipCount = 0; + while (true) { + int cp = input.peekChar(); + if (StringUtils.isWhitespace(cp)) { + skipCount++; + input.readCodePoint(); + } else { return skipCount; } - skipCount++; } } + private int skipOneOptionalBidi() { + int c = input.peekChar(); + if (StringUtils.isBidi(c)) { + // Consume it + input.readCodePoint(); + return 1; + } + return 0; + } + + // abnf: complex-message = o *(declaration o) complex-body o private MFDataModel.Message getComplexMessage() throws MFParseException { List declarations = new ArrayList<>(); boolean foundMatch = false; @@ -545,12 +592,13 @@ private MFDataModel.Message getComplexMessage() throws MFParseException { } declarations.add(declaration); } + // abnf: complex-body = quoted-pattern / matcher if (foundMatch) { return getMatch(declarations); } else { // Expect {{...}} or end of message + // abnf: complex-message = o *(declaration o) complex-body o skipOptionalWhitespaces(); int cp = input.peekChar(); - // abnf: complex-message = [s] *(declaration [s]) complex-body [s] checkCondition(cp != EOF, "Expected a quoted pattern or .match; got end-of-input"); MFDataModel.Pattern pattern = getQuotedPattern(); skipOptionalWhitespaces(); // Trailing whitespace is allowed @@ -559,10 +607,10 @@ private MFDataModel.Message getComplexMessage() throws MFParseException { } } - // abnf: matcher = match-statement 1*([s] variant) - // abnf: match-statement = match 1*([s] selector) - // abnf: selector = expression - // abnf: variant = key *(s key) [s] quoted-pattern + // abnf: matcher = match-statement s variant *(o variant) + // abnf: match-statement = match 1*(s selector) + // abnf: selector = variable + // abnf: variant = key *(s key) o quoted-pattern // abnf: key = literal / "*" // abnf: match = %s".match" private MFDataModel.SelectMessage getMatch(List declarations) @@ -571,17 +619,18 @@ private MFDataModel.SelectMessage getMatch(List declara // Look for selectors List expressions = new ArrayList<>(); while (true) { - // Whitespace not required between selectors: - // match 1*([s] selector) - // Whitespace not required before first variant: - // matcher = match-statement 1*([s] variant) - skipOptionalWhitespaces(); - MFDataModel.Expression expression = getPlaceholder(); - if (expression == null) { + // Whitespace required between selectors but not required before first variant. + skipRequiredWhitespaces(); + int cp = input.peekChar(); + if (cp != '$') { break; } - checkCondition( - !(expression instanceof MFDataModel.Markup), "Cannot do selection on markup"); + MFDataModel.VariableRef variableRef = getVariableRef(); + if (variableRef == null) { + break; + } + MFDataModel.Expression expression = + new MFDataModel.VariableExpression(variableRef, null, new ArrayList<>()); expressions.add(expression); } @@ -600,11 +649,10 @@ private MFDataModel.SelectMessage getMatch(List declara return new MFDataModel.SelectMessage(declarations, expressions, variants); } - // abnf: variant = key *(s key) [s] quoted-pattern + // abnf: variant = key *(s key) o quoted-pattern // abnf: key = literal / "*" private MFDataModel.Variant getVariant() throws MFParseException { List keys = new ArrayList<>(); - // abnf: variant = key *(s key) [s] quoted-pattern while (true) { // Space is required between keys MFDataModel.LiteralOrCatchallKey key = getKey(!keys.isEmpty()); @@ -633,7 +681,7 @@ private MFDataModel.LiteralOrCatchallKey getKey(boolean requireSpaces) throws MF } int skipCount = 0; if (requireSpaces) { - skipCount = skipMandatoryWhitespaces(); + skipCount = skipRequiredWhitespaces(); } else { skipCount = skipOptionalWhitespaces(); } @@ -647,7 +695,7 @@ private MFDataModel.LiteralOrCatchallKey getKey(boolean requireSpaces) throws MF input.backup(skipCount); return null; } - return getLiteral(); + return getLiteral(true); } private static class MatchDeclaration implements MFDataModel.Declaration { @@ -655,8 +703,9 @@ private static class MatchDeclaration implements MFDataModel.Declaration { // There is no such thing in the data model. } - // abnf: input-declaration = input [s] variable-expression - // abnf: local-declaration = local s variable [s] "=" [s] expression + // abnf: declaration = input-declaration / local-declaration + // abnf: input-declaration = input o variable-expression + // abnf: local-declaration = local s variable o "=" o expression private MFDataModel.Declaration getDeclaration() throws MFParseException { int position = input.getPosition(); skipOptionalWhitespaces(); @@ -671,6 +720,7 @@ private MFDataModel.Declaration getDeclaration() throws MFParseException { MFDataModel.Expression expression; switch (declName) { case "input": + // abnf: input = %s".input" skipOptionalWhitespaces(); expression = getPlaceholder(); String inputVarName = null; @@ -680,8 +730,9 @@ private MFDataModel.Declaration getDeclaration() throws MFParseException { return new MFDataModel.InputDeclaration(inputVarName, (MFDataModel.VariableExpression) expression); case "local": - // abnf: local-declaration = local s variable [s] "=" [s] expression - skipMandatoryWhitespaces(); + // abnf: local = %s".local" + // abnf: local-declaration = local s variable o "=" o expression + skipRequiredWhitespaces(); MFDataModel.LiteralOrVariableRef varName = getVariableRef(); skipOptionalWhitespaces(); cp = input.readCodePoint(); @@ -701,8 +752,9 @@ private MFDataModel.Declaration getDeclaration() throws MFParseException { return null; } - // quoted-pattern = "{{" pattern "}}" - private MFDataModel.Pattern getQuotedPattern() throws MFParseException { // {{ ... }} + // abnf: quoted-pattern = "{{" pattern "}}" + private MFDataModel.Pattern getQuotedPattern() throws MFParseException { + // abnf: quoted-pattern = "{{" pattern "}}" int cp = input.readCodePoint(); checkCondition(cp == '{', "Expected { for a complex body"); cp = input.readCodePoint(); @@ -715,12 +767,15 @@ private MFDataModel.Pattern getQuotedPattern() throws MFParseException { // {{ . return pattern; } + // abnf: name = [bidi] name-start *name-char [bidi] private String getName() throws MFParseException { + int savedPosition = input.getPosition(); StringBuilder result = new StringBuilder(); + skipOneOptionalBidi(); int cp = input.readCodePoint(); checkCondition(cp != EOF, "Expected name or namespace."); if (!StringUtils.isNameStart(cp)) { - input.backup(1); + input.gotoPosition(savedPosition); return null; } result.appendCodePoint(cp); @@ -735,7 +790,8 @@ private String getName() throws MFParseException { break; } } - return result.toString(); + skipOneOptionalBidi(); + return StringUtils.toNfc(result.toString()); } private void checkCondition(boolean condition, String message) throws MFParseException { @@ -769,8 +825,7 @@ private void error(String message) throws MFParseException { private String peekWithRegExp(Pattern pattern) { StringView sv = new StringView(input.buffer, input.getPosition()); Matcher m = pattern.matcher(sv); - boolean found = m.find(); - if (found) { + if (m.find()) { input.skip(m.group().length()); return m.group(); } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFSerializer.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFSerializer.java index 4ce31109f4f2..adc7e0de657b 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFSerializer.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MFSerializer.java @@ -7,12 +7,11 @@ import java.util.Map; import java.util.regex.Matcher; -import com.ibm.icu.message2.MFDataModel.Annotation; import com.ibm.icu.message2.MFDataModel.Attribute; import com.ibm.icu.message2.MFDataModel.CatchallKey; import com.ibm.icu.message2.MFDataModel.Declaration; import com.ibm.icu.message2.MFDataModel.Expression; -import com.ibm.icu.message2.MFDataModel.FunctionAnnotation; +import com.ibm.icu.message2.MFDataModel.Function; import com.ibm.icu.message2.MFDataModel.FunctionExpression; import com.ibm.icu.message2.MFDataModel.InputDeclaration; import com.ibm.icu.message2.MFDataModel.Literal; @@ -79,7 +78,15 @@ private void selectMessageToString(SelectMessage message) { result.append(".match"); for (Expression selector : message.selectors) { result.append(' '); - expressionToString(selector); + if (selector instanceof VariableExpression) { + VariableExpression ve = (VariableExpression) selector; + literalOrVariableRefToString(ve.arg); + } else { + // TODO: we have a (valid?) data model, so do we really want to fail? + // It is very close to release, so I am a bit reluctant to add a throw. + // I tried, and none of the unit tests fail (as expected). But still feels unsafe. + expressionToString(selector); + } } for (Variant variant : message.variants) { variantToString(variant); @@ -152,7 +159,7 @@ private void optionsToString(Map options) { private void functionExpressionToString(FunctionExpression fe) { result.append('{'); - annotationToString(fe.annotation); + functionToString(fe.function); attributesToString(fe.attributes); result.append('}'); } @@ -172,17 +179,17 @@ private void attributesToString(List attributes) { } } - private void annotationToString(Annotation annotation) { - if (annotation == null) { + private void functionToString(Function function) { + if (function == null) { return; } - if (annotation instanceof FunctionAnnotation) { + if (function instanceof Function) { addSpaceIfNeeded(); result.append(":"); - result.append(((FunctionAnnotation) annotation).name); - optionsToString(((FunctionAnnotation) annotation).options); + result.append(((Function) function).name); + optionsToString(((Function) function).options); } else { - errorType("Annotation", annotation); + errorType("Function", function); } } @@ -193,7 +200,7 @@ private void variableExpressionToString(VariableExpression ve) { result.append('{'); literalOrVariableRefToString(ve.arg); needSpace = true; - annotationToString(ve.annotation); + functionToString(ve.function); attributesToString(ve.attributes); result.append('}'); needSpace = false; @@ -251,7 +258,7 @@ private void literalExpressionToString(LiteralExpression le) { result.append('{'); literalOrVariableRefToString(le.arg); needSpace = true; - annotationToString(le.annotation); + functionToString(le.function); attributesToString(le.attributes); result.append('}'); } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MessageFormatter.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MessageFormatter.java index 2289d84f6c26..940f98492abe 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/MessageFormatter.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/MessageFormatter.java @@ -144,6 +144,7 @@ public class MessageFormatter { private final Locale locale; private final String pattern; private final ErrorHandlingBehavior errorHandlingBehavior; + private final BidiIsolation bidiIsolation; private final MFFunctionRegistry functionRegistry; private final MFDataModel.Message dataModel; private final MFDataModelFormatter modelFormatter; @@ -152,6 +153,7 @@ private MessageFormatter(Builder builder) { this.locale = builder.locale; this.functionRegistry = builder.functionRegistry; this.errorHandlingBehavior = builder.errorHandlingBehavior; + this.bidiIsolation = builder.bidiIsolation; if ((builder.pattern == null && builder.dataModel == null) || (builder.pattern != null && builder.dataModel != null)) { throw new IllegalArgumentException( @@ -173,7 +175,7 @@ private MessageFormatter(Builder builder) { + "Error: " + pe.getMessage() + "\n"); } } - modelFormatter = new MFDataModelFormatter(dataModel, locale, errorHandlingBehavior, functionRegistry); + modelFormatter = new MFDataModelFormatter(dataModel, locale, errorHandlingBehavior, bidiIsolation, functionRegistry); } /** @@ -217,6 +219,20 @@ public ErrorHandlingBehavior getErrorHandlingBehavior() { return errorHandlingBehavior; } + /** + * Get the {@link BidiIsolation} algorithm to use when formatting mixed + * message parts with mixed direction. + * + * @return the bidi isolation algorithm. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + public BidiIsolation getBidiIsolation() { + return bidiIsolation; + } + /** * Get the pattern (the serialized message in MessageFormat 2 syntax) of * the current {@code MessageFormatter}. @@ -320,6 +336,38 @@ public static enum ErrorHandlingBehavior { STRICT } + /** + * Determines how the mixtures of bidirectional text are converted to string. + * + *

They can be either ignored, or will implement the default algorithm + * described in the MessageFormat 2 specification.

+ * + *

Used in conjunction with the + * {@link MessageFormatter.Builder#setBidiIsolation(BidiIsolation)} method.

+ * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + public static enum BidiIsolation { + /** + * Ignore any text direction mixture, don't introduce bidi control characters in the formatted result. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + NONE, + /** + * Wrap direction mixtures in bidi control characters as described in the MessageFormat 2 specification. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + DEFAULT + } + /** * A {@code Builder} used to build instances of {@link MessageFormatter}. * @@ -331,6 +379,7 @@ public static class Builder { private Locale locale = Locale.getDefault(Locale.Category.FORMAT); private String pattern = null; private ErrorHandlingBehavior errorHandlingBehavior = ErrorHandlingBehavior.BEST_EFFORT; + private BidiIsolation bidiIsolation = BidiIsolation.NONE; private MFFunctionRegistry functionRegistry = MFFunctionRegistry.builder().build(); private MFDataModel.Message dataModel = null; @@ -374,7 +423,7 @@ public Builder setPattern(String pattern) { * *

The default value is {@code ErrorHandlingBehavior.BEST_EFFORT}, trying to fallback.

* - * @param the error handling behavior to use. + * @param errorHandlingBehavior the error handling behavior to use. * @return the builder, for fluent use. * * @internal ICU 76 technology preview @@ -386,6 +435,24 @@ public Builder setErrorHandlingBehavior(ErrorHandlingBehavior errorHandlingBehav return this; } + /** + * Sets the {@link BidiIsolation} to introduce bidi control characters / tags + * as described in the MessageFormat 2 specification. + * + *

The default value is {@code BidiIsolation.NONE}.

+ * + * @param bidiIsolation the bidi isolation algorithm to use. + * @return the builder, for fluent use. + * + * @internal ICU 77 technology preview + * @deprecated This API is for technology preview only. + */ + @Deprecated + public Builder setBidiIsolation(BidiIsolation bidiIsolation) { + this.bidiIsolation = bidiIsolation; + return this; + } + /** * Sets an instance of {@link MFFunctionRegistry} that should register any * custom functions used by the message. diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/NumberFormatterFactory.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/NumberFormatterFactory.java index 6cb6c56c45eb..3f97552d576b 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/NumberFormatterFactory.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/NumberFormatterFactory.java @@ -9,23 +9,25 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.regex.Pattern; import com.ibm.icu.math.BigDecimal; +import com.ibm.icu.message2.MFDataModel.CatchallKey; import com.ibm.icu.number.FormattedNumber; import com.ibm.icu.number.LocalizedNumberFormatter; import com.ibm.icu.number.Notation; import com.ibm.icu.number.NumberFormatter; import com.ibm.icu.number.NumberFormatter.GroupingStrategy; import com.ibm.icu.number.NumberFormatter.SignDisplay; +import com.ibm.icu.number.NumberFormatter.UnitWidth; import com.ibm.icu.number.Precision; -import com.ibm.icu.number.Scale; import com.ibm.icu.number.UnlocalizedNumberFormatter; import com.ibm.icu.text.FormattedValue; import com.ibm.icu.text.NumberingSystem; import com.ibm.icu.text.PluralRules; import com.ibm.icu.text.PluralRules.PluralType; +import com.ibm.icu.util.Currency; import com.ibm.icu.util.CurrencyAmount; -import com.ibm.icu.util.MeasureUnit; /** * Creates a {@link Formatter} doing numeric formatting, similar to {exp, number} @@ -38,6 +40,8 @@ public NumberFormatterFactory(String kind) { switch (kind) { case "number": // $FALL-THROUGH$ case "integer": + case "currency": + case "math": break; default: // Default to number @@ -79,15 +83,13 @@ static class NumberFormatterImpl implements Formatter { private final Map fixedOptions; private final LocalizedNumberFormatter icuFormatter; private final String kind; - final boolean advanced; NumberFormatterImpl(Locale locale, Map fixedOptions, String kind) { - this.locale = locale; + this.locale = OptUtils.getBestLocale(fixedOptions, locale); this.fixedOptions = new HashMap<>(fixedOptions); String skeleton = OptUtils.getString(fixedOptions, "icu:skeleton"); boolean fancy = skeleton != null; this.icuFormatter = formatterForOptions(locale, fixedOptions, kind); - this.advanced = fancy; this.kind = kind; } @@ -108,11 +110,12 @@ public String formatToString(Object toFormat, Map variableOption */ @Override public FormattedPlaceholder format(Object toFormat, Map variableOptions) { + boolean reportErrors = OptUtils.reportErrors(fixedOptions) || OptUtils.reportErrors(variableOptions); LocalizedNumberFormatter realFormatter; + Map mergedOptions = new HashMap<>(fixedOptions); if (variableOptions.isEmpty()) { realFormatter = this.icuFormatter; } else { - Map mergedOptions = new HashMap<>(fixedOptions); mergedOptions.putAll(variableOptions); // This is really wasteful, as we don't use the existing // formatter if even one option is variable. @@ -120,57 +123,105 @@ public FormattedPlaceholder format(Object toFormat, Map variable realFormatter = formatterForOptions(locale, mergedOptions, kind); } - Integer offset = OptUtils.getInteger(variableOptions, "icu:offset"); + Integer offset = OptUtils.getInteger(variableOptions, reportErrors, "icu:offset"); if (offset == null && fixedOptions != null) { - offset = OptUtils.getInteger(fixedOptions, "icu:offset"); + offset = OptUtils.getInteger(fixedOptions, reportErrors, "icu:offset"); } if (offset == null) { offset = 0; } + Double mathOperand = null; + if ("math".equals(kind)) { + ResolvedMathOptions resolvedMathOptions = ResolvedMathOptions.of(fixedOptions); + mathOperand = resolvedMathOptions.operand; + } + + if (kind.equals("currency")) { + String currencyCode = getCurrency(mergedOptions); + if (currencyCode == null && !(toFormat instanceof CurrencyAmount)) { + // Error, we need a currency code, either from the message, + // with the {... :currency currency=}, or from the thing to format + throw new IllegalArgumentException( + "bad-option: the `currency` must be an ISO 4217 code."); + } + } + + boolean isInt = kind.equals("integer"); FormattedValue result = null; if (toFormat == null) { // This is also what MessageFormat does. throw new NullPointerException("Argument to format can't be null"); } else if (toFormat instanceof Double) { - result = realFormatter.format((double) toFormat - offset); + if (isInt) toFormat = Math.floor((double) toFormat); + double toFormatAdjusted =(double) toFormat - offset; + if (mathOperand != null) { + toFormatAdjusted += mathOperand; + } + result = realFormatter.format(toFormatAdjusted); } else if (toFormat instanceof Long) { - result = realFormatter.format((long) toFormat - offset); + if (mathOperand != null) { + result = realFormatter.format((long) toFormat - offset + mathOperand); + } else { + result = realFormatter.format((long) toFormat - offset); + } } else if (toFormat instanceof Integer) { - result = realFormatter.format((int) toFormat - offset); + if (mathOperand != null) { + result = realFormatter.format((int) toFormat - offset + mathOperand); + } else { + result = realFormatter.format((int) toFormat - offset); + } } else if (toFormat instanceof BigDecimal) { BigDecimal bd = (BigDecimal) toFormat; - result = realFormatter.format(bd.subtract(BigDecimal.valueOf(offset))); + if (isInt) toFormat = bd.longValue(); + bd = bd.subtract(BigDecimal.valueOf(offset)); + if (mathOperand != null) { + bd = bd.add(BigDecimal.valueOf(mathOperand)); + } + result = realFormatter.format(bd); } else if (toFormat instanceof Number) { - result = realFormatter.format(((Number) toFormat).doubleValue() - offset); + if (isInt) toFormat = Math.floor(((Number) toFormat).doubleValue()); + double toFormatAdjusted = ((Number) toFormat).doubleValue() - offset; + if (mathOperand != null) { + toFormatAdjusted += mathOperand; + } + result = realFormatter.format(toFormatAdjusted); } else if (toFormat instanceof CurrencyAmount) { result = realFormatter.format((CurrencyAmount) toFormat); } else { // The behavior is not in the spec, will be in the registry. // We can return "NaN", or try to parse the string as a number String strValue = Objects.toString(toFormat); - Number nrValue = OptUtils.asNumber(strValue); + Number nrValue = OptUtils.asNumber(reportErrors, "argument", strValue); if (nrValue != null) { - result = realFormatter.format(nrValue.doubleValue() - offset); + if (isInt) toFormat = Math.floor(nrValue.doubleValue()); + double toFormatAdjusted = nrValue.doubleValue() - offset; + if (mathOperand != null) { + toFormatAdjusted += mathOperand; + } + result = realFormatter.format(toFormatAdjusted); } else { result = new PlainStringFormattedValue("{|" + strValue + "|}"); } } - return new FormattedPlaceholder(toFormat, result); + Directionality dir = OptUtils.getBestDirectionality(variableOptions, locale); + return new FormattedPlaceholder(toFormat, result, dir, false); } } private static class PluralSelectorImpl implements Selector { private static final String NO_MATCH = "\uFFFDNO_MATCH\uFFFE"; // Unlikely to show in a key private final PluralRules rules; - private Map fixedOptions; - private LocalizedNumberFormatter icuFormatter; + private final Map fixedOptions; + private final LocalizedNumberFormatter icuFormatter; + private final String kind; private PluralSelectorImpl( Locale locale, PluralRules rules, Map fixedOptions, String kind) { this.rules = rules; this.fixedOptions = fixedOptions; this.icuFormatter = formatterForOptions(locale, fixedOptions, kind); + this.kind = kind; } /** @@ -210,10 +261,10 @@ private static int pluralComparator(String o1, String o2) { return -1; } // * sorts last - if ("*".equals(o1)) { + if (CatchallKey.isCatchAll(o1)) { return 1; } - if ("*".equals(o2)) { + if (CatchallKey.isCatchAll(o2)) { return -1; } // Numbers sort first @@ -229,13 +280,14 @@ private static int pluralComparator(String o1, String o2) { } private boolean matches(Object value, String key, Map variableOptions) { - if ("*".equals(key)) { + if (CatchallKey.isCatchAll(key)) { return true; } - Integer offset = OptUtils.getInteger(variableOptions, "icu:offset"); + boolean reportErrors = OptUtils.reportErrors(fixedOptions); + Integer offset = OptUtils.getInteger(variableOptions, reportErrors, "icu:offset"); if (offset == null && fixedOptions != null) { - offset = OptUtils.getInteger(fixedOptions, "icu:offset"); + offset = OptUtils.getInteger(fixedOptions, reportErrors, "icu:offset"); } if (offset == null) { offset = 0; @@ -249,9 +301,14 @@ private boolean matches(Object value, String key, Map variableOp if (value instanceof Number) { valToCheck = ((Number) value).doubleValue(); + } else if (value instanceof CharSequence) { + return value.equals(key); } else { return false; } + if ("integer".equals(kind)) { + valToCheck = valToCheck.longValue(); + } Number keyNrVal = OptUtils.asNumber(key); if (keyNrVal != null && valToCheck.doubleValue() == keyNrVal.doubleValue()) { @@ -261,14 +318,20 @@ private boolean matches(Object value, String key, Map variableOp FormattedNumber formatted = icuFormatter.format(valToCheck.doubleValue() - offset); String match = rules.select(formatted); if (match.equals("other")) { - match = "*"; + match = CatchallKey.AS_KEY_STRING; } return match.equals(key); } } + // Currency ISO code + private final static Pattern CURRENCY_ISO_CODE = + Pattern.compile("^[A-Z][A-Z][A-Z]$", Pattern.CASE_INSENSITIVE); + private static LocalizedNumberFormatter formatterForOptions( Locale locale, Map fixedOptions, String kind) { + boolean reportErrors = OptUtils.reportErrors(fixedOptions); + UnlocalizedNumberFormatter nf; String skeleton = OptUtils.getString(fixedOptions, "icu:skeleton"); if (skeleton != null) { @@ -308,19 +371,16 @@ private static LocalizedNumberFormatter formatterForOptions( nf = nf.notation(notation); strOption = OptUtils.getString(fixedOptions, "style", "decimal"); - if (strOption.equals("percent")) { - nf = nf.unit(MeasureUnit.PERCENT).scale(Scale.powerOfTen(2)); - } - option = OptUtils.getInteger(fixedOptions, "minimumFractionDigits"); + option = OptUtils.getInteger(fixedOptions, reportErrors, "minimumFractionDigits"); if (option != null) { nf = nf.precision(Precision.minFraction(option)); } - option = OptUtils.getInteger(fixedOptions, "maximumFractionDigits"); + option = OptUtils.getInteger(fixedOptions, reportErrors, "maximumFractionDigits"); if (option != null) { nf = nf.precision(Precision.maxFraction(option)); } - option = OptUtils.getInteger(fixedOptions, "minimumSignificantDigits"); + option = OptUtils.getInteger(fixedOptions, reportErrors, "minimumSignificantDigits"); if (option != null) { nf = nf.precision(Precision.minSignificantDigits(option)); } @@ -335,11 +395,11 @@ private static LocalizedNumberFormatter formatterForOptions( } // The options below apply to both `:number` and `:integer` - option = OptUtils.getInteger(fixedOptions, "minimumIntegerDigits"); + option = OptUtils.getInteger(fixedOptions, reportErrors, "minimumIntegerDigits"); if (option != null) { // TODO! Ask Shane. nf.integerWidth(null) ? } - option = OptUtils.getInteger(fixedOptions, "maximumSignificantDigits"); + option = OptUtils.getInteger(fixedOptions, reportErrors, "maximumSignificantDigits"); if (option != null) { nf = nf.precision(Precision.maxSignificantDigits(option)); } @@ -386,7 +446,96 @@ private static LocalizedNumberFormatter formatterForOptions( if (kind.equals("integer")) { nf = nf.precision(Precision.integer()); } + if (kind.equals("currency")) { + strOption = getCurrency(fixedOptions); + if (strOption != null) { + nf = nf.unit(Currency.getInstance(strOption)); + } + strOption = OptUtils.getString(fixedOptions, "currencySign", "standard"); + switch (strOption) { + case "accounting": + case "standard": + break; + } + strOption = OptUtils.getString(fixedOptions, "currencyDisplay", "symbol"); + UnitWidth width; + switch (strOption) { + case "narrowSymbol": + width = UnitWidth.NARROW; + break; + case "symbol": + width = UnitWidth.SHORT; + break; + case "name": + width = UnitWidth.FULL_NAME; + break; + case "code": + width = UnitWidth.ISO_CODE; + break; + case "formalSymbol": + width = UnitWidth.FORMAL; + break; + case "never": + width = UnitWidth.HIDDEN; + break; + default: + width = UnitWidth.SHORT; + } + nf = nf.unitWidth(width); + } return nf.locale(locale); } + + static String getCurrency(Map options) { + String value = OptUtils.getString(options, "currency", null); + if (value != null) { + if (CURRENCY_ISO_CODE.matcher(value).find()) { + return value; + } else { + if (OptUtils.reportErrors(options)) { + throw new IllegalArgumentException( + "bad-option: the `currency` must be an ISO 4217 code."); + } + } + } + return null; + } + + private static class ResolvedMathOptions { + final Double operand; + final boolean reportErrors; + + ResolvedMathOptions(Double operand, boolean reportErrors) { + this.operand = operand; + this.reportErrors = reportErrors; + } + + static ResolvedMathOptions of(Map options) { + boolean reportErrors = OptUtils.reportErrors(options); + + Double operand = null; + String addOption = OptUtils.getString(options, "add", null); + String subtractOption = OptUtils.getString(options, "subtract"); + + if (addOption == null) { + if (subtractOption == null) { // both null + throw new IllegalArgumentException( + "bad-option: :math function needs an `add` or `subtract` option."); + } else { + operand = -OptUtils.asNumber(reportErrors, "subtract", subtractOption).doubleValue(); + } + } else { + if (subtractOption == null) { + operand = OptUtils.asNumber(reportErrors, "add", addOption).doubleValue(); + } else { // both set + throw new IllegalArgumentException( + "bad-option: :math function can't have both `add` and `subtract` options."); + } + } + + return new ResolvedMathOptions(operand, reportErrors); + } + } + } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/OptUtils.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/OptUtils.java index 7013984a19a4..b01742c4db1e 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/OptUtils.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/OptUtils.java @@ -3,9 +3,32 @@ package com.ibm.icu.message2; +import java.util.Locale; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.icu.util.ULocale; class OptUtils { + // abnf: ; number-literal matches JSON number (https://www.rfc-editor.org/rfc/rfc8259#section-6) + // abnf: number-literal = ["-"] (%x30 / (%x31-39 *DIGIT)) ["." 1*DIGIT] [%i"e" ["-" / "+"] 1*DIGIT] + // + // WARNING: this is different from the one in StringUtils by having a $ at the end. + // If there is an update to the spec, update StringUtils, then the one here is the same but add + // a "$" at the end + // + // That one is used to match the input up to a point, but continue (not an error). + // For example parsing `|3.14|` will match a `|`, then RE_NUMBER_LITERAL, then a `|` again. + // Not an error. + // The one here is used to validate options and arguments, for example `maxDigits=|1.|`, + // or `{|01| :number}` and by the time it gets to the checking the string literal was extracted + // by the parser and we only see "1." and "01". + // + // TBD: a way to reuse? + private static final Pattern RE_NUMBER_LITERAL = + Pattern.compile("^-?(0|[1-9][0-9]*)(\\.[0-9]+)?([eE][+\\-]?[0-9]+)?$"); + private OptUtils() {} static Number asNumber(Object value) { @@ -14,20 +37,43 @@ static Number asNumber(Object value) { } if (value instanceof CharSequence) { try { - return Double.parseDouble(value.toString()); + Matcher m = RE_NUMBER_LITERAL.matcher(value.toString()); + if (m.find()) { + return Double.parseDouble(value.toString()); + } } catch (NumberFormatException e) { - /* just ignore, we want to try more */ + /* just ignore, we continue and report */ } } return null; } - static Integer getInteger(Map options, String key) { + static Number asNumber(boolean reportErrors, String keyName, Object value) { + if (value instanceof Number) { + return (Number) value; + } + if (value instanceof CharSequence) { + try { + Matcher m = RE_NUMBER_LITERAL.matcher(value.toString()); + if (m.find()) { + return Double.parseDouble(value.toString()); + } + } catch (NumberFormatException e) { + /* just ignore, we continue and report */ + } + } + if (reportErrors) { + throw new IllegalArgumentException("bad-operand: " + keyName + " must be numeric"); + } + return null; + } + + static Integer getInteger(Map options, boolean reportErrors, String key) { Object value = options.get(key); if (value == null) { return null; } - Number nrValue = asNumber(value); + Number nrValue = asNumber(reportErrors, key, value); if (nrValue != null) { return nrValue.intValue(); } @@ -45,4 +91,70 @@ static String getString(Map options, String key, String defaultV static String getString(Map options, String key) { return getString(options, key, null); } + + static boolean reportErrors(Map options) { + String reportErrors = getString(options, "icu:impl:errorPolicy"); + return "STRICT".equals(reportErrors); + } + + static boolean reportErrors(Map fixedOptions, Map variableOptions) { + return reportErrors(fixedOptions) || reportErrors(variableOptions); + } + + static Locale getBestLocale(Map options, Locale defaultValue) { + Locale result = null; + String localeOverride = getString(options, "u:locale"); + if (localeOverride != null) { + try { + result = Locale.forLanguageTag(localeOverride.replace('_', '-')); + } catch (Exception e) { + if (reportErrors(options)) { + throw new IllegalArgumentException("bad-operand: u:locale must be a valid BCP 47 language tag"); + } + } + } + if (result == null) { + if (defaultValue == null) { + result = Locale.getDefault(); + } else { + result = defaultValue; + } + } + return result; + } + + static Directionality getBestDirectionality(Map options, Locale locale) { + Directionality result = getDirectionality(options); + return result == Directionality.UNKNOWN ? Directionality.of(ULocale.forLocale(locale)) : result; + } + + static Directionality getDirectionality(Map options) { + String value = getString(options, "u:dir"); + if (value == null) { + return Directionality.UNKNOWN; + } + Directionality result; + switch (value) { + case "rtl": + result = Directionality.RTL; + break; + case "ltr": + result = Directionality.LTR; + break; + case "auto": + result = Directionality.AUTO; + break; + case "inherit": + result = Directionality.INHERIT; + break; + default: + result = Directionality.UNKNOWN; + break; + } + return result; + } + + static String getUId(Map options) { + return getString(options, "u:id"); + } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/StringUtils.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/StringUtils.java index d90c4dc8d483..9b269b05c9ae 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/StringUtils.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/StringUtils.java @@ -3,6 +3,8 @@ package com.ibm.icu.message2; +import com.ibm.icu.text.Normalizer2; + class StringUtils { /* @@ -13,8 +15,8 @@ class StringUtils { * abnf: / %x2F-3F ; omit @ (%x40) * abnf: / %x41-5B ; omit \ (%x5C) * abnf: / %x5D-7A ; omit { | } (%x7B-7D) - * abnf: / %x7E-D7FF ; omit surrogates - * abnf: / %xE000-10FFFF + * abnf: / %x7E-2FFF ; omit IDEOGRAPHIC SPACE (%x3000) + * abnf: / %x3001-10FFFF ; allowing surrogates is intentional */ static boolean isContentChar(int cp) { return (cp >= 0x0001 && cp <= 0x0008) // omit HTAB (%x09) and LF (%x0A) @@ -24,11 +26,11 @@ static boolean isContentChar(int cp) { || (cp >= 0x002F && cp <= 0x003F) // omit @ (%x40) || (cp >= 0x0041 && cp <= 0x005B) // omit \ (%x5C) || (cp >= 0x005D && cp <= 0x007A) // omit { | } (%x7B-7D) - || (cp >= 0x007E && cp <= 0xD7FF) // omit surrogates - || (cp >= 0xE000 && cp <= 0x10FFFF); + || (cp >= 0x007E && cp <= 0x2FFF) // omit IDEOGRAPHIC SPACE (%x3000) + || (cp >= 0x3001 && cp <= 0x10FFFF); //allowing surrogates is intentional } - // abnf: text-char = content-char / s / "." / "@" / "|" + // abnf: text-char = content-char / ws / "." / "@" / "|" static boolean isTextChar(int cp) { return isContentChar(cp) || isWhitespace(cp) || cp == '.' || cp == '@' || cp == '|'; } @@ -40,16 +42,25 @@ static boolean isBackslash(int cp) { /* * ; Whitespace - * abnf: s = 1*( SP / HTAB / CR / LF / %x3000 ) + * abnf: ws = SP / HTAB / CR / LF / %x3000 */ static boolean isWhitespace(int cp) { - return cp == ' ' || cp == '\t' || cp == '\r' || cp == '\n' || cp == '\u3000'; + return cp == ' ' || cp == '\t' || cp == '\r' || cp == '\n' || cp == 0x3000; + } + + /* + * ; Bidirectional marks and isolates + * ; ALM / LRM / RLM / LRI, RLI, FSI & PDI + * abnf: bidi = %x061C / %x200E / %x200F / %x2066-2069 + */ + static boolean isBidi(int cp) { + return cp == 0x061C || cp == 0x200E || cp == 0x200F || (cp >= 0x2066 && cp <= 0x2069); } /* * abnf: name-start = ALPHA / "_" * abnf: / %xC0-D6 / %xD8-F6 / %xF8-2FF - * abnf: / %x370-37D / %x37F-1FFF / %x200C-200D + * abnf: / %x370-37D / %x37F-61B / %x61D-1FFF / %x200C-200D * abnf: / %x2070-218F / %x2C00-2FEF / %x3001-D7FF * abnf: / %xF900-FDCF / %xFDF0-FFFC / %x10000-EFFFF */ @@ -60,7 +71,8 @@ static boolean isNameStart(int cp) { || (cp >= 0x00D8 && cp <= 0x00F6) || (cp >= 0x00F8 && cp <= 0x02FF) || (cp >= 0x0370 && cp <= 0x037D) - || (cp >= 0x037F && cp <= 0x1FFF) + || (cp >= 0x037F && cp <= 0x061B) + || (cp >= 0x061D && cp <= 0x1FFF) || (cp >= 0x200C && cp <= 0x200D) || (cp >= 0x2070 && cp <= 0x218F) || (cp >= 0x2C00 && cp <= 0x2FEF) @@ -84,7 +96,7 @@ static boolean isNameChar(int cp) { || (cp >= 0x203F && cp <= 0x2040); } - // abnf: quoted-char = content-char / s / "." / "@" / "{" / "}" + // abnf: quoted-char = content-char / ws / "." / "@" / "{" / "}" static boolean isQuotedChar(int cp) { return isContentChar(cp) || isWhitespace(cp) @@ -117,4 +129,10 @@ static boolean isDigit(int cp) { static boolean isFunctionSigil(int cp) { return cp == ':'; } + + final private static Normalizer2 NFC_NORMALIZER = Normalizer2.getNFCInstance(); + + static String toNfc(CharSequence value) { + return value == null ? null : NFC_NORMALIZER.normalize(value); + } } diff --git a/icu4j/main/core/src/main/java/com/ibm/icu/message2/TextSelectorFactory.java b/icu4j/main/core/src/main/java/com/ibm/icu/message2/TextSelectorFactory.java index d08f62f77626..29591af370bc 100644 --- a/icu4j/main/core/src/main/java/com/ibm/icu/message2/TextSelectorFactory.java +++ b/icu4j/main/core/src/main/java/com/ibm/icu/message2/TextSelectorFactory.java @@ -9,6 +9,8 @@ import java.util.Map; import java.util.Objects; +import com.ibm.icu.message2.MFDataModel.CatchallKey; + /** * Creates a {@link Selector} doing literal selection, similar to {exp, select} * in {@link com.ibm.icu.text.MessageFormat}. @@ -32,6 +34,9 @@ public List matches( Object value, List keys, Map variableOptions) { List result = new ArrayList<>(); if (value == null) { + if (OptUtils.reportErrors(variableOptions)) { + throw new IllegalArgumentException("unresolved-variable: argument to match on can't be null"); + } return result; } for (String key : keys) { @@ -45,7 +50,7 @@ public List matches( @SuppressWarnings("static-method") private boolean matches(Object value, String key) { - if ("*".equals(key)) { + if (CatchallKey.isCatchAll(key)) { return true; } return key.equals(Objects.toString(value)); diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java index 63e3eefddf1d..13079c5d78b6 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CoreTest.java @@ -14,37 +14,47 @@ @SuppressWarnings({"static-method", "javadoc"}) @RunWith(JUnit4.class) public class CoreTest extends CoreTestFmwk { - private static final String[] JSON_FILES = {"alias-selector-annotations.json", - "duplicate-declarations.json", - "icu-parser-tests.json", - "icu-test-functions.json", - "icu-test-previous-release.json", - "icu-test-selectors.json", - "invalid-number-literals-diagnostics.json", - "invalid-options.json", - "markup.json", - "matches-whitespace.json", - "more-data-model-errors.json", - "more-functions.json", - "resolution-errors.json", - "runtime-errors.json", - "spec/data-model-errors.json", - "spec/syntax-errors.json", - "spec/syntax.json", - "spec/functions/date.json", - "spec/functions/datetime.json", - "spec/functions/integer.json", - "spec/functions/number.json", - "spec/functions/string.json", - "spec/functions/time.json", - "syntax-errors-diagnostics.json", - "syntax-errors-diagnostics-multiline.json", - "syntax-errors-end-of-input.json", - "syntax-errors-reserved.json", - "tricky-declarations.json", - "unsupported-expressions.json", - "unsupported-statements.json", - "valid-tests.json"}; + + private static final String[] JSON_FILES = { + "alias-selector-annotations.json", + "duplicate-declarations.json", + "icu-parser-tests.json", + "icu-test-functions.json", + "icu-test-previous-release.json", + "icu-test-selectors.json", + "invalid-number-literals-diagnostics.json", + "invalid-options.json", + "markup.json", + "matches-whitespace.json", + "more-data-model-errors.json", + "more-functions.json", + "normalization.json", + "resolution-errors.json", + "runtime-errors.json", + "spec/bidi.json", + "spec/data-model-errors.json", + "spec/syntax-errors.json", + "spec/syntax.json", + "spec/fallback.json", + "spec/functions/currency.json", + "spec/functions/date.json", + "spec/functions/datetime.json", + "spec/functions/integer.json", + "spec/functions/math.json", // FAILS 2 / 16, chaining to select + "spec/functions/number.json", + "spec/functions/string.json", + "spec/functions/time.json", + "spec/pattern-selection.json", + "spec/u-options.json", // FAILS 1 / 11, `:u:` on markup, issue #1005 + "syntax-errors-diagnostics.json", + "syntax-errors-diagnostics-multiline.json", + "syntax-errors-end-of-input.json", + "syntax-errors-reserved.json", + "tricky-declarations.json", + "unsupported-expressions.json", + "unsupported-statements.json", + "valid-tests.json" + }; @Test public void test() throws Exception { diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterMessageRefTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterMessageRefTest.java index 362ff351e170..524ede919d8c 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterMessageRefTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterMessageRefTest.java @@ -79,9 +79,9 @@ public String formatToString(Object toFormat, Map variableOption @BeforeClass public static void beforeClass() { - PROPERTIES.put("firefox", ".match {$gcase :string} genitive {{Firefoxin}} * {{Firefox}}"); - PROPERTIES.put("chrome", ".match {$gcase :string} genitive {{Chromen}} * {{Chrome}}"); - PROPERTIES.put("safari", ".match {$gcase :string} genitive {{Safarin}} * {{Safari}}"); + PROPERTIES.put("firefox", ".input {$gcase :string} .match $gcase genitive {{Firefoxin}} * {{Firefox}}"); + PROPERTIES.put("chrome", ".input {$gcase :string} .match $gcase genitive {{Chromen}} * {{Chrome}}"); + PROPERTIES.put("safari", ".input {$gcase :string} .match $gcase genitive {{Safarin}} * {{Safari}}"); } @Test diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java index 97dcfb3f287d..93baa18cc5b8 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/CustomFormatterPersonTest.java @@ -146,11 +146,13 @@ public void testCustomFunctionsComplexMessage() { Person malePerson = new Person("Mr.", "John", "Doe"); Person unknownPerson = new Person("Mr./Ms.", "Anonymous", "Doe"); String message = "" + + ".input {$hostGender :string}\n" + + ".input {$guestCount :number}\n" + ".local $hostName = {$host :person length=long}\n" + ".local $guestName = {$guest :person length=long}\n" + ".local $guestsOther = {$guestCount :number icu:offset=1}\n" // + "\n" - + ".match {$hostGender :icu:gender} {$guestCount :number}\n" + + ".match $hostGender $guestCount\n" // + "\n" + " female 0 {{{$hostName} does not give a party.}}\n" + " female 1 {{{$hostName} invites {$guestName} to her party.}}\n" diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DefaultTestProperties.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DefaultTestProperties.java index 21cc5a363a32..22ebc8630022 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DefaultTestProperties.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/DefaultTestProperties.java @@ -8,12 +8,31 @@ // Class corresponding to the json test files. // Since this is serialized by Gson, the field names should match the keys in the .json files. class DefaultTestProperties { + private static final ExpErrors NO_ERRORS = new ExpErrors(false); // Unused fields ignored - final String locale; - final Object[] expErrors; + private final String locale; + private final ExpErrors expErrors; + private final String bidiIsolation; - DefaultTestProperties(String locale, Object[] expErrors) { + DefaultTestProperties() { + this("en-US", new ExpErrors(false), "none"); + } + + DefaultTestProperties(String locale, ExpErrors expErrors, String bidiIsolation) { this.locale = locale; this.expErrors = expErrors; + this.bidiIsolation = bidiIsolation; + } + + String getLocale() { + return this.locale; + } + + ExpErrors getExpErrors() { + return expErrors == null ? NO_ERRORS : expErrors; + } + + String getBidiIsolation() { + return this.bidiIsolation; } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpErrors.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpErrors.java new file mode 100644 index 000000000000..7adba38ce28c --- /dev/null +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpErrors.java @@ -0,0 +1,44 @@ +// © 2025 and later: Unicode, Inc. and others. +// License & terms of use: https://www.unicode.org/copyright.html + +package com.ibm.icu.dev.test.message2; + +import java.util.ArrayList; +import java.util.List; + +// Class corresponding to the json test files. +// See Unit.java and StringToListAdapter.java for how this is used. +// Workaround for not being able to get the class of a generic type. + +class ExpErrors { + public final static String ANY_ERROR = "*-any"; + final List errors; + final boolean hasErrors; + + ExpErrors(boolean hasErrors) { + this.hasErrors = hasErrors; + this.errors = new ArrayList<>(); + if (hasErrors) { + // This is problematic if we start testing for the exact error we expect. + // Unlikely, since in Java we only report the error by throwing, + // without a good way to specify the exact error type. + // If we get there we might change this to an enum, so there is some + // refactoring to be done anyway. + this.errors.add(ANY_ERROR); + } + } + + ExpErrors(List errors) { + this.errors = errors == null ? new ArrayList<>() : errors; + this.hasErrors = errors != null && !errors.isEmpty(); + } + + boolean expectErrors() { + return this.hasErrors; + } + + @Override + public String toString() { + return ("[" + errors.toString() + "]"); + } +} diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpectedErrorAdapter.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpectedErrorAdapter.java new file mode 100644 index 000000000000..0b51e9abc7ef --- /dev/null +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/ExpectedErrorAdapter.java @@ -0,0 +1,68 @@ +// © 2025 and later: Unicode, Inc. and others. +// License & terms of use: https://www.unicode.org/copyright.html + +package com.ibm.icu.dev.test.message2; + +import java.io.IOException; +import java.util.ArrayList; + +import com.google.gson.TypeAdapter; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; + +/* Helper class that converts an array of objects with key named "type" + * and value `String` to a List. + * so that the ExpErrors property can be either a boolean or an array + * of strings objects. + * + * Example (json): + * ``` + * "expErrors": false, + * "expErrors": true, + * "expErrors": [], + * "expErrors": [{ "type": "syntax-error" }, { "type": "unknown-function" }] + * + * Used in the TestUtils class. + */ + +// Uses ArrayList instead of List so that when registering, it's possible +// to get ArrayList.class +public class ExpectedErrorAdapter extends TypeAdapter { + public ExpErrors read(JsonReader reader) throws IOException { + if (reader.peek() == JsonToken.NULL) { + reader.nextNull(); + return null; + } + if (reader.peek() == JsonToken.BEGIN_ARRAY) { + ArrayList result = new ArrayList(); + reader.beginArray(); + while (reader.hasNext()) { + reader.beginObject(); + String name = reader.nextName(); + assert(name.equals("type")); + String value = reader.nextString(); + result.add(value); + reader.endObject(); + } + reader.endArray(); + return new ExpErrors(result); + } + if (reader.peek() == JsonToken.BOOLEAN) { + return new ExpErrors(reader.nextBoolean()); + } + throw new IOException(); + } + + public void write(JsonWriter writer, ExpErrors value) throws IOException { + writer.beginArray(); + for (String s : value.errors) { + writer.beginObject(); + writer.name("type"); + writer.value(s); + writer.endObject(); + } + writer.endArray(); + } +} + diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java index 08cc9f0e1de9..892117798dc5 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/MessageFormat2Test.java @@ -129,7 +129,8 @@ public void testDateFormat() { @Test public void testPlural() { String message = "" - + ".match {$count :number}\n" + + ".input {$count :number}\n" + + ".match $count\n" + " 1 {{You have one notification.}}\n" + " * {{You have {$count} notifications.}}"; @@ -147,7 +148,8 @@ public void testPlural() { @Test public void testPluralOrdinal() { String message = "" - + ".match {$place :number select=ordinal}\n" + + ".input {$place :number select=ordinal}\n" + + ".match $place\n" + " 1 {{You got the gold medal}}\n" + " 2 {{You got the silver medal}}\n" + " 3 {{You got the bronze medal}}\n" @@ -203,7 +205,7 @@ public Formatter createFormatter(Locale locale, Map fixedOptions return new TemperatureFormatterImpl(nf, this); } - static private class TemperatureFormatterImpl implements Formatter { + private static class TemperatureFormatterImpl implements Formatter { private final TemperatureFormatterFactory formatterFactory; private final LocalizedNumberFormatter nf; private final Map cachedFormatters = @@ -332,7 +334,8 @@ public void testFormatterIsCreatedOnce() { @Test public void testPluralWithOffset() { String message = "" - + ".match {$count :number icu:offset=2}\n" + + ".input {$count :number icu:offset=2}\n" + + ".match $count\n" + " 1 {{Anna}}\n" + " 2 {{Anna and Bob}}\n" + " one {{Anna, Bob, and {$count :number icu:offset=2} other guest}}\n" @@ -361,7 +364,7 @@ public void testPluralWithOffset() { public void testPluralWithOffsetAndLocalVar() { String message = "" + ".local $foo = {$count :number icu:offset=2}" - + ".match {$foo :number}\n" // should "inherit" the offset + + ".match $foo\n" // should "inherit" the offset + " 1 {{Anna}}\n" + " 2 {{Anna and Bob}}\n" + " one {{Anna, Bob, and {$foo} other guest}}\n" @@ -390,7 +393,7 @@ public void testPluralWithOffsetAndLocalVar() { public void testPluralWithOffsetAndLocalVar2() { String message = "" + ".local $foo = {$amount :number icu:skeleton=|.00/w|}\n" - + ".match {$foo :number}\n" // should "inherit" the offset + + ".match $foo\n" // should "inherit" the offset + " 1 {{Last dollar}}\n" + " one {{{$foo} dollar}}\n" + " * {{{$foo} dollars}}"; @@ -412,7 +415,7 @@ public void testPluralWithOffsetAndLocalVar2() { public void testPluralWithOffsetAndLocalVar2Options() { String message = "" + ".local $foo = {$amount :number minumumFractionalDigits=2}\n" - + ".match {$foo :number}\n" // should "inherit" the offset + + ".match $foo\n" // should "inherit" the offset + " 1 {{Last dollar}}\n" + " one {{{$foo} dollar}}\n" + " * {{{$foo} dollars}}"; @@ -447,7 +450,8 @@ public void testLoopOnLocalVars() { @Test public void testVariableOptionsInSelector() { String messageVar = "" - + ".match {$count :number icu:offset=$delta}\n" + + ".input {$count :number icu:offset=$delta}\n" + + ".match $count\n" + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$count :number icu:offset=$delta} more character}}\n" @@ -465,7 +469,8 @@ public void testVariableOptionsInSelector() { mfVar.formatToString(Args.of("count", 7, "delta", 2))); String messageVar2 = "" - + ".match {$count :number icu:offset=$delta}\n" + + ".input {$count :number icu:offset=$delta}\n" + + ".match $count\n" + " 1 {{Exactly 1}}\n" + " 2 {{Exactly 2}}\n" + " * {{Count = {$count :number icu:offset=$delta} and delta={$delta}.}}"; @@ -505,7 +510,7 @@ public void testVariableOptionsInSelector() { public void testVariableOptionsInSelectorWithLocalVar() { String messageFix = "" + ".local $offCount = {$count :number icu:offset=2}" - + ".match {$offCount :number}\n" + + ".match $offCount\n" + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$offCount} more character}}\n" @@ -520,7 +525,7 @@ public void testVariableOptionsInSelectorWithLocalVar() { String messageVar = "" + ".local $offCount = {$count :number icu:offset=$delta}" - + ".match {$offCount :number}\n" + + ".match $offCount\n" + " 1 {{A}}\n" + " 2 {{A and B}}\n" + " one {{A, B, and {$offCount} more character}}\n" @@ -539,7 +544,7 @@ public void testVariableOptionsInSelectorWithLocalVar() { String messageVar2 = "" + ".local $offCount = {$count :number icu:offset=$delta}" - + ".match {$offCount :number}\n" + + ".match $offCount\n" + " 1 {{Exactly 1}}\n" + " 2 {{Exactly 2}}\n" + " * {{Count = {$count}, OffCount = {$offCount}, and delta={$delta}.}}"; diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Mf2IcuTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Mf2IcuTest.java index b72f4db0a6cc..915c08436d50 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Mf2IcuTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Mf2IcuTest.java @@ -74,7 +74,8 @@ public void testSimpleFormat() { @Test public void testSelectFormatToPattern() { String pattern = "" - + ".match {$userGender :string}\n" + + ".input {$userGender :string}\n" + + ".match $userGender\n" + " female {{{$userName} est all\u00E9e \u00E0 Paris.}}" + " * {{{$userName} est all\u00E9 \u00E0 Paris.}}" ; diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Param.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Param.java index fea1daaa466c..562bb1438514 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Param.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Param.java @@ -16,4 +16,11 @@ class Param { this.name = name; this.value = value; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Param { name:'").append(name).append("' value:'").append(value).append("' }"); + return builder.toString(); + } } diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java index 465b7cae8131..4cc18ec11c84 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/SerializationTest.java @@ -28,13 +28,13 @@ public void test() throws Exception { ".input {$a :number} {{Hello world!}}", ".local $b = {$a :number} {{Hello world!}}", ".local $c = {1 :number} {{Hello {userName}}}", - ".match {$count :number}\n" + ".input {$count :number} .match $count\n" + "one {{You deleted {$count} file}}\n" + "* {{You deleted {$count} files}}", - ".match {$count :number}\n" + ".input {$count :number} .match $count\n" + "one {{You deleted {$count} file}}\n" + "* {{You deleted {$count} files}}", - ".match {$place :number select=ordinal}\n" + ".input {$place :number select=ordinal} .match $place\n" + "* {{You fininshed in the {$place}th place}}\n" + "two {{You fininshed in the {$place}nd place}}\n" + "one {{You fininshed in the {$place}st place}}\n" @@ -42,7 +42,8 @@ public void test() throws Exception { + "2 {{You got the silver medal}}\n" + "3 {{You got the bronze medal}}\n" + "few {{You fininshed in the {$place}rd place}}", - ".match {$fileCount :number} {$folderCount :number}\n" + ".input {$fileCount :number}\n.input {$folderCount :number}\n" + + ".match $fileCount $folderCount\n" + "* * {{You deleted {$fileCount} files in {$folderCount} folders}}\n" + "one one {{You deleted {$fileCount} file in {$folderCount} folder}}\n" + "one * {{You deleted {$fileCount} file in {$folderCount} folders}}\n" @@ -52,26 +53,26 @@ public void test() throws Exception { "{|3.1415| :number minimumFractionDigits=5} dollars", "{|3.1415| :number maximumFractionDigits=2} dollars", ".local $c = {$count :number minimumFractionDigits=2}\n" - + ".match {$c}\n" + + ".match $c\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", ".local $c = {1 :number minimumFractionDigits=2}\n" - + ".match {$c}\n" + + ".match $c\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", ".local $c = {1 :number}\n" - + ".match {$c}\n" + + ".match $c\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", ".local $c = {1.25 :number}\n" - + ".match {$c}\n" + + ".match $c\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", ".local $c = {1.25 :number maximumFractionDigits=0}\n" - + ".match {$c}\n" + + ".match $c\n" + "one {{{$c} dollar}}\n" + "* {{{$c} dollars}}", - ".match {$count :number} 1 {{one}} * {{other}}", + ".input {$count :number} .match $count 1 {{one}} * {{other}}", }; for (String test : testStrings) { checkOneString(test); diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/StringToListAdapter.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/StringToListAdapter.java index bfe4be2cc260..70b86e0999ac 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/StringToListAdapter.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/StringToListAdapter.java @@ -40,6 +40,7 @@ public Sources read(JsonReader reader) throws IOException { } throw new IOException(); } + public void write(JsonWriter writer, Sources value) throws IOException { writer.beginArray(); for (String s : value.sources) { diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestFunctionFactory.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestFunctionFactory.java new file mode 100644 index 000000000000..74d40e063dee --- /dev/null +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestFunctionFactory.java @@ -0,0 +1,252 @@ +// © 2025 and later: Unicode, Inc. and others. +// License & terms of use: https://www.unicode.org/copyright.html + +package com.ibm.icu.dev.test.message2; + +import java.security.InvalidParameterException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.ibm.icu.message2.FormattedPlaceholder; +import com.ibm.icu.message2.Formatter; +import com.ibm.icu.message2.FormatterFactory; +import com.ibm.icu.message2.MFDataModel.CatchallKey; +import com.ibm.icu.message2.PlainStringFormattedValue; +import com.ibm.icu.message2.Selector; +import com.ibm.icu.message2.SelectorFactory; +import com.ibm.icu.text.FormattedValue; + +/** + * Locale-independent functions for formatting and selection. + * Implements the functionality required by `:test:function`, `:test:format`, and `:test:select`. + * Used only for testing (see test/README.md in the MF2 repository). + */ +public class TestFunctionFactory implements FormatterFactory, SelectorFactory { + private final String kind; + + public TestFunctionFactory(String kind) { + this.kind = kind; + } + + @Override + public Formatter createFormatter(Locale locale, Map fixedOptions) { + return new TestFormatterImpl(kind, fixedOptions); + } + + @Override + public Selector createSelector(Locale locale, Map fixedOptions) { + return new TestSelectorImpl(kind, fixedOptions); + } + + private static class TestFormatterImpl implements Formatter { + private final String kind; + private final ParsedOptions parsedOptions; + + public TestFormatterImpl(String kind, Map fixedOptions) { + this.kind = kind; + this.parsedOptions = ParsedOptions.of(fixedOptions); + } + + @Override + public String formatToString(Object toFormat, Map variableOptions) { + if (!"select".equals(kind) && parsedOptions.failsFormat) { + throw new InvalidParameterException("ALWAYS FAIL"); + } + return format(toFormat, variableOptions).toString(); + } + + @Override + public FormattedPlaceholder format(Object toFormat, Map variableOptions) { + return TestFunctionFactory.formatImpl(toFormat, parsedOptions); + } + } + + private static class TestSelectorImpl implements Selector { + private static final String NO_MATCH = "\uFFFDNO_MATCH\uFFFE"; // Unlikely to show in a key + private final String kind; + private final ParsedOptions parsedOptions; + + public TestSelectorImpl(String kind, Map fixedOptions) { + this.kind = kind; + this.parsedOptions = ParsedOptions.of(fixedOptions); + } + + @Override + public List matches(Object value, List keys, Map variableOptions) { +// ParsedOptions parsedOptions = ParsedOptions.of(variableOptions); + if (parsedOptions.failsSelect) { + throw new InvalidParameterException("Expected the test to always fail."); + } + + FormattedPlaceholder foo = TestFunctionFactory.formatImpl(value, parsedOptions); + List result = new ArrayList<>(); + for (String key : keys) { + if (CatchallKey.isCatchAll(key) || key.equals(foo.getFormattedValue().toString())) { + result.add(key); + } else { + result.add(NO_MATCH); + } + } + + result.sort(TestSelectorImpl::testComparator); + return result; + } + + private static int testComparator(String o1, String o2) { + if (o1.equals(o2)) { + return 0; + } + if (NO_MATCH.equals(o1)) { + return 1; + } + if (NO_MATCH.equals(o2)) { + return -1; + } + // * sorts last + if ("*".equals(o1)) { + return 1; + } + if ("*".equals(o2)) { + return -1; + } + // At this point they are both strings + // We should never get here, so the order does not really matter + return o1.compareTo(o2); + } + } + + private static class ParsedOptions { + final boolean reportErrors; + final boolean failsFormat; + final boolean failsSelect; + final int decimalPlaces; + + ParsedOptions(boolean reportErrors, boolean failsFormat, boolean failsSelect, int decimalPlaces) { + this.failsFormat = failsFormat; + this.failsSelect = failsSelect; + this.decimalPlaces = decimalPlaces; + this.reportErrors = reportErrors; + } + + static ParsedOptions of(Map options) { + boolean reportErrors = false; + // fail = "never" (default) + boolean failsFormat = false; + boolean failsSelect = false; + int decimalPlaces = 0; + if (options == null) { + return new ParsedOptions(reportErrors, failsFormat, failsSelect, decimalPlaces); + } + + String option = getStringOption(options, "icu:impl:errorPolicy", null); + reportErrors= "STRICT".equals(option); + + option = getStringOption(options, "fails", "never"); + if (option == null) { + System.out.println("WTF?"); + } + switch (option) { + case "never": + // both options are already set to false, all good + break; + case "select": + failsSelect = true; + break; + case "format": + failsFormat = true; + break; + case "always": + failsFormat = true; + failsSelect = true; + break; + default: + // "All other _options_ and their values are ignored." (spec, `test/README.md`) + // 1. Emit "bad-option" _Resolution Error_. + if (reportErrors) { + throw new InvalidParameterException("bad-option-value"); + } + } + option = getStringOption(options, "decimalPlaces", "0"); + switch (option) { + case "0": + decimalPlaces = 0; + break; + case "1": + decimalPlaces = 1; + break; + default: + // "All other _options_ and their values are ignored." (spec, `test/README.md`) + // 1. Emit "bad-option" _Resolution Error_. + // 1. Use a _fallback value_ as the _resolved value_ of the _expression_. + if (reportErrors) { + throw new InvalidParameterException("bad-option"); + } + } + return new ParsedOptions(reportErrors, failsFormat, failsSelect, decimalPlaces); + } + } + + private static String getStringOption(Map options, String key, String defaultVal) { + if (options == null) { + return defaultVal; + } + Object value = options.get(key); + if (value == null) { + return defaultVal; + } + return value.toString(); + } + + static FormattedPlaceholder formatImpl(Object toFormat, ParsedOptions parsedOptions) { + FormattedValue result = null; + Double dblToFormat = null; + if (toFormat == null) { + // This is also what MessageFormat does. + throw new NullPointerException("Argument to format can't be null"); + } else if (toFormat instanceof Double) { + dblToFormat = (Double) toFormat; + } else if (toFormat instanceof CharSequence) { + try { + dblToFormat = Double.valueOf(toFormat.toString()); + } catch (NumberFormatException e) { + if (parsedOptions.reportErrors) { + throw new IllegalArgumentException("bad-operand: argument must be numeric"); + } + } + } + + if (parsedOptions.failsFormat) { + if (parsedOptions.reportErrors) { + throw new InvalidParameterException("Expected the test to always fail."); + } + } + if (dblToFormat == null) { + if (parsedOptions.reportErrors) { + throw new NullPointerException("unresolved-variable: argument to format can't be null"); + } + result = new PlainStringFormattedValue("{|" + toFormat + "|}"); + } else { + StringBuffer buffer = new StringBuffer(); + if (dblToFormat < 0) { + // 1. If `Input` is less than 0, the character `-` U+002D Hyphen-Minus. + buffer.append('-'); + dblToFormat = -dblToFormat; + } + // 1. The truncated absolute integer value of `Input`, i.e. floor(abs(`Input`)), + // formatted as a sequence of decimal digit characters (U+0030...U+0039). + buffer.append(dblToFormat.intValue()); + if (parsedOptions.decimalPlaces == 1) { + // 1. If `DecimalPlaces` is 1, then + // 1. The character `.` U+002E Full Stop. + buffer.append('.'); + // 1. The single decimal digit character representing the value floor((abs(`Input`) - floor(abs(`Input`))) \* 10) + buffer.append((int) ((dblToFormat - dblToFormat.intValue()) * 10)); + } + result = new PlainStringFormattedValue(buffer.toString()); + } + + return new FormattedPlaceholder(toFormat, result); + } +} diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java index 71b9708dd7b7..80f6776d94c8 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/TestUtils.java @@ -14,6 +14,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Date; import java.util.Locale; import java.util.Map; @@ -25,6 +26,8 @@ import com.google.gson.GsonBuilder; import com.ibm.icu.message2.MFFunctionRegistry; import com.ibm.icu.message2.MessageFormatter; +import com.ibm.icu.message2.MessageFormatter.BidiIsolation; +import com.ibm.icu.message2.MessageFormatter.ErrorHandlingBehavior; /** Utility class, has no test methods. */ @Ignore("Utility class, has no test methods.") @@ -33,8 +36,17 @@ public class TestUtils { static final Gson GSON = new GsonBuilder() .setDateFormat("yyyy-MM-dd HH:mm:ss") .registerTypeAdapter(Sources.class, new StringToListAdapter()) + .registerTypeAdapter(ExpErrors.class, new ExpectedErrorAdapter()) .create(); + private static final MFFunctionRegistry TEST_REGISTRY = MFFunctionRegistry.builder() + .setFormatter("test:function", new TestFunctionFactory("function")) + .setFormatter("test:format", new TestFunctionFactory("format")) + .setFormatter("test:select", new TestFunctionFactory("select")) + .setSelector("test:function", new TestFunctionFactory("function")) + .setSelector("test:select", new TestFunctionFactory("select")) + .build(); + // ======= Legacy TestCase utilities, no json-compatible ======== static void runTestCase(TestCase testCase) { @@ -128,8 +140,8 @@ static Map paramsToMap(Param[] params) { } static boolean expectsErrors(DefaultTestProperties defaults, Unit unit) { - return (unit.expErrors != null && !unit.expErrors.isEmpty()) - || (defaults.expErrors != null && defaults.expErrors.length > 0); + return (unit.expErrors != null && unit.expErrors.expectErrors()) + || (defaults.getExpErrors().expectErrors()); } static void runTestCase(DefaultTestProperties defaults, Unit unit) { @@ -137,7 +149,7 @@ static void runTestCase(DefaultTestProperties defaults, Unit unit) { } static void runTestCase(DefaultTestProperties defaults, Unit unit, Param[] params) { - if (unit.ignoreJava != null) { + if (unit == null || unit.ignoreJava != null) { return; } @@ -150,15 +162,32 @@ static void runTestCase(DefaultTestProperties defaults, Unit unit, Param[] param // We can call the "complete" constructor with null values, but we want to test that // all constructors work properly. - MessageFormatter.Builder mfBuilder = - MessageFormatter.builder().setPattern(pattern.toString()); + MessageFormatter.Builder mfBuilder = MessageFormatter.builder() + .setPattern(pattern.toString()) + .setFunctionRegistry(TEST_REGISTRY); + + if (expectsErrors(defaults, unit)) { + mfBuilder.setErrorHandlingBehavior(ErrorHandlingBehavior.STRICT); + } if (unit.locale != null && !unit.locale.isEmpty()) { mfBuilder.setLocale(Locale.forLanguageTag(unit.locale)); - } else if (defaults.locale != null) { - mfBuilder.setLocale(Locale.forLanguageTag(defaults.locale)); + } else if (defaults.getLocale() != null) { + mfBuilder.setLocale(Locale.forLanguageTag(defaults.getLocale())); } else { mfBuilder.setLocale(Locale.US); } + if (defaults.getBidiIsolation() != null) { + switch (defaults.getBidiIsolation()) { + case "none": + mfBuilder.setBidiIsolation(BidiIsolation.NONE); + break; + case "default": + mfBuilder.setBidiIsolation(BidiIsolation.DEFAULT); + break; + default: + // Nothing + } + } try { MessageFormatter mf = mfBuilder.build(); @@ -171,7 +200,9 @@ static void runTestCase(DefaultTestProperties defaults, Unit unit, Param[] param if (expectsErrors(defaults, unit)) { fail(reportCase(unit) + "\nExpected error, but it didn't happen.\n" - + "Result: '" + result + "'"); + + "Result: '" + result + "'\n" + + "Params: " + Arrays.toString(params) + "\n" + + "Errors expected: " + unit.expErrors); } else { if (unit.exp != null) { assertEquals(reportCase(unit), unit.exp, result); @@ -201,4 +232,5 @@ private static Path getTestFile(Class cls, String fileName) throws URISyntaxE Path filePath = Paths.get(getPath); Path json = Paths.get(fileName); return filePath.resolve(json); - }} + } +} diff --git a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java index ff90ce0ccf9a..a190f802c9ca 100644 --- a/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java +++ b/icu4j/main/core/src/test/java/com/ibm/icu/dev/test/message2/Unit.java @@ -3,7 +3,6 @@ package com.ibm.icu.dev.test.message2; -import java.util.List; import java.util.StringJoiner; // Class corresponding to the json test files. @@ -15,7 +14,7 @@ class Unit { final Param[] params; final String exp; final String ignoreJava; - final List expErrors; + final ExpErrors expErrors; Unit( Sources src, @@ -23,7 +22,7 @@ class Unit { Param[] params, String exp, String ignoreJava, - List expErrors) { + ExpErrors expErrors) { this.src = src; this.locale = locale; this.params = params; @@ -32,7 +31,7 @@ class Unit { this.expErrors = expErrors; } - class Error { + static class Error { final String name; final String type; @@ -62,6 +61,9 @@ public String toString() { if (exp != null) { result.add("exp=" + escapeString(exp)); } + if (ignoreJava != null) { + result.add("ignoreJava=" + ignoreJava); + } return result.toString(); } @@ -79,7 +81,7 @@ public Unit merge(Unit other) { Param[] newParams = other.params != null ? other.params : this.params; String newExp = other.exp != null ? other.exp : this.exp; String newIgnore = other.ignoreJava != null ? other.ignoreJava : this.ignoreJava; - List newExpErrors = other.expErrors != null ? other.expErrors : this.expErrors; + ExpErrors newExpErrors = other.expErrors != null ? other.expErrors : this.expErrors; return new Unit(newSrc, newLocale, newParams, newExp, newIgnore, newExpErrors); } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/alias-selector-annotations.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/alias-selector-annotations.json index c064d40d4c82..d5307c5b1192 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/alias-selector-annotations.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/alias-selector-annotations.json @@ -6,11 +6,11 @@ }, "tests": [ { - "src": ".local $one = {|The one| :string}\n .match {$one}\n 1 {{Value is one}}\n * {{Value is not one}}", + "src": ".local $one = {|The one| :string}\n .match $one\n 1 {{Value is one}}\n * {{Value is not one}}", "exp": "Value is not one" }, { - "src": ".local $one = {|The one| :string}\n .local $two = {$one}\n .match {$two}\n 1 {{Value is one}}\n * {{Value is not one}}", + "src": ".local $one = {|The one| :string}\n .local $two = {$one}\n .match $two\n 1 {{Value is one}}\n * {{Value is not one}}", "exp": "Value is not one" } ] diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json index a97446addf0e..7121ad2e2c9d 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-functions.json @@ -193,10 +193,6 @@ "src": "Numbering system {123456 :number numberingSystem=deva}", "exp": "Numbering system १२३,४५६" }, - { - "src": "Percent {0.1416 :number style=percent}", - "exp": "Percent 14.16%" - }, { "src": "Scientific {123456789.97531 :number notation=scientific}", "exp": "Scientific 1.234568E8" diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json index 0a1e27dff6e2..74fc41fdfcea 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-previous-release.json @@ -31,50 +31,50 @@ "exp": "bar foo" }, { - "src": ".match {$foo :number} 1 {{one}} * {{other}}", + "src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}", "params": [{ "name": "foo", "value": "1" }], "exp": "one", "ignoreJava": "See ICU-22809" }, { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", + "src": ".input {$foo :string} .match $foo 1 {{one}} * {{other}}", "params": [{ "name": "foo", "value": "1" }], "exp": "one" }, { - "src": ".match {$foo :number} 1 {{one}} * {{other}}", + "src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}", "params": [{ "name": "foo", "value": 1 }], "exp": "one" }, { "ignoreJava": "Can't pass null in a map", "ignoreCpp": "Same as Java", - "src": ".match {$foo} 1 {{one}} * {{other}}", + "src": ".match $foo 1 {{one}} * {{other}}", "params": [{ "name": "foo", "value": null }], "exp": "other" }, { - "src": ".match {$foo :number} 1 {{one}} * {{other}}", + "src": ".input {$foo :number} .match $foo 1 {{one}} * {{other}}", "exp": "other", "expErrors": [{ "type": "unresolved-variable" }] }, { - "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", + "src": ".local $foo = {$bar :number} .match $foo one {{one}} * {{other}}", "params": [{ "name": "bar", "value": 1 }], "exp": "one" }, { - "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", + "src": ".local $foo = {$bar :number} .match $foo one {{one}} * {{other}}", "params": [{ "name": "bar", "value": 2 }], "exp": "other" }, { - "src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}", + "src": ".local $bar = {$none} .input {$foo :number} .match $foo one {{one}} * {{{$bar}}}", "params": [{ "name": "foo", "value": 1 }, {"name": "none", "value": "" }], "exp": "one" }, { - "src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}", + "src": ".local $bar = {$none :number} .input {$foo :string} .match $foo one {{one}} * {{{$bar}}}", "params": [{ "name": "foo", "value": 2 }], "exp": "{$none}", "expErrors": [{ "type": "unresolved-variable" }], @@ -120,17 +120,17 @@ "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/703" }, { - "src": ".match {|foo| :string} *{{foo}}", + "src": ".local $f = {|foo| :string} .match $f *{{foo}}", "exp": "foo" }, { - "src": ".match {$foo :string} * * {{foo}}", + "src": ".input {$foo :string} .match $foo * * {{foo}}", "exp": "foo", "expErrors": [{ "type": "variant-key-mismatch" }, { "type": "unresolved-variable" }], "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735" }, { - "src": ".match {$foo :string} {$bar :string} * {{foo}}", + "src": ".input {$foo :string} .input {$bar :string} .match $foo $bar * {{foo}}", "exp": "foo", "expErrors": [{ "type": "variant-key-mismatch" }, { "type": "unresolved-variable" }], "ignoreCpp": "Fallback is unclear. See https://github.com/unicode-org/message-format-wg/issues/735" diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json index 102bdfd88f50..7998f1359fb2 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/icu-test-selectors.json @@ -8,7 +8,8 @@ { "comment": "Testing simple plural", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -18,7 +19,8 @@ { "comment": "Testing simple plural", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -28,7 +30,8 @@ { "comment": "Testing simple plural", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -39,7 +42,8 @@ "comment": "Testing simple plural", "locale": "fr", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -50,7 +54,8 @@ "comment": "Testing simple plural", "locale": "fr", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -61,7 +66,8 @@ "comment": "Testing simple plural", "locale": "fr", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", "one {{{$count} file}}\n", " * {{{$count} files}}" ], @@ -71,7 +77,8 @@ { "comment": "Testing simple plural, but swap variant order", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", " * {{You deleted {$count} files}}\n", "one {{You deleted {$count} file}}" ], @@ -81,7 +88,8 @@ { "comment": "Testing simple plural, but swap variant order", "src": [ - ".match {$count :number}\n", + ".input {$count :number}\n", + ".match $count\n", " * {{You deleted {$count} files}}\n", "one {{You deleted {$count} file}}" ], @@ -91,7 +99,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -106,7 +115,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -121,7 +131,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -136,7 +147,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -151,7 +163,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -166,7 +179,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -181,7 +195,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -196,7 +211,8 @@ { "comment": "Ordinal, with mixed order and exact matches", "src": [ - ".match {$place :number select=ordinal}\n", + ".input {$place :number select=ordinal}\n", + ".match $place\n", "* {{You finished in the {$place}th place}}\n", "two {{You finished in the {$place}nd place}}\n", "one {{You finished in the {$place}st place}}\n", @@ -211,7 +227,8 @@ { "comment": "Plural combinations, mixed order", "src": [ - ".match {$fileCount :number} {$folderCount :number}\n", + ".input {$fileCount :number} .input {$folderCount :number}\n", + ".match $fileCount $folderCount\n", " * * {{You found {$fileCount} files in {$folderCount} folders}}\n", " one one {{You found {$fileCount} file in {$folderCount} folder}}\n", " one * {{You found {$fileCount} file in {$folderCount} folders}}\n", @@ -224,7 +241,8 @@ { "comment": "Plural combinations, mixed order", "src": [ - ".match {$fileCount :number} {$folderCount :number}\n", + ".input {$fileCount :number} .input {$folderCount :number}\n", + ".match $fileCount $folderCount\n", " * * {{You found {$fileCount} files in {$folderCount} folders}}\n", " one one {{You found {$fileCount} file in {$folderCount} folder}}\n", " one * {{You found {$fileCount} file in {$folderCount} folders}}\n", @@ -237,7 +255,8 @@ { "comment": "Plural combinations, mixed order", "src": [ - ".match {$fileCount :number} {$folderCount :number}\n", + ".input {$fileCount :number} .input {$folderCount :number}\n", + ".match $fileCount $folderCount\n", " * * {{You found {$fileCount} files in {$folderCount} folders}}\n", " one one {{You found {$fileCount} file in {$folderCount} folder}}\n", " one * {{You found {$fileCount} file in {$folderCount} folders}}\n", @@ -250,7 +269,8 @@ { "comment": "Plural combinations, mixed order", "src": [ - ".match {$fileCount :number} {$folderCount :number}\n", + ".input {$fileCount :number} .input {$folderCount :number}\n", + ".match $fileCount $folderCount\n", " * * {{You found {$fileCount} files in {$folderCount} folders}}\n", " one one {{You found {$fileCount} file in {$folderCount} folder}}\n", " one * {{You found {$fileCount} file in {$folderCount} folders}}\n", @@ -264,7 +284,7 @@ "comment": "Test that the selection honors the formatting option (`1.00 dollars`)", "src": [ ".local $c = {$price :number minimumFractionDigits=$minF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -276,7 +296,7 @@ "comment": "Test that the selection honors the formatting option (`1.00 dollars`)", "src": [ ".local $c = {$price :number minimumFractionDigits=$minF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -288,7 +308,7 @@ "comment": "Test that the selection honors the formatting option (`1.00 dollars`)", "src": [ ".local $c = {$price :number maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -300,7 +320,7 @@ "comment": "Test that the selection honors the formatting option (`1.00 dollars`)", "src": [ ".local $c = {$price :number maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -312,7 +332,7 @@ "comment": "Test that the selection honors the `:integer` over options", "src": [ ".local $c = {$price :integer maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -324,7 +344,7 @@ "comment": "Test that the selection honors the `:integer` over options", "src": [ ".local $c = {$price :integer maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -336,7 +356,7 @@ "comment": "Test that the selection honors the `:integer` over options", "src": [ ".local $c = {$price :integer maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -348,7 +368,7 @@ "comment": "Test that the selection honors the `:integer` over options", "src": [ ".local $c = {$price :integer maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], @@ -360,7 +380,7 @@ "comment": "Test that the selection honors the `:integer` over options", "src": [ ".local $c = {$price :integer maximumFractionDigits=$maxF}\n", - ".match {$c}\n", + ".match $c\n", " one {{{$c} dollar}}\n", " * {{{$c} dollars}}" ], diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/matches-whitespace.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/matches-whitespace.json index a0af4c4d143e..fd4b2b2ced15 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/matches-whitespace.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/matches-whitespace.json @@ -5,29 +5,14 @@ "locale": "en-US" }, "tests": [ - { "src": ".match {one :string} {bar :string} one * {{one}} * * {{other}}", - "exp": "one" }, - { "src": ".match {foo :string} {bar :string}one * {{one}} * * {{other}}", - "exp": "other" - }, - { "src": ".match {foo :string}{bar :string} one * {{one}} * * {{other}}", - "exp": "other" - }, - { "src": ".match {one :string}{bar :string}one * {{one}} * * {{other}}", - "exp": "one" - }, - { "src": ".match{foo :string} {bar :string} one * {{one}} * * {{other}}", - "exp": "other" - }, - { "src": ".match {foo :string} {bar :string} one * {{one}}* * {{other}}", + { "src": ".local $one = {1 :string} .local $bar = {bar :string} .match $one $bar one * {{one}} * * {{other}}", "exp": "other" }, - { "src": ".match {foo :string} {bar :string}one * {{one}}* * {{other}}", - "exp": "other" - }, - { "src": ".match {foo :string} {bar :string} one *{{one}} * * {{foo}}", + { "src": ".local $foo = {foo :string} .local $bar = {bar :string} .match $foo $bar one * {{one}}* * {{other}}", + "exp": "other" }, + { "src": ".local $foo = {foo :string} .local $bar = {bar :string} .match $foo $bar one *{{one}} * * {{foo}}", "exp": "foo" }, - { "src": ".match {foo :string} {bar :string} one * {{one}} * * {{foo}}", + { "src": ".local $foo = {foo :string} .local $bar = {bar :string} .match $foo $bar one * {{one}} * * {{foo}}", "exp": "foo" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/more-data-model-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/more-data-model-errors.json index 32744083af1c..36b528bc2715 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/more-data-model-errors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/more-data-model-errors.json @@ -5,7 +5,7 @@ }, "tests": [ { - "src": ".match {$foo :number} {$bar :number} one{{one}}", + "src": ".input {$foo :number} .input {$bar :number} .match $foo $bar one{{one}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -14,7 +14,7 @@ }, { - "src": ".match {$foo :number} {$bar :number} one {{one}}", + "src": ".input {$foo :number} .input {$bar :number} .match $foo $bar one {{one}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -23,7 +23,7 @@ }, { - "src": ".match {$foo :number} {$bar :number} one {{one}}", + "src": ".input {$foo :number} .input {$bar :number} .match $foo $bar one {{one}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -32,7 +32,7 @@ }, { - "src": ".match {$foo :number} * * {{foo}}", + "src": ".input {$foo :number} .match $foo * * {{foo}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -41,7 +41,7 @@ }, { - "src": ".match {$one :number}\n 1 2 {{Too many}}\n * {{Otherwise}}", + "src": ".input {$one :number} .match $one\n 1 2 {{Too many}}\n * {{Otherwise}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -50,7 +50,7 @@ }, { - "src": ".match {$one :number} {$two :number}\n 1 2 {{Two keys}}\n * {{Missing a key}}\n * * {{Otherwise}}", + "src": ".input {$one :number} .input {$two :number} .match $one $two \n 1 2 {{Two keys}}\n * {{Missing a key}}\n * * {{Otherwise}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -59,7 +59,7 @@ }, { - "src": ".match {$foo :x} {$bar :x} * {{foo}}", + "src": ".input {$foo :x} .input {$bar :X} .match $foo $bar * {{foo}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -68,7 +68,7 @@ }, { - "src": ".match {$one :number}\n 1 {{Value is one}}\n 2 {{Value is two}}", + "src": ".input {$one :number} .match $one\n 1 {{Value is one}}\n 2 {{Value is two}}", "expErrors": [ { "type": "missing-fallback-variant" @@ -77,7 +77,7 @@ }, { - "src": ".match {$one :number} {$two :number}\n 1 * {{First is one}}\n * 1 {{Second is one}}", + "src": ".input {$one :number} .input {$two :number} .match $one $two\n 1 * {{First is one}}\n * 1 {{Second is one}}", "expErrors": [ { "type": "missing-fallback-variant" @@ -86,7 +86,7 @@ }, { - "src": ".match {$one}\n 1 {{Value is one}}\n * {{Value is not one}}", + "src": ".match $one\n 1 {{Value is one}}\n * {{Value is not one}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -95,7 +95,7 @@ }, { - "src": ".local $one = {|The one|}\n .match {$one}\n 1 {{Value is one}}\n * {{Value is not one}}", + "src": ".local $one = {|The one|}\n .match $one\n 1 {{Value is one}}\n * {{Value is not one}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -103,7 +103,7 @@ ] }, { - "src": ".input {$foo} .match {$foo} one {{one}} * {{other}}", + "src": ".input {$foo} .match $foo one {{one}} * {{other}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -112,7 +112,7 @@ }, { - "src": ".local $foo = {$bar} .match {$foo} one {{one}} * {{other}}", + "src": ".local $foo = {$bar} .match $foo one {{one}} * {{other}}", "expErrors": [ { "type": "missing-selector-annotation" diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/normalization.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/normalization.json index 283e3dac8a82..da3bdca87eb3 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/normalization.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/normalization.json @@ -45,22 +45,22 @@ }, { "description": "NFC: keys are normalized", - "src": ".local $x = {\u1E0C\u0307 :string} .match {$x} \u1E0A\u0323 {{Right}} * {{Wrong}}", + "src": ".local $x = {\u1E0C\u0307 :string} .match $x \u1E0A\u0323 {{Right}} * {{Wrong}}", "exp": "Right" }, { "description": "NFC: keys are normalized (unquoted)", - "src": ".local $x = {\u1E0A\u0323 :string} .match {$x} \u1E0A\u0323 {{Not normalized}} \u1E0C\u0307 {{Normalized}} * {{Wrong}}", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x \u1E0A\u0323 {{Not normalized}} \u1E0C\u0307 {{Normalized}} * {{Wrong}}", "expErrors": [{"type": "duplicate-variant"}] }, { "description": "NFC: keys are normalized (quoted)", - "src": ".local $x = {\u1E0A\u0323 :string} .match {$x} |\u1E0A\u0323| {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x |\u1E0A\u0323| {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", "expErrors": [{"type": "duplicate-variant"}] }, { "description": "NFC: keys are normalized (mixed)", - "src": ".local $x = {\u1E0A\u0323 :string} .match {$x} \u1E0A\u0323 {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x \u1E0A\u0323 {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", "expErrors": [{"type": "duplicate-variant"}] } ] diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/resolution-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/resolution-errors.json index 8f1ac2fb9364..736e1cb64f05 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/resolution-errors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/resolution-errors.json @@ -7,8 +7,8 @@ "tests": [ { "src": "{$oops}", "exp": "{$oops}", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "ICU4J doesn't signal unresolved variable errors?"}, { "src": ".input {$x :number} {{{$x}}}", "exp": "{$x}", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, - { "src": ".local $foo = {$bar} .match {$foo :number} one {{one}} * {{other}}", "exp": "other", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, - { "src": ".local $bar = {$none :number} .match {$foo :string} one {{one}} * {{{$bar}}}", "exp": "{$none}", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": ".local $foo = {$bar} .local $f = {$foo :number} .match $f one {{one}} * {{other}}", "exp": "other", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, + { "src": ".local $bar = {$none :number} .local $f = {$foo :string} .match $f one {{one}} * {{{$bar}}}", "exp": "{$none}", "expErrors": [{ "type": "unresolved-variable" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"}, { "src": "The value is {horse :func}.", "exp": "The value is {|horse|}.", "expErrors": [{ "type": "unknown-function" }], "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782"} ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/runtime-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/runtime-errors.json index b1bb0cd491a0..ffeb081fab0d 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/runtime-errors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/runtime-errors.json @@ -6,19 +6,19 @@ }, "tests": [ { - "src": ".match {|horse| :date}\n 1 {{The value is one.}}\n * {{Formatter used as selector.}}", + "src": ".local $h = {|horse| :date} .match $h\n 1 {{The value is one.}}\n * {{Formatter used as selector.}}", "exp": "Formatter used as selector.", "expErrors": [{"type": "bad-selector"}], "ignoreJava": "ICU4J doesn't signal runtime errors?" }, { - "src": ".match {|horse| :number}\n 1 {{The value is one.}}\n * {{horse is not a number.}}", + "src": ".local $h = {|horse| :number} .match $h\n 1 {{The value is one.}}\n * {{horse is not a number.}}", "exp": "horse is not a number.", "expErrors": [{"type": "bad-selector"}], "ignoreJava": "ICU4J doesn't signal runtime errors?" }, { - "src": ".local $sel = {|horse| :number}\n .match {$sel}\n 1 {{The value is one.}}\n * {{horse is not a number.}}", + "src": ".local $sel = {|horse| :number}\n .match $sel\n 1 {{The value is one.}}\n * {{horse is not a number.}}", "exp": "horse is not a number.", "expErrors": [{"type": "bad-selector"}], "ignoreJava": "ICU4J doesn't signal runtime errors?" diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/bidi.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/bidi.json new file mode 100644 index 000000000000..2d650a3e34d6 --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/bidi.json @@ -0,0 +1,146 @@ +{ + "scenario": "Bidi support", + "description": "Tests for correct parsing of messages with bidirectional marks and isolates", + "defaultTestProperties": { + "bidiIsolation": "default", + "locale": "en-US" + }, + "tests": [ + { + "description": "simple-message = o [simple-start pattern]", + "src": " \u061C Hello world!", + "exp": " \u061C Hello world!" + }, + { + "description": "complex-message = o *(declaration o) complex-body o", + "src": "\u200E .local $x = {1} {{ {$x}}}", + "exp": " \u20681\u2069" + }, + { + "description": "complex-message = o *(declaration o) complex-body o", + "src": ".local $x = {1} \u200F {{ {$x}}}", + "exp": " \u20681\u2069" + }, + { + "description": "complex-message = o *(declaration o) complex-body o", + "src": ".local $x = {1} {{ {$x}}} \u2066", + "exp": " \u20681\u2069" + }, + { + "description": "input-declaration = input o variable-expression", + "src": ".input \u2067 {$x :number} {{hello}}", + "params": [{"name": "x", "value": "1"}], + "exp": "hello" + }, + { + "description": "local s variable o \"=\" o expression", + "src": ".local $x \u2068 = \u2069 {1} {{hello}}", + "exp": "hello" + }, + { + "description": "local s variable o \"=\" o expression", + "src": ".local \u2067 $x = {1} {{hello}}", + "exp": "hello" + }, + { + "description": "local s variable o \"=\" o expression", + "src": ".local\u2067 $x = {1} {{hello}}", + "exp": "hello" + }, + { + "description": "o \"{{\" pattern \"}}\"", + "src": "\u2067 {{hello}}", + "exp": "hello" + }, + { + "description": "match-statement s variant *(o variant)", + "src": ".local $x = {1 :number}\n.match $x\n1 {{one}}\n\u061C * {{other}}", + "exp": "one" + }, + { + "description": "match-statement s variant *(o variant)", + "src": ".local $x = {1 :number}.match $x \u061c1 {{one}}* {{other}}", + "exp": "one" + }, + { + "description": "match-statement s variant *(o variant)", + "src": ".local $x = {1 :number}.match $x\u061c1 {{one}}* {{other}}", + "expErrors": [{"type": "syntax-error"}] + }, + { + "description": "variant = key *(s key) quoted-pattern", + "src": ".local $x = {1 :number} .local $y = {$x :number}.match $x $y\n1 \u200E 1 {{one}}* * {{other}}", + "exp": "one" + }, + { + "description": "variant = key *(s key) quoted-pattern", + "src": ".local $x = {1 :number} .local $y = {$x :number}.match $x $y\n1\u200E 1 {{one}}* * {{other}}", + "exp": "one" + }, + { + "description": "literal-expression = \"{\" o literal [s function] *(s attribute) o \"}\"", + "src": "{\u200E hello \u200F}", + "exp": "\u2068hello\u2069" + }, + { + "description": "variable-expression = \"{\" o variable [s function] *(s attribute) o \"}\"", + "src": ".local $x = {1} {{ {\u200E $x \u200F} }}", + "exp": " \u20681\u2069 " + }, + { + "description": "function-expression = \"{\" o function *(s attribute) o \"}\"", + "src": "{1 \u200E :number \u200F}", + "exp": "1" + }, + { + "description": "markup = \"{\" o \"#\" identifier *(s option) *(s attribute) o [\"/\"] \"}\"", + "src": "{\u200F #b \u200E }", + "exp": "" + }, + { + "description": "markup = \"{\" o \"/\" identifier *(s option) *(s attribute) o \"}\"", + "src": "{\u200F /b \u200E }", + "exp": "" + }, + { + "description": "option = identifier o \"=\" o (literal / variable)", + "src": "{1 :number minimumFractionDigits\u200F=\u200E1 }", + "exp": "1.0" + }, + { + "description": "attribute = \"@\" identifier [o \"=\" o (literal / variable)]", + "src": "{1 :number @locale\u200F=\u200Een }", + "exp": "1" + }, + { + "description": " name... excludes U+FFFD and U+061C -- this pases as name -> [bidi] name-start *name-char", + "src": ".local $\u061Cfoo = {1} {{ {$\u061Cfoo} }}", + "exp": " \u20681\u2069 " + }, + { + "description": " name matches https://www.w3.org/TR/REC-xml-names/#NT-NCName but excludes U+FFFD and U+061C", + "src": ".local $foo\u061Cbar = {2} {{ }}", + "expErrors": [{"type": "syntax-error"}] + }, + { + "description": "name = [bidi] name-start *name-char [bidi]", + "src": ".local $\u200Efoo\u200F = {3} {{{$\u200Efoo\u200F}}}", + "exp": "\u20683\u2069" + }, + { + "description": "name = [bidi] name-start *name-char [bidi]", + "src": ".local $foo = {4} {{{$\u200Efoo\u200F}}}", + "exp": "\u20684\u2069" + }, + { + "description": "name = [bidi] name-start *name-char [bidi]", + "src": ".local $\u200Efoo\u200F = {5} {{{$foo}}}", + "exp": "\u20685\u2069" + }, + { + "description": "name = [bidi] name-start *name-char [bidi]", + "src": ".local $foo\u200Ebar = {5} {{{$foo\u200Ebar}}}", + "expErrors": [{"type": "syntax-error"}] + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/data-model-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/data-model-errors.json index 86a674c43961..f1f54cabe7c2 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/data-model-errors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/data-model-errors.json @@ -6,7 +6,7 @@ }, "tests": [ { - "src": ".match {$foo :x} * * {{foo}}", + "src": ".input {$foo :x} .match $foo * * {{foo}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -14,7 +14,7 @@ ] }, { - "src": ".match {$foo :x} {$bar :x} * {{foo}}", + "src": ".input {$foo :x} .input {$bar :x} .match $foo $bar * {{foo}}", "expErrors": [ { "type": "variant-key-mismatch" @@ -22,7 +22,7 @@ ] }, { - "src": ".match {:foo} 1 {{_}}", + "src": ".input {$foo :x} .match $foo 1 {{_}}", "expErrors": [ { "type": "missing-fallback-variant" @@ -30,7 +30,7 @@ ] }, { - "src": ".match {:foo} other {{_}}", + "src": ".input {$foo :x} .match $foo other {{_}}", "expErrors": [ { "type": "missing-fallback-variant" @@ -38,7 +38,7 @@ ] }, { - "src": ".match {:foo} {:bar} * 1 {{_}} 1 * {{_}}", + "src": ".input {$foo :x} .input {$bar :x} .match $foo $bar * 1 {{_}} 1 * {{_}}", "expErrors": [ { "type": "missing-fallback-variant" @@ -46,7 +46,7 @@ ] }, { - "src": ".match {$foo} one {{one}} * {{other}}", + "src": ".input {$foo} .match $foo one {{one}} * {{other}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -54,7 +54,7 @@ ] }, { - "src": ".input {$foo} .match {$foo} one {{one}} * {{other}}", + "src": ".local $foo = {$bar} .match $foo one {{one}} * {{other}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -62,7 +62,7 @@ ] }, { - "src": ".local $foo = {$bar} .match {$foo} one {{one}} * {{other}}", + "src": ".input {$bar} .local $foo = {$bar} .match $foo one {{one}} * {{other}}", "expErrors": [ { "type": "missing-selector-annotation" @@ -166,7 +166,7 @@ ] }, { - "src": ".match {$var :string} * {{The first default}} * {{The second default}}", + "src": ".input {$var :string} .match $var * {{The first default}} * {{The second default}}", "expErrors": [ { "type": "duplicate-variant" @@ -174,12 +174,16 @@ ] }, { - "src": ".match {$x :string} {$y :string} * foo {{The first foo variant}} bar * {{The bar variant}} * |foo| {{The second foo variant}} * * {{The default variant}}", + "src": ".input {$x :string} .input {$y :string} .match $x $y * foo {{The first foo variant}} bar * {{The bar variant}} * |foo| {{The second foo variant}} * * {{The default variant}}", "expErrors": [ { "type": "duplicate-variant" } ] + }, + { + "src": ".local $star = {star :string} .match $star |*| {{Literal star}} * {{The default}}", + "exp": "The default" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/fallback.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/fallback.json new file mode 100644 index 000000000000..fd1429c9b664 --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/fallback.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", + "scenario": "Fallback", + "description": "Test cases for fallback behaviour.", + "defaultTestProperties": { + "bidiIsolation": "none", + "locale": "en-US", + "expErrors": true + }, + "tests": [ + { + "description": "function with unquoted literal operand", + "src": "{42 :test:function fails=format}", + "exp": "{|42|}" + }, + { + "description": "function with quoted literal operand", + "src": "{|C:\\\\| :test:function fails=format}", + "exp": "{|C:\\\\|}" + }, + { + "description": "unannotated implicit input variable", + "src": "{$var}", + "exp": "{$var}" + }, + { + "description": "annotated implicit input variable", + "src": "{$var :number}", + "exp": "{$var}" + }, + { + "description": "local variable with unknown function in declaration", + "src": ".local $var = {|val| :test:undefined} {{{$var}}}", + "exp": "{$var}" + }, + { + "description": "function with local variable operand with unknown function in declaration", + "src": ".local $var = {|val| :test:undefined} {{{$var :test:function}}}", + "exp": "{$var}" + }, + { + "description": "local variable with unknown function in placeholder", + "src": ".local $var = {|val|} {{{$var :test:undefined}}}", + "exp": "{$var}" + }, + { + "description": "function with no operand", + "src": "{:test:undefined}", + "exp": "{:test:undefined}" + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/currency.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/currency.json new file mode 100644 index 000000000000..b844fa69eaa0 --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/currency.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", + "scenario": "Currency function", + "description": "The built-in formatter and selector for currencies.", + "defaultTestProperties": { + "bidiIsolation": "none", + "locale": "en-US" + }, + "tests": [ + { + "src": "{:currency}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": "{foo :currency}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": "{42 :currency}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": ".local $n = {42 :number} {{{$n :currency}}}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": "{42 :currency currency=EUR}", + "expErrors": false + }, + { + "src": ".local $n = {42 :number} {{{$n :currency currency=EUR}}}", + "expErrors": false + }, + { + "src": ".local $n = {42 :integer} {{{$n :currency currency=EUR}}}", + "expErrors": false + }, + { + "src": ".local $n = {42 :currency currency=EUR} {{{$n :currency}}}", + "expErrors": false + }, + { + "src": "{42 :currency currency=EUR fractionDigits=auto}", + "expErrors": false + }, + { + "src": "{42 :currency currency=EUR fractionDigits=2}", + "expErrors": false + }, + { + "src": "{$x :currency currency=EUR}", + "params": [{ "name": "x", "value": 41 }], + "expErrors": false + }, + { + "src": ".local $n = {42 :currency currency=EUR} .match $n * {{other}}", + "exp": "other", + "expErrors": [{ "type": "bad-selector" }] + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/date.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/date.json index dd14e6785fb6..625eb9712e46 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/date.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/date.json @@ -1,9 +1,11 @@ { + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", "scenario": "Date function", "description": "The built-in formatter for dates.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US", - "expErrors": [] + "expErrors": false }, "tests": [ { @@ -13,8 +15,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{horse :date}", @@ -23,8 +24,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{|2006-01-02| :date}" @@ -36,11 +36,10 @@ "src": "{|2006-01-02| :date style=long}" }, { - "src": ".local $d = {|2006-01-02| :date style=long} {{{$d :date}}}" + "src": ".local $d = {|2006-01-02| :date style=long} {{{$d}}}" }, { - "src": ".local $t = {|2006-01-02T15:04:06| :time} {{{$t :date}}}", - "ignoreJava": "ICU4J doesn't support this kind of composition" + "src": ".local $d = {|2006-01-02| :datetime dateStyle=long timeStyle=long} {{{$d :date}}}" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/datetime.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/datetime.json index bdfea3096cda..a5f3bd00bc9d 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/datetime.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/datetime.json @@ -1,9 +1,11 @@ { + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", "scenario": "Datetime function", "description": "The built-in formatter for datetimes.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US", - "expErrors": [] + "expErrors": false }, "tests": [ { @@ -13,8 +15,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{$x :datetime}", @@ -29,8 +30,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{horse :datetime}", @@ -39,8 +39,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{|2006-01-02T15:04:06| :datetime}" diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/integer.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/integer.json index c8e75077a221..f249cb27e500 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/integer.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/integer.json @@ -3,6 +3,7 @@ "scenario": "Integer function", "description": "The built-in formatter for integers.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US" }, "tests": [ @@ -19,14 +20,18 @@ "exp": "hello 4" }, { - "src": ".match {$foo :integer} one {{one}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1.2 - } - ], - "exp": "one" + "src": ".input {$foo :integer} .match $foo 1 {{=1}} * {{other}}", + "params": [{ "name": "foo", "value": 1.2 }], + "exp": "=1" + }, + { + "src": ".input {$foo :integer} .match $foo 1 {{=1}} one {{one}} * {{other}}", + "params": [{ "name": "foo", "value": 1.2 }], + "exp": "=1" + }, + { + "src": ".local $x = {1.25 :integer} .local $y = {$x :number} {{{$y}}}", + "exp": "1" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/math.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/math.json new file mode 100644 index 000000000000..dcaefc35a7bf --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/math.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", + "scenario": "Math function", + "description": "The built-in formatter and selector for addition and subtraction.", + "defaultTestProperties": { + "bidiIsolation": "none", + "locale": "en-US" + }, + "tests": [ + { + "src": "{:math add=13}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": "{foo :math add=13}", + "expErrors": [{ "type": "bad-operand" }] + }, + { + "src": "{42 :math}", + "expErrors": [{ "type": "bad-option" }] + }, + { + "src": "{42 :math add=foo}", + "expErrors": [{ "type": "bad-option" }] + }, + { + "src": "{42 :math subtract=foo}", + "expErrors": [{ "type": "bad-option" }] + }, + { + "src": "{42 :math foo=13}", + "expErrors": [{ "type": "bad-option" }] + }, + { + "src": "{42 :math add=13 subtract=13}", + "expErrors": [{ "type": "bad-option" }] + }, + { + "src": "{41 :math add=1}", + "exp": "42" + }, + { + "src": "{52 :math subtract=10}", + "exp": "42" + }, + { + "src": "{41 :math add=1 foo=13}", + "exp": "42" + }, + { + "src": ".local $x = {41 :integer signDisplay=always} {{{$x :math add=1}}}", + "exp": "+42" + }, + { + "src": ".local $x = {52 :number signDisplay=always} {{{$x :math subtract=10}}}", + "exp": "+42" + }, + { + "src": "{$x :math add=1}", + "params": [{ "name": "x", "value": 41 }], + "exp": "42" + }, + { + "src": "{$x :math subtract=10}", + "params": [{ "name": "x", "value": 52 }], + "exp": "42" +// }, +// { +// "src": ".local $x = {1 :math add=1} .match $x 1 {{=1}} 2 {{=2}} * {{other}}", +// "exp": "=2" +// }, +// { +// "src": ".local $x = {10 :integer} .local $y = {$x :math subtract=6} .match $y 10 {{=10}} 4 {{=4}} * {{other}}", +// "exp": "=4" + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/number.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/number.json index 1b81c705622b..dee7b9c01376 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/number.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/number.json @@ -1,7 +1,9 @@ { + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", "scenario": "Number function", "description": "The built-in formatter for numbers.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US" }, "tests": [ @@ -24,8 +26,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "invalid number literal {|.1| :number}", @@ -34,8 +35,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "invalid number literal {|1.| :number}", @@ -44,8 +44,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "invalid number literal {|01| :number}", @@ -54,8 +53,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "invalid number literal {|+1| :number}", @@ -64,8 +62,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "invalid number literal {|0x1| :number}", @@ -74,8 +71,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "hello {:number}", @@ -84,8 +80,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "hello {4.2 :number minimumFractionDigits=2}", @@ -137,35 +132,14 @@ }, { "src": ".local $foo = {$bar :number minimumFractionDigits=foo} {{bar {$foo}}}", - "params": [ - { - "name": "bar", - "value": 4.2 - } - ], - "exp": "bar {$bar}", - "expErrors": [ - { - "type": "bad-option" - } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "params": [{ "name": "bar", "value": 4.2 }], + "expErrors": [{ "type": "bad-option" }] }, { "src": ".local $foo = {$bar :number} {{bar {$foo}}}", - "params": [ - { - "name": "bar", - "value": "foo" - } - ], - "exp": "bar {$bar}", - "expErrors": [ - { - "type": "bad-operand" - } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "params": [{ "name": "bar", "value": "foo" }], + "exp": "bar {$foo}", + "expErrors": [{ "type": "bad-operand" }] }, { "src": ".input {$foo :number} {{bar {$foo}}}", @@ -189,19 +163,8 @@ }, { "src": ".input {$foo :number minimumFractionDigits=foo} {{bar {$foo}}}", - "params": [ - { - "name": "foo", - "value": 4.2 - } - ], - "exp": "bar {$foo}", - "expErrors": [ - { - "type": "bad-option" - } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "params": [{ "name": "foo", "value": 4.2 }], + "expErrors": [{ "type": "bad-option" }] }, { "src": ".input {$foo :number} {{bar {$foo}}}", @@ -216,176 +179,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" - }, - { - "src": ".match {$foo :number} one {{one}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".match {$foo :number} 1 {{=1}} one {{one}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "=1" - }, - { - "src": ".match {$foo :number} one {{one}} 1 {{=1}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "=1" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - }, - { - "name": "bar", - "value": 1 - } - ], - "exp": "one one" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - }, - { - "name": "bar", - "value": 2 - } - ], - "exp": "one other" - }, - { - "src": ".match {$foo :number} {$bar :number} one one {{one one}} one * {{one other}} * * {{other}}", - "params": [ - { - "name": "foo", - "value": 2 - }, - { - "name": "bar", - "value": 2 - } - ], - "exp": "other" - }, - { - "src": ".input {$foo :number} .match {$foo} one {{one}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".local $foo = {$bar :number} .match {$foo} one {{one}} * {{other}}", - "params": [ - { - "name": "bar", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".input {$foo :number} .local $bar = {$foo} .match {$bar} one {{one}} * {{other}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".input {$bar :number} .match {$bar} one {{one}} * {{other}}", - "params": [ - { - "name": "bar", - "value": 2 - } - ], - "exp": "other" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": [ - { - "name": "bar", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".input {$bar} .match {$bar :number} one {{one}} * {{other}}", - "params": [ - { - "name": "bar", - "value": 2 - } - ], - "exp": "other" - }, - { - "src": ".input {$none} .match {$foo :number} one {{one}} * {{{$none}}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}", - "params": [ - { - "name": "foo", - "value": 1 - } - ], - "exp": "one" - }, - { - "src": ".local $bar = {$none} .match {$foo :number} one {{one}} * {{{$bar}}}", - "params": [ - { - "name": "foo", - "value": 2 - } - ], - "exp": "{$none}", - "expErrors": [ - { - "type": "unresolved-variable" - } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{42 :number @foo @bar=13}", diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/string.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/string.json index 5858079b99a6..06d0255ce538 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/string.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/string.json @@ -1,12 +1,14 @@ { + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", "scenario": "String function", "description": "The built-in formatter for strings.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US" }, "tests": [ { - "src": ".match {$foo :string} |1| {{one}} * {{other}}", + "src": ".input {$foo :string} .match $foo |1| {{one}} * {{other}}", "params": [ { "name": "foo", @@ -16,36 +18,58 @@ "exp": "one" }, { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", + "src": ".input {$foo :string} .match $foo 1 {{one}} * {{other}}", "params": [ { "name": "foo", - "value": 1 + "value": "1" } ], - "exp": "one", - "ignoreJava": ":string doesn't stringify numbers?" + "exp": "one" }, { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", + "src": ".input {$foo :string} .match $foo 1 {{one}} * {{other}}", "params": [ { "name": "foo", - "value": null + "value": "2" } ], - "exp": "other", - "ignoreCpp": "Can't handle null value for input variable" + "exp": "other" }, { - "src": ".match {$foo :string} 1 {{one}} * {{other}}", + "src": ".input {$foo :string} .match $foo 1 {{one}} * {{other}}", "exp": "other", "expErrors": [ { "type": "unresolved-variable" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] + }, + { + "description": "NFC: keys are normalized (unquoted)", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x \u1E0A\u0323 {{Not normalized}} \u1E0C\u0307 {{Normalized}} * {{Wrong}}", + "expErrors": [{"type": "duplicate-variant"}] + }, + { + "description": "NFC: keys are normalized (quoted)", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x |\u1E0A\u0323| {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", + "expErrors": [{"type": "duplicate-variant"}] + }, + { + "description": "NFC: keys are normalized (mixed)", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x \u1E0A\u0323 {{Not normalized}} |\u1E0C\u0307| {{Normalized}} * {{Wrong}}", + "expErrors": [{"type": "duplicate-variant"}] + }, + { + "description": "NFC: :string normalizes the comparison value (un-normalized selector, normalized key)", + "src": ".local $x = {\u1E0A\u0323 :string} .match $x \u1E0C\u0307 {{Right}} * {{Wrong}}", + "exp": "Right" + }, + { + "description": "NFC: keys are normalized (normalized selector, un-normalized key)", + "src": ".local $x = {\u1E0C\u0307 :string} .match $x \u1E0A\u0323 {{Right}} * {{Wrong}}", + "exp": "Right" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/time.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/time.json index 845934a5e16a..1f6cf2293132 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/time.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/functions/time.json @@ -1,9 +1,11 @@ { + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", "scenario": "Time function", "description": "The built-in formatter for times.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US", - "expErrors": [] + "expErrors": false }, "tests": [ { @@ -13,8 +15,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{horse :time}", @@ -23,8 +24,7 @@ { "type": "bad-operand" } - ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + ] }, { "src": "{|2006-01-02T15:04:06| :time}" @@ -33,11 +33,10 @@ "src": "{|2006-01-02T15:04:06| :time style=medium}" }, { - "src": ".local $t = {|2006-01-02T15:04:06| :time style=medium} {{{$t :time}}}" + "src": ".local $t = {|2006-01-02T15:04:06| :time style=medium} {{{$t}}}" }, { - "src": ".local $d = {|2006-01-02T15:04:06| :date} {{{$d :time}}}", - "ignoreJava": "ICU4J doesn't support this kind of composition" + "src": ".local $t = {|2006-01-02T15:04:06| :datetime dateStyle=long timeStyle=long} {{{$t :time}}}" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/pattern-selection.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/pattern-selection.json new file mode 100644 index 000000000000..29dc146c1907 --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/pattern-selection.json @@ -0,0 +1,120 @@ +{ + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", + "scenario": "Pattern selection", + "description": "Tests for pattern selection", + "defaultTestProperties": { + "locale": "und" + }, + "tests": [ + { + "src": ".local $x = {1 :test:select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "1" + }, + { + "src": ".local $x = {0 :test:select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "other" + }, + { + "src": ".input {$x :test:select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 1 }], + "exp": "1" + }, + { + "src": ".input {$x :test:select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 2 }], + "exp": "other" + }, + { + "src": ".input {$x :test:select} .local $y = {$x} .match $y 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 1 }], + "exp": "1" + }, + { + "src": ".input {$x :test:select} .local $y = {$x} .match $y 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 2 }], + "exp": "other" + }, + { + "src": ".local $x = {1 :test:select decimalPlaces=1} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "1.0" + }, + { + "src": ".local $x = {1 :test:select decimalPlaces=1} .match $x 1 {{1}} 1.0 {{1.0}} * {{other}}", + "exp": "1.0" + }, + { + "src": ".local $x = {1 :test:select decimalPlaces=9} .match $x 1.0 {{1.0}} 1 {{1}} * {{bad-option-value}}", + "exp": "bad-option-value", + "expErrors": [{ "type": "bad-option" }, { "type": "bad-selector" }] + }, + { + "src": ".input {$x :test:select} .local $y = {$x :test:select decimalPlaces=1} .match $y 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 1 }], + "exp": "1.0" + }, + { + "src": ".input {$x :test:select decimalPlaces=1} .local $y = {$x :test:select} .match $y 1.0 {{1.0}} 1 {{1}} * {{other}}", + "params": [{ "name": "x", "value": 1 }], + "exp": "1.0" + }, + { + "src": ".input {$x :test:select decimalPlaces=9} .local $y = {$x :test:select decimalPlaces=1} .match $y 1.0 {{1.0}} 1 {{1}} * {{bad-option-value}}", + "params": [{ "name": "x", "value": 1 }], + "exp": "bad-option-value", + "expErrors": [ + { "type": "bad-option" }, + { "type": "bad-operand" }, + { "type": "bad-selector" } + ] + }, + { + "src": ".local $x = {1 :test:select fails=select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "other", + "expErrors": [{ "type": "bad-selector" }] + }, + { + "src": ".local $x = {1 :test:select fails=format} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "1" + }, + { + "src": ".local $x = {1 :test:format} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "other", + "expErrors": [{ "type": "bad-selector" }] + }, + { + "src": ".input {$x :test:select} .match $x 1.0 {{1.0}} 1 {{1}} * {{other}}", + "exp": "other", + "expErrors": [ + { "type": "unresolved-variable" }, + { "type": "bad-operand" }, + { "type": "bad-selector" } + ] + }, + { + "src": ".local $x = {1 :test:select} .local $y = {1 :test:select} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "1,1" + }, + { + "src": ".local $x = {1 :test:select} .local $y = {0 :test:select} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "1,*" + }, + { + "src": ".local $x = {0 :test:select} .local $y = {1 :test:select} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "*,1" + }, + { + "src": ".local $x = {0 :test:select} .local $y = {0 :test:select} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "*,*" + }, + { + "src": ".local $x = {1 :test:select fails=select} .local $y = {1 :test:select} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "*,1", + "expErrors": [{ "type": "bad-selector" }] + }, + { + "src": ".local $x = {1 :test:select} .local $y = {1 :test:format} .match $x $y 1 1 {{1,1}} 1 * {{1,*}} * 1 {{*,1}} * * {{*,*}}", + "exp": "1,*", + "expErrors": [{ "type": "bad-selector" }] + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax-errors.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax-errors.json index 34d9aa484572..00d0420f46f7 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax-errors.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax-errors.json @@ -122,6 +122,9 @@ { "src": "bad {:placeholder @attribute=@foo}" }, + { + "src": "bad {:placeholder @attribute=$foo}" + }, { "src": "{ @misplaced = attribute }" }, @@ -155,26 +158,90 @@ { "src": ".local $bar = |foo| {{_}}" }, - { - "src": ".match {#foo} * {{foo}}" - }, - { - "src": ".match {} * {{foo}}" - }, - { - "src": ".match {|foo| :x} {|bar| :x} ** {{foo}}" - }, - { - "src": ".match * {{foo}}" - }, - { - "src": ".match {|x| :x} * foo" - }, - { - "src": ".match {|x| :x} * {{foo}} extra" - }, - { - "src": ".match |x| * {{foo}}" - } + { "src": ".match {{foo}}" }, + { "src": ".match * {{foo}}" }, + { "src": ".match x * {{foo}}" }, + { "src": ".match |x| * {{foo}}" }, + { "src": ".match :x * {{foo}}" }, + { "src": ".match {$foo} * {{foo}}" }, + { "src": ".match {#foo} * {{foo}}" }, + { "src": ".input {$x :x} .match {$x} * {{foo}}" }, + { "src": ".input {$x :x} .match$x * {{foo}}" }, + { "src": ".input {$x :x} .match $x* {{foo}}" }, + { "src": ".input {$x :x} .match $x|x| {{foo}} * {{foo}}" }, + { "src": ".input {$x :x} .local $y = {y :y} .match $x$y * * {{foo}}" }, + { "src": ".input {$x :x} .local $y = {y :y} .match $x $y ** {{foo}}" }, + { "src": ".input {$x :x} .match $x" }, + { "src": ".input {$x :x} .match $x *" }, + { "src": ".input {$x :x} .match $x * foo" }, + { "src": ".input {$x :x} .match $x * {{foo}} extra" }, + { "src": ".n{a}{{}}" }, + { "src": "{^}" }, + { "src": "{!}" }, + { "src": ".n .{a}{{}}" }, + { "src": ".n. {a}{{}}" }, + { "src": ".n.{a}{b}{{}}" }, + { "src": "{!.}" }, + { "src": "{! .}" }, + { "src": "{%}" }, + { "src": "{*}" }, + { "src": "{+}" }, + { "src": "{<}" }, + { "src": "{>}" }, + { "src": "{?}" }, + { "src": "{~}" }, + { "src": "{^.}" }, + { "src": "{^ .}" }, + { "src": "{&}" }, + { "src": "{!.\\{}" }, + { "src": "{!. \\{}" }, + { "src": "{!|a|}" }, + { "src": "foo {+reserved}" }, + { "src": "foo {&private}" }, + { "src": "foo {?reserved @a @b=c}" }, + { "src": ".foo {42} {{bar}}" }, + { "src": ".foo{42}{{bar}}" }, + { "src": ".foo |}lit{| {42}{{bar}}" }, + { "src": ".i {1} {{}}" }, + { "src": ".l $y = {|bar|} {{}}" }, + { "src": ".l $x.y = {|bar|} {{}}" }, + { "src": "hello {|4.2| %number}" }, + { "src": "hello {|4.2| %n|um|ber}" }, + { "src": "{+42}" }, + { "src": "hello {|4.2| &num|be|r}" }, + { "src": "hello {|4.2| ^num|be|r}" }, + { "src": "hello {|4.2| +num|be|r}" }, + { "src": "hello {|4.2| ?num|be||r|s}" }, + { "src": "hello {|foo| !number}" }, + { "src": "hello {|foo| *number}" }, + { "src": "hello {?number}" }, + { "src": "{xyzz }" }, + { "src": "hello {$foo ~xyzz }" }, + { "src": "hello {$x xyzz }" }, + { "src": "{ !xyzz }" }, + { "src": "{~xyzz }" }, + { "src": "{ num x \\\\ abcde |aaa||3.14||42| r }" }, + { "src": "hello {$foo >num x \\\\ abcde |aaa||3.14| |42| r }" }, + { "src" : ".input{ $n ~ }{{{$n}}}" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax.json index 558fc64062b4..d03024bc81a6 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/syntax.json @@ -3,6 +3,7 @@ "scenario": "Syntax", "description": "Test cases that do not depend on any registry definitions.", "defaultTestProperties": { + "bidiIsolation": "none", "locale": "en-US" }, "tests": [ @@ -26,6 +27,11 @@ "src": "\\\\", "exp": "\\" }, + { + "description": "message -> simple-message -> simple-start pattern -> 1*escaped-char", + "src": "\\\\\\{\\|\\}", + "exp": "\\{|}" + }, { "description": "message -> simple-message -> simple-start pattern -> simple-start-char pattern -> ... -> simple-start-char *text-char placeholder", "src": "hello {world}", @@ -68,8 +74,7 @@ "type": "unresolved-variable" } ], - "exp": "hello {$place}", - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "exp": "hello {$place}" }, { "description": "message -> simple-message -> simple-start pattern -> placeholder -> expression -> literal-expression -> \"{\" literal \"}\"", @@ -80,50 +85,43 @@ "description": "... -> literal-expression -> \"{\" literal s annotation \"}\" -> \"{\" literal s function \"}\" -> \"{\" literal s \":\" identifier \"}\" -> \"{\" literal s \":\" name \"}\"", "src": "{a :f}", "exp": "{|a|}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... -> \"{\" literal s \":\" namespace \":\" name \"}\"", "src": "{a :u:f}", "exp": "{|a|}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "message -> simple-message -> simple-start pattern -> placeholder -> expression -> variable-expression -> \"{\" variable \"}\"", "src": "{$x}", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }] }, { "description": "... -> variable-expression -> \"{\" variable s annotation \"}\" -> \"{\" variable s function \"}\" -> \"{\" variable s \":\" identifier \"}\" -> \"{\" variable s \":\" name \"}\"", "src": "{$x :f}", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }, { "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }, { "type": "unknown-function" }] }, { "description": "... -> \"{\" variable s \":\" namespace \":\" name \"}\"", "src": "{$x :u:f}", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }, { "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }, { "type": "unknown-function" }] }, { "description": "... -> annotation-expression -> function -> \"{\" \":\" namespace \":\" name \"}\"", "src": "{:u:f}", "exp": "{:u:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... -> annotation-expression -> function -> \"{\" \":\" name \"}\"", "src": "{:f}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "message -> complex-message -> complex-body -> quoted-pattern -> \"{{\" pattern \"}}\" -> \"{{\"\"}}\"", @@ -173,11 +171,10 @@ "exp": "" }, { - "description": "message -> complex-message -> complex-body -> matcher -> match-statement variant -> match selector key quoted-pattern -> \".match\" expression literal quoted-pattern", - "src": ".match{a :f}a{{}}*{{}}", + "description": "message -> complex-message -> complex-body -> ... -> matcher -> match-statement variant -> match selector key quoted-pattern -> \".match\" variable literal quoted-pattern", + "src": ".local $a={a :f}.match $a a{{}}*{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }, { "type": "bad-selector" }] }, { "description": "... input-declaration -> input s variable-expression ...", @@ -199,40 +196,57 @@ "src": ".local $x = {a}{{}}", "exp": "" }, + { + "description": "input-declaration-like content in complex-message", + "src": "{{.input {$x}}}", + "params": [{ "name": "x", "value": "X" }], + "exp": ".input X" + }, + { + "description": "local-declaration-like content in complex-message with leading whitespace", + "src": "{{ .local $x = {$y}}}", + "params": [{ "name": "y", "value": "Y" }], + "exp": " .local $x = Y" + }, { "description": "... matcher -> match-statement [s] variant -> match 1*([s] selector) variant -> match selector selector variant -> match selector selector variant key s key quoted-pattern", - "src": ".match{a :f}{b :f}a b{{}}* *{{}}", + "src": ".local $a={a :f}.local $b={b :f}.match $a $b a b{{}}* *{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [ + { "type": "unknown-function" }, + { "type": "bad-selector" }, + { "type": "unknown-function" }, + { "type": "bad-selector" } + ] }, { "description": "... matcher -> match-statement [s] variant -> match 1*([s] selector) variant -> match selector variant variant ...", - "src": ".match{a :f}a{{}}b{{}}*{{}}", + "src": ".local $a={a :f}.match $a a{{}}b{{}}*{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }, { "type": "bad-selector" }] }, { "description": "... variant -> key s quoted-pattern -> ...", - "src": ".match{a :f}a {{}}*{{}}", + "src": ".local $a={a :f}.match $a a {{}}*{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }, { "type": "bad-selector" }] }, { "description": "... variant -> key s key s quoted-pattern -> ...", - "src": ".match{a :f}{b :f}a b {{}}* *{{}}", + "src": ".local $a={a :f}.local $b={b :f}.match $a $b a b {{}}* *{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [ + { "type": "unknown-function" }, + { "type": "bad-selector" }, + { "type": "unknown-function" }, + { "type": "bad-selector" } + ] }, { "description": "... key -> \"*\" ...", - "src": ".match{a :f}*{{}}", + "src": ".local $a={a :f}.match $a *{{}}", "exp": "", - "expErrors": [ { "type": "unknown-function" } ], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }, { "type": "bad-selector" }] }, { "description": "simple-message -> simple-start pattern -> placeholder -> expression -> literal-expression -> \"{\" s literal \"}\"", @@ -253,43 +267,37 @@ "description": "simple-message -> simple-start pattern -> placeholder -> expression -> variable-expression -> \"{\" s variable \"}\"", "src": "{ $x}", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }] }, { "description": "... variable-expression -> \"{\" variable s attribute \"}\" -> \"{\" variable s \"@\" identifier \"}\"", "src": "{$x @c}", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }] }, { "description": "... -> variable-expression -> \"{\" variable s \"}\"", "src": "{$x }", "exp": "{$x}", - "expErrors": [{ "type": "unresolved-variable" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unresolved-variable" }] }, { "description": "simple-message -> simple-start pattern -> placeholder -> expression -> annotation-expression -> \"{\" s annotation \"}\"", "src": "{ :f}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... annotation-expression -> \"{\" annotation s attribute \"}\" -> \"{\" annotation s \"@\" identifier \"}\"", "src": "{:f @c}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... -> annotation-expression -> \"{\" annotation s \"}\"", "src": "{:f }", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "message -> simple-message -> simple-start pattern -> placeholder -> markup -> \"{\" s \"#\" identifier \"}\"", @@ -360,29 +368,25 @@ "description": "... annotation-expression -> function -> \":\" identifier option", "src": "{:f k=v}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... option -> identifier s \"=\" literal", "src": "{:f k =v}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... option -> identifier \"=\" s literal", "src": "{:f k= v}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... option -> identifier s \"=\" s literal", "src": "{:f k = v}", "exp": "{:f}", - "expErrors": [{ "type": "unknown-function" }], - "ignoreJava": "See https://github.com/unicode-org/message-format-wg/issues/782" + "expErrors": [{ "type": "unknown-function" }] }, { "description": "... attribute -> \"@\" identifier \"=\" literal ...", @@ -405,8 +409,8 @@ "exp": "a" }, { - "description": "... attribute -> \"@\" identifier s \"=\" s variable ...", - "src": "{42 @foo=$bar}", + "description": "... attribute -> \"@\" identifier s \"=\" s quoted-literal ...", + "src": "{42 @foo=|bar|}", "exp": "42", "expParts": [ { @@ -432,9 +436,9 @@ "exp": "\\" }, { - "description": "... quoted-literal -> \"|\" quoted-char escaped-char \"|\"", - "src": "{|a\\\\|}", - "exp": "a\\" + "description": "... quoted-literal -> \"|\" quoted-char 1*escaped-char \"|\"", + "src": "{|a\\\\\\{\\|\\}|}", + "exp": "a\\{|}" }, { "description": "... unquoted-literal -> number-literal -> %x30", @@ -694,6 +698,43 @@ { "src": "{{trailing whitespace}} \n", "exp": "trailing whitespace" + }, + { + "description": "NFC: text is not normalized", + "src": "\u1E0A\u0323", + "exp": "\u1E0A\u0323" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is non-normalized, use is", + "src": ".local $\u0044\u0323\u0307 = {foo} {{{$\u1E0c\u0307}}}", + "exp": "foo" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is normalized, use isn't", + "src": ".local $\u1E0c\u0307 = {foo} {{{$\u0044\u0323\u0307}}}", + "exp": "foo" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is normalized, use isn't", + "src": ".input {$\u1E0c\u0307} {{{$\u0044\u0323\u0307}}}", + "params": [{"name": "\u1E0c\u0307", "value": "foo"}], + "exp": "foo" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is non-normalized, use is", + "src": ".input {$\u0044\u0323\u0307} {{{$\u1E0c\u0307}}}", + "params": [{"name": "\u0044\u0323\u0307", "value": "foo"}], + "exp": "foo" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is non-normalized, use is; reordering", + "src": ".local $\u0044\u0307\u0323 = {foo} {{{$\u1E0c\u0307}}}", + "exp": "foo" + }, + { + "description": "NFC: variables are compared to each other as-if normalized; decl is non-normalized, use is; special case mapping", + "src": ".local $\u0041\u030A\u0301 = {foo} {{{$\u01FA}}}", + "exp": "foo" } ] } diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/u-options.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/u-options.json new file mode 100644 index 000000000000..f0efaa1b59aa --- /dev/null +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/spec/u-options.json @@ -0,0 +1,148 @@ +{ + "$schema": "https://raw.githubusercontent.com/unicode-org/message-format-wg/main/test/schemas/v0/tests.schema.json", + "scenario": "u: Options", + "description": "Common options affecting the function context", + "defaultTestProperties": { + "bidiIsolation": "default", + "locale": "en-US" + }, + "tests": [ + { + "src": "{#tag u:id=x}content{/ns:tag u:id=x}", + "exp": "content", + "expParts": [ + { + "type": "markup", + "kind": "open", + "id": "x", + "name": "tag" + }, + { + "type": "literal", + "value": "content" + }, + { + "type": "markup", + "kind": "close", + "id": "x", + "name": "ns:tag" + } + ] + }, + { + "src": "{#tag u:dir=rtl u:locale=ar}content{/ns:tag}", + "exp": "content", +// "expErrors": [{ "type": "bad-option" }, { "type": "bad-option" }], + "expParts": [ + { + "type": "markup", + "kind": "open", + "name": "tag" + }, + { + "type": "literal", + "value": "content" + }, + { + "type": "markup", + "kind": "close", + "name": "ns:tag" + } + ] + }, + { + "src": "hello {4.2 :number u:locale=fr}", + "exp": "hello 4,2" + }, + { + "src": "hello {world :string u:dir=ltr u:id=foo}", + "exp": "hello \u2066world\u2069", + "expParts": [ + { + "type": "literal", + "value": "hello " + }, + { "type": "bidiIsolation", "value": "\u2066" }, + { + "type": "string", + "source": "|world|", + "dir": "ltr", + "id": "foo", + "value": "world" + }, + { "type": "bidiIsolation", "value": "\u2069" } + + ] + }, + { + "src": "hello {world :string u:dir=rtl}", + "exp": "hello \u2067world\u2069", + "expParts": [ + { "type": "literal", "value": "hello " }, + { "type": "bidiIsolation", "value": "\u2067" }, + { + "type": "string", + "source": "|world|", + "dir": "rtl", + "locale": "en-US", + "value": "world" + }, + { "type": "bidiIsolation", "value": "\u2069" } + ] + }, + { + "src": "hello {world :string u:dir=auto}", + "exp": "hello \u2068world\u2069", + "expParts": [ + { "type": "literal", "value": "hello " }, + { "type": "bidiIsolation", "value": "\u2068" }, + { + "type": "string", + "source": "|world|", + "locale": "en-US", + "value": "world" + }, + { "type": "bidiIsolation", "value": "\u2069" } + ] + }, + { + "src": ".local $world = {world :string u:dir=ltr u:id=foo} {{hello {$world}}}", + "exp": "hello \u2066world\u2069", + "expParts": [ + { + "type": "literal", + "value": "hello " + }, + { "type": "bidiIsolation", "value": "\u2066" }, + { + "type": "string", + "source": "|world|", + "dir": "ltr", + "id": "foo", + "value": "world" + }, + { "type": "bidiIsolation", "value": "\u2069" } + ] + }, + { + "locale": "ar", + "src": "أهلاً {بالعالم :string u:dir=rtl}", + "exp": "أهلاً \u2067بالعالم\u2069" + }, + { + "locale": "ar", + "src": "أهلاً {بالعالم :string u:dir=auto}", + "exp": "أهلاً \u2068بالعالم\u2069" + }, + { + "locale": "ar", + "src": "أهلاً {world :string u:dir=ltr}", + "exp": "أهلاً \u2066world\u2069" + }, + { + "locale": "ar", + "src": "أهلاً {بالعالم :string}", + "exp": "أهلاً \u2068بالعالم\u2069" + } + ] +} diff --git a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/valid-tests.json b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/valid-tests.json index 4b1ce8d67fb6..0f062116b733 100644 --- a/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/valid-tests.json +++ b/icu4j/main/core/src/test/resources/com/ibm/icu/dev/test/message2/valid-tests.json @@ -141,7 +141,7 @@ }, { "comment": "Trailing whitespace after match is valid", - "src": ".match {1 :string} * {{}} ", + "src": ".local $x = {1 :string} .match $x * {{}} ", "exp": "" }, {