[EN DateTimeV2] Fixed "one morning" incorrectly extracted as 1:00 AM (…

…#2681) (#2746) * Fixed "one morning" incorrectly extracted as 1:00 AM (#2681) * Added Duration AmbiguityFilter as per review Co-authored-by: LionbridgeCS2 <[email protected]>
microsoft · Nov 4, 2021 · 52c16ac · 52c16ac
1 parent 70270a0
commit 52c16ac
Show file tree

Hide file tree

Showing 17 changed files with 419 additions and 1 deletion.
diff --git a/.NET/Microsoft.Recognizers.Definitions.Common/English/DateTimeDefinitions.cs b/.NET/Microsoft.Recognizers.Definitions.Common/English/DateTimeDefinitions.cs
@@ -755,6 +755,14 @@ public static class DateTimeDefinitions
             { @"^(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)$", @"([$%£&!?@#])(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)|(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)([$%£&@#])" },
             { @"^(to\s+date)$", @"\b((equals?|up)\s+to\s+date)\b" }
         };
+      public static readonly Dictionary<string, string> AmbiguityTimeFiltersDict = new Dictionary<string, string>
+        {
+            { @"^(\p{L}+|\d{1,2})(\s+(morning|afternoon|evening|night))?$", @"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|\d{1,2})\s+(morning|afternoon|evening|night)\b" }
+        };
+      public static readonly Dictionary<string, string> AmbiguityDurationFiltersDict = new Dictionary<string, string>
+        {
+            { @"night$", @"\bnight(\s*|-)(club|light|market|shift|work(er)?)s?\b" }
+        };
       public static readonly IList<string> MorningTermList = new List<string>
         {
             @"morning"

diff --git a/...osoft.Recognizers.Text.DateTime/Arabic/Extractors/ArabicDurationExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/Arabic/Extractors/ArabicDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -127,5 +128,7 @@ public ArabicDurationExtractorConfiguration(IDateTimeOptionsConfiguration config
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...crosoft.Recognizers.Text.DateTime/Dutch/Extractors/DutchDurationExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/Dutch/Extractors/DutchDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -126,5 +127,7 @@ public DutchDurationExtractorConfiguration(IDateTimeOptionsConfiguration config)
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...oft.Recognizers.Text.DateTime/English/Extractors/EnglishDurationExtractorConfiguration.cs b/...oft.Recognizers.Text.DateTime/English/Extractors/EnglishDurationExtractorConfiguration.cs
@@ -1,10 +1,12 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
 using Microsoft.Recognizers.Definitions.English;
+using Microsoft.Recognizers.Definitions.Utilities;
 using Microsoft.Recognizers.Text.Number;
 
 namespace Microsoft.Recognizers.Text.DateTime.English
@@ -127,5 +129,7 @@ public EnglishDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => DefinitionLoader.LoadAmbiguityFilters(DateTimeDefinitions.AmbiguityDurationFiltersDict);
     }
 }
diff --git a/...crosoft.Recognizers.Text.DateTime/English/Extractors/EnglishTimeExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/English/Extractors/EnglishTimeExtractorConfiguration.cs
@@ -5,6 +5,7 @@
 using System.Text.RegularExpressions;
 
 using Microsoft.Recognizers.Definitions.English;
+using Microsoft.Recognizers.Definitions.Utilities;
 
 namespace Microsoft.Recognizers.Text.DateTime.English
 {
@@ -146,6 +147,6 @@ public EnglishTimeExtractorConfiguration(IDateTimeOptionsConfiguration config)
 
         public string TimeTokenPrefix => DateTimeDefinitions.TimeTokenPrefix;
 
-        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => DefinitionLoader.LoadAmbiguityFilters(DateTimeDefinitions.AmbiguityTimeFiltersDict);
     }
 }
diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/BaseDurationExtractor.cs b/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/BaseDurationExtractor.cs
@@ -3,6 +3,7 @@
 
 using System.Collections.Generic;
 using System.Linq;
+using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Text.Utilities;
 using DateObject = System.DateTime;
 
@@ -38,6 +39,9 @@ public List<ExtractResult> Extract(string text, DateObject reference)
 
             var rets = Token.MergeAllTokens(tokens, text, ExtractorName);
 
+            // Remove common ambiguous cases
+            rets = FilterAmbiguity(rets, text);
+
             // First MergeMultipleDuration then ResolveMoreThanOrLessThanPrefix so cases like "more than 4 days and less than 1 week" will not be merged into one "multipleDuration"
             if (this.merge)
             {
@@ -355,5 +359,26 @@ private List<ExtractResult> MergeMultipleDuration(string text, List<ExtractResul
 
             return results;
         }
+
+        private List<ExtractResult> FilterAmbiguity(List<ExtractResult> extractResults, string text)
+        {
+            if (this.config.AmbiguityFiltersDict != null)
+            {
+                foreach (var regex in this.config.AmbiguityFiltersDict)
+                {
+                    foreach (var extractResult in extractResults)
+                    {
+                        if (regex.Key.IsMatch(extractResult.Text))
+                        {
+                            var matches = regex.Value.Matches(text).Cast<Match>();
+                            extractResults = extractResults.Where(er => !matches.Any(m => m.Index < er.Start + er.Length && m.Index + m.Length > er.Start))
+                                .ToList();
+                        }
+                    }
+                }
+            }
+
+            return extractResults;
+        }
     }
 }
diff --git a/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/IDurationExtractorConfiguration.cs b/.NET/Microsoft.Recognizers.Text.DateTime/Extractors/IDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -52,5 +53,7 @@ public interface IDurationExtractorConfiguration : IDateTimeOptionsConfiguration
 
         IImmutableDictionary<string, long> UnitValueMap { get; }
 
+        Dictionary<Regex, Regex> AmbiguityFiltersDict { get; }
+
     }
 }
diff --git a/...osoft.Recognizers.Text.DateTime/French/Extractors/FrenchDurationExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/French/Extractors/FrenchDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -127,5 +128,7 @@ public FrenchDurationExtractorConfiguration(IDateTimeOptionsConfiguration config
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...osoft.Recognizers.Text.DateTime/German/Extractors/GermanDurationExtractorConfiguration.cs b/...osoft.Recognizers.Text.DateTime/German/Extractors/GermanDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Definitions.German;
@@ -127,5 +128,7 @@ public GermanDurationExtractorConfiguration(IDateTimeOptionsConfiguration config
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...crosoft.Recognizers.Text.DateTime/Hindi/Extractors/HindiDurationExtractorConfiguration.cs b/...crosoft.Recognizers.Text.DateTime/Hindi/Extractors/HindiDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -116,5 +117,7 @@ public HindiDurationExtractorConfiguration(IDateTimeOptionsConfiguration config)
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...oft.Recognizers.Text.DateTime/Italian/Extractors/ItalianDurationExtractorConfiguration.cs b/...oft.Recognizers.Text.DateTime/Italian/Extractors/ItalianDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -116,5 +117,7 @@ public ItalianDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...cognizers.Text.DateTime/Portuguese/Extractors/PortugueseDurationExtractorConfiguration.cs b/...cognizers.Text.DateTime/Portuguese/Extractors/PortugueseDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Definitions.Portuguese;
@@ -130,5 +131,7 @@ public PortugueseDurationExtractorConfiguration(IDateTimeOptionsConfiguration co
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...oft.Recognizers.Text.DateTime/Spanish/Extractors/SpanishDurationExtractorConfiguration.cs b/...oft.Recognizers.Text.DateTime/Spanish/Extractors/SpanishDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 using Microsoft.Recognizers.Definitions.Spanish;
@@ -130,5 +131,7 @@ public SpanishDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/...oft.Recognizers.Text.DateTime/Turkish/Extractors/TurkishDurationExtractorConfiguration.cs b/...oft.Recognizers.Text.DateTime/Turkish/Extractors/TurkishDurationExtractorConfiguration.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Text.RegularExpressions;
 
@@ -116,5 +117,7 @@ public TurkishDurationExtractorConfiguration(IDateTimeOptionsConfiguration confi
         Regex IDurationExtractorConfiguration.ModPrefixRegex => ModPrefixRegex;
 
         Regex IDurationExtractorConfiguration.ModSuffixRegex => ModSuffixRegex;
+
+        public Dictionary<Regex, Regex> AmbiguityFiltersDict => null;
     }
 }
diff --git a/Patterns/English/English-DateTime.yaml b/Patterns/English/English-DateTime.yaml
@@ -1181,6 +1181,14 @@ AmbiguityFiltersDict: !dictionary
     '^\d+m$': '^\d+m$'
     '^(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)$': '([$%£&!?@#])(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)|(apr|aug|dec|feb|jan|jul|jun|mar|may|nov|oct|sept?)([$%£&@#])'
     '^(to\s+date)$': '\b((equals?|up)\s+to\s+date)\b'
+AmbiguityTimeFiltersDict: !dictionary
+  types: [ string, string ]
+  entries:
+    '^(\p{L}+|\d{1,2})(\s+(morning|afternoon|evening|night))?$': '\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|\d{1,2})\s+(morning|afternoon|evening|night)\b'
+AmbiguityDurationFiltersDict: !dictionary
+  types: [ string, string ]
+  entries:
+    'night$': '\bnight(\s*|-)(club|light|market|shift|work(er)?)s?\b'
 # For TimeOfDay resolution
 MorningTermList: !list
   types: [ string ]