Skip to content

Commit

Permalink
[* DateTimeV2] Fix time resolution not consistent across cultures (#2936
Browse files Browse the repository at this point in the history 
) (#2961)

Co-authored-by: aitelint <[email protected]>
  • Loading branch information
@aitelint
aitelint and aitelint authored May 25, 2022
1 parent 10ac5d7 commit a1d01af
Show file tree
Hide file tree
Showing 25 changed files with 617 additions and 125 deletions.
6 changes: 3 additions & 3 deletions .NET/Microsoft.Recognizers.Definitions.Common/English/DateTimeDefinitions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public static class DateTimeDefinitions
public const string MealTimeRegex = @"\b(at\s+)?(?<mealTime>breakfast|brunch|lunch(\s*time)?|dinner(\s*time)?|supper)\b";
public static readonly string UnspecificTimePeriodRegex = $@"({MealTimeRegex})";
public static readonly string TimeOfDayRegex = $@"\b(?<timeOfDay>((((in\s+the\s+){LaterEarlyRegex}?(morning|afternoon|night(-?time)?|evening)s)|((in\s+the\s+)?{LaterEarlyRegex}?(in(\s+the)?\s+)?(morning|afternoon|night(-?time)?|evening)))|{MealTimeRegex}|(((in\s+(the)?\s+)?)(daytime|business\s+hours?))))\b";
public static readonly string SpecificTimeOfDayRegex = $@"\b(({StrictRelativeRegex}\s+{TimeOfDayRegex})\b|\btoni(ght|te))s?\b";
public static readonly string SpecificTimeOfDayRegex = $@"\b(({StrictRelativeRegex}\s+{TimeOfDayRegex})\b|\b(?<pm>toni(ght|te)))s?\b";
public static readonly string TimeFollowedUnit = $@"^\s*{TimeUnitRegex}";
public static readonly string TimeNumberCombinedWithUnit = $@"\b(?<num>\d+(\.\d*)?){TimeUnitRegex}";
public static readonly string[] BusinessHourSplitStrings = { @"business", @"hour" };
Expand All @@ -213,8 +213,8 @@ public static class DateTimeDefinitions
public const string SpecificEndOfRegex = @"(the\s+)?end of(\s+the)?\s*$";
public const string UnspecificEndOfRegex = @"\b(the\s+)?(eod|(end\s+of\s+day))\b";
public const string UnspecificEndOfRangeRegex = @"\b(eoy)\b";
public static readonly string PeriodTimeOfDayRegex = $@"\b((in\s+(the)?\s+)?{LaterEarlyRegex}?(this\s+)?{DateTimeTimeOfDayRegex})\b";
public static readonly string PeriodSpecificTimeOfDayRegex = $@"\b({LaterEarlyRegex}?this\s+{DateTimeTimeOfDayRegex}|({StrictRelativeRegex}\s+{PeriodTimeOfDayRegex})\b|\btoni(ght|te))\b";
public static readonly string PeriodTimeOfDayRegex = $@"\b((in\s+(the)?\s+)?{LaterEarlyRegex}?((this\s+)?{DateTimeTimeOfDayRegex}|(?<timeOfDay>(?<pm>tonight))))\b";
public static readonly string PeriodSpecificTimeOfDayRegex = $@"\b({LaterEarlyRegex}?this\s+{DateTimeTimeOfDayRegex}|({StrictRelativeRegex}\s+{PeriodTimeOfDayRegex})\b|\b(?<pm>toni(ght|te)))\b";
public static readonly string PeriodTimeOfDayWithDateRegex = $@"\b(({PeriodTimeOfDayRegex}(\s+(on|of))?))\b";
public const string LessThanRegex = @"\b(less\s+than)\b";
public const string MoreThanRegex = @"\b(more\s+than)\b";
Expand Down
4 changes: 4 additions & 0 deletions .NET/Microsoft.Recognizers.Text.DateTime/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ public static class Constants
// Hours is a half mid-day-duration
public const int HalfMidDayDurationHourCount = 2;

// Minutes in an hour
public const int HourMinuteCount = 60;

// Char length of four digits year, e.g., 2018
public const int FourDigitsYearLength = 4;

Expand Down Expand Up @@ -261,6 +264,7 @@ public static class Constants
public const string TimexHour = "H";
public const string TimexMinute = "M";
public const string TimexSecond = "S";
public const string TimexNow = "PRESENT_REF";
public const char TimexFuzzy = 'X';
public const string TimexFuzzyYear = "XXXX";
public const string TimexFuzzyMonth = "XX";
Expand Down
45 changes: 25 additions & 20 deletions .NET/Microsoft.Recognizers.Text.DateTime/Parsers/BaseDateTimePeriodParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1097,55 +1097,60 @@ private DateTimeResolutionResult MergeTwoTimePoints(string text, DateObject refe
DateObject pastBegin = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue,
pastEnd = (DateObject)((DateTimeResolutionResult)pr2.Value).PastValue;

if (bothHaveDates)
// If one side contains "ampm" while the other doesn't, shift the time appropriately
var ampmStr1 = ((DateTimeResolutionResult)pr1.Value).Comment;
var ampmStr2 = ((DateTimeResolutionResult)pr2.Value).Comment;
if (ampmStr1 is Constants.Comment_AmPm ^ ampmStr2 is Constants.Comment_AmPm)
{
if (futureBegin > futureEnd)
if (futureBegin > futureEnd && futureBegin.Date == futureEnd.Date)
{
futureBegin = pastBegin;
futureEnd = futureEnd.AddHours(Constants.HalfDayHourCount);
}

if (pastEnd < pastBegin)
if (pastBegin > pastEnd && pastBegin.Date == pastEnd.Date)
{
pastEnd = futureEnd;
pastEnd = pastEnd.AddHours(Constants.HalfDayHourCount);
}
}

var leftTimex = pr1.TimexStr;
var rightTimex = pr2.TimexStr;

if (bothHaveDates)
{
var duration = futureEnd - futureBegin;
var durationStr = Convert.ToInt32(duration.TotalHours) != 0 ? $"{Convert.ToInt32(duration.TotalHours)}H" :
$"{Convert.ToInt32(duration.TotalMinutes)}M";
ret.Timex = $"({pr1.TimexStr},{pr2.TimexStr},PT{durationStr})";
if (futureBegin > futureEnd)
{
futureBegin = pastBegin;
}

// Do nothing
if (pastEnd < pastBegin)
{
pastEnd = futureEnd;
}
}
else if (beginHasDate)
{
futureEnd = DateObject.MinValue.SafeCreateFromValue(
futureBegin.Year, futureBegin.Month, futureBegin.Day, futureEnd.Hour, futureEnd.Minute, futureEnd.Second);

pastEnd = DateObject.MinValue.SafeCreateFromValue(
pastBegin.Year, pastBegin.Month, pastBegin.Day, pastEnd.Hour, pastEnd.Minute, pastEnd.Second);

var dateStr = pr1.TimexStr.Split('T')[0];
var durationStr = DateTimeFormatUtil.LuisTimeSpan(futureEnd - futureBegin);
ret.Timex = $"({pr1.TimexStr},{dateStr + pr2.TimexStr},{durationStr})";
rightTimex = pr1.TimexStr.Equals(Constants.TimexNow, StringComparison.Ordinal) ? DateTimeFormatUtil.LuisDateShortTime(futureEnd) :
pr1.TimexStr.Split(Constants.TimeTimexPrefix[0])[0] + pr2.TimexStr;
}
else if (endHasDate)
{
futureBegin = DateObject.MinValue.SafeCreateFromValue(
futureEnd.Year, futureEnd.Month, futureEnd.Day, futureBegin.Hour, futureBegin.Minute, futureBegin.Second);

pastBegin = DateObject.MinValue.SafeCreateFromValue(
pastEnd.Year, pastEnd.Month, pastEnd.Day, pastBegin.Hour, pastBegin.Minute, pastBegin.Second);

var dateStr = pr2.TimexStr.Split('T')[0];
var durationStr = DateTimeFormatUtil.LuisTimeSpan(pastEnd - pastBegin);
ret.Timex = $"({dateStr + pr1.TimexStr},{pr2.TimexStr},{durationStr})";
leftTimex = pr2.TimexStr.Equals(Constants.TimexNow, StringComparison.Ordinal) ? DateTimeFormatUtil.LuisDateShortTime(pastBegin) :
pr2.TimexStr.Split(Constants.TimeTimexPrefix[0])[0] + pr1.TimexStr;
}

var ampmStr1 = ((DateTimeResolutionResult)pr1.Value).Comment;
var ampmStr2 = ((DateTimeResolutionResult)pr2.Value).Comment;
ret.Timex = TimexUtility.GenerateDateTimePeriodTimex(leftTimex, rightTimex, futureEnd - futureBegin);

if (!string.IsNullOrEmpty(ampmStr1) && ampmStr1.EndsWith(Constants.Comment_AmPm, StringComparison.Ordinal) &&
!string.IsNullOrEmpty(ampmStr2) && ampmStr2.EndsWith(Constants.Comment_AmPm, StringComparison.Ordinal))
{
Expand Down
52 changes: 10 additions & 42 deletions .NET/Microsoft.Recognizers.Text.DateTime/Parsers/CJK/BaseCJKDateTimePeriodParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,42 +232,24 @@ private DateTimeResolutionResult MergeTwoTimePoints(string text, DateObject refe
DateObject futureBegin = (DateObject)((DateTimeResolutionResult)pr1.Value).FutureValue,
futureEnd = (DateObject)((DateTimeResolutionResult)pr2.Value).FutureValue;

DateObject pastBegin = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue,
pastEnd = (DateObject)((DateTimeResolutionResult)pr2.Value).PastValue;
DateObject pastBegin = (DateObject)((DateTimeResolutionResult)pr1.Value).PastValue;

if (futureBegin > futureEnd)
{
futureBegin = pastBegin;
}

if (pastEnd < pastBegin)
{
pastEnd = futureEnd;
}

if (bothHaveDates)
{
rightTime = DateObject.MinValue.SafeCreateFromValue(futureEnd.Year, futureEnd.Month, futureEnd.Day);
leftTime = DateObject.MinValue.SafeCreateFromValue(futureBegin.Year, futureBegin.Month, futureBegin.Day);
}
else if (beginHasDate)
{
// TODO: Handle "明天下午两点到五点"
futureEnd = DateObject.MinValue.SafeCreateFromValue(
futureBegin.Year, futureBegin.Month, futureBegin.Day, futureEnd.Hour, futureEnd.Minute, futureEnd.Second);
pastEnd = DateObject.MinValue.SafeCreateFromValue(
pastBegin.Year, pastBegin.Month, pastBegin.Day, pastEnd.Hour, pastEnd.Minute, pastEnd.Second);

leftTime = DateObject.MinValue.SafeCreateFromValue(futureBegin.Year, futureBegin.Month, futureBegin.Day);
}
else if (endHasDate)
{
// TODO: Handle "明天下午两点到五点"
futureBegin = DateObject.MinValue.SafeCreateFromValue(
futureEnd.Year, futureEnd.Month, futureEnd.Day, futureBegin.Hour, futureBegin.Minute, futureBegin.Second);
pastBegin = DateObject.MinValue.SafeCreateFromValue(
pastEnd.Year, pastEnd.Month, pastEnd.Day, pastBegin.Hour, pastBegin.Minute, pastBegin.Second);

rightTime = DateObject.MinValue.SafeCreateFromValue(futureEnd.Year, futureEnd.Month, futureEnd.Day);
}

Expand All @@ -276,27 +258,18 @@ private DateTimeResolutionResult MergeTwoTimePoints(string text, DateObject refe
var leftResultTime = (DateObject)leftResult.FutureValue;
var rightResultTime = (DateObject)rightResult.FutureValue;

int day = referenceTime.Day,
month = referenceTime.Month,
year = referenceTime.Year;

// check if the right time is smaller than the left time, if yes, add one day
int hour = leftResultTime.Hour > 0 ? leftResultTime.Hour : 0,
min = leftResultTime.Minute > 0 ? leftResultTime.Minute : 0,
second = leftResultTime.Second > 0 ? leftResultTime.Second : 0;

leftTime = leftTime.AddHours(hour);
leftTime = leftTime.AddMinutes(min);
leftTime = leftTime.AddSeconds(second);
DateObject.MinValue.SafeCreateFromValue(year, month, day, hour, min, second);
leftTime = leftTime.AddHours(hour).AddMinutes(min).AddSeconds(second);

hour = rightResultTime.Hour > 0 ? rightResultTime.Hour : 0;
min = rightResultTime.Minute > 0 ? rightResultTime.Minute : 0;
second = rightResultTime.Second > 0 ? rightResultTime.Second : 0;

rightTime = rightTime.AddHours(hour);
rightTime = rightTime.AddMinutes(min);
rightTime = rightTime.AddSeconds(second);
rightTime = rightTime.AddHours(hour).AddMinutes(min).AddSeconds(second);

// the right side time contains "ampm", while the left side doesn't
if (rightResult.Comment is Constants.Comment_AmPm &&
Expand All @@ -312,23 +285,18 @@ private DateTimeResolutionResult MergeTwoTimePoints(string text, DateObject refe

ret.FutureValue = ret.PastValue = new Tuple<DateObject, DateObject>(leftTime, rightTime);

var leftTimex = string.Empty;
var rightTimex = string.Empty;

// "X" is timex token for not determined time
if (!pr1.TimexStr.Contains("X") && !pr2.TimexStr.Contains("X"))
var leftTimex = pr1.TimexStr;
var rightTimex = pr2.TimexStr;
if (beginHasDate)
{
leftTimex = DateTimeFormatUtil.LuisDateTime(leftTime);
rightTimex = DateTimeFormatUtil.LuisDateTime(rightTime);
rightTimex = DateTimeFormatUtil.LuisDateShortTime(rightTime, pr2.TimexStr);
}
else
else if (endHasDate)
{
leftTimex = pr1.TimexStr;
rightTimex = pr2.TimexStr;
leftTimex = DateTimeFormatUtil.LuisDateShortTime(leftTime, pr1.TimexStr);
}

ret.Timex = $"({leftTimex},{rightTimex},PT{Convert.ToInt32((rightTime - leftTime).TotalHours)}H)";

ret.Timex = TimexUtility.GenerateDateTimePeriodTimex(leftTimex, rightTimex, rightTime - leftTime);
ret.Success = true;
return ret;
}
Expand Down
25 changes: 22 additions & 3 deletions .NET/Microsoft.Recognizers.Text.DateTime/Utilities/DateTimeFormatUtil.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,14 +157,24 @@ public static string LuisDateTime(DateObject time)
return $"{LuisDate(time)}{Constants.TimeTimexPrefix}{LuisTime(time.Hour, time.Minute, time.Second)}";
}

// Only handle TimeSpan which is less than one day
// If a timex is given and it contains minutes and seconds, the result also includes minutes and seconds.
// Otherwise the result does not include minutes and seconds if they are zero.
public static string LuisDateShortTime(DateObject time, string timex = null)
{
var hasMin = timex != null ? timex.Contains(Constants.TimeTimexConnector) : false;
var hasSec = timex != null ? timex.Split(Constants.TimeTimexConnector[0]).Length > 2 : false;

return $"{LuisDate(time)}{FormatShortTime(time, hasMin, hasSec)}";
}

// Also handle TimeSpans which are more than one day
public static string LuisTimeSpan(System.TimeSpan timeSpan)
{
var timexBuilder = new StringBuilder($"{Constants.GeneralPeriodPrefix}{Constants.TimeTimexPrefix}");

if (timeSpan.Hours > 0)
if (timeSpan.Days > 0 || timeSpan.Hours > 0)
{
timexBuilder.Append($"{timeSpan.Hours}H");
timexBuilder.Append($"{(timeSpan.Days * Constants.DayHourCount) + timeSpan.Hours}H");
}

if (timeSpan.Minutes > 0)
Expand All @@ -190,6 +200,15 @@ public static string FormatTime(DateObject time)
return string.Join(Constants.TimeTimexConnector, time.Hour.ToString("D2", CultureInfo.InvariantCulture), time.Minute.ToString("D2", CultureInfo.InvariantCulture), time.Second.ToString("D2", CultureInfo.InvariantCulture));
}

// Does not return minutes and seconds if they are zero
public static string FormatShortTime(DateObject time, bool keepMin = false, bool keepSec = false)
{
int hour = time.Hour,
min = (keepMin || time.Minute > 0) ? time.Minute : Constants.InvalidMinute,
sec = (keepSec || time.Second > 0) ? time.Second : Constants.InvalidSecond;
return ShortTime(hour, min, sec);
}

public static string FormatDateTime(DateObject datetime)
{
return $"{FormatDate(datetime)} {FormatTime(datetime)}";
Expand Down
6 changes: 6 additions & 0 deletions .NET/Microsoft.Recognizers.Text.DateTime/Utilities/TimexUtility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -463,6 +463,12 @@ public static string GenerateDateTimePeriodTimex(string beginTimex, string endTi
return $"({beginTimex},{endTimex},{durationTimex})";
}

public static string GenerateDateTimePeriodTimex(string beginTimex, string endTimex, TimeSpan duration)
{
var durationTimex = DateTimeFormatUtil.LuisTimeSpan(duration);
return GenerateDateTimePeriodTimex(beginTimex, endTimex, durationTimex);
}

public static string GenerateDateTimePeriodTimex(DateObject beginDateTime, DateObject endDateTime, string durationTimex)
{
return GenerateDateTimePeriodTimex(DateTimeFormatUtil.LuisDateTime(beginDateTime),
Expand Down
6 changes: 3 additions & 3 deletions JavaScript/packages/recognizers-date-time/src/dateTime/baseDateTimePeriod.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,19 +571,19 @@ export class BaseDateTimePeriodParser implements IDateTimeParser {
if (pastEnd < pastBegin) {
pastEnd = futureEnd;
}
result.timex = `(${prs.begin.timexStr},${prs.end.timexStr},PT${DateUtils.totalHours(futureEnd, futureBegin)}H)`;
result.timex = `(${prs.begin.timexStr},${prs.end.timexStr},${DateTimeFormatUtil.luisTimeSpan(futureBegin, futureEnd)})`;
}
else if (beginHasDate) {
futureEnd = DateUtils.safeCreateFromMinValue(futureBegin.getFullYear(), futureBegin.getMonth(), futureBegin.getDate(), futureEnd.getHours(), futureEnd.getMinutes(), futureEnd.getSeconds());
pastEnd = DateUtils.safeCreateFromMinValue(pastBegin.getFullYear(), pastBegin.getMonth(), pastBegin.getDate(), pastEnd.getHours(), pastEnd.getMinutes(), pastEnd.getSeconds());
let dateStr = prs.begin.timexStr.split('T').pop();
result.timex = `(${prs.begin.timexStr},${dateStr}${prs.end.timexStr},PT${DateUtils.totalHours(futureEnd, futureBegin)}H)`;
result.timex = `(${prs.begin.timexStr},${dateStr}${prs.end.timexStr},${DateTimeFormatUtil.luisTimeSpan(futureBegin, futureEnd)})`;
}
else if (endHasDate) {
futureBegin = DateUtils.safeCreateFromMinValue(futureEnd.getFullYear(), futureEnd.getMonth(), futureEnd.getDate(), futureBegin.getHours(), futureBegin.getMinutes(), futureBegin.getSeconds());
pastBegin = DateUtils.safeCreateFromMinValue(pastEnd.getFullYear(), pastEnd.getMonth(), pastEnd.getDate(), pastBegin.getHours(), pastBegin.getMinutes(), pastBegin.getSeconds());
let dateStr = prs.end.timexStr.split('T')[0];
result.timex = `(${dateStr}${prs.begin.timexStr},${prs.end.timexStr},PT${DateUtils.totalHours(futureEnd, futureBegin)}H)`;
result.timex = `(${dateStr}${prs.begin.timexStr},${prs.end.timexStr},${DateTimeFormatUtil.luisTimeSpan(futureBegin, futureEnd)})`;
}
if (!StringUtility.isNullOrEmpty(begin.comment) && begin.comment.endsWith('ampm') && !StringUtility.isNullOrEmpty(end.comment) && end.comment.endsWith('ampm')) {
result.comment = 'ampm';
Expand Down
15 changes: 10 additions & 5 deletions ...Script/packages/recognizers-date-time/src/dateTime/chinese/dateTimePeriodConfiguration.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,12 +556,17 @@ export class ChineseDateTimePeriodParser extends BaseDateTimePeriodParser {
result.futureValue = [leftTime, rightTime];
result.pastValue = [leftTime, rightTime];

let hasFuzzyTimex = prs.begin.timexStr.includes('X') || prs.end.timexStr.includes('X');
let leftTimex = hasFuzzyTimex ? prs.begin.timexStr : DateTimeFormatUtil.luisDateTime(leftTime);
let rightTimex = hasFuzzyTimex ? prs.end.timexStr : DateTimeFormatUtil.luisDateTime(rightTime);
let hoursBetween = DateUtils.totalHours(rightTime, leftTime);
let leftTimex = prs.begin.timexStr;
let rightTimex = prs.end.timexStr;
if (beginHasDate) {
rightTimex = DateTimeFormatUtil.luisDateShortTime(rightTime, rightTimex);
}
else if (endHasDate) {
leftTimex = DateTimeFormatUtil.luisDateShortTime(leftTime, leftTimex);
}
let durationTimex = DateTimeFormatUtil.luisTimeSpan(leftTime, rightTime);

result.timex = `(${leftTimex},${rightTimex},PT${hoursBetween}H)`;
result.timex = `(${leftTimex},${rightTimex},${durationTimex})`;
result.success = true;

return result;
Expand Down
Loading

0 comments on commit a1d01af

Please sign in to comment.