Skip to content

Commit

Permalink
Adding parsing of rill iso extensions
Browse files Browse the repository at this point in the history
  • Loading branch information
@AdityaHegde
AdityaHegde committed Nov 8, 2023
1 parent cc61866 commit 41b42e7
Show file tree
Hide file tree
Showing 8 changed files with 260 additions and 162 deletions.
61 changes: 51 additions & 10 deletions runtime/pkg/duration/duration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,20 @@ import (
"fmt"
"regexp"
"strconv"
"strings"
"time"

runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
)

// Duration represents an ISO8601 duration with Rill-specific extensions.
type Duration interface {
Add(t time.Time) time.Time
Sub(t time.Time) time.Time
}

// StandardDuration represents an ISO8601 duration with Rill-specific extensions.
// See ParseISO8601 for details.
type Duration struct {
type StandardDuration struct {
// If Inf is true, the other components should be ignored
Inf bool
// Date component
Expand All @@ -26,8 +34,21 @@ type Duration struct {

// Regexes used by ParseISO8601
var (
infPattern = regexp.MustCompile("^(?i)inf$")
durationPattern = regexp.MustCompile(`^P((?P<year>\d+)Y)?((?P<month>\d+)M)?((?P<week>\d+)W)?((?P<day>\d+)D)?(T((?P<hour>\d+)H)?((?P<minute>\d+)M)?((?P<second>\d+)S)?)?$`)
infPattern = regexp.MustCompile("^(?i)inf$")
durationPattern = regexp.MustCompile(`^P((?P<year>\d+)Y)?((?P<month>\d+)M)?((?P<week>\d+)W)?((?P<day>\d+)D)?(T((?P<hour>\d+)H)?((?P<minute>\d+)M)?((?P<second>\d+)S)?)?$`)
daxToDateNotations = map[string]runtimev1.TimeGrain{
// Pulled from https://www.daxpatterns.com/standard-time-related-calculations/
// Add more here once support in UI has been added
"MTD": runtimev1.TimeGrain_TIME_GRAIN_MONTH,
"QTD": runtimev1.TimeGrain_TIME_GRAIN_QUARTER,
"YTD": runtimev1.TimeGrain_TIME_GRAIN_YEAR,
}
daxOffsetNotations = map[string]StandardDuration{
"PP": {},
"PM": {Month: 1},
"PQ": {Month: 3},
"PY": {Year: 1},
}
)

// ParseISO8601 parses an ISO8601 duration as well as some Rill-specific extensions.
Expand All @@ -36,15 +57,23 @@ var (
func ParseISO8601(from string) (Duration, error) {
// Try parsing for "inf"
if infPattern.MatchString(from) {
return Duration{Inf: true}, nil
return StandardDuration{Inf: true}, nil
}

rillDur := strings.Replace(from, "rill-", "", 1)
if a, ok := daxToDateNotations[rillDur]; ok {
return ToDateDuration{anchor: a}, nil
}
if o, ok := daxOffsetNotations[rillDur]; ok {
return o, nil
}

// Parse as a regular ISO8601 duration
if !durationPattern.MatchString(from) {
return Duration{}, fmt.Errorf("string %q is not a valid ISO 8601 duration", from)
return StandardDuration{}, fmt.Errorf("string %q is not a valid ISO 8601 duration", from)
}

var d Duration
var d StandardDuration
match := durationPattern.FindStringSubmatch(from)
for i, name := range durationPattern.SubexpNames() {
part := match[i]
Expand All @@ -54,7 +83,7 @@ func ParseISO8601(from string) (Duration, error) {

val, err := strconv.Atoi(part)
if err != nil {
return Duration{}, err
return StandardDuration{}, err
}
switch name {
case "year":
Expand All @@ -80,7 +109,7 @@ func ParseISO8601(from string) (Duration, error) {
}

// Add adds the duration to a timestamp
func (d Duration) Add(t time.Time) time.Time {
func (d StandardDuration) Add(t time.Time) time.Time {
if d.Inf {
return time.Time{}
}
Expand All @@ -93,7 +122,7 @@ func (d Duration) Add(t time.Time) time.Time {
}

// Sub subtracts the duration from a timestamp
func (d Duration) Sub(t time.Time) time.Time {
func (d StandardDuration) Sub(t time.Time) time.Time {
if d.Inf {
return time.Time{}
}
Expand All @@ -104,3 +133,15 @@ func (d Duration) Sub(t time.Time) time.Time {
td := time.Duration(d.Second)*time.Second + time.Duration(d.Minute)*time.Minute + time.Duration(d.Hour)*time.Hour
return t.Add(-td)
}

type ToDateDuration struct {
anchor runtimev1.TimeGrain
}

func (d ToDateDuration) Add(t time.Time) time.Time {
return time.Time{}
}

func (d ToDateDuration) Sub(t time.Time) time.Time {
return TruncateTime(t, d.anchor, t.Location(), 1, 1) // TODO: get first day and month
}
10 changes: 5 additions & 5 deletions runtime/pkg/duration/duration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@ import (
func TestParseISO8601(t *testing.T) {
tests := []struct {
from string
expected Duration
expected StandardDuration
err bool
}{
{from: "P2W", expected: Duration{Week: 2}},
{from: "P1Y2WT5M", expected: Duration{Year: 1, Week: 2, Minute: 5}},
{from: "P2W", expected: StandardDuration{Week: 2}},
{from: "P1Y2WT5M", expected: StandardDuration{Year: 1, Week: 2, Minute: 5}},
{from: "P1X", err: true},
{from: "inf", expected: Duration{Inf: true}},
{from: "Inf", expected: Duration{Inf: true}},
{from: "inf", expected: StandardDuration{Inf: true}},
{from: "Inf", expected: StandardDuration{Inf: true}},
{from: "infinity", err: true},
}
for _, tt := range tests {
Expand Down
70 changes: 70 additions & 0 deletions runtime/pkg/duration/timeutil.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
package duration

import (
"time"
// Load IANA time zone data
_ "time/tzdata"

runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
)

func TruncateTime(start time.Time, tg runtimev1.TimeGrain, tz *time.Location, firstDay, firstMonth int) time.Time {
switch tg {
case runtimev1.TimeGrain_TIME_GRAIN_MILLISECOND:
return start.Truncate(time.Millisecond)
case runtimev1.TimeGrain_TIME_GRAIN_SECOND:
return start.Truncate(time.Second)
case runtimev1.TimeGrain_TIME_GRAIN_MINUTE:
return start.Truncate(time.Minute)
case runtimev1.TimeGrain_TIME_GRAIN_HOUR:
start = start.In(tz)
start = time.Date(start.Year(), start.Month(), start.Day(), start.Hour(), 0, 0, 0, tz)
return start.In(time.UTC)
case runtimev1.TimeGrain_TIME_GRAIN_DAY:
start = start.In(tz)
start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz)
return start.In(time.UTC)
case runtimev1.TimeGrain_TIME_GRAIN_WEEK:
start = start.In(tz)
weekday := int(start.Weekday())
if weekday == 0 {
weekday = 7
}
if firstDay < 1 {
firstDay = 1
}
if firstDay > 7 {
firstDay = 7
}

daysToSubtract := -(weekday - firstDay)
if weekday < firstDay {
daysToSubtract = -7 + daysToSubtract
}
start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, tz)
start = start.AddDate(0, 0, daysToSubtract)
return start.In(time.UTC)
case runtimev1.TimeGrain_TIME_GRAIN_MONTH:
start = start.In(tz)
start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz)
start = start.In(time.UTC)
return start
case runtimev1.TimeGrain_TIME_GRAIN_QUARTER:
monthsToSubtract := (3 + int(start.Month()) - firstMonth%3) % 3
start = start.In(tz)
start = time.Date(start.Year(), start.Month(), 1, 0, 0, 0, 0, tz)
start = start.AddDate(0, -monthsToSubtract, 0)
return start.In(time.UTC)
case runtimev1.TimeGrain_TIME_GRAIN_YEAR:
start = start.In(tz)
year := start.Year()
if int(start.Month()) < firstMonth {
year = start.Year() - 1
}

start = time.Date(year, time.Month(firstMonth), 1, 0, 0, 0, 0, tz)
return start.In(time.UTC)
}

return start
}
89 changes: 89 additions & 0 deletions runtime/pkg/duration/timeutil_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package duration

import (
"testing"
"time"

runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
"github.com/stretchr/testify/require"
)

func TestTruncateTime(t *testing.T) {
require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-07T04:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-07T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2023-10-09T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-04-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, time.UTC, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, time.UTC, 1, 1))
}

func TestTruncateTime_Kathmandu(t *testing.T) {
tz, err := time.LoadLocation("Asia/Kathmandu")
require.NoError(t, err)
require.Equal(t, parseTestTime(t, "2019-01-07T04:20:07Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07.29Z"), runtimev1.TimeGrain_TIME_GRAIN_SECOND, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-07T04:20:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:07Z"), runtimev1.TimeGrain_TIME_GRAIN_MINUTE, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-07T04:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_HOUR, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-06T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-01-07T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_DAY, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2023-10-08T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2019-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_MONTH, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2019-03-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-05-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 1, 1))
require.Equal(t, parseTestTime(t, "2018-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2019-02-07T01:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 1, 1))
}

func TestTruncateTime_UTC_first_day(t *testing.T) {
tz := time.UTC
require.Equal(t, parseTestTime(t, "2023-10-08T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1))
require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
require.Equal(t, parseTestTime(t, "2023-10-10T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T00:01:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
}

func TestTruncateTime_Kathmandu_first_day(t *testing.T) {
tz, err := time.LoadLocation("Asia/Kathmandu")
require.NoError(t, err)
require.Equal(t, parseTestTime(t, "2023-10-07T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 7, 1))
require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-10T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-11T04:20:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
require.Equal(t, parseTestTime(t, "2023-10-09T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-09T18:16:01Z"), runtimev1.TimeGrain_TIME_GRAIN_WEEK, tz, 2, 1))
}

func TestTruncateTime_UTC_first_month(t *testing.T) {
tz := time.UTC
require.Equal(t, parseTestTime(t, "2023-08-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2))
require.Equal(t, parseTestTime(t, "2023-11-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5))
require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2023-09-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6))
require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6))

require.Equal(t, parseTestTime(t, "2023-02-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2))
require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2023-03-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-03-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2022-12-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12))
require.Equal(t, parseTestTime(t, "2023-01-01T00:00:00Z"), TruncateTime(parseTestTime(t, "2023-01-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1))
}

func TestTruncateTime_Kathmandu_first_month(t *testing.T) {
tz, err := time.LoadLocation("Asia/Kathmandu")
require.NoError(t, err)
require.Equal(t, parseTestTime(t, "2023-07-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 2))
require.Equal(t, parseTestTime(t, "2023-10-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 5))
require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2023-08-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-11-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6))
require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-02-01T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_QUARTER, tz, 2, 6))

require.Equal(t, parseTestTime(t, "2023-01-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 2))
require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2023-02-28T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-03-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 3))
require.Equal(t, parseTestTime(t, "2022-11-30T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-10-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 12))
require.Equal(t, parseTestTime(t, "2022-12-31T18:15:00Z"), TruncateTime(parseTestTime(t, "2023-01-02T00:20:00Z"), runtimev1.TimeGrain_TIME_GRAIN_YEAR, tz, 2, 1))
}

func parseTestTime(tst *testing.T, t string) time.Time {
ts, err := time.Parse(time.RFC3339, t)
require.NoError(tst, err)
return ts
}
5 changes: 2 additions & 3 deletions runtime/queries/metricsview_comparison_toplist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,14 @@ import (
"fmt"
"io"
"strings"
// Load IANA time zone data
_ "time/tzdata"

runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
"github.com/rilldata/rill/runtime"
"github.com/rilldata/rill/runtime/drivers"
"github.com/rilldata/rill/runtime/pkg/pbutil"
"google.golang.org/protobuf/types/known/structpb"

// Load IANA time zone data
_ "time/tzdata"
)

type MetricsViewComparison struct {
Expand Down
3 changes: 2 additions & 1 deletion runtime/queries/metricsview_timeseries.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
runtimev1 "github.com/rilldata/rill/proto/gen/rill/runtime/v1"
"github.com/rilldata/rill/runtime"
"github.com/rilldata/rill/runtime/drivers"
"github.com/rilldata/rill/runtime/pkg/duration"
"github.com/rilldata/rill/runtime/pkg/pbutil"
"google.golang.org/protobuf/types/known/structpb"
"google.golang.org/protobuf/types/known/timestamppb"
Expand Down Expand Up @@ -161,7 +162,7 @@ func (q *MetricsViewTimeSeries) Resolve(ctx context.Context, rt *runtime.Runtime

if zeroTime.Equal(start) {
if q.TimeStart != nil {
start = TruncateTime(q.TimeStart.AsTime(), q.TimeGranularity, tz, int(fdow), int(fmoy))
start = duration.TruncateTime(q.TimeStart.AsTime(), q.TimeGranularity, tz, int(fdow), int(fmoy))
data = addNulls(data, nullRecords, start, t, q.TimeGranularity, tz)
}
} else {
Expand Down
Loading

0 comments on commit 41b42e7

Please sign in to comment.