Skip to content

Commit

Permalink
feat: consolidate fastFromFlux parser changes to fromFlux (#773)
Browse files Browse the repository at this point in the history
  • Loading branch information
@asalem1
asalem1 authored Jul 14, 2022
1 parent 66466ac commit e6213f4
Show file tree
Hide file tree
Showing 6 changed files with 182 additions and 199 deletions.
4 changes: 1 addition & 3 deletions giraffe/package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@influxdata/giraffe",
"version": "2.30.0",
"version": "2.30.1",
"main": "dist/index.js",
"module": "dist/index.js",
"license": "MIT",
Expand Down Expand Up @@ -46,7 +46,6 @@
"@testing-library/react": "^11.2.2",
"@types/d3-array": "^2.0.0",
"@types/d3-color": "^1.2.2",
"@types/d3-dsv": "^1.0.36",
"@types/d3-format": "^1.3.1",
"@types/d3-interpolate": "^1.3.1",
"@types/d3-scale": "^2.1.1",
Expand All @@ -65,7 +64,6 @@
"css-modules-typescript-loader": "^4.0.0",
"d3-array": "^2.0.3",
"d3-color": "^1.2.3",
"d3-dsv": "^1.1.1",
"d3-format": "^1.3.2",
"d3-interpolate": "^1.3.2",
"d3-scale": "^3.0.0",
Expand Down
5 changes: 0 additions & 5 deletions giraffe/src/utils/assert.ts
View file

This file was deleted.

103 changes: 79 additions & 24 deletions giraffe/src/utils/fromFlux.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,7 @@ describe('fromFlux', () => {
,result,table,_start,_stop,_time,_value,_field,_measurement,code,id,magType,net,title\r
,,5,2022-06-28T13:22:34.9161857Z,2022-06-29T13:22:34.9161857Z,2022-06-28T15:09:54.052Z,3.1,cdi,earthquake,7000hkux,us7000hkux,mb,us,"M 4.6 - 107 km NNW of Te Anau, New Zealand"\r
`

/* eslint-disable */
const expected = {
_field: {
Expand Down Expand Up @@ -400,7 +401,7 @@ describe('fromFlux', () => {
type: 'number',
},
'_value (string)': {
data: [, , , ,'green', ,],
data: [, , , , 'green', ,],
fluxDataType: 'string',
name: '_value',
type: 'string',
Expand Down Expand Up @@ -485,14 +486,21 @@ describe('fromFlux', () => {
type: 'number',
},
title: {
data: [, , , ,'M 5.4 - 44 km W of Hengchun, Taiwan', 'M 4.6 - 107 km NNW of Te Anau, New Zealand'],
data: [
,
,
,
,
'M 5.4 - 44 km W of Hengchun, Taiwan',
'M 4.6 - 107 km NNW of Te Anau, New Zealand',
],
fluxDataType: 'string',
name: 'title',
type: 'string',
},
}
/* eslint-enable */

/* eslint-enable */
const expectedGroupKeys = [
'_start',
'_stop',
Expand All @@ -517,6 +525,36 @@ describe('fromFlux', () => {
expect(columns).toStrictEqual(expected)
expect(fluxGroupKeyUnion).toStrictEqual(expectedGroupKeys)
})
it('parses query with newlines and hashtags', () => {
const CSV = `#group,false,false,true,false
#datatype,string,long,long,string
#default,_result,,,
,result,table,_time_reverse,_value
,,0,-1652887800000000000,"Row 1
#newline #somehashTags https://a_link.com/giraffe"
,,0,-1652887811000000000,Row 2
,,0,-1652888700000000000,"Row 3: 👇👇👇 // emojis
,Mew line!"
,,0,-1652889550000000000,"Row 4:
New line 1!
New line 2!
multiple new lines!`
const {table} = fromFlux(CSV)

const expectedColumns = [
`Row 1
#newline #somehashTags https://a_link.com/giraffe`,
'Row 2',
`Row 3: 👇👇👇 // emojis
,Mew line!`,
`Row 4:
New line 1!
New line 2!
multiple new lines!`,
]

expect(table.getColumn('_value')).toStrictEqual(expectedColumns)
})

it('can parse a Flux CSV with mismatched schemas', () => {
const CSV = `#group,false,false,true,true,false,true,true,true,true,true
Expand Down Expand Up @@ -630,7 +668,7 @@ describe('fromFlux', () => {
'#group,false,false,true,true,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true'

expect(() => {
fastFromFlux(CSV)
fromFlux(CSV)
}).not.toThrow()
})

Expand Down Expand Up @@ -860,35 +898,45 @@ there",5

expect(table.getColumn('time')).toEqual([1610972402582])
})

it('should parse CSV with hashtags, commas and newlines', () => {
const CSV = `#group,false,false,true,false
#datatype,string,long,long,string
#default,_result,,,
,result,table,_time_reverse,_value
,,0,-1652887800000000000,"[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/
,#hashtag, #another, hash tag"
,,0,-1652888700000000000,"[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇
, new line
another new line"`
it('should parse JSON data as part of the table body correctly', () => {
const CSV = `#group,false,false,true,true,false,false,true,true,true,true,true,true,true,true,true,true
#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,string,string,string,string,string,string,string,string,string,string,string
#default,_result,,,,,,,,,,,,,,,
,result,table,_start,_stop,_time,_value,_field,_measurement,env,error,errorCode,errorType,orgID,ot_trace_sampled,role,source
,,0,2022-06-15T19:05:02.361293138Z,2022-06-15T20:05:02.361293138Z,2022-06-15T19:05:05.145623698Z,"{""request"":{""organization_id"":""fc0e1bf81e62ea27"",""jwttoken"":""REDACTED"",""compiler"":{""Now"":""2022-06-15T19:05:00Z"",""query"":""import ""influxdata/influxdb/monitor""\nimport ""experimental""\nimport ""influxdata/influxdb/v1""\n\ndata = from(bucket: ""Website Monitoring Bucket"")\n |\u003e range(start: -10m)\n |\u003e filter(fn: (r) =\u003e r[""_measurement""] == ""http_response"")\n |\u003e filter(fn: (r) =\u003e r[""_field""] == ""result_code"")\n |\u003e filter(fn: (r) =\u003e r[""method""] == ""HEAD"")\n |\u003e filter(fn: (r) =\u003e r[""result""] == ""success"")\n |\u003e filter(fn: (r) =\u003e r[""server""] == ""https://influxdata.com"")\n\noption task = {name: ""Name this Check"", every: 1m, offset: 0s}\n\ncheck = {_check_id: ""0854d93f9225d000"", _check_name: ""Name this Check"", _type: ""deadman"", tags: {}}\ncrit = (r) =\u003e r[""dead""]\nmessageFn = (r) =\u003e ""Check: $\{r._check_name} is: $\{r._level}""\n\ndata |\u003e v1[""fieldsAsCols""]() |\u003e monitor[""deadman""](t: experimental[""subDuration""](from: now(), d: 90s))\n |\u003e monitor[""check""](data: check, messageFn: messageFn, crit: crit)""},""source"":""tasks"",""parameters"":null,""UseIOx"":false,""compiler_type"":""flux""},""dialect"":{},""dialect_type"":""no-content""}",request,query_log,prod01-eu-central-1,"failed to initialize execute state: could not find bucket ""Website Monitoring Bucket""",not found,user,fc0e1bf81e62ea27,false,queryd-pull-internal,tasks\
`

const {table} = fromFlux(CSV)

const valueColumn = table.getColumn('_value')

const expectedValueColumn = [
`[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/
,#hashtag, #another, hash tag`,
`[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇
, new line
another new line`,
'{"request":{"organization_id":"fc0e1bf81e62ea27","jwttoken":"REDACTED","compiler":{"Now":"2022-06-15T19:05:00Z","query":"import "influxdata/influxdb/monitor"\n' +
'import "experimental"\n' +
'import "influxdata/influxdb/v1"\n' +
'\n' +
'data = from(bucket: "Website Monitoring Bucket")\n' +
' |> range(start: -10m)\n' +
' |> filter(fn: (r) => r["_measurement"] == "http_response")\n' +
' |> filter(fn: (r) => r["_field"] == "result_code")\n' +
' |> filter(fn: (r) => r["method"] == "HEAD")\n' +
' |> filter(fn: (r) => r["result"] == "success")\n' +
' |> filter(fn: (r) => r["server"] == "https://influxdata.com")\n' +
'\n' +
'option task = {name: "Name this Check", every: 1m, offset: 0s}\n' +
'\n' +
'check = {_check_id: "0854d93f9225d000", _check_name: "Name this Check", _type: "deadman", tags: {}}\n' +
'crit = (r) => r["dead"]\n' +
'messageFn = (r) => "Check: ${r._check_name} is: ${r._level}"\n' +
'\n' +
'data |> v1["fieldsAsCols"]() |> monitor["deadman"](t: experimental["subDuration"](from: now(), d: 90s))\n' +
' |> monitor["check"](data: check, messageFn: messageFn, crit: crit)"},"source":"tasks","parameters":null,"UseIOx":false,"compiler_type":"flux"},"dialect":{},"dialect_type":"no-content"}',
]

expect(valueColumn).toEqual(expectedValueColumn)
})
})

describe('fastFromFlux', () => {
it('should always pass for stability checks', () => {
const resp = `#group,false,false,true,true,false,false,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true
Expand Down Expand Up @@ -1290,7 +1338,7 @@ describe('fastFromFlux', () => {
type: 'number',
},
'_value (string)': {
data: [, , , ,'green', ,],
data: [, , , , 'green', ,],
fluxDataType: 'string',
name: '_value',
type: 'string',
Expand Down Expand Up @@ -1375,7 +1423,14 @@ describe('fastFromFlux', () => {
type: 'number',
},
title: {
data: [, , , ,'M 5.4 - 44 km W of Hengchun, Taiwan', 'M 4.6 - 107 km NNW of Te Anau, New Zealand'],
data: [
,
,
,
,
'M 5.4 - 44 km W of Hengchun, Taiwan',
'M 4.6 - 107 km NNW of Te Anau, New Zealand',
],
fluxDataType: 'string',
name: 'title',
type: 'string',
Expand Down
Loading

0 comments on commit e6213f4

Please sign in to comment.