feat: handle hash symbol and use papaparse to parse csv (#767)

giraffe/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@influxdata/giraffe",
-  "version": "2.27.0",
+  "version": "2.27.1",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "license": "MIT",

giraffe/src/utils/escapeCSVFieldWithSpecialCharacters.ts

@@ -0,0 +1,9 @@
+// escape the string containing commas and newlines by wrapping it in double quotes.
+// See: https://stackoverflow.com/a/4617967
+export const escapeCSVFieldWithSpecialCharacters = (str: string) => {
+  if (str.includes(',') || str.includes('\n')) {
+    return `"${str}"` // wrap in double quotes
+  } else {
+    return str
+  }
+}

giraffe/src/utils/escapeSpecialCharacters.test.ts

@@ -0,0 +1,31 @@
+import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters'
+
+describe('escapeSpecialCharacters', () => {
+  it('should escape a string with commas', function() {
+    const stringWithCommas = 'this is a string, with comma, and another one'
+
+    expect(escapeCSVFieldWithSpecialCharacters(stringWithCommas)).toEqual(
+      `"${stringWithCommas}"`
+    )
+  })
+
+  it('should escape a string with newLines', function() {
+    const stringWithNewline = `this is a string 
+    with a newline
+    and another one`
+
+    expect(escapeCSVFieldWithSpecialCharacters(stringWithNewline)).toEqual(
+      `"${stringWithNewline}"`
+    )
+  })
+
+  it('should escape a string with both commas and newLines', function() {
+    const stringWithNewlineAndComma = `this is a string 
+    with a newline, a comma,
+    and another newline and comma`
+
+    expect(
+      escapeCSVFieldWithSpecialCharacters(stringWithNewlineAndComma)
+    ).toEqual(`"${stringWithNewlineAndComma}"`)
+  })
+})

giraffe/src/utils/fluxParsing.ts

@@ -4,6 +4,7 @@ import uuid from 'uuid'
 import {FluxTable} from '../types'
 import {get} from './get'
 import {groupBy} from './groupBy'
+import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters'
 
 export const parseResponseError = (response: string): FluxTable[] => {
   const data = Papa.parse(response.trim()).data as string[][]
@@ -45,9 +46,10 @@ export const parseChunks = (response: string): string[] => {
   //
   // [0]: https://github.com/influxdata/influxdb/issues/15017
 
+  // use regex lookahead
   const chunks = trimmedResponse
-    .split(/\n\s*\n#/)
-    .map((s, i) => (i === 0 ? s : `#${s}`))
+    .split(/\n\s*\n#(?=datatype|group|default)/)
+    .map((chunk, chunkNumber) => (chunkNumber === 0 ? chunk : `#${chunk}`)) // Add back the `#` characters that were removed by splitting
 
   return chunks
 }
@@ -62,7 +64,10 @@ export const parseResponse = (response: string): FluxTable[] => {
 }
 
 export const parseTables = (responseChunk: string): FluxTable[] => {
-  const lines = responseChunk.split('\n')
+  const linesData = Papa.parse(responseChunk).data
+  const lines: string[] = linesData.map(line =>
+    line.map(escapeCSVFieldWithSpecialCharacters).join(',')
+  )
   const annotationLines: string = lines
     .filter(line => line.startsWith('#'))
     .join('\n')

giraffe/src/utils/fromFlux.test.ts

@@ -641,6 +641,34 @@ there",5
 
     expect(table.getColumn('time')).toEqual([1610972402582])
   })
+
+  it('should parse CSV with hashtags, commas and newlines', () => {
+    const CSV = `#group,false,false,true,false
+#datatype,string,long,long,string
+#default,_result,,,
+,result,table,_time_reverse,_value
+,,0,-1652887800000000000,"[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/
+
+,#hashtag, #another, hash tag"
+,,0,-1652888700000000000,"[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇
+, new line
+another new line"`
+
+    const {table} = fromFlux(CSV)
+
+    const valueColumn = table.getColumn('_value')
+
+    const expectedValueColumn = [
+      `[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/
+
+,#hashtag, #another, hash tag`,
+      `[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇
+, new line
+another new line`,
+    ]
+
+    expect(valueColumn).toEqual(expectedValueColumn)
+  })
 })
 describe('fastFromFlux', () => {
   it('should always pass for stability checks', () => {

giraffe/src/utils/fromFlux.ts

@@ -3,6 +3,8 @@ import {Table, ColumnType, FluxDataType} from '../types'
 import {assert} from './assert'
 import {newTable} from './newTable'
 import {RESULT} from '../constants/columnKeys'
+import Papa from 'papaparse'
+import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters'
 export interface FromFluxResult {
   error?: Error
 
@@ -110,7 +112,7 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => {
       const prevIndex = currentIndex
       const nextIndex = fluxCSV
         .substring(currentIndex, fluxCSV.length)
-        .search(/\n\s*\n#/)
+        .search(/\n\s*\n#(?=datatype|group|default)/)
       if (nextIndex === -1) {
         chunks.push([prevIndex, fluxCSV.length])
         currentIndex = -1
@@ -132,7 +134,10 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => {
 
     for (const [start, end] of chunks) {
       chunk = fluxCSV.substring(start, end)
-      const splittedChunk = chunk.split('\n')
+      const parsedChunkData = Papa.parse(chunk).data
+      const splittedChunk: string[] = parsedChunkData.map(line =>
+        line.map(escapeCSVFieldWithSpecialCharacters).join(',')
+      )
 
       const tableTexts = []
       const annotationTexts = []

stories/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@influxdata/giraffe-stories",
-  "version": "2.27.0",
+  "version": "2.27.1",
   "license": "MIT",
   "repository": {
     "type": "git",