diff --git a/giraffe/package.json b/giraffe/package.json index a2aeaee2..b4a43b69 100644 --- a/giraffe/package.json +++ b/giraffe/package.json @@ -1,6 +1,6 @@ { "name": "@influxdata/giraffe", - "version": "2.27.0", + "version": "2.27.1", "main": "dist/index.js", "module": "dist/index.js", "license": "MIT", diff --git a/giraffe/src/utils/escapeCSVFieldWithSpecialCharacters.ts b/giraffe/src/utils/escapeCSVFieldWithSpecialCharacters.ts new file mode 100644 index 00000000..1d13cdeb --- /dev/null +++ b/giraffe/src/utils/escapeCSVFieldWithSpecialCharacters.ts @@ -0,0 +1,9 @@ +// escape the string containing commas and newlines by wrapping it in double quotes. +// See: https://stackoverflow.com/a/4617967 +export const escapeCSVFieldWithSpecialCharacters = (str: string) => { + if (str.includes(',') || str.includes('\n')) { + return `"${str}"` // wrap in double quotes + } else { + return str + } +} diff --git a/giraffe/src/utils/escapeSpecialCharacters.test.ts b/giraffe/src/utils/escapeSpecialCharacters.test.ts new file mode 100644 index 00000000..3fe09be4 --- /dev/null +++ b/giraffe/src/utils/escapeSpecialCharacters.test.ts @@ -0,0 +1,31 @@ +import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters' + +describe('escapeSpecialCharacters', () => { + it('should escape a string with commas', function() { + const stringWithCommas = 'this is a string, with comma, and another one' + + expect(escapeCSVFieldWithSpecialCharacters(stringWithCommas)).toEqual( + `"${stringWithCommas}"` + ) + }) + + it('should escape a string with newLines', function() { + const stringWithNewline = `this is a string + with a newline + and another one` + + expect(escapeCSVFieldWithSpecialCharacters(stringWithNewline)).toEqual( + `"${stringWithNewline}"` + ) + }) + + it('should escape a string with both commas and newLines', function() { + const stringWithNewlineAndComma = `this is a string + with a newline, a comma, + and another newline and comma` + + expect( + escapeCSVFieldWithSpecialCharacters(stringWithNewlineAndComma) + ).toEqual(`"${stringWithNewlineAndComma}"`) + }) +}) diff --git a/giraffe/src/utils/fluxParsing.ts b/giraffe/src/utils/fluxParsing.ts index 13421345..b30d18a8 100644 --- a/giraffe/src/utils/fluxParsing.ts +++ b/giraffe/src/utils/fluxParsing.ts @@ -4,6 +4,7 @@ import uuid from 'uuid' import {FluxTable} from '../types' import {get} from './get' import {groupBy} from './groupBy' +import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters' export const parseResponseError = (response: string): FluxTable[] => { const data = Papa.parse(response.trim()).data as string[][] @@ -45,9 +46,10 @@ export const parseChunks = (response: string): string[] => { // // [0]: https://github.com/influxdata/influxdb/issues/15017 + // use regex lookahead const chunks = trimmedResponse - .split(/\n\s*\n#/) - .map((s, i) => (i === 0 ? s : `#${s}`)) + .split(/\n\s*\n#(?=datatype|group|default)/) + .map((chunk, chunkNumber) => (chunkNumber === 0 ? chunk : `#${chunk}`)) // Add back the `#` characters that were removed by splitting return chunks } @@ -62,7 +64,10 @@ export const parseResponse = (response: string): FluxTable[] => { } export const parseTables = (responseChunk: string): FluxTable[] => { - const lines = responseChunk.split('\n') + const linesData = Papa.parse(responseChunk).data + const lines: string[] = linesData.map(line => + line.map(escapeCSVFieldWithSpecialCharacters).join(',') + ) const annotationLines: string = lines .filter(line => line.startsWith('#')) .join('\n') diff --git a/giraffe/src/utils/fromFlux.test.ts b/giraffe/src/utils/fromFlux.test.ts index 33cf38f6..4d58c352 100644 --- a/giraffe/src/utils/fromFlux.test.ts +++ b/giraffe/src/utils/fromFlux.test.ts @@ -641,6 +641,34 @@ there",5 expect(table.getColumn('time')).toEqual([1610972402582]) }) + + it('should parse CSV with hashtags, commas and newlines', () => { + const CSV = `#group,false,false,true,false +#datatype,string,long,long,string +#default,_result,,, +,result,table,_time_reverse,_value +,,0,-1652887800000000000,"[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/ + +,#hashtag, #another, hash tag" +,,0,-1652888700000000000,"[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇 +, new line +another new line"` + + const {table} = fromFlux(CSV) + + const valueColumn = table.getColumn('_value') + + const expectedValueColumn = [ + `[2022-05-18 15:30:00 UTC] @textAndCommas: Visit https://a.link/ + +,#hashtag, #another, hash tag`, + `[2022-05-18 15:45:00 UTC] @emojis: 👇👇👇 +, new line +another new line`, + ] + + expect(valueColumn).toEqual(expectedValueColumn) + }) }) describe('fastFromFlux', () => { it('should always pass for stability checks', () => { diff --git a/giraffe/src/utils/fromFlux.ts b/giraffe/src/utils/fromFlux.ts index db09050e..c4f9f9b9 100644 --- a/giraffe/src/utils/fromFlux.ts +++ b/giraffe/src/utils/fromFlux.ts @@ -3,6 +3,8 @@ import {Table, ColumnType, FluxDataType} from '../types' import {assert} from './assert' import {newTable} from './newTable' import {RESULT} from '../constants/columnKeys' +import Papa from 'papaparse' +import {escapeCSVFieldWithSpecialCharacters} from './escapeCSVFieldWithSpecialCharacters' export interface FromFluxResult { error?: Error @@ -110,7 +112,7 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => { const prevIndex = currentIndex const nextIndex = fluxCSV .substring(currentIndex, fluxCSV.length) - .search(/\n\s*\n#/) + .search(/\n\s*\n#(?=datatype|group|default)/) if (nextIndex === -1) { chunks.push([prevIndex, fluxCSV.length]) currentIndex = -1 @@ -132,7 +134,10 @@ export const fromFlux = (fluxCSV: string): FromFluxResult => { for (const [start, end] of chunks) { chunk = fluxCSV.substring(start, end) - const splittedChunk = chunk.split('\n') + const parsedChunkData = Papa.parse(chunk).data + const splittedChunk: string[] = parsedChunkData.map(line => + line.map(escapeCSVFieldWithSpecialCharacters).join(',') + ) const tableTexts = [] const annotationTexts = [] diff --git a/stories/package.json b/stories/package.json index bbd328a1..f64fa157 100644 --- a/stories/package.json +++ b/stories/package.json @@ -1,6 +1,6 @@ { "name": "@influxdata/giraffe-stories", - "version": "2.27.0", + "version": "2.27.1", "license": "MIT", "repository": { "type": "git",