diff --git a/src/array-joins.ts b/src/array-joins.ts index 50d60ce..e50813a 100644 --- a/src/array-joins.ts +++ b/src/array-joins.ts @@ -1,36 +1,79 @@ +export type FieldSelectorFunction = (item: any) => string; + +function fieldSelector(input: string | string[] | FieldSelectorFunction): FieldSelectorFunction { + if (typeof input === "function") { + return input; + } else if (typeof input === "string") { + return (item) => item[input]; + } else if (Array.isArray(input)) { + return (item) => input.map(r => item[r]).join('|'); + } else { + throw Error(`Unknown input. Can't create a fieldSelector`) + } +} + +/** + * leftJoin returns all elements from the left array (leftArray), and the matched elements from the right array (rightArray). + * The result is NULL from the right side, if there is no match. + * @param leftArray : array for left side in a join + * @param rightArray : array for right side in a join + * @param leftKey : A key from left side array. What can be as a fieldName, multiple fields or key Selector + * @param rightKey : A key from right side array. what can be as a fieldName, multiple fields or key Selector + * @param resultSelector A callback function that returns result value + */ export function leftJoin( leftArray: any[], rightArray: any[], - leftKeySelector: (item: any) => string, - rightKeySelector: (item: any) => string, + leftKeySelector: string | string[] | FieldSelectorFunction, + rightKeySelector: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): any[] { return leftOrInnerJoin(false, leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); } +/** + * innerJoin - Joins two arrays together by selecting elements that have matching values in both arrays. + * If there are elements in any array that do not have matches in other array, these elements will not be shown! + * @param leftArray : array for left side in a join + * @param rightArray : array for right side in a join + * @param leftKey : A key from left side array. What can be as a fieldName, multiple fields or key Selector + * @param rightKey : A key from right side array. what can be as a fieldName, multiple fields or key Selector + * @param resultSelector A callback function that returns result value + */ export function innerJoin( leftArray: any[], rightArray: any[], - leftKeySelector: (item: any) => string, - rightKeySelector: (item: any) => string, + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): any[] { - return leftOrInnerJoin(true, leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); + return leftOrInnerJoin(true, leftArray, rightArray, leftKey, rightKey, resultSelector); } +/** + * fullJoin returns all elements from the left array (leftArray), and all elements from the right array (rightArray). + * The result is NULL from the right/left side, if there is no match. + * @param leftArray : array for left side in a join + * @param rightArray : array for right side in a join + * @param leftKey : A key from left side array. What can be as a fieldName, multiple fields or key Selector + * @param rightKey : A key from right side array. what can be as a fieldName, multiple fields or key Selector + * @param resultSelector A callback function that returns result value + */ export function fullJoin( leftArray: any[], rightArray: any[], - leftKeySelector: (item: any) => string, - rightKeySelector: (item: any) => string, + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): any[] { + const leftKeySelector = fieldSelector(leftKey); + const rightKeySelector = fieldSelector(rightKey); verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); // build a lookup maps for both arrays. - // so, both jeys have to be unique, otherwise it will flatter result + // so, both of them have to be unique, otherwise it will flattern result const leftArrayMap = Object.create(null); for (let item of rightArray) { leftArrayMap[leftKeySelector(item)] = item; @@ -67,6 +110,50 @@ export function fullJoin( return result; } +/** + * merges elements from two arrays. It appends source element or overrides to target array based on matching keys provided + * @param targetArray target array + * @param sourceArray source array + * @param targetKey tartget key field, arry of fields or field serlector + * @param sourceKey source key field, arry of fields or field serlector + */ +export function merge( + targetArray: any[], + sourceArray: any[], + targetKey: string | string[] | FieldSelectorFunction, + sourceKey: string | string[] | FieldSelectorFunction +): any[] { + + const targetKeySelector = fieldSelector(targetKey); + const sourceKeySelector = fieldSelector(sourceKey); + verifyJoinArgs(targetArray, sourceArray, targetKeySelector, sourceKeySelector, () => { }); + + // build a lookup maps for both arrays. + // so, both of them have to be unique, otherwise it will flattern result + const targetArrayMap = Object.create(null); + for (let item of sourceArray) { + targetArrayMap[targetKeySelector(item)] = item; + } + + const sourceArrayMap = Object.create(null); + for (let item of sourceArray) { + sourceArrayMap[sourceKeySelector(item)] = item; + } + + for (let sourceItemKey of Object.keys(sourceArrayMap)) { + const sourceItem = sourceArrayMap[sourceItemKey]; + if(!targetArrayMap[sourceItemKey]){ + targetArray.push(sourceItem); + } else { + // merge properties in + Object.assign(targetArrayMap[sourceItemKey], sourceItem); + } + } + + return targetArray; +} + + function verifyJoinArgs( leftArray: any[], rightArray: any[], @@ -98,10 +185,12 @@ function leftOrInnerJoin( isInnerJoin: boolean, leftArray: any[], rightArray: any[], - leftKeySelector: (item: any) => string, - rightKeySelector: (item: any) => string, + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): any[] { + const leftKeySelector = fieldSelector(leftKey); + const rightKeySelector = fieldSelector(rightKey); verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); diff --git a/src/array.spec.ts b/src/array.spec.ts index ab4e790..2edc88d 100644 --- a/src/array.spec.ts +++ b/src/array.spec.ts @@ -1,4 +1,5 @@ import * as pipeFuncs from './array'; +import { leftJoin } from './array-joins'; export const data = [ { name: "John", country: "US" }, { name: "Joe", country: "US" }, { name: "Bill", country: "US" }, { name: "Adam", country: "UK" }, @@ -17,7 +18,7 @@ describe('Test array methods', () => { const testAnyPrimitiveArrayMin = 0; const testAnyPrimitiveArrayMax = 33; - const testObjArray = testNumberArray.map(value => ({value})); + const testObjArray = testNumberArray.map(value => ({ value })); const dates = [new Date('10/01/12'), new Date('10/01/10'), new Date('10/01/09'), new Date('10/01/11')] it('count', () => { @@ -104,8 +105,8 @@ describe('Test array methods', () => { }); it('joinArray', () => { - const countries = [{code: 'US', capital: 'Washington'}, {code: 'UK', capital: 'London'}]; - const joinedArray = pipeFuncs.joinArray(data, countries, i => i.country, i2 => i2.code); + const countries = [{ code: 'US', capital: 'Washington' }, { code: 'UK', capital: 'London' }]; + const joinedArray = leftJoin(data, countries, i => i.country, i2 => i2.code, (l, r) => ({ ...r, ...l })); expect(joinedArray.length).toBe(8); const item = pipeFuncs.first(joinedArray, i => i.country === 'US'); expect(item.country).toBe('US'); diff --git a/src/data-pipe.ts b/src/data-pipe.ts index 30f43bf..fe6f7b8 100644 --- a/src/data-pipe.ts +++ b/src/data-pipe.ts @@ -1,8 +1,9 @@ -import { sum, avg, count, min, max, first, last, groupBy, flatten, countBy, joinArray } from './array'; +import { sum, avg, count, min, max, first, last, groupBy, flatten, countBy } from './array'; import { Selector, Predicate } from './models'; import { fromTable, toTable } from './table'; -import { ParsingOptions, fromCsv } from './dsv-parser'; -import { leftJoin, innerJoin, fullJoin } from './array-joins'; +import { ParsingOptions, parseCsv } from './dsv-parser'; +import { leftJoin, innerJoin, fullJoin, merge, FieldSelectorFunction } from './array-joins'; + export class DataPipe { private data: Array; @@ -125,56 +126,47 @@ export class DataPipe { return countBy(this.data, elementSelector); } - innerJoin(rightArray: any[], leftKeyField: string, rightKeyField: string, - resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; - - innerJoin(rightArray: any[], leftKeySelector: (item: any) => string, rightKeySelector: (item: any) => string, - resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; - - innerJoin(rightArray: any[], leftKey: any, rightKey: any, + /** + * Joins two arrays together by selecting elements that have matching values in both arrays + * @param rightArray array of elements to join + * @param leftKey left Key + * @param rightKey + * @param resultSelector + */ + innerJoin(rightArray: any[], + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): DataPipe { - const leftKeySelector: (item: any) => string = typeof leftKey === "function" ? leftKey : (item) => item[String(leftKey)]; - const rightKeySelector: (item: any) => string = typeof rightKey === "function" ? rightKey : (item) => item[String(rightKey)]; - - this.data = innerJoin(this.data, rightArray, leftKeySelector, rightKeySelector, resultSelector); + this.data = innerJoin(this.data, rightArray, leftKey, rightKey, resultSelector); return this; } - leftJoin(rightArray: any[], leftKeyField: string, rightKeyField: string, - resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; - - leftJoin(rightArray: any[], leftKeySelector: (item: any) => string, rightKeySelector: (item: any) => string, - resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; - - leftJoin(rightArray: any[], leftKey: any, rightKey: any, + leftJoin(rightArray: any[], + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any ): DataPipe { - const leftKeySelector: (item: any) => string = typeof leftKey === "function" ? leftKey : (item) => item[String(leftKey)]; - const rightKeySelector: (item: any) => string = typeof rightKey === "function" ? rightKey : (item) => item[String(rightKey)]; - this.data = leftJoin(this.data, rightArray, leftKeySelector, rightKeySelector, resultSelector); + this.data = leftJoin(this.data, rightArray, leftKey, rightKey, resultSelector); return this; } - fullJoin(rightArray: any[], leftKeyField: string, rightKeyField: string, + fullJoin(rightArray: any[], + leftKey: string | string[] | FieldSelectorFunction, + rightKey: string | string[] | FieldSelectorFunction, resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; - - fullJoin(rightArray: any[], leftKeySelector: (item: any) => string, rightKeySelector: (item: any) => string, - resultSelector: (leftItem: any, rightItem: any) => any - ): DataPipe; + ): DataPipe { + this.data = fullJoin(this.data, rightArray, leftKey, rightKey, resultSelector); + return this; + } - fullJoin(rightArray: any[], leftKey: any, rightKey: any, - resultSelector: (leftItem: any, rightItem: any) => any + merge( + sourceArray: any[], + targetKey: string | string[] | FieldSelectorFunction, + sourceKey: string | string[] | FieldSelectorFunction ): DataPipe { - const leftKeySelector: (item: any) => string = typeof leftKey === "function" ? leftKey : (item) => item[String(leftKey)]; - const rightKeySelector: (item: any) => string = typeof rightKey === "function" ? rightKey : (item) => item[String(rightKey)]; - this.data = fullJoin(this.data, rightArray, leftKeySelector, rightKeySelector, resultSelector); + this.data = merge(this.data, sourceArray, targetKey, sourceKey); return this; } @@ -198,7 +190,7 @@ export class DataPipe { filter = this.where.bind(this); fromCsv(content: string, options?: ParsingOptions): DataPipe { - this.data = fromCsv(content, options); + this.data = parseCsv(content, options); return this; } @@ -232,4 +224,6 @@ export class DataPipe { this.data = Array.from(new Set(this.data)); return this; } + + } diff --git a/src/dsv-parser.spec.ts b/src/dsv-parser.spec.ts index d1eb46d..3309207 100644 --- a/src/dsv-parser.spec.ts +++ b/src/dsv-parser.spec.ts @@ -1,10 +1,10 @@ import { dataPipe } from './index'; -import { fromCsv, ParsingOptions } from './dsv-parser'; +import { ParsingOptions, parseCsv, toCsv } from './dsv-parser'; describe('Dsv Parser specification', () => { it('simple numbers', () => { const csv = ["F1,F2", "1,2"].join('\n') - const result = fromCsv(csv); + const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(2); @@ -12,14 +12,14 @@ describe('Dsv Parser specification', () => { it('simple numders and strings', () => { const csv = ["F1,F2,F3", `1,2,"Test, comma"`].join('\n') - const result = fromCsv(csv); + const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F3).toBe('Test, comma'); }) it('Empty should be null', () => { const csv = ["F1,F2,F3", `1,,"Test, comma"`].join('\n') - const result = fromCsv(csv); + const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(null); }) @@ -29,7 +29,7 @@ describe('Dsv Parser specification', () => { multi-line string`; const csv = ["F1,F2,F3", `1,"${multiLineString}","Test, comma"`].join('\n') - const result = fromCsv(csv); + const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(multiLineString); }) @@ -39,27 +39,27 @@ describe('Dsv Parser specification', () => { multi-line string`; const csv = ["F1\tF2\tF3", `1\t"${multiLineString}"\t"Test, comma"`].join('\n') - const result = fromCsv(csv, { separator: '\t' }); + const result = parseCsv(csv, { delimiter: '\t' }); expect(result.length).toBe(1); expect(result[0].F2).toBe(multiLineString); }) it('DSV with comma numbers', () => { const csv = ["F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') - const result = fromCsv(csv, { separator: '\t' }); + const result = parseCsv(csv, { delimiter: '\t' }); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); }) it('skip rows', () => { const csv = ["", "", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') - const result = fromCsv(csv, { separator: '\t', skipRows: 2 }); + const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 }); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); }) it('skip rows not empty rows', () => { const csv = ["", " * not Empty *", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') - const result = fromCsv(csv, { separator: '\t', skipRows: 2 }); + const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 }); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); }) @@ -67,20 +67,20 @@ describe('Dsv Parser specification', () => { it('skipUntil', () => { const csv = ["", " * not Empty *", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') var options = new ParsingOptions(); - options.separator = '\t'; + options.delimiter = '\t'; options.skipUntil = t => t && t.length > 1; - const result = fromCsv(csv, options); + const result = parseCsv(csv, options); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); }) it('empty values', () => { - const csv = ["", "",, "\t\t\t", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`, "\t\t"].join('\n') + const csv = ["", "", , "\t\t\t", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`, "\t\t"].join('\n') var options = new ParsingOptions(); - options.separator = '\t'; - - const result = fromCsv(csv, options); + options.delimiter = '\t'; + + const result = parseCsv(csv, options); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); }) @@ -88,13 +88,18 @@ describe('Dsv Parser specification', () => { it('Date Fields', () => { const csv = ["F1\tF2\tF3", `2020-02-11\t1,000.32\t"Test, comma"`].join('\n') var options = new ParsingOptions(); - options.separator = '\t'; + options.delimiter = '\t'; options.dateFields = ['F1'] - - const result = fromCsv(csv, options); + + const result = parseCsv(csv, options); expect(result.length).toBe(1); expect(result[0].F1 instanceof Date).toBe(true); - }) + }); + it('ToCsv', () => { + const csv = ["F1,F2,F3", `2020-02-11,1,tt`].join('\n') + const result = toCsv(parseCsv(csv)); + expect(result).toBe(csv); + }); }) diff --git a/src/dsv-parser.ts b/src/dsv-parser.ts index 6d6b58b..f2846ef 100644 --- a/src/dsv-parser.ts +++ b/src/dsv-parser.ts @@ -1,7 +1,7 @@ import { parseNumberOrNull, parseDatetimeOrNull } from "./utils"; export class ParsingOptions { - separator: string = ','; + delimiter: string = ','; skipRows: number = 0; dateFields: string[] = []; numberFields: string[] = []; @@ -12,6 +12,51 @@ export class ParsingOptions { elementSelector?: (headers: string[], tokens: string[]) => any; } +export function parseCsv(content: string, options?: ParsingOptions): any[] { + const result: any[] = []; + content = content || ''; + + if (!content.length) { + return result; + } + + return getLineTokens(content, options || new ParsingOptions()); +} + +export function toCsv(array: any[], delimiter: string = ','): string { + array = array || []; + + var result = ""; + const headers: string[] = []; + + // workout all headers + for (const item of array) { + for (const name in item) { + if (headers.indexOf(name) < 0) { headers.push(name); } + } + } + + // create a csv string + const lines = array.map(item => { + const values: string[] = []; + for (const name of headers) { + let value: any = item[name]; + if (value instanceof Date) { + value = parseDatetimeOrNull(value); + } else if (typeof value === "string" && value.indexOf(delimiter) >= 0) { + value = '"' + value + '"'; + } + + values.push(value || '') + } + return values.join(delimiter); + + }); + lines.unshift(headers.join(delimiter)) + + return lines.join('\n') +} + type ParsingContext = { content: string, currentIndex: number @@ -28,7 +73,7 @@ function getObjectElement(fieldNames: string[], tokens: string[], options: Parsi value = parseDatetimeOrNull(value); } else if (options.numberFields && options.numberFields.indexOf(fieldName) >= 0) { value = parseNumberOrNull(value); - } else if (options.booleanFields && options.booleanFields.indexOf(fieldName) >= 0) { + } else if (options.booleanFields && options.booleanFields.indexOf(fieldName) >= 0) { value = !!value; } else { const num = parseNumberOrNull(value); @@ -46,7 +91,7 @@ function getLineTokens(content: string, options: ParsingOptions): string[][] { currentIndex: 0 }; content = content || ''; - const delimiter = options.separator || ','; + const delimiter = options.delimiter || ','; const result = []; let lineNumber = 0; @@ -101,17 +146,6 @@ function getLineTokens(content: string, options: ParsingOptions): string[][] { return result; } -export function fromCsv(content: string, options?: ParsingOptions): any[] { - const result: any[] = []; - content = content || ''; - - if (!content.length) { - return result; - } - - return getLineTokens(content, options || new ParsingOptions()); -} - function nextLineTokens(context: ParsingContext, delimiter: string = ','): string[] { const tokens: string[] = []; let token = '';