From edec931e9b273fd08cd800e62e97eb37751abe5c Mon Sep 17 00:00:00 2001 From: Pavlo Date: Tue, 5 Apr 2022 10:43:41 +0100 Subject: [PATCH] Pretified and fixed datetime miliseconds issue --- .prettierrc | 8 + package.json | 2 +- src/_internals.ts | 28 +- src/array/index.ts | 8 +- src/array/joins.ts | 281 +++++++++--------- src/array/stats.ts | 78 +++-- src/array/transform.ts | 46 ++- src/array/utils.ts | 56 ++-- src/data-pipe.ts | 89 ++++-- src/index.ts | 4 +- src/string/index.ts | 2 +- src/string/stringUtils.ts | 34 +-- src/tests/array.spec.ts | 173 +++++++----- src/tests/data-pipe.spec.ts | 26 +- src/tests/dsv-parser.spec.ts | 169 +++++------ src/tests/string-utils.spec.ts | 70 ++--- src/tests/table.spec.ts | 29 +- src/tests/utils-pipe.spec.ts | 56 ++-- src/types.ts | 1 - src/utils/dsv-parser.ts | 503 +++++++++++++++++---------------- src/utils/helpers.ts | 342 +++++++++++++++------- src/utils/index.ts | 6 +- src/utils/table.ts | 37 +-- 23 files changed, 1192 insertions(+), 856 deletions(-) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..ebba38f --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "printWidth": 100, + "trailingComma": "none", + "arrowParens": "avoid", + "parser": "typescript", + "singleQuote": true, + "tabWidth": 2 +} diff --git a/package.json b/package.json index b41bd73..8587860 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "datapipe-js", - "version": "0.3.21", + "version": "0.3.22", "description": "dataPipe is a data processing and data analytics library for JavaScript. Inspired by LINQ (C#) and Pandas (Python)", "main": "dist/cjs/data-pipe.js", "module": "dist/esm/data-pipe.mjs", diff --git a/src/_internals.ts b/src/_internals.ts index cedbe37..64c52a5 100644 --- a/src/_internals.ts +++ b/src/_internals.ts @@ -1,14 +1,16 @@ -import { Selector } from "./types"; -import { dateToString } from "./utils"; +import { Selector } from './types'; +import { dateToString } from './utils'; -export function fieldSelector(input: string | string[] | Selector): Selector { - if (typeof input === "function") { - return input; - } else if (typeof input === "string") { - return (item): any => item[input] instanceof Date ? dateToString(item[input]) : item[input]; - } else if (Array.isArray(input)) { - return (item): any => input.map(r => item[r]).join('|'); - } else { - throw Error(`Unknown input. Can't create a fieldSelector`) - } -} \ No newline at end of file +export function fieldSelector( + input: string | string[] | Selector +): Selector { + if (typeof input === 'function') { + return input; + } else if (typeof input === 'string') { + return (item): any => (item[input] instanceof Date ? dateToString(item[input]) : item[input]); + } else if (Array.isArray(input)) { + return (item): any => input.map(r => item[r]).join('|'); + } else { + throw Error(`Unknown input. Can't create a fieldSelector`); + } +} diff --git a/src/array/index.ts b/src/array/index.ts index 1d990bc..2720854 100644 --- a/src/array/index.ts +++ b/src/array/index.ts @@ -1,4 +1,4 @@ -export * from './joins' -export * from './stats' -export * from './transform' -export * from './utils' +export * from './joins'; +export * from './stats'; +export * from './transform'; +export * from './utils'; diff --git a/src/array/joins.ts b/src/array/joins.ts index 8175ebc..55c178d 100644 --- a/src/array/joins.ts +++ b/src/array/joins.ts @@ -1,164 +1,172 @@ -import { Selector } from "../types"; -import { fieldSelector } from "../_internals"; +import { Selector } from '../types'; +import { fieldSelector } from '../_internals'; function verifyJoinArgs( - leftArray: any[], - rightArray: any[], - leftKeySelector: (item: any) => string, - rightKeySelector: (item: any) => string, - resultSelector: (leftItem: any, rightItem: any) => any + leftArray: any[], + rightArray: any[], + leftKeySelector: (item: any) => string, + rightKeySelector: (item: any) => string, + resultSelector: (leftItem: any, rightItem: any) => any ): void { - if (!leftArray || !Array.isArray(leftArray)) { - throw Error('leftArray is not provided or not a valid') - } - if (!rightArray || !Array.isArray(rightArray)) { - throw Error('rightArray is not provided or not a valid') - } - - if (typeof leftKeySelector !== 'function') { - throw Error('leftKeySelector is not provided or not a valid function') - } - - if (typeof rightKeySelector !== 'function') { - throw Error('rightKeySelector is not provided or not a valid function') - } - - if (typeof resultSelector !== 'function') { - throw Error('resultSelector is not provided or not a valid function') - } + if (!leftArray || !Array.isArray(leftArray)) { + throw Error('leftArray is not provided or not a valid'); + } + if (!rightArray || !Array.isArray(rightArray)) { + throw Error('rightArray is not provided or not a valid'); + } + + if (typeof leftKeySelector !== 'function') { + throw Error('leftKeySelector is not provided or not a valid function'); + } + + if (typeof rightKeySelector !== 'function') { + throw Error('rightKeySelector is not provided or not a valid function'); + } + + if (typeof resultSelector !== 'function') { + throw Error('resultSelector is not provided or not a valid function'); + } } function leftOrInnerJoin( - isInnerJoin: boolean, - leftArray: any[], - rightArray: any[], - leftKey: string | string[] | Selector, - rightKey: string | string[] | Selector, - resultSelector: (leftItem: any, rightItem: any) => any + isInnerJoin: boolean, + leftArray: any[], + rightArray: any[], + leftKey: string | string[] | Selector, + rightKey: string | string[] | Selector, + resultSelector: (leftItem: any, rightItem: any) => any ): any[] { - const leftKeySelector = fieldSelector(leftKey); - const rightKeySelector = fieldSelector(rightKey); + const leftKeySelector = fieldSelector(leftKey); + const rightKeySelector = fieldSelector(rightKey); - verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); + verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); - // build a lookup map - const rightArrayMap = Object.create(null); - for (const item of rightArray) { - rightArrayMap[rightKeySelector(item)] = item; - } + // build a lookup map + const rightArrayMap = Object.create(null); + for (const item of rightArray) { + rightArrayMap[rightKeySelector(item)] = item; + } - const result: any[] = []; - for (const leftItem of leftArray) { - const leftKey = leftKeySelector(leftItem); - const rightItem = rightArrayMap[leftKey] || null; + const result: any[] = []; + for (const leftItem of leftArray) { + const leftKey = leftKeySelector(leftItem); + const rightItem = rightArrayMap[leftKey] || null; - if (isInnerJoin && !rightItem) { continue; } + if (isInnerJoin && !rightItem) { + continue; + } - const resultItem = resultSelector(leftItem, rightItem); + const resultItem = resultSelector(leftItem, rightItem); - // if result is null then probably a left item was modified - result.push(resultItem || leftItem); - } + // if result is null then probably a left item was modified + result.push(resultItem || leftItem); + } - return result; + return result; } - /** * leftJoin returns all elements from the left array (leftArray), and the matched elements from the right array (rightArray). * The result is NULL from the right side, if there is no match. - * @param leftArray array for left side in a join + * @param leftArray array for left side in a join * @param rightArray array for right side in a join * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector * @param resultSelector A callback function that returns result value */ export function leftJoin( - leftArray: any[], - rightArray: any[], - leftKeySelector: string | string[] | Selector, - rightKeySelector: string | string[] | Selector, - resultSelector: (leftItem: any, rightItem: any) => any + leftArray: any[], + rightArray: any[], + leftKeySelector: string | string[] | Selector, + rightKeySelector: string | string[] | Selector, + resultSelector: (leftItem: any, rightItem: any) => any ): any[] { - return leftOrInnerJoin(false, leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); + return leftOrInnerJoin( + false, + leftArray, + rightArray, + leftKeySelector, + rightKeySelector, + resultSelector + ); } /** * innerJoin - Joins two arrays together by selecting elements that have matching values in both arrays. * If there are elements in any array that do not have matches in other array, these elements will not be shown! - * @param leftArray array for left side in a join + * @param leftArray array for left side in a join * @param rightArray array for right side in a join * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector * @param resultSelector A callback function that returns result value */ export function innerJoin( - leftArray: any[], - rightArray: any[], - leftKey: string | string[] | Selector, - rightKey: string | string[] | Selector, - resultSelector: (leftItem: any, rightItem: any) => any + leftArray: any[], + rightArray: any[], + leftKey: string | string[] | Selector, + rightKey: string | string[] | Selector, + resultSelector: (leftItem: any, rightItem: any) => any ): any[] { - return leftOrInnerJoin(true, leftArray, rightArray, leftKey, rightKey, resultSelector); + return leftOrInnerJoin(true, leftArray, rightArray, leftKey, rightKey, resultSelector); } /** * fullJoin returns all elements from the left array (leftArray), and all elements from the right array (rightArray). * The result is NULL from the right/left side, if there is no match. - * @param leftArray array for left side in a join + * @param leftArray array for left side in a join * @param rightArray array for right side in a join * @param leftKey A key from left side array. What can be as a fieldName, multiple fields or key Selector * @param rightKey A key from right side array. what can be as a fieldName, multiple fields or key Selector * @param resultSelector A callback function that returns result value */ export function fullJoin( - leftArray: any[], - rightArray: any[], - leftKey: string | string[] | Selector, - rightKey: string | string[] | Selector, - resultSelector: (leftItem: any, rightItem: any) => any + leftArray: any[], + rightArray: any[], + leftKey: string | string[] | Selector, + rightKey: string | string[] | Selector, + resultSelector: (leftItem: any, rightItem: any) => any ): any[] { - const leftKeySelector = fieldSelector(leftKey); - const rightKeySelector = fieldSelector(rightKey); - - verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); - - // build a lookup maps for both arrays. - // so, both of them have to be unique, otherwise it will flattern result - const leftArrayMap = Object.create(null); - for (const item of rightArray) { - leftArrayMap[leftKeySelector(item)] = item; + const leftKeySelector = fieldSelector(leftKey); + const rightKeySelector = fieldSelector(rightKey); + + verifyJoinArgs(leftArray, rightArray, leftKeySelector, rightKeySelector, resultSelector); + + // build a lookup maps for both arrays. + // so, both of them have to be unique, otherwise it will flattern result + const leftArrayMap = Object.create(null); + for (const item of rightArray) { + leftArrayMap[leftKeySelector(item)] = item; + } + + const rightArrayMap = Object.create(null); + for (const item of rightArray) { + rightArrayMap[rightKeySelector(item)] = item; + } + + const result: any[] = []; + for (const leftItem of leftArray) { + const leftKey = leftKeySelector(leftItem); + const rightItem = rightArrayMap[leftKey] || null; + + const resultItem = resultSelector(leftItem, rightItem); + + // if result is null then probably a left item was modified + result.push(resultItem || leftItem); + if (rightItem) { + delete rightArrayMap[leftKey]; } + } - const rightArrayMap = Object.create(null); - for (const item of rightArray) { - rightArrayMap[rightKeySelector(item)] = item; - } - - const result: any[] = []; - for (const leftItem of leftArray) { - const leftKey = leftKeySelector(leftItem); - const rightItem = rightArrayMap[leftKey] || null; + // add remaining right items + for (const rightItemKey in rightArrayMap) { + const rightItem = rightArrayMap[rightItemKey]; + const resultItem = resultSelector(null, rightItem); - const resultItem = resultSelector(leftItem, rightItem); - - // if result is null then probably a left item was modified - result.push(resultItem || leftItem); - if (rightItem) { - delete rightArrayMap[leftKey]; - } - } - - // add remaining right items - for (const rightItemKey in rightArrayMap) { - const rightItem = rightArrayMap[rightItemKey]; - const resultItem = resultSelector(null, rightItem); - - // if result is null then probably a left item was modified - result.push(resultItem || rightItem); - } + // if result is null then probably a left item was modified + result.push(resultItem || rightItem); + } - return result; + return result; } /** @@ -169,37 +177,36 @@ export function fullJoin( * @param sourceKey source key field, arry of fields or field serlector */ export function merge( - targetArray: any[], - sourceArray: any[], - targetKey: string | string[] | Selector, - sourceKey: string | string[] | Selector + targetArray: any[], + sourceArray: any[], + targetKey: string | string[] | Selector, + sourceKey: string | string[] | Selector ): any[] { - - const targetKeySelector = fieldSelector(targetKey); - const sourceKeySelector = fieldSelector(sourceKey); - verifyJoinArgs(targetArray, sourceArray, targetKeySelector, sourceKeySelector, () => false); - - // build a lookup maps for both arrays. - // so, both of them have to be unique, otherwise it will flattern result - const targetArrayMap = Object.create(null); - for (const item of sourceArray) { - targetArrayMap[targetKeySelector(item)] = item; - } - - const sourceArrayMap = Object.create(null); - for (const item of sourceArray) { - sourceArrayMap[sourceKeySelector(item)] = item; - } - - for (const sourceItemKey of Object.keys(sourceArrayMap)) { - const sourceItem = sourceArrayMap[sourceItemKey]; - if (!targetArrayMap[sourceItemKey]) { - targetArray.push(sourceItem); - } else { - // merge properties in - Object.assign(targetArrayMap[sourceItemKey], sourceItem); - } + const targetKeySelector = fieldSelector(targetKey); + const sourceKeySelector = fieldSelector(sourceKey); + verifyJoinArgs(targetArray, sourceArray, targetKeySelector, sourceKeySelector, () => false); + + // build a lookup maps for both arrays. + // so, both of them have to be unique, otherwise it will flattern result + const targetArrayMap = Object.create(null); + for (const item of sourceArray) { + targetArrayMap[targetKeySelector(item)] = item; + } + + const sourceArrayMap = Object.create(null); + for (const item of sourceArray) { + sourceArrayMap[sourceKeySelector(item)] = item; + } + + for (const sourceItemKey of Object.keys(sourceArrayMap)) { + const sourceItem = sourceArrayMap[sourceItemKey]; + if (!targetArrayMap[sourceItemKey]) { + targetArray.push(sourceItem); + } else { + // merge properties in + Object.assign(targetArrayMap[sourceItemKey], sourceItem); } + } - return targetArray; + return targetArray; } diff --git a/src/array/stats.ts b/src/array/stats.ts index 1b3c4d9..962468d 100644 --- a/src/array/stats.ts +++ b/src/array/stats.ts @@ -1,12 +1,14 @@ -import { Selector, Predicate } from "../types"; -import { parseNumber } from "../utils"; -import { isArrayEmptyOrNull } from "./utils"; +import { Selector, Predicate } from '../types'; +import { parseNumber } from '../utils'; +import { isArrayEmptyOrNull } from './utils'; function fieldSelector(field?: string | Selector): Selector { if (!field) { return (item: any): any => item; } - return typeof field === 'function' ? field as Selector : (item: any): any => item[String(field)]; + return typeof field === 'function' + ? (field as Selector) + : (item: any): any => item[String(field)]; } function fieldComparator(field?: string | Selector): (a: any, b: any) => number { @@ -23,12 +25,14 @@ function fieldComparator(field?: string | Selector): (a: any, b: any) => number } return aVal - bVal >= 0 ? 1 : -1; - } + }; } function getNumberValuesArray(array: any[], field?: string | Selector): number[] { const elementSelector = fieldSelector(field); - return array.map(item => parseNumber(item, elementSelector)).filter(v => v !== undefined) as number[]; + return array + .map(item => parseNumber(item, elementSelector)) + .filter(v => v !== undefined) as number[]; } /** @@ -43,7 +47,9 @@ function getNumberValuesArray(array: any[], field?: string | Selector): number[] * sum([{ val: 1 }, { val: 5 }], i => i.val); // 6 */ export function sum(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null } + if (isArrayEmptyOrNull(array)) { + return null; + } const elementSelector = fieldSelector(field); @@ -52,7 +58,7 @@ export function sum(array: any[], field?: Selector | string): number | null { const numberVal = parseNumber(item, elementSelector); if (numberVal) { sum += numberVal; - } + } } return sum; @@ -67,7 +73,9 @@ export function sum(array: any[], field?: Selector | string): number | null { * avg([1, 5, 3]); // 3 */ export function avg(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null } + if (isArrayEmptyOrNull(array)) { + return null; + } const elementSelector = fieldSelector(field); @@ -84,7 +92,9 @@ export function avg(array: any[], field?: Selector | string): number | null { export function min(array: any[], field?: Selector | string): number | Date | null { const elementSelector = fieldSelector(field); const numberArray = getNumberValuesArray(array, elementSelector); - if (isArrayEmptyOrNull(numberArray)) { return null } + if (isArrayEmptyOrNull(numberArray)) { + return null; + } const min = Math.min(...numberArray); const item = elementSelector ? elementSelector(array[0]) : array[0]; if (item instanceof Date) { @@ -102,7 +112,9 @@ export function min(array: any[], field?: Selector | string): number | Date | nu export function max(array: any[], field?: Selector | string): number | Date | null { const elementSelector = fieldSelector(field); const numberArray = getNumberValuesArray(array, elementSelector); - if (isArrayEmptyOrNull(numberArray)) { return null } + if (isArrayEmptyOrNull(numberArray)) { + return null; + } const max = Math.max(...numberArray); const item = elementSelector ? elementSelector(array[0]) : array[0]; if (item instanceof Date) { @@ -132,7 +144,9 @@ export function count(array: any[], predicate?: Predicate): number | null { * @param predicate Predicate function invoked per iteration. */ export function first(array: T[], predicate?: Predicate): T | null { - if (isArrayEmptyOrNull(array)) { return null } + if (isArrayEmptyOrNull(array)) { + return null; + } if (!predicate) { return array[0]; @@ -151,7 +165,9 @@ export function first(array: T[], predicate?: Predicate): T | null { * @param predicate Predicate function invoked per iteration. */ export function last(array: T[], predicate?: Predicate): T | null { - if (isArrayEmptyOrNull(array)) { return null } + if (isArrayEmptyOrNull(array)) { + return null; + } let lastIndex = array.length - 1; if (!predicate) { @@ -173,7 +189,9 @@ export function last(array: T[], predicate?: Predicate): T | null { * @param elementSelector Function invoked per iteration. */ export function countBy(array: any[], elementSelector: Selector): Record { - if (!array || !Array.isArray(array)) { throw Error('No array provided') } + if (!array || !Array.isArray(array)) { + throw Error('No array provided'); + } const results: Record = {}; const length = array.length; @@ -194,14 +212,16 @@ export function countBy(array: any[], elementSelector: Selector): Record r[columnField] === columnValue) - .map((r) => r[dataField]); + .filter(r => r[columnField] === columnValue) + .map(r => r[dataField]); item[columnValue] = aggFunction(dataArray); } @@ -152,21 +149,21 @@ export function pivot( */ export function transpose(data: any[]): any[] { if (!Array.isArray(data)) { - throw Error("An array is not provided"); + throw Error('An array is not provided'); } if (!data.length) { return data; } - return Object.keys(data[0]).map((key) => { + return Object.keys(data[0]).map(key => { const res: { [key: string]: any } = {}; data.forEach((item, i) => { if (i === 0) { res.fieldName = key; } - res["row" + i] = item[key]; + res['row' + i] = item[key]; }); return res; }); @@ -177,12 +174,9 @@ export function transpose(data: any[]): any[] { * @param array The array to process. * @param elementSelector Function invoked per iteration. */ -export function select( - data: any[], - selector: string | string[] | Selector -): any[] { +export function select(data: any[], selector: string | string[] | Selector): any[] { if (!Array.isArray(data)) { - throw Error("An array is not provided"); + throw Error('An array is not provided'); } if (!data.length) { @@ -199,7 +193,7 @@ export function select( */ export function where(data: any[], predicate: Predicate): any[] { if (!Array.isArray(data)) { - throw Error("An array is not provided"); + throw Error('An array is not provided'); } return data.filter(predicate); diff --git a/src/array/utils.ts b/src/array/utils.ts index 76720d4..a2d2e1f 100644 --- a/src/array/utils.ts +++ b/src/array/utils.ts @@ -1,6 +1,6 @@ -import { parseNumber, parseDatetimeOrNull, dateToString } from "../utils"; -import { ScalarType, Selector } from ".."; -import { fieldSelector } from "../_internals"; +import { parseNumber, parseDatetimeOrNull, dateToString } from '../utils'; +import { ScalarType, Selector } from '..'; +import { fieldSelector } from '../_internals'; function compareStrings(a: string, b: any): number { return a.localeCompare(b); @@ -44,10 +44,14 @@ function compare(a: any, b: any, { field, asc }: any): number { } switch (typeof valA) { - case 'number': return order * compareNumbers(valA, valB); - case 'string': return order * compareStrings(valA, valB); - case 'object': return order * compareObjects(valA, valB); - case 'undefined': return valB === undefined ? 0 : (-1 * order); + case 'number': + return order * compareNumbers(valA, valB); + case 'string': + return order * compareStrings(valA, valB); + case 'object': + return order * compareObjects(valA, valB); + case 'undefined': + return valB === undefined ? 0 : -1 * order; } return 0; } @@ -86,8 +90,9 @@ export function isArrayEmptyOrNull(array: any[]): boolean { * sort(array, 'name ASC', 'age DESC'); */ export function sort(array: any[], ...fields: string[]): any[] { - - if (!array || !Array.isArray(array)) { throw Error('Array is not provided'); } + if (!array || !Array.isArray(array)) { + throw Error('Array is not provided'); + } if (!fields?.length) { // just a default sort @@ -110,40 +115,48 @@ export function sort(array: any[], ...fields: string[]): any[] { * @param array to be converted * @param keyField a selector or field name for a property name */ -export function toObject(array: any[], keyField: string | string[] | Selector): Record { +export function toObject( + array: any[], + keyField: string | string[] | Selector +): Record { const result = {} as Record; for (const item of array) { let key = fieldSelector(keyField)(item); - if (key as any instanceof Date) { - key = dateToString(key as any) + if ((key as any) instanceof Date) { + key = dateToString(key as any); } - result[key] = item + result[key] = item; } return result; } /** - * Convert array of items to into series array or series record. + * Convert array of items to into series array or series record. * @param array Array to be converted - * @param propertyName optional parameter to define a property to be unpacked. + * @param propertyName optional parameter to define a property to be unpacked. * If it is string the array with values will be returned, otherwise an object with a list of series map */ -export function toSeries(array: any[], propertyName?: string | string[]): Record | ScalarType[] { - if (!array?.length) { return {}; } +export function toSeries( + array: any[], + propertyName?: string | string[] +): Record | ScalarType[] { + if (!array?.length) { + return {}; + } // a single property if (typeof propertyName == 'string') { - return array.map(r => r[propertyName] === undefined ? null : r[propertyName]); + return array.map(r => (r[propertyName] === undefined ? null : r[propertyName])); } const seriesRecord: Record = {}; - const seriesNames = (Array.isArray(propertyName) && propertyName.length) ? propertyName : Object.keys(array[0]); + const seriesNames = + Array.isArray(propertyName) && propertyName.length ? propertyName : Object.keys(array[0]); for (let i = 0; i < array.length; i++) { - for (let j = 0; j < seriesNames.length; j++) { const seriesName = seriesNames[j]; if (!seriesRecord[seriesName]) { @@ -152,10 +165,7 @@ export function toSeries(array: any[], propertyName?: string | string[]): Record const value = array[i][seriesName]; seriesRecord[seriesName].push(value === undefined ? null : value); } - } return seriesRecord; - } - diff --git a/src/data-pipe.ts b/src/data-pipe.ts index bfb3383..c695678 100644 --- a/src/data-pipe.ts +++ b/src/data-pipe.ts @@ -1,8 +1,40 @@ -import { sum, avg, count, min, max, first, last, mean, quantile, variance, median, stdev } from './array/stats'; -import { Selector, Predicate, ParsingOptions, FieldDescription, PrimitiveType, TableDto, DataTypeName, ScalarType } from './types'; +import { + sum, + avg, + count, + min, + max, + first, + last, + mean, + quantile, + variance, + median, + stdev +} from './array/stats'; +import { + Selector, + Predicate, + ParsingOptions, + FieldDescription, + PrimitiveType, + TableDto, + DataTypeName, + ScalarType +} from './types'; import { parseCsv, fromTable, toTable, getFieldsInfo, toCsv } from './utils'; -import { leftJoin, innerJoin, fullJoin, merge, groupBy, sort, pivot, transpose, toObject, toSeries } from './array'; - +import { + leftJoin, + innerJoin, + fullJoin, + merge, + groupBy, + sort, + pivot, + transpose, + toObject, + toSeries +} from './array'; export class DataPipe { private data: any[]; @@ -27,8 +59,11 @@ export class DataPipe { * @param fieldNames fieldNames what correspond to the rows * @param fieldDataTypes fieldNames what correspond to the rows */ - fromTable(rowsOrTable: PrimitiveType[][] | TableDto, fieldNames?: string[], - fieldDataTypes?: DataTypeName[]): DataPipe { + fromTable( + rowsOrTable: PrimitiveType[][] | TableDto, + fieldNames?: string[], + fieldDataTypes?: DataTypeName[] + ): DataPipe { this.data = fromTable(rowsOrTable, fieldNames, fieldDataTypes); return this; } @@ -43,10 +78,10 @@ export class DataPipe { /** * Outputs Pipe value as CSV content - * @param delimiter + * @param delimiter */ toCsv(delimiter = ','): string { - return toCsv(this.data, delimiter) + return toCsv(this.data, delimiter); } /** @@ -58,10 +93,10 @@ export class DataPipe { } /** - * Convert array of items to into series array or series record. - * @param propertyName optional parameter to define a property to be unpacked. - * If it is string the array with values will be returned, otherwise an object with a list of series map - */ + * Convert array of items to into series array or series record. + * @param propertyName optional parameter to define a property to be unpacked. + * If it is string the array with values will be returned, otherwise an object with a list of series map + */ toSeries(propertyName?: string | string[]): Record | ScalarType[] { return toSeries(this.data, propertyName); } @@ -74,10 +109,9 @@ export class DataPipe { } // end of output functions - /** * This method allows you to examine a state of the data during pipe execution. - * @param dataFunc + * @param dataFunc */ tap(dataFunc: (d: any[]) => void): DataPipe { if (typeof dataFunc === 'function') { @@ -194,7 +228,6 @@ export class DataPipe { return median(this.data, field); } - // Data Transformation functions /** * Groups array items based on elementSelector function @@ -212,7 +245,8 @@ export class DataPipe { * @param rightKey * @param resultSelector */ - innerJoin(rightArray: any[], + innerJoin( + rightArray: any[], leftKey: string | string[] | Selector, rightKey: string | string[] | Selector, resultSelector: (leftItem: any, rightItem: any) => any @@ -221,17 +255,18 @@ export class DataPipe { return this; } - leftJoin(rightArray: any[], + leftJoin( + rightArray: any[], leftKey: string | string[] | Selector, rightKey: string | string[] | Selector, resultSelector: (leftItem: any, rightItem: any) => any ): DataPipe { - this.data = leftJoin(this.data, rightArray, leftKey, rightKey, resultSelector); return this; } - fullJoin(rightArray: any[], + fullJoin( + rightArray: any[], leftKey: string | string[] | Selector, rightKey: string | string[] | Selector, resultSelector: (leftItem: any, rightItem: any) => any @@ -263,10 +298,14 @@ export class DataPipe { return this; } - pivot(rowFields: string | string[], columnField: string, dataField: string, - aggFunction?: (array: any[]) => any | null, columnValues?: string[]): DataPipe { - - this.data = pivot(this.data, rowFields, columnField, dataField, aggFunction, columnValues) + pivot( + rowFields: string | string[], + columnField: string, + dataField: string, + aggFunction?: (array: any[]) => any | null, + columnValues?: string[] + ): DataPipe { + this.data = pivot(this.data, rowFields, columnField, dataField, aggFunction, columnValues); return this; } @@ -308,7 +347,7 @@ export class DataPipe { */ sort(...fields: string[]): DataPipe { sort(this.data, ...fields); - return this + return this; } // end of transformation functions @@ -318,6 +357,6 @@ export class DataPipe { * if any properties are Objects, it would use JSON.stringify to calculate maxSize field. */ getFieldsInfo(): FieldDescription[] { - return getFieldsInfo(this.data) + return getFieldsInfo(this.data); } } diff --git a/src/index.ts b/src/index.ts index 09df02e..cea92dc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,6 @@ -import { DataPipe } from "./data-pipe"; +import { DataPipe } from './data-pipe'; -export * from "./types" +export * from './types'; /** * Data Pipeline factory function what creates DataPipe diff --git a/src/string/index.ts b/src/string/index.ts index a4fa13b..d647b37 100644 --- a/src/string/index.ts +++ b/src/string/index.ts @@ -1 +1 @@ -export * from './stringUtils' \ No newline at end of file +export * from './stringUtils'; diff --git a/src/string/stringUtils.ts b/src/string/stringUtils.ts index 4f9c1d0..1beba05 100644 --- a/src/string/stringUtils.ts +++ b/src/string/stringUtils.ts @@ -1,16 +1,12 @@ -export function formatCamelStr(str = ""): string { +export function formatCamelStr(str = ''): string { return str - .replace(/^\w/, (c) => c.toUpperCase()) - .replace(/([a-z0-9])([A-Z])/g, "$1 $2") - .replace(/_/g, " "); + .replace(/^\w/, c => c.toUpperCase()) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/_/g, ' '); } -export function replaceAll( - text: string, - searchValue: string, - replaceValue = "" -): string { - return text.replace(new RegExp(searchValue, "g"), replaceValue || ""); +export function replaceAll(text: string, searchValue: string, replaceValue = ''): string { + return text.replace(new RegExp(searchValue, 'g'), replaceValue || ''); } /** @@ -19,7 +15,7 @@ export function replaceAll( * @param characters character to trim * @returns trimmed string */ -export function trimStart(text: string, characters = " \n\t\r"): string { +export function trimStart(text: string, characters = ' \n\t\r'): string { let startIndex = 0; while (characters.indexOf(text.charAt(startIndex)) >= 0) { startIndex++; @@ -33,7 +29,7 @@ export function trimStart(text: string, characters = " \n\t\r"): string { * @param characters character to trim * @returns trimmed string */ -export function trimEnd(text: string, characters = " \n\t\r"): string { +export function trimEnd(text: string, characters = ' \n\t\r'): string { let endIndex = text.length; while (characters.indexOf(text.charAt(endIndex - 1)) >= 0) { endIndex--; @@ -47,7 +43,7 @@ export function trimEnd(text: string, characters = " \n\t\r"): string { * @param characters character to trim * @returns trimmed string */ -export function trim(text: string, characters = " \n\t\r"): string { +export function trim(text: string, characters = ' \n\t\r'): string { return trimStart(trimEnd(text, characters), characters); } @@ -61,11 +57,7 @@ export function trim(text: string, characters = " \n\t\r"): string { * @param openClose * @returns */ -export function split( - text: string, - separator = ",", - openClose?: string[] -): string[] { +export function split(text: string, separator = ',', openClose?: string[]): string[] { const res: string[] = []; if (!text) { @@ -75,11 +67,11 @@ export function split( openClose = openClose || []; let index = -1; - let token = ""; + let token = ''; while (++index < text.length) { let currentChar = text[index]; - const oIndex = openClose.findIndex((s) => s[0] === currentChar); + const oIndex = openClose.findIndex(s => s[0] === currentChar); if (oIndex >= 0) { token += text[index]; let innerBrackets = 0; @@ -105,7 +97,7 @@ export function split( if (separator.includes(currentChar)) { res.push(token); - token = ""; + token = ''; } else { token += currentChar; } diff --git a/src/tests/array.spec.ts b/src/tests/array.spec.ts index 396ca91..94e4861 100644 --- a/src/tests/array.spec.ts +++ b/src/tests/array.spec.ts @@ -1,19 +1,33 @@ -import * as pipeFuncs from '../array' -import { leftJoin, pivot, avg, sum, quantile, mean, variance, stdev, median, first, fullJoin, innerJoin, toObject, toSeries } from '../array'; +import * as pipeFuncs from '../array'; +import { + leftJoin, + pivot, + avg, + sum, + quantile, + mean, + variance, + stdev, + median, + first, + fullJoin, + innerJoin, + toObject, + toSeries +} from '../array'; export const data = [ - { name: "John", country: "US", age: 32 }, - { name: "Joe", country: "US", age: 24 }, - { name: "Bill", country: "US", age: 27 }, - { name: "Adam", country: "UK", age: 18 }, - { name: "Scott", country: "UK", age: 45 }, - { name: "Diana", country: "UK" }, - { name: "Marry", country: "FR", age: 18 }, - { name: "Luc", country: "FR", age: null } -] + { name: 'John', country: 'US', age: 32 }, + { name: 'Joe', country: 'US', age: 24 }, + { name: 'Bill', country: 'US', age: 27 }, + { name: 'Adam', country: 'UK', age: 18 }, + { name: 'Scott', country: 'UK', age: 45 }, + { name: 'Diana', country: 'UK' }, + { name: 'Marry', country: 'FR', age: 18 }, + { name: 'Luc', country: 'FR', age: null } +]; describe('Test array methods', () => { - const testNumberArray = [2, 6, 3, 7, 11, 7, -1]; const testNumberArraySum = 35; const testNumberArrayAvg = 5; @@ -25,20 +39,25 @@ describe('Test array methods', () => { const testAnyPrimitiveArrayMax = 33; const testObjArray = testNumberArray.map(value => ({ value })); - const dates = [new Date('10/01/12'), new Date('10/01/10'), new Date('10/01/09'), new Date('10/01/11')] + const dates = [ + new Date('10/01/12'), + new Date('10/01/10'), + new Date('10/01/09'), + new Date('10/01/11') + ]; it('count', () => { expect(pipeFuncs.count(testNumberArray)).toBe(testNumberArray.length); expect(pipeFuncs.count(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArray.length); expect(pipeFuncs.count(testObjArray)).toBe(testObjArray.length); expect(pipeFuncs.count(data, i => i.country === 'US')).toBe(3); - }) + }); it('sum', () => { expect(pipeFuncs.sum(testNumberArray)).toBe(testNumberArraySum); expect(pipeFuncs.sum(testAnyPrimitiveArray)).toBe(testAnyPrimitiveArraySum); expect(pipeFuncs.sum(testObjArray, obj => obj.value)).toBe(testNumberArraySum); - }) + }); it('avg', () => { expect(pipeFuncs.avg(testNumberArray)).toBe(testNumberArrayAvg); @@ -50,7 +69,7 @@ describe('Test array methods', () => { } else { throw Error('testAnyPrimitiveArray failed'); } - }) + }); it('min', () => { expect(pipeFuncs.min(testNumberArray)).toBe(Math.min(...testNumberArray)); @@ -60,9 +79,9 @@ describe('Test array methods', () => { const mindate = pipeFuncs.min(dates); expect(mindate).toBeInstanceOf(Date); if (mindate instanceof Date) { - expect(mindate.getFullYear()).toBe(2009) + expect(mindate.getFullYear()).toBe(2009); } - }) + }); it('max', () => { expect(pipeFuncs.max(testNumberArray)).toBe(Math.max(...testNumberArray)); @@ -72,14 +91,14 @@ describe('Test array methods', () => { const maxdate = pipeFuncs.max(dates); expect(maxdate).toBeInstanceOf(Date); if (maxdate instanceof Date) { - expect(maxdate.getFullYear()).toBe(2012) + expect(maxdate.getFullYear()).toBe(2012); } - }) + }); it('first', () => { expect(pipeFuncs.first(testNumberArray)).toBe(testNumberArray[0]); expect(pipeFuncs.first(testNumberArray, v => v > 6)).toBe(7); - }) + }); it('last', () => { const last = pipeFuncs.last(data, item => item.country === 'UK'); @@ -88,12 +107,12 @@ describe('Test array methods', () => { } else { throw Error('Not found'); } - }) + }); it('groupBy', () => { const groups = pipeFuncs.groupBy(data, item => item.country); expect(groups.length).toBe(3); - }) + }); it('flatten', () => { const testArray = [1, 4, [2, [5, 5, [9, 7]], 11], 0, [], []]; @@ -103,7 +122,7 @@ describe('Test array methods', () => { const testArray2 = [testArray, [data], [], [testAnyPrimitiveArray]]; const flatten2 = pipeFuncs.flatten(testArray2); expect(flatten2.length).toBe(9 + data.length + testAnyPrimitiveArray.length); - }) + }); it('countBy', () => { const countriesCount = pipeFuncs.countBy(data, i => i.country); @@ -132,17 +151,25 @@ describe('Test array methods', () => { expect(pipeFuncs.quantile([], 0)).toBe(null); }); - it('leftJoin', () => { - const countries = [{ code: 'US', capital: 'Washington' }, { code: 'UK', capital: 'London' }]; - const joinedArray = leftJoin(data, countries, i => i.country, i2 => i2.code, (l, r) => ({ ...r, ...l })); + const countries = [ + { code: 'US', capital: 'Washington' }, + { code: 'UK', capital: 'London' } + ]; + const joinedArray = leftJoin( + data, + countries, + i => i.country, + i2 => i2.code, + (l, r) => ({ ...r, ...l }) + ); expect(joinedArray.length).toBe(8); const item = pipeFuncs.first(joinedArray, i => i.country === 'US'); expect(item.country).toBe('US'); expect(item.code).toBe('US'); expect(item.capital).toBe('Washington'); expect(item.name).toBeTruthy(); - }) + }); it('leftJoin 2', () => { const arr1 = [ @@ -159,12 +186,18 @@ describe('Test array methods', () => { { keyField: 'k8', value2: 38 } ]; - const joinedArray = leftJoin(arr1, arr2, l => l.keyField, r => r.keyField, (l, r) => ({ ...r, ...l })); + const joinedArray = leftJoin( + arr1, + arr2, + l => l.keyField, + r => r.keyField, + (l, r) => ({ ...r, ...l }) + ); expect(joinedArray.length).toBe(4); expect(joinedArray[1].keyField).toBe('k2'); expect(joinedArray[1].value1).toBe(22); expect(joinedArray[1].value2).toBe(32); - }) + }); it('innerJoin', () => { const arr1 = [ @@ -181,12 +214,18 @@ describe('Test array methods', () => { { keyField: 'k8', value2: 38 } ]; - const joinedArray = innerJoin(arr1, arr2, l => l.keyField, r => r.keyField, (l, r) => ({ ...r, ...l })); + const joinedArray = innerJoin( + arr1, + arr2, + l => l.keyField, + r => r.keyField, + (l, r) => ({ ...r, ...l }) + ); expect(joinedArray.length).toBe(2); expect(joinedArray[1].keyField).toBe('k2'); expect(joinedArray[1].value1).toBe(22); expect(joinedArray[1].value2).toBe(32); - }) + }); it('fullJoin', () => { const arr1 = [ @@ -203,27 +242,32 @@ describe('Test array methods', () => { { keyField: 'k8', value2: 38 } ]; - const joinedArray = fullJoin(arr1, arr2, l => l.keyField, r => r.keyField, (l, r) => ({ ...r, ...l })); + const joinedArray = fullJoin( + arr1, + arr2, + l => l.keyField, + r => r.keyField, + (l, r) => ({ ...r, ...l }) + ); expect(joinedArray.length).toBe(6); expect(joinedArray[1].keyField).toBe('k2'); expect(joinedArray[1].value1).toBe(22); expect(joinedArray[1].value2).toBe(32); - }) - + }); it('simple pivot', () => { const arr = [ { product: 'P1', year: '2018', sale: '11' }, { product: 'P1', year: '2019', sale: '12' }, { product: 'P2', year: '2018', sale: '21' }, - { product: 'P2', year: '2019', sale: '22' }, + { product: 'P2', year: '2019', sale: '22' } ]; - const res = pivot(arr, 'product', 'year', 'sale') + const res = pivot(arr, 'product', 'year', 'sale'); expect(res.length).toBe(2); expect(res.filter(r => r.product === 'P1')[0]['2018']).toBe(11); expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); - }) + }); it('pivot with default sum', () => { const arr = [ @@ -231,14 +275,14 @@ describe('Test array methods', () => { { product: 'P1', year: '2019', sale: '12' }, { product: 'P1', year: '2019', sale: '22' }, { product: 'P2', year: '2018', sale: '21' }, - { product: 'P2', year: '2019', sale: '22' }, + { product: 'P2', year: '2019', sale: '22' } ]; - const res = pivot(arr, 'product', 'year', 'sale') + const res = pivot(arr, 'product', 'year', 'sale'); expect(res.length).toBe(2); expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(34); expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); - }) + }); it('pivot with specified AVG', () => { const arr = [ @@ -246,15 +290,14 @@ describe('Test array methods', () => { { product: 'P1', year: '2019', sale: '12' }, { product: 'P1', year: '2019', sale: '22' }, { product: 'P2', year: '2018', sale: '21' }, - { product: 'P2', year: '2019', sale: '22' }, + { product: 'P2', year: '2019', sale: '22' } ]; - const res = pivot(arr, 'product', 'year', 'sale', avg) + const res = pivot(arr, 'product', 'year', 'sale', avg); expect(res.length).toBe(2); expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(17); expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); - }) - + }); it('pivot with null value', () => { const arr = [ @@ -263,16 +306,16 @@ describe('Test array methods', () => { { product: 'P1', year: '2019', sale: '22' }, { product: 'P2', year: '2018', sale: '21' }, { product: 'P2', year: '2019', sale: '22' }, - { product: 'P3', year: '2019', sale: '33' }, + { product: 'P3', year: '2019', sale: '33' } ]; - const res = pivot(arr, 'product', 'year', 'sale', avg) + const res = pivot(arr, 'product', 'year', 'sale', avg); expect(res.length).toBe(3); expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(17); expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe(33); expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); - }) + }); it('pivot with null value with sum', () => { const arr = [ @@ -281,16 +324,16 @@ describe('Test array methods', () => { { product: 'P1', year: '2019', sale: '22' }, { product: 'P2', year: '2018', sale: '21' }, { product: 'P2', year: '2019', sale: '22' }, - { product: 'P3', year: '2019', sale: '33' }, + { product: 'P3', year: '2019', sale: '33' } ]; - const res = pivot(arr, 'product', 'year', 'sale', sum) + const res = pivot(arr, 'product', 'year', 'sale', sum); expect(res.length).toBe(3); expect(res.filter(r => r.product === 'P1')[0]['2019']).toBe(34); expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe(21); expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe(33); expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); - }) + }); it('pivot not string data value', () => { const arr = [ @@ -298,7 +341,7 @@ describe('Test array methods', () => { { product: 'P1', year: '2019', notAString: 'Data12' }, { product: 'P2', year: '2018', notAString: 'Data21' }, { product: 'P2', year: '2019', notAString: 'Data22' }, - { product: 'P3', year: '2019', notAString: 'Data33' }, + { product: 'P3', year: '2019', notAString: 'Data33' } ]; const res = pivot(arr, 'year', 'product', 'notAString', first); @@ -391,7 +434,8 @@ describe('Test array methods', () => { { name: 'Tom', age: 7 }, { name: 'Bob', age: 10 }, { age: 5 }, - { name: 'Jerry', age: 3 }]; + { name: 'Jerry', age: 3 } + ]; arr = pipeFuncs.sort(arrWithUndefinedProps, 'name ASC') || []; expect(arr[0].age).toBe(5); arr = pipeFuncs.sort(arrWithUndefinedProps, 'name DESC') || []; @@ -406,16 +450,16 @@ describe('Test array methods', () => { { name: 'Jerry', age: 3 } ]; - const obj1 = toObject(array, "name") - expect(Object.keys(obj1).length).toBe(array.length) - expect(obj1['Bob'].age).toBe(10) - expect(obj1['undefined'].age).toBe(5) + const obj1 = toObject(array, 'name'); + expect(Object.keys(obj1).length).toBe(array.length); + expect(obj1['Bob'].age).toBe(10); + expect(obj1['undefined'].age).toBe(5); - const obj2 = toObject(array, i => i.name) - expect(Object.keys(obj2).length).toBe(array.length) + const obj2 = toObject(array, i => i.name); + expect(Object.keys(obj2).length).toBe(array.length); // make sure both are thesame - expect(JSON.stringify(obj1)).toBe(JSON.stringify(obj2)) + expect(JSON.stringify(obj1)).toBe(JSON.stringify(obj2)); }); it('toObject NOT string', () => { @@ -426,11 +470,11 @@ describe('Test array methods', () => { { name: 'Jerry', age: 3, date: new Date(2020, 0, 4) } ]; - expect(toObject(array, i => i.age)['7'].name).toBe('Tom') - expect(toObject(array, 'age')['7'].name).toBe('Tom') + expect(toObject(array, i => i.age)['7'].name).toBe('Tom'); + expect(toObject(array, 'age')['7'].name).toBe('Tom'); - expect(toObject(array, i => i.date)['2020-01-02'].name).toBe('Bob') - expect(toObject(array, 'date')['2020-01-02'].name).toBe('Bob') + expect(toObject(array, i => i.date)['2020-01-02'].name).toBe('Bob'); + expect(toObject(array, 'date')['2020-01-02'].name).toBe('Bob'); }); it('toSeries > ', () => { @@ -447,5 +491,4 @@ describe('Test array methods', () => { expect((toSeries(array) as any)['age'].length).toBe(4); expect(Object.keys(toSeries(array, ['name', 'age'])).length).toBe(2); }); - }); diff --git a/src/tests/data-pipe.spec.ts b/src/tests/data-pipe.spec.ts index 56aaa3e..a616fd4 100644 --- a/src/tests/data-pipe.spec.ts +++ b/src/tests/data-pipe.spec.ts @@ -12,18 +12,26 @@ describe('DataPipe specification', () => { const arr = dataPipe(['US']).toArray(); expect(arr instanceof Array).toBeTruthy(); expect(arr).toEqual(['US']); - }) + }); it('select/map', () => { const arr = [{ country: 'US' }]; - expect(dataPipe(arr).select(i => i.country).toArray()).toEqual(['US']); - expect(dataPipe(arr).map(i => i.country).toArray()).toEqual(['US']); - }) + expect( + dataPipe(arr) + .select(i => i.country) + .toArray() + ).toEqual(['US']); + expect( + dataPipe(arr) + .map(i => i.country) + .toArray() + ).toEqual(['US']); + }); it('groupBy', () => { - const dp = dataPipe(data).groupBy(i => i.country) + const dp = dataPipe(data).groupBy(i => i.country); expect(dp.toArray().length).toBe(3); - }) + }); it('fromTable/toTable', () => { const tData = dataPipe() @@ -31,10 +39,10 @@ describe('DataPipe specification', () => { .filter(r => r.country !== 'US') .toTable(); expect(tData.rows.length).toBe(5); - }) + }); it('unique', () => { const arr = [1, '5', 3, 5, 3, 4, 3, 1]; - expect(dataPipe(arr).unique().toArray().length).toBe(5) + expect(dataPipe(arr).unique().toArray().length).toBe(5); }); -}) +}); diff --git a/src/tests/dsv-parser.spec.ts b/src/tests/dsv-parser.spec.ts index e2b41b5..ebafaa0 100644 --- a/src/tests/dsv-parser.spec.ts +++ b/src/tests/dsv-parser.spec.ts @@ -3,168 +3,168 @@ import { ParsingOptions, DataTypeName } from '../types'; describe('Dsv Parser specification', () => { it('simple numbers', () => { - const csv = ["F1,F2", "1,2"].join('\n') + const csv = ['F1,F2', '1,2'].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(2); - }) + }); it('simple numbers (double)', () => { - const csv = ["F1,F2", "1,2.5"].join('\n') + const csv = ['F1,F2', '1,2.5'].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(2.5); - }) + }); it('simple numbers (double) with thousand', () => { - const csv = ["F1,F2", `1,"2,000.5"`].join('\n') + const csv = ['F1,F2', `1,"2,000.5"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(2000.5); - }) + }); it('simple numbers (double) negative', () => { - const csv = ["F1,F2", `1,-2000.5`].join('\n') + const csv = ['F1,F2', `1,-2000.5`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(-2000.5); - }) + }); it('simple numbers (double) with thousand', () => { - const csv = ["F1,F2", `1,"-2,000.5"`].join('\n') + const csv = ['F1,F2', `1,"-2,000.5"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(1); expect(result[0].F2).toBe(-2000.5); - }) + }); it('simple numders and strings', () => { - const csv = ["F1,F2,F3", `1,2,"Test, comma"`].join('\n') + const csv = ['F1,F2,F3', `1,2,"Test, comma"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F3).toBe('Test, comma'); - }) + }); it('String with quotes 1', () => { - const csv = ["F1,F2", `1,"T ""k"" c"`].join('\n') + const csv = ['F1,F2', `1,"T ""k"" c"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe('T "k" c'); - }) + }); it('String with quotes 2', () => { - const csv = ["F1,F2,F3", `1,"T ""k"" c",77`].join('\n') + const csv = ['F1,F2,F3', `1,"T ""k"" c",77`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe('T "k" c'); expect(result[0].F3).toBe(77); - }) + }); it('String with quotes 3', () => { - const csv = ["F1,F2,F3", `1,"T """,77`].join('\n') + const csv = ['F1,F2,F3', `1,"T """,77`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe('T "'); expect(result[0].F3).toBe(77); - }) + }); it('String with quotes 4', () => { - const csv = ["F1,F2", `1,"T """`].join('\n') + const csv = ['F1,F2', `1,"T """`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe('T "'); - }) + }); it('String with quotes 5 empty " " ', () => { - const csv = ["F1,F2", `1," "`].join('\n') + const csv = ['F1,F2', `1," "`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(' '); - }) + }); it('String with quotes 6 empty " " ', () => { - const csv = ["F1,F2", `1," "`].join('\n') + const csv = ['F1,F2', `1," "`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(' '); - }) + }); it('String with quotes 7 empty "" ', () => { - const csv = ["F1,F2", `1,"dd"`, `1,""`].join('\n') + const csv = ['F1,F2', `1,"dd"`, `1,""`].join('\n'); const result = parseCsv(csv); expect(result[1].F2).toBe(''); - }) + }); it('simple numders and strings with spaces', () => { - const csv = ["F1,F2 ,F3", `1,2,"Test, comma"`].join('\n') + const csv = ['F1,F2 ,F3', `1,2,"Test, comma"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(2); - }) + }); it('simple numders and zeros', () => { - const csv = ["F1,F2,F3", `0,2,0`].join('\n') + const csv = ['F1,F2,F3', `0,2,0`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F1).toBe(0); expect(result[0].F2).toBe(2); expect(result[0].F3).toBe(0); - }) + }); it('Empty should be null', () => { - const csv = ["F1,F2,F3", `1,,"Test, comma"`].join('\n') + const csv = ['F1,F2,F3', `1,,"Test, comma"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(null); - }) + }); it('Multiline string', () => { const multiLineString = `this is , 5 - , multi-line string`; - const csv = ["F1,F2,F3", `1,"${multiLineString}","Test, comma"`].join('\n') + const csv = ['F1,F2,F3', `1,"${multiLineString}","Test, comma"`].join('\n'); const result = parseCsv(csv); expect(result.length).toBe(1); expect(result[0].F2).toBe(multiLineString); - }) + }); it('Multiline string DSV', () => { const multiLineString = `this is , 5 - , multi-line string`; - const csv = ["F1\tF2\tF3", `1\t"${multiLineString}"\t"Test, comma"`].join('\n') + const csv = ['F1\tF2\tF3', `1\t"${multiLineString}"\t"Test, comma"`].join('\n'); const result = parseCsv(csv, { delimiter: '\t' } as ParsingOptions); expect(result.length).toBe(1); expect(result[0].F2).toBe(multiLineString); - }) + }); it('DSV with comma numbers', () => { - const csv = ["F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') + const csv = ['F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); const result = parseCsv(csv, { delimiter: '\t' } as ParsingOptions); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); - }) + }); it('skip rows', () => { - const csv = ["", "", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') + const csv = ['', '', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 } as ParsingOptions); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); - }) + }); it('skip rows not empty rows', () => { - const csv = ["", " * not Empty *", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') + const csv = ['', ' * not Empty *', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); const result = parseCsv(csv, { delimiter: '\t', skipRows: 2 } as ParsingOptions); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); - }) + }); it('skipUntil', () => { - const csv = ["", " * not Empty *", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') + const csv = ['', ' * not Empty *', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`].join('\n'); const options = new ParsingOptions(); options.delimiter = '\t'; options.skipUntil = (t: string[]): boolean => t && t.length > 1; @@ -172,23 +172,23 @@ describe('Dsv Parser specification', () => { const result = parseCsv(csv, options); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); - }) + }); it('empty values', () => { - const csv = ["", "", "\t\t\t", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`, "\t\t"].join('\n') + const csv = ['', '', '\t\t\t', 'F1\tF2\tF3', `1\t1,000.32\t"Test, comma"`, '\t\t'].join('\n'); const options = new ParsingOptions(); options.delimiter = '\t'; const result = parseCsv(csv, options); expect(result.length).toBe(1); expect(result[0].F2).toBe(1000.32); - }) + }); it('Date Fields', () => { - const csv = ["F1\tF2\tF3", `2020-02-11\t1,000.32\t"Test, comma"`].join('\n') + const csv = ['F1\tF2\tF3', `2020-02-11\t1,000.32\t"Test, comma"`].join('\n'); const options = new ParsingOptions(); options.delimiter = '\t'; - options.dateFields = ['F1'] + options.dateFields = ['F1']; const result = parseCsv(csv, options); expect(result.length).toBe(1); @@ -196,33 +196,31 @@ describe('Dsv Parser specification', () => { }); it('Still string. Because second row is not a number or date', () => { - const csv = ["F1\tF2\tF3", `2020-02-11\t1,000.32\t"Test, comma"`, "nn\tnn\tnn"].join('\n') + const csv = ['F1\tF2\tF3', `2020-02-11\t1,000.32\t"Test, comma"`, 'nn\tnn\tnn'].join('\n'); const options = new ParsingOptions(); options.delimiter = '\t'; const result = parseCsv(csv, options); expect(result.length).toBe(2); - expect(typeof result[0].F1 === "string").toBe(true); - expect(typeof result[0].F2 === "string").toBe(true); + expect(typeof result[0].F1 === 'string').toBe(true); + expect(typeof result[0].F2 === 'string').toBe(true); expect(result[0].F2).toBe('1,000.32'); }); it('ToCsv', () => { - const csv = ["F1,F2,F3", `2020-02-11,1,tt`].join('\n') + const csv = ['F1,F2,F3', `2020-02-11,1,tt`].join('\n'); const result = toCsv(parseCsv(csv)); expect(result).toBe(csv); }); - }); -describe ('Parse Csv To Table', () => { - +describe('Parse Csv To Table', () => { it('simple numbers', () => { - const csv = ["F1,F2", "1,2"].join('\n') + const csv = ['F1,F2', '1,2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(1); expect(result.fieldDescriptions.length).toBe(2); - expect(result.fieldDescriptions[0].fieldName).toBe("F1"); + expect(result.fieldDescriptions[0].fieldName).toBe('F1'); expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.WholeNumber); expect(result.fieldDescriptions[0].isUnique).toBe(true); expect(result.fieldDescriptions[0].isNullable).toBe(false); @@ -231,11 +229,11 @@ describe ('Parse Csv To Table', () => { }); it('double and non unique', () => { - const csv = ["F1,F2", "1,2", "1.3,2"].join('\n') + const csv = ['F1,F2', '1,2', '1.3,2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions.length).toBe(2); - expect(result.fieldDescriptions[0].fieldName).toBe("F1"); + expect(result.fieldDescriptions[0].fieldName).toBe('F1'); expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.FloatNumber); expect(result.fieldDescriptions[0].isUnique).toBe(true); expect(result.fieldDescriptions[1].isUnique).toBe(false); @@ -245,43 +243,43 @@ describe ('Parse Csv To Table', () => { }); it('Date parse', () => { - const csv = ["F1,F2", "1,06/02/2020", "1.3,06/07/2020"].join('\n') - const result = parseCsv(csv, {dateFields: [['F2','MM/dd/yyyy']]} as ParsingOptions); + const csv = ['F1,F2', '1,06/02/2020', '1.3,06/07/2020'].join('\n'); + const result = parseCsv(csv, { dateFields: [['F2', 'MM/dd/yyyy']] } as ParsingOptions); expect(result.length).toBe(2); expect(dateToString(result[0].F2 as Date)).toBe('2020-06-02'); expect(dateToString(result[1].F2 as Date)).toBe('2020-06-07'); }); it('Date parse 2', () => { - const csv = ["F1,F2", "1,20200602", "1.3,20200607"].join('\n') - const result = parseCsv(csv, {dateFields: [['F2','yyyyMMdd']]} as ParsingOptions); + const csv = ['F1,F2', '1,20200602', '1.3,20200607'].join('\n'); + const result = parseCsv(csv, { dateFields: [['F2', 'yyyyMMdd']] } as ParsingOptions); expect(result.length).toBe(2); expect(dateToString(result[0].F2 as Date)).toBe('2020-06-02'); expect(dateToString(result[1].F2 as Date)).toBe('2020-06-07'); }); it('Date parse 3', () => { - const csv = ["F1,F2", "1,202006", "1.3,202005"].join('\n') - const result = parseCsv(csv, {dateFields: [['F2','yyyyMM']]} as ParsingOptions); + const csv = ['F1,F2', '1,202006', '1.3,202005'].join('\n'); + const result = parseCsv(csv, { dateFields: [['F2', 'yyyyMM']] } as ParsingOptions); expect(result.length).toBe(2); expect(dateToString(result[0].F2 as Date)).toBe('2020-06-01'); expect(dateToString(result[1].F2 as Date)).toBe('2020-05-01'); }); it('Date parse 4', () => { - const csv = ["F1,F2", "1,06/02/2020", "1.3,06/07/2020"].join('\n') - const result = parseCsv(csv, {dateFields: [['F2','dd/MM/yyyy']]} as ParsingOptions); + const csv = ['F1,F2', '1,06/02/2020', '1.3,06/07/2020'].join('\n'); + const result = parseCsv(csv, { dateFields: [['F2', 'dd/MM/yyyy']] } as ParsingOptions); expect(result.length).toBe(2); expect(dateToString(result[0].F2 as Date)).toBe('2020-02-06'); expect(dateToString(result[1].F2 as Date)).toBe('2020-07-06'); }); it('boolean test', () => { - const csv = ["F1,F2", "true,2", "TRUE,2"].join('\n') + const csv = ['F1,F2', 'true,2', 'TRUE,2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions.length).toBe(2); - expect(result.fieldDescriptions[0].fieldName).toBe("F1"); + expect(result.fieldDescriptions[0].fieldName).toBe('F1'); expect(result.fieldDescriptions[0].dataTypeName).toBe(DataTypeName.Boolean); expect(result.fieldDescriptions[0].isUnique).toBe(true); expect(result.fieldDescriptions[1].isUnique).toBe(false); @@ -292,7 +290,7 @@ describe ('Parse Csv To Table', () => { }); it('nullable boolean test', () => { - const csv = ["F1,F2", "true,2", ",2"].join('\n') + const csv = ['F1,F2', 'true,2', ',2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions[0].isNullable).toBe(true); @@ -301,7 +299,7 @@ describe ('Parse Csv To Table', () => { }); it('nullable date test', () => { - const csv = ["F1,F2", "2020-02-02,2", ",2"].join('\n') + const csv = ['F1,F2', '2020-02-02,2', ',2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions[0].isNullable).toBe(true); @@ -312,7 +310,7 @@ describe ('Parse Csv To Table', () => { }); it('nullable date test 2', () => { - const csv = ["F1,F2", ",2", "2020-02-02,2"].join('\n') + const csv = ['F1,F2', ',2', '2020-02-02,2'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions[0].isNullable).toBe(true); @@ -320,10 +318,10 @@ describe ('Parse Csv To Table', () => { expect(result.rows[1][result.fieldDescriptions[0].index]).toBe('2020-02-02'); expect(result.rows[0][result.fieldDescriptions[0].index]).toBe(null); - }); + }); it('nullable float test', () => { - const csv = ["F1,F2", ",2", ",-2020.98"].join('\n') + const csv = ['F1,F2', ',2', ',-2020.98'].join('\n'); const result = parseCsvToTable(csv); expect(result.rows.length).toBe(2); expect(result.fieldDescriptions[0].isNullable).toBe(true); @@ -332,24 +330,29 @@ describe ('Parse Csv To Table', () => { expect(result.rows[1][result.fieldDescriptions[1].index]).toBe('-2020.98'); expect(result.rows[0][result.fieldDescriptions[0].index]).toBe(null); - }); + }); it('header smoothing', () => { - const csv = ["F 1,F 2", "11,12", "21,22"].join('\n') + const csv = ['F 1,F 2', '11,12', '21,22'].join('\n'); const result = parseCsvToTable(csv); expect(result.fieldNames.length).toBe(2); expect(result.fieldNames[0]).toBe('F_1'); expect(result.fieldNames[1]).toBe('F_2'); - }); + }); it('toCsv/parse', () => { - expect(toCsv(parseCsv("F 1,F 2\n11,12\n21,22"))).toBe("F_1,F_2\n11,12\n21,22"); - expect(toCsv(parseCsv("F 1,F 2\n11,12\n21,22", {keepOriginalHeaders: true} as ParsingOptions))).toBe("F 1,F 2\n11,12\n21,22"); - }); + expect(toCsv(parseCsv('F 1,F 2\n11,12\n21,22'))).toBe('F_1,F_2\n11,12\n21,22'); + expect( + toCsv(parseCsv('F 1,F 2\n11,12\n21,22', { keepOriginalHeaders: true } as ParsingOptions)) + ).toBe('F 1,F 2\n11,12\n21,22'); + }); it('toCsv', () => { - const obj = [{f1: 1, f2: "test"}, {f1: 2, f2: "test\""}, {f1: 3, f2: "te,st"} ]; + const obj = [ + { f1: 1, f2: 'test' }, + { f1: 2, f2: 'test"' }, + { f1: 3, f2: 'te,st' } + ]; expect(toCsv(obj)).toBe(`f1,f2\n1,test\n2,"test"""\n3,"te,st"`); - }); - + }); }); diff --git a/src/tests/string-utils.spec.ts b/src/tests/string-utils.spec.ts index 8c606fb..48b9ee6 100644 --- a/src/tests/string-utils.spec.ts +++ b/src/tests/string-utils.spec.ts @@ -1,59 +1,59 @@ -import { trimStart, trimEnd, split } from "../string"; +import { trimStart, trimEnd, split } from '../string'; -describe("string utils", () => { - it("ltrim", () => { - expect(trimStart(".net", ".")).toBe("net"); - expect(trimStart("...net", ".")).toBe("net"); - expect(trimStart(".+net", ".+")).toBe("net"); +describe('string utils', () => { + it('ltrim', () => { + expect(trimStart('.net', '.')).toBe('net'); + expect(trimStart('...net', '.')).toBe('net'); + expect(trimStart('.+net', '.+')).toBe('net'); }); - it("rtrim", () => { - expect(trimEnd("net.", ".")).toBe("net"); - expect(trimEnd("net..", ".")).toBe("net"); - expect(trimEnd("net++", ".+")).toBe("net"); + it('rtrim', () => { + expect(trimEnd('net.', '.')).toBe('net'); + expect(trimEnd('net..', '.')).toBe('net'); + expect(trimEnd('net++', '.+')).toBe('net'); }); }); -describe("string split", () => { - it("split => simple", () => { - const t = split("a,b", ","); +describe('string split', () => { + it('split => simple', () => { + const t = split('a,b', ','); expect(t.length).toBe(2); - expect(t[0]).toBe("a"); - expect(t[1]).toBe("b"); + expect(t[0]).toBe('a'); + expect(t[1]).toBe('b'); }); - it("split => simple2", () => { - const t = split("a2,b2", ","); + it('split => simple2', () => { + const t = split('a2,b2', ','); expect(t.length).toBe(2); - expect(t[0]).toBe("a2"); - expect(t[1]).toBe("b2"); + expect(t[0]).toBe('a2'); + expect(t[1]).toBe('b2'); }); - it("split => with brackets 1", () => { - const t = split("a(s,d)t,b2", ",", ["()"]); + it('split => with brackets 1', () => { + const t = split('a(s,d)t,b2', ',', ['()']); expect(t.length).toBe(2); - expect(t[0]).toBe("a(s,d)t"); - expect(t[1]).toBe("b2"); + expect(t[0]).toBe('a(s,d)t'); + expect(t[1]).toBe('b2'); }); - it("split => with brackets 2", () => { - const t = split("a(s,d),b2", ",", ["()"]); + it('split => with brackets 2', () => { + const t = split('a(s,d),b2', ',', ['()']); expect(t.length).toBe(2); - expect(t[0]).toBe("a(s,d)"); - expect(t[1]).toBe("b2"); + expect(t[0]).toBe('a(s,d)'); + expect(t[1]).toBe('b2'); }); - it("split => with brackets 3", () => { - const t = split("a(s,ff(e,r)d),b2", ",", ["()"]); + it('split => with brackets 3', () => { + const t = split('a(s,ff(e,r)d),b2', ',', ['()']); expect(t.length).toBe(2); - expect(t[0]).toBe("a(s,ff(e,r)d)"); - expect(t[1]).toBe("b2"); + expect(t[0]).toBe('a(s,ff(e,r)d)'); + expect(t[1]).toBe('b2'); }); - it("split => with brackets 4", () => { - const t = split("a(s,ff(e,r)d),b2ff(e,r)", ",", ["()"]); + it('split => with brackets 4', () => { + const t = split('a(s,ff(e,r)d),b2ff(e,r)', ',', ['()']); expect(t.length).toBe(2); - expect(t[0]).toBe("a(s,ff(e,r)d)"); - expect(t[1]).toBe("b2ff(e,r)"); + expect(t[0]).toBe('a(s,ff(e,r)d)'); + expect(t[1]).toBe('b2ff(e,r)'); }); }); diff --git a/src/tests/table.spec.ts b/src/tests/table.spec.ts index 070c664..f161a6e 100644 --- a/src/tests/table.spec.ts +++ b/src/tests/table.spec.ts @@ -1,21 +1,20 @@ - -import { data } from "./array.spec"; -import { fromTable, toTable } from "../utils"; -import { ScalarObject } from "../types"; +import { data } from './array.spec'; +import { fromTable, toTable } from '../utils'; +import { ScalarObject } from '../types'; export const table = { fields: ['name', 'country'], rows: [ - [ 'John', 'US' ], - [ 'Joe', 'US' ], - [ 'Bill', 'US' ], - [ 'Adam', 'UK' ], - [ 'Scott', 'UK' ], - [ 'Diana', 'UK' ], - [ 'Marry', 'FR' ], - [ 'Luc', 'FR' ] + ['John', 'US'], + ['Joe', 'US'], + ['Bill', 'US'], + ['Adam', 'UK'], + ['Scott', 'UK'], + ['Diana', 'UK'], + ['Marry', 'FR'], + ['Luc', 'FR'] ] -} +}; describe('Test table methods', () => { it('fromTable', () => { @@ -23,7 +22,7 @@ describe('Test table methods', () => { expect(list[0]).toHaveProperty('name'); expect(list[0]).toHaveProperty('country'); expect(list.length).toBe(table.rows.length); - }) + }); it('toTable', () => { const tableData = toTable(data as ScalarObject[]); @@ -32,5 +31,5 @@ describe('Test table methods', () => { expect(tableData.rows.length).toBe(data.length); expect(tableData.fieldNames[0]).toBe('name'); expect(tableData.fieldNames[1]).toBe('country'); - }) + }); }); diff --git a/src/tests/utils-pipe.spec.ts b/src/tests/utils-pipe.spec.ts index cd1e970..7ec5e3b 100644 --- a/src/tests/utils-pipe.spec.ts +++ b/src/tests/utils-pipe.spec.ts @@ -1,6 +1,11 @@ -import { parseDatetimeOrNull, parseNumberOrNull, getFieldsInfo, dateToString, addBusinessDays } from "../utils"; -import { FieldDescription, DataTypeName } from "../types"; - +import { + parseDatetimeOrNull, + parseNumberOrNull, + getFieldsInfo, + dateToString, + addBusinessDays +} from '../utils'; +import { FieldDescription, DataTypeName } from '../types'; describe('Test dataUtils', () => { it('parseDate', () => { @@ -25,24 +30,37 @@ describe('Test dataUtils', () => { expect(dateToString(parseDatetimeOrNull(strDate) as Date)).toBe(strDate); }); + it('parseDateTime with larger miliseconds', () => { + const dt = parseDatetimeOrNull('2020-06-08T13:49:15.16789'); + expect(dt).toBeInstanceOf(Date); + expect(dateToString(dt as Date)).toBe('2020-06-08T13:49:15.167Z'); + const strDate = '2020-02-21T13:49:15.167Z'; + expect(dateToString(parseDatetimeOrNull(strDate) as Date)).toBe(strDate); + }); + it('parseDateTime with format', () => { const dt = parseDatetimeOrNull('20200608', 'yyyyMMdd'); expect(dt).toBeInstanceOf(Date); - expect(dateToString(dt as Date)).toBe('2020-06-08'); + expect(dateToString(dt as Date)).toBe('2020-06-08'); expect(dateToString(parseDatetimeOrNull('202006', 'yyyyMM') as Date)).toBe('2020-06-01'); - expect(dateToString(parseDatetimeOrNull('06/02/2020', 'MM/dd/yyyy') as Date)).toBe('2020-06-02'); - expect(dateToString(parseDatetimeOrNull('06/02/2020', 'dd/MM/yyyy') as Date)).toBe('2020-02-06'); - expect(dateToString(parseDatetimeOrNull('2020-06-02', 'yyyy-mm-dd') as Date)).toBe('2020-06-02'); + expect(dateToString(parseDatetimeOrNull('06/02/2020', 'MM/dd/yyyy') as Date)).toBe( + '2020-06-02' + ); + expect(dateToString(parseDatetimeOrNull('06/02/2020', 'dd/MM/yyyy') as Date)).toBe( + '2020-02-06' + ); + expect(dateToString(parseDatetimeOrNull('2020-06-02', 'yyyy-mm-dd') as Date)).toBe( + '2020-06-02' + ); }); it('last business date', () => { const dt = parseDatetimeOrNull('20210111', 'yyyyMMdd'); expect(dt).toBeInstanceOf(Date); - expect(dateToString(dt as Date, "yyyyMMdd")).toBe('20210111'); - expect(dateToString(addBusinessDays(dt as Date, -1), "yyyyMMdd")).toBe('20210108'); + expect(dateToString(dt as Date, 'yyyyMMdd')).toBe('20210111'); + expect(dateToString(addBusinessDays(dt as Date, -1), 'yyyyMMdd')).toBe('20210108'); }); - it('parseNumber', () => { expect(parseNumberOrNull('')).toBe(null); expect(parseNumberOrNull('11')).toBe(11); @@ -50,7 +68,7 @@ describe('Test dataUtils', () => { expect(parseNumberOrNull('-11.1')).toBe(-11.1); expect(parseNumberOrNull(11.1)).toBe(11.1); expect(parseNumberOrNull(NaN)).toBe(NaN); - }) + }); it('getFieldsInfo', () => { const arr = [2, 4, 5].map(r => ({ val1: r })); @@ -101,8 +119,12 @@ describe('Test dataUtils', () => { expect(fdFn(['2019-01-01', 'NOT A DATE', '2019-01-02']).dataTypeName).toBe(DataTypeName.String); expect(fdFn(['2019-01-01', 76, '2019-01-02']).dataTypeName).toBe(DataTypeName.String); expect(fdFn(['2019-01-01', 76, false, '2019-01-02']).dataTypeName).toBe(DataTypeName.String); - expect(fdFn(['2019-01-01', 76, false, null, '2019-01-02']).dataTypeName).toBe(DataTypeName.String); - expect(fdFn([new Date(2001, 1, 1), 'NOT A DATE', new Date()]).dataTypeName).toBe(DataTypeName.String); + expect(fdFn(['2019-01-01', 76, false, null, '2019-01-02']).dataTypeName).toBe( + DataTypeName.String + ); + expect(fdFn([new Date(2001, 1, 1), 'NOT A DATE', new Date()]).dataTypeName).toBe( + DataTypeName.String + ); }); it('getFieldsInfo size check', () => { @@ -118,7 +140,7 @@ describe('Test dataUtils', () => { }); it('check value types array', () => { - const fields = getFieldsInfo([1,2,3]) + const fields = getFieldsInfo([1, 2, 3]); expect(fields.length).toBe(1); expect(fields[0].dataTypeName).toBe(DataTypeName.WholeNumber); @@ -129,9 +151,5 @@ describe('Test dataUtils', () => { expect(getFieldsInfo(['2021-06-02', '2021-06-02']).length).toBe(1); expect(getFieldsInfo(['2021-06-02', '2021-06-02'])[0].dataTypeName).toBe(DataTypeName.Date); - }); - - -}) - +}); diff --git a/src/types.ts b/src/types.ts index d7179ba..36b72d2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,7 +53,6 @@ export interface Table { rows: T[][]; } - /** * A simple data table structure what provides a most efficient way * to send data across the wire diff --git a/src/utils/dsv-parser.ts b/src/utils/dsv-parser.ts index ad5bd86..baafcdc 100644 --- a/src/utils/dsv-parser.ts +++ b/src/utils/dsv-parser.ts @@ -1,291 +1,320 @@ -import { parseNumberOrNull, parseDatetimeOrNull, workoutDataType, parseBooleanOrNull, dateToString } from "./helpers"; -import { ParsingOptions, ScalarType, ScalarObject, StringsDataTable, FieldDescription, DataTypeName } from "../types"; +import { + parseNumberOrNull, + parseDatetimeOrNull, + workoutDataType, + parseBooleanOrNull, + dateToString +} from './helpers'; +import { + ParsingOptions, + ScalarType, + ScalarObject, + StringsDataTable, + FieldDescription, + DataTypeName +} from '../types'; type ParsingContext = { - content: string; - currentIndex: number; + content: string; + currentIndex: number; }; const EmptySymbol = '_#EMPTY#_'; -function getObjectElement(fieldDescs: FieldDescription[], tokens: string[], options: ParsingOptions): ScalarObject { - const obj = Object.create(null); - for (let i = 0; i < fieldDescs.length; i++) { - const fieldDesc = fieldDescs[i]; - const fieldName = fieldDesc.fieldName; - let value: ScalarType = tokens[i]; - const dateFields = options?.dateFields?.map(r => Array.isArray(r) ? r[0] : r) || []; - - if (options.textFields && options.textFields?.indexOf(fieldName) >= 0) { - value = tokens[i]; - } else if ((fieldDesc.dataTypeName === DataTypeName.DateTime || fieldDesc.dataTypeName === DataTypeName.Date) - || (dateFields.indexOf(fieldName) >= 0)) { - - const ind = dateFields.indexOf(fieldName) - const dtField = ind >= 0 ? options?.dateFields[ind] : []; - const format = Array.isArray(dtField) ? dtField[1] : null; - value = parseDatetimeOrNull(value as string, format || null); - } else if (fieldDesc.dataTypeName === DataTypeName.WholeNumber - || fieldDesc.dataTypeName === DataTypeName.FloatNumber - || fieldDesc.dataTypeName === DataTypeName.BigIntNumber - || (options.numberFields && options.numberFields.indexOf(fieldName) >= 0)) { - value = parseNumberOrNull(value as string); - } else if (fieldDesc.dataTypeName === DataTypeName.Boolean - || (options.booleanFields && options.booleanFields.indexOf(fieldName) >= 0)) { - value = parseBooleanOrNull(value as string); - } - - obj[fieldName] = value === EmptySymbol ? '' : value; +function getObjectElement( + fieldDescs: FieldDescription[], + tokens: string[], + options: ParsingOptions +): ScalarObject { + const obj = Object.create(null); + for (let i = 0; i < fieldDescs.length; i++) { + const fieldDesc = fieldDescs[i]; + const fieldName = fieldDesc.fieldName; + let value: ScalarType = tokens[i]; + const dateFields = options?.dateFields?.map(r => (Array.isArray(r) ? r[0] : r)) || []; + + if (options.textFields && options.textFields?.indexOf(fieldName) >= 0) { + value = tokens[i]; + } else if ( + fieldDesc.dataTypeName === DataTypeName.DateTime || + fieldDesc.dataTypeName === DataTypeName.Date || + dateFields.indexOf(fieldName) >= 0 + ) { + const ind = dateFields.indexOf(fieldName); + const dtField = ind >= 0 ? options?.dateFields[ind] : []; + const format = Array.isArray(dtField) ? dtField[1] : null; + value = parseDatetimeOrNull(value as string, format || null); + } else if ( + fieldDesc.dataTypeName === DataTypeName.WholeNumber || + fieldDesc.dataTypeName === DataTypeName.FloatNumber || + fieldDesc.dataTypeName === DataTypeName.BigIntNumber || + (options.numberFields && options.numberFields.indexOf(fieldName) >= 0) + ) { + value = parseNumberOrNull(value as string); + } else if ( + fieldDesc.dataTypeName === DataTypeName.Boolean || + (options.booleanFields && options.booleanFields.indexOf(fieldName) >= 0) + ) { + value = parseBooleanOrNull(value as string); } - return obj; + + obj[fieldName] = value === EmptySymbol ? '' : value; + } + return obj; } function nextLineTokens(context: ParsingContext, delimiter = ','): string[] { - const tokens: string[] = []; - let token = ''; + const tokens: string[] = []; + let token = ''; - function elementAtOrNull(arr: string, index: number): string | null { - return (arr.length > index) ? arr[index] : null; - } + function elementAtOrNull(arr: string, index: number): string | null { + return arr.length > index ? arr[index] : null; + } - do { - const currentChar = context.content[context.currentIndex]; - if (currentChar === '\r') { - continue; - } + do { + const currentChar = context.content[context.currentIndex]; + if (currentChar === '\r') { + continue; + } - if (currentChar === '\n') { - if (context.content[context.currentIndex + 1] === '\r') { context.currentIndex++; } - break; - } + if (currentChar === '\n') { + if (context.content[context.currentIndex + 1] === '\r') { + context.currentIndex++; + } + break; + } - if (token.length === 0 && currentChar === '"') { - - if (elementAtOrNull(context.content, context.currentIndex + 1) === '"' - && elementAtOrNull(context.content, context.currentIndex + 2) !== '"') { - // just empty string - token = EmptySymbol; - context.currentIndex++; - } - else { - // enumerate till the end of quote - while (context.content[++context.currentIndex] !== '"') { - token += context.content[context.currentIndex]; - - // check if we need to escape "" - if (elementAtOrNull(context.content, context.currentIndex + 1) === '"' - && elementAtOrNull(context.content, context.currentIndex + 2) === '"') { - token += '"'; - context.currentIndex += 2 - } - } - } - - } else if (currentChar === delimiter) { - tokens.push(token); - token = ''; - } else { - token += currentChar; + if (token.length === 0 && currentChar === '"') { + if ( + elementAtOrNull(context.content, context.currentIndex + 1) === '"' && + elementAtOrNull(context.content, context.currentIndex + 2) !== '"' + ) { + // just empty string + token = EmptySymbol; + context.currentIndex++; + } else { + // enumerate till the end of quote + while (context.content[++context.currentIndex] !== '"') { + token += context.content[context.currentIndex]; + + // check if we need to escape "" + if ( + elementAtOrNull(context.content, context.currentIndex + 1) === '"' && + elementAtOrNull(context.content, context.currentIndex + 2) === '"' + ) { + token += '"'; + context.currentIndex += 2; + } } + } + } else if (currentChar === delimiter) { + tokens.push(token); + token = ''; + } else { + token += currentChar; } - while (++context.currentIndex < context.content.length) + } while (++context.currentIndex < context.content.length); - tokens.push(token); - return tokens; + tokens.push(token); + return tokens; } function parseLineTokens(content: string, options: ParsingOptions): StringsDataTable { - const ctx = { - content: content, - currentIndex: 0 - } as ParsingContext; - content = content || ''; - const delimiter = options.delimiter || ','; - - const result = { - fieldDescriptions: [] as FieldDescription[], - rows: [] as ScalarType[][] - } as StringsDataTable; - let lineNumber = 0; - let fieldNames: string[] | null = null; - const uniqueValues: string[][] = []; - - do { - const rowTokens = nextLineTokens(ctx, delimiter); - - // skip if all tokens are empty - if (rowTokens.filter(f => !f || !f.length).length === rowTokens.length) { - lineNumber++; - continue; - } + const ctx = { + content: content, + currentIndex: 0 + } as ParsingContext; + content = content || ''; + const delimiter = options.delimiter || ','; + + const result = { + fieldDescriptions: [] as FieldDescription[], + rows: [] as ScalarType[][] + } as StringsDataTable; + let lineNumber = 0; + let fieldNames: string[] | null = null; + const uniqueValues: string[][] = []; + + do { + const rowTokens = nextLineTokens(ctx, delimiter); + + // skip if all tokens are empty + if (rowTokens.filter(f => !f || !f.length).length === rowTokens.length) { + lineNumber++; + continue; + } - // skip rows based skipRows value - if (lineNumber < options.skipRows) { - lineNumber++; - continue; - } + // skip rows based skipRows value + if (lineNumber < options.skipRows) { + lineNumber++; + continue; + } - // skip rows based on skipUntil call back - if (!fieldNames && typeof options.skipUntil === "function" && !options.skipUntil(rowTokens)) { - lineNumber++; - continue; - } + // skip rows based on skipUntil call back + if (!fieldNames && typeof options.skipUntil === 'function' && !options.skipUntil(rowTokens)) { + lineNumber++; + continue; + } - if (!fieldNames) { - // fieldName is used as indicator on whether data rows handling started - fieldNames = []; - const fieldDescriptions = []; + if (!fieldNames) { + // fieldName is used as indicator on whether data rows handling started + fieldNames = []; + const fieldDescriptions = []; - for (let i = 0; i < rowTokens.length; i++) { - // if empty then _ - let token = rowTokens[i].trim().length ? rowTokens[i].trim() : '_'; + for (let i = 0; i < rowTokens.length; i++) { + // if empty then _ + let token = rowTokens[i].trim().length ? rowTokens[i].trim() : '_'; - if (!options.keepOriginalHeaders) { - token = token.replace(/\W/g, '_'); - } + if (!options.keepOriginalHeaders) { + token = token.replace(/\W/g, '_'); + } - // just to ensure no dublicated field names - fieldNames.push(fieldNames.indexOf(token) >= 0 ? `${token}_${i}` : token) + // just to ensure no dublicated field names + fieldNames.push(fieldNames.indexOf(token) >= 0 ? `${token}_${i}` : token); - fieldDescriptions.push({ - fieldName: fieldNames[fieldNames.length - 1], - isNullable: false, - isUnique: true, - index: i - } as FieldDescription) + fieldDescriptions.push({ + fieldName: fieldNames[fieldNames.length - 1], + isNullable: false, + isUnique: true, + index: i + } as FieldDescription); - uniqueValues.push([]); - } + uniqueValues.push([]); + } - result.fieldDescriptions = fieldDescriptions; - result.fieldNames = fieldNames; + result.fieldDescriptions = fieldDescriptions; + result.fieldNames = fieldNames; - lineNumber++; - continue; - } + lineNumber++; + continue; + } - if (typeof options.takeWhile === "function" && fieldNames && !options.takeWhile(rowTokens)) { - break; + if (typeof options.takeWhile === 'function' && fieldNames && !options.takeWhile(rowTokens)) { + break; + } + const rowValues: string[] = []; + + // analyze each cell in a row + for (let i = 0; i < Math.min(rowTokens.length, result.fieldDescriptions.length); i++) { + const fDesc = result.fieldDescriptions[i]; + let value: string | null = rowTokens[i]; + + if (value === null || value === undefined || value.length === 0) { + fDesc.isNullable = true; + } else if (value !== EmptySymbol) { + const newType = workoutDataType(value, fDesc.dataTypeName); + if (newType !== fDesc.dataTypeName) { + fDesc.dataTypeName = newType; } - const rowValues: string[] = []; - - // analyze each cell in a row - for (let i = 0; i < Math.min(rowTokens.length, result.fieldDescriptions.length); i++) { - const fDesc = result.fieldDescriptions[i]; - let value: string | null = rowTokens[i]; - - if (value === null || value === undefined || value.length === 0) { - fDesc.isNullable = true - } else if (value !== EmptySymbol) { - const newType = workoutDataType(value, fDesc.dataTypeName); - if (newType !== fDesc.dataTypeName) { - fDesc.dataTypeName = newType; - } - - if ( - (fDesc.dataTypeName == DataTypeName.String || fDesc.dataTypeName == DataTypeName.LargeString) - && String(value).length > (fDesc.maxSize || 0) - ) { - fDesc.maxSize = String(value).length; - } - } - - if (fDesc.isUnique) { - if (uniqueValues[i].indexOf(value) >= 0) { - fDesc.isUnique = false; - } else { - uniqueValues[i].push(value); - } - } - - if (value === EmptySymbol) { - value = (fDesc.dataTypeName === DataTypeName.String || fDesc.dataTypeName === DataTypeName.LargeString) ? - '' : null - } else if (!value.length) { - value = null; - } - rowValues.push(value as string); + + if ( + (fDesc.dataTypeName == DataTypeName.String || + fDesc.dataTypeName == DataTypeName.LargeString) && + String(value).length > (fDesc.maxSize || 0) + ) { + fDesc.maxSize = String(value).length; } + } - // no need for null or empty objects - result.rows.push(rowValues); - lineNumber++; + if (fDesc.isUnique) { + if (uniqueValues[i].indexOf(value) >= 0) { + fDesc.isUnique = false; + } else { + uniqueValues[i].push(value); + } + } + + if (value === EmptySymbol) { + value = + fDesc.dataTypeName === DataTypeName.String || + fDesc.dataTypeName === DataTypeName.LargeString + ? '' + : null; + } else if (!value.length) { + value = null; + } + rowValues.push(value as string); } - while (++ctx.currentIndex < ctx.content.length) - result.fieldDataTypes = result.fieldDescriptions.map(f => f.dataTypeName as DataTypeName); - return result; + // no need for null or empty objects + result.rows.push(rowValues); + lineNumber++; + } while (++ctx.currentIndex < ctx.content.length); + + result.fieldDataTypes = result.fieldDescriptions.map(f => f.dataTypeName as DataTypeName); + return result; } export function parseCsv(content: string, options?: ParsingOptions): ScalarObject[] { - content = content || ''; - options = Object.assign(new ParsingOptions(), options || {}); - if (!content.length) { - return []; + content = content || ''; + options = Object.assign(new ParsingOptions(), options || {}); + if (!content.length) { + return []; + } + + const table = parseLineTokens(content, options || new ParsingOptions()); + + const result: ScalarObject[] = []; + for (let i = 0; i < table.rows.length; i++) { + const obj = + typeof options.elementSelector === 'function' + ? options.elementSelector(table.fieldDescriptions, table.rows[i] as string[]) + : getObjectElement(table.fieldDescriptions, table.rows[i] as string[], options); + + if (obj) { + // no need for null or empty objects + result.push(obj); } + } - const table = parseLineTokens(content, options || new ParsingOptions()); - - const result: ScalarObject[] = []; - for (let i = 0; i < table.rows.length; i++) { - const obj = (typeof options.elementSelector === "function") ? - options.elementSelector(table.fieldDescriptions, table.rows[i] as string[]) - : getObjectElement(table.fieldDescriptions, table.rows[i] as string[], options) - - if (obj) { - // no need for null or empty objects - result.push(obj); - } - - } - - return result; + return result; } export function parseCsvToTable(content: string, options?: ParsingOptions): StringsDataTable { - content = content || ''; + content = content || ''; - if (!content.length) { - return {} as StringsDataTable; - } + if (!content.length) { + return {} as StringsDataTable; + } - return parseLineTokens(content, options || new ParsingOptions()); + return parseLineTokens(content, options || new ParsingOptions()); } export function toCsv(array: ScalarObject[], delimiter = ','): string { - array = array || []; + array = array || []; - const headers: string[] = []; + const headers: string[] = []; - // workout all headers - for (const item of array) { - for (const name in item) { - if (headers.indexOf(name) < 0) { headers.push(name); } - } + // workout all headers + for (const item of array) { + for (const name in item) { + if (headers.indexOf(name) < 0) { + headers.push(name); + } } + } + + // create a csv string + const lines = array.map(item => { + const values: string[] = []; + for (const name of headers) { + let value: ScalarType = item[name]; + if (value instanceof Date) { + value = dateToString(value); + } else if ( + typeof value === 'string' && + (value.indexOf(delimiter) >= 0 || value.indexOf('"') >= 0) + ) { + // excel style csv + value = value.replace(new RegExp('"', 'g'), '""'); + value = `"${value}"`; + } + value = value !== null && value !== undefined ? value : ''; + values.push(String(value)); + } + return values.join(delimiter); + }); + lines.unshift(headers.join(delimiter)); - // create a csv string - const lines = array.map(item => { - const values: string[] = []; - for (const name of headers) { - let value: ScalarType = item[name]; - if (value instanceof Date) { - value = dateToString(value); - } else if (typeof value === "string" && ( - value.indexOf(delimiter) >= 0 - || value.indexOf('"') >= 0 - )) { - // excel style csv - value = value.replace(new RegExp('"', 'g'), '""'); - value = `"${value}"`; - } - value = (value !== null && value !== undefined) ? value : ''; - values.push(String(value)) - } - return values.join(delimiter); - - }); - lines.unshift(headers.join(delimiter)) - - return lines.join('\n') + return lines.join('\n'); } diff --git a/src/utils/helpers.ts b/src/utils/helpers.ts index 0d50d2c..f8b3f72 100644 --- a/src/utils/helpers.ts +++ b/src/utils/helpers.ts @@ -1,4 +1,4 @@ -import { Selector, FieldDescription, DataTypeName, ScalarType, PrimitiveType } from "../types"; +import { Selector, FieldDescription, DataTypeName, ScalarType, PrimitiveType } from '../types'; /** * Formats selected value to number. @@ -21,8 +21,10 @@ export function parseNumber(val: ScalarType, elementSelector?: Selector): number } break; } - case 'boolean': return Number(val); - case 'number': return isNaN(val) ? undefined : val; + case 'boolean': + return Number(val); + case 'number': + return isNaN(val) ? undefined : val; } } @@ -42,8 +44,7 @@ export function parseNumberOrNull(value: string | number): number | null { const d = value.charCodeAt(i); if (d < 48 || d > 57) { // '.' - 46 ',' - 44 '-' - 45(but only first char) - if (d !== 46 && d !== 44 && (d !== 45 || i !== 0)) - return null; + if (d !== 46 && d !== 44 && (d !== 45 || i !== 0)) return null; } } @@ -55,17 +56,26 @@ export function parseNumberOrNull(value: string | number): number | null { * More wider datetime parser * @param value */ -export function parseDatetimeOrNull(value: string | Date, format: string | null = null): Date | null { +export function parseDatetimeOrNull( + value: string | Date, + format: string | null = null +): Date | null { format = (format || '').toLowerCase(); - if (!value) { return null; } - if (value instanceof Date && !isNaN(value.valueOf())) { return value; } + if (!value) { + return null; + } + if (value instanceof Date && !isNaN(value.valueOf())) { + return value; + } // only string values can be converted to Date - if (typeof value !== 'string') { return null; } - - + if (typeof value !== 'string') { + return null; + } const strValue = String(value); - if (!strValue.length) { return null; } + if (!strValue.length) { + return null; + } const parseMonth = (mm: string): number => { if (!mm || !mm.length) { @@ -78,18 +88,42 @@ export function parseDatetimeOrNull(value: string | Date, format: string | null } // make sure english months are coming through - if (mm.startsWith('jan')) { return 0; } - if (mm.startsWith('feb')) { return 1; } - if (mm.startsWith('mar')) { return 2; } - if (mm.startsWith('apr')) { return 3; } - if (mm.startsWith('may')) { return 4; } - if (mm.startsWith('jun')) { return 5; } - if (mm.startsWith('jul')) { return 6; } - if (mm.startsWith('aug')) { return 7; } - if (mm.startsWith('sep')) { return 8; } - if (mm.startsWith('oct')) { return 9; } - if (mm.startsWith('nov')) { return 10; } - if (mm.startsWith('dec')) { return 11; } + if (mm.startsWith('jan')) { + return 0; + } + if (mm.startsWith('feb')) { + return 1; + } + if (mm.startsWith('mar')) { + return 2; + } + if (mm.startsWith('apr')) { + return 3; + } + if (mm.startsWith('may')) { + return 4; + } + if (mm.startsWith('jun')) { + return 5; + } + if (mm.startsWith('jul')) { + return 6; + } + if (mm.startsWith('aug')) { + return 7; + } + if (mm.startsWith('sep')) { + return 8; + } + if (mm.startsWith('oct')) { + return 9; + } + if (mm.startsWith('nov')) { + return 10; + } + if (mm.startsWith('dec')) { + return 11; + } return NaN; }; @@ -102,55 +136,131 @@ export function parseDatetimeOrNull(value: string | Date, format: string | null } }; - const validDateOrNull = - (yyyy: number, month: number, day: number, hours: number, mins: number, ss: number, ms: number): Date | null => { - if (month > 11 || day > 31 || hours >= 60 || mins >= 60 || ss >= 60) { return null; } + const validDateOrNull = ( + yyyy: number, + month: number, + day: number, + hours: number, + mins: number, + ss: number, + ms: number + ): Date | null => { + if (month > 11 || day > 31 || hours >= 60 || mins >= 60 || ss >= 60) { + return null; + } - const dd = new Date(yyyy, month, day, hours, mins, ss, ms); - return !isNaN(dd.valueOf()) ? dd : null; - }; + if (ms > 1000) { + ms = parseInt(String(ms).substring(0, 3)); + } + const dd = new Date(yyyy, month, day, hours, mins, ss, ms); + return !isNaN(dd.valueOf()) ? dd : null; + }; - const strTokens = strValue.replace('T', ' ').replace('.', ' ').toLowerCase().split(/[: /-]/); + const strTokens = strValue + .replace('T', ' ') + .replace('.', ' ') + .toLowerCase() + .split(/[: /-]/); const dt = strTokens.map(parseFloat); let d: Date | null = null; - if (format.startsWith('mm/dd/yy') || format.startsWith('mmm/dd/yy') - || format.startsWith('mm-dd-yy') || format.startsWith('mmm-dd-yy')) { + if ( + format.startsWith('mm/dd/yy') || + format.startsWith('mmm/dd/yy') || + format.startsWith('mm-dd-yy') || + format.startsWith('mmm-dd-yy') + ) { // handle US format - return validDateOrNull(correctYear(dt[2]), parseMonth(strTokens[0]), dt[1], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); + return validDateOrNull( + correctYear(dt[2]), + parseMonth(strTokens[0]), + dt[1], + dt[3] || 0, + dt[4] || 0, + dt[5] || 0, + dt[6] || 0 + ); } else if (format.startsWith('yyyymm')) { return validDateOrNull( parseInt(value.substring(0, 4)), parseInt(value.substring(4, 6)) - 1, - (value.length > 6) ? parseInt(value.substring(6, 8)) : 1, 0, 0, 0, 0) - } else if (format.startsWith('dd/mm/yy') || format.startsWith('dd/mmm/yy') - || format.startsWith('dd-mm-yy') || format.startsWith('dd-mmm-yy')) { - return validDateOrNull(correctYear(dt[2]), parseMonth(strTokens[1]), dt[0], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); + value.length > 6 ? parseInt(value.substring(6, 8)) : 1, + 0, + 0, + 0, + 0 + ); + } else if ( + format.startsWith('dd/mm/yy') || + format.startsWith('dd/mmm/yy') || + format.startsWith('dd-mm-yy') || + format.startsWith('dd-mmm-yy') + ) { + return validDateOrNull( + correctYear(dt[2]), + parseMonth(strTokens[1]), + dt[0], + dt[3] || 0, + dt[4] || 0, + dt[5] || 0, + dt[6] || 0 + ); } else if (format.startsWith('yyyy-mm')) { - return validDateOrNull(dt[0], parseMonth(strTokens[1]), dt[2] || 1, dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); + return validDateOrNull( + dt[0], + parseMonth(strTokens[1]), + dt[2] || 1, + dt[3] || 0, + dt[4] || 0, + dt[5] || 0, + dt[6] || 0 + ); } else if (format.length) { throw new Error(`Unrecognized format '${format}'`); } - // try ISO first d = validDateOrNull(dt[0], dt[1] - 1, dt[2], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); - if (d) { return d; } + if (d) { + return d; + } // then UK - d = validDateOrNull(correctYear(dt[2]), parseMonth(strTokens[1]), dt[0], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); - if (d) { return d; } + d = validDateOrNull( + correctYear(dt[2]), + parseMonth(strTokens[1]), + dt[0], + dt[3] || 0, + dt[4] || 0, + dt[5] || 0, + dt[6] || 0 + ); + if (d) { + return d; + } // then US guess - return validDateOrNull(correctYear(dt[2]), parseMonth(strTokens[0]), dt[1], dt[3] || 0, dt[4] || 0, dt[5] || 0, dt[6] || 0); + return validDateOrNull( + correctYear(dt[2]), + parseMonth(strTokens[0]), + dt[1], + dt[3] || 0, + dt[4] || 0, + dt[5] || 0, + dt[6] || 0 + ); return null; } export function parseBooleanOrNull(val: boolean | string): boolean | null { - if (!val) { return null; } - if (typeof val === 'boolean') { return val; } + if (!val) { + return null; + } + if (typeof val === 'boolean') { + return val; + } const trulyVals = ['1', 'yes', 'true', 'on']; const falsyVals = ['0', 'no', 'false', 'off']; @@ -178,7 +288,11 @@ export function addDays(dt: Date, daysOffset: number): Date { export function addBusinessDays(dt: Date, bDaysOffset: number, holidays?: (Date | string)[]): Date { const date = parseDatetimeOrNull(dt); - const holidayDates = holidays?.map(d => parseDatetimeOrNull(d))?.filter(d => !!d)?.map(d => d?.toDateString()) || []; + const holidayDates = + holidays + ?.map(d => parseDatetimeOrNull(d)) + ?.filter(d => !!d) + ?.map(d => d?.toDateString()) || []; if (!date) { throw new Error(`A first parameter to 'addBusinessdays' must be Date`); } @@ -215,7 +329,7 @@ export function dateToString(d: Date, format?: string): string { yy: t[0].slice(2), mm: t[1], dd: t[2] - } + }; if (format.toLowerCase() === 'dd/mm/yyyy') { return `${f.dd}/${f.mm}/${f.yyyy}`; @@ -242,7 +356,6 @@ export function dateToString(d: Date, format?: string): string { } export function deepClone(obj: any): any { - if (obj == null || obj == undefined) { return obj; } @@ -252,7 +365,7 @@ export function deepClone(obj: any): any { } if (Array.isArray(obj)) { - return obj.map(v => deepClone(v)) + return obj.map(v => deepClone(v)); } if (typeof obj === 'object') { @@ -266,7 +379,7 @@ export function deepClone(obj: any): any { clone[propName] = new Date(propValue.getTime()); } else if (Array.isArray(propValue)) { clone[propName] = propValue.map(v => deepClone(v)); - } else if (typeof (propValue) == "object") { + } else if (typeof propValue == 'object') { clone[propName] = deepClone(propValue); } else { clone[propName] = propValue; @@ -278,13 +391,14 @@ export function deepClone(obj: any): any { return obj; } -export function workoutDataType(value: ScalarType, inType: DataTypeName | undefined): DataTypeName | undefined { - +export function workoutDataType( + value: ScalarType, + inType: DataTypeName | undefined +): DataTypeName | undefined { function getRealType(val: ScalarType): DataTypeName { - function processNumber(num: number): DataTypeName { if (num % 1 === 0) { - return (num > 2147483647) ? DataTypeName.BigIntNumber : DataTypeName.WholeNumber; + return num > 2147483647 ? DataTypeName.BigIntNumber : DataTypeName.WholeNumber; } else { return DataTypeName.FloatNumber; } @@ -295,38 +409,52 @@ export function workoutDataType(value: ScalarType, inType: DataTypeName | undefi let dt = null; switch (typeof val) { - case 'boolean': return DataTypeName.Boolean; + case 'boolean': + return DataTypeName.Boolean; case 'number': - return processNumber(val) + return processNumber(val); case 'object': if (val instanceof Date) { const dt = val; - return (dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0) ? DataTypeName.Date : DataTypeName.DateTime; + return dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0 + ? DataTypeName.Date + : DataTypeName.DateTime; } return DataTypeName.String; case 'string': dt = parseDatetimeOrNull(val); if (dt) { - return (dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0) ? DataTypeName.Date : DataTypeName.DateTime; + return dt.getHours() === 0 && dt.getMinutes() === 0 && dt.getSeconds() === 0 + ? DataTypeName.Date + : DataTypeName.DateTime; } num = parseNumberOrNull(val); - if (num !== null) { return processNumber(num); } + if (num !== null) { + return processNumber(num); + } bl = parseBooleanOrNull(val); - if (bl !== null) { return DataTypeName.Boolean; } + if (bl !== null) { + return DataTypeName.Boolean; + } - return (val.length > 4000) ? DataTypeName.LargeString : DataTypeName.String; + return val.length > 4000 ? DataTypeName.LargeString : DataTypeName.String; - default: return DataTypeName.LargeString + default: + return DataTypeName.LargeString; } } - if (value == null || value == undefined) { return undefined; } + if (value == null || value == undefined) { + return undefined; + } // no point to proceed, string is most common type - if (inType === DataTypeName.LargeString) { return DataTypeName.LargeString; } + if (inType === DataTypeName.LargeString) { + return DataTypeName.LargeString; + } const realType = getRealType(value); @@ -334,27 +462,36 @@ export function workoutDataType(value: ScalarType, inType: DataTypeName | undefi return realType; } else { // normal case. Means all values in column are the same - if (inType === realType) { return inType; } + if (inType === realType) { + return inType; + } // date / datetime case if ( - (inType === DataTypeName.Date && realType === DataTypeName.DateTime) - || (inType === DataTypeName.DateTime && realType === DataTypeName.Date) + (inType === DataTypeName.Date && realType === DataTypeName.DateTime) || + (inType === DataTypeName.DateTime && realType === DataTypeName.Date) ) { return DataTypeName.DateTime; } - // if any of items are string, then it must be string - if (realType === DataTypeName.String) { return DataTypeName.String; } - if (inType === DataTypeName.String && realType !== DataTypeName.LargeString) { return DataTypeName.String; } + if (realType === DataTypeName.String) { + return DataTypeName.String; + } + if (inType === DataTypeName.String && realType !== DataTypeName.LargeString) { + return DataTypeName.String; + } - if (inType === DataTypeName.FloatNumber) { return DataTypeName.FloatNumber; } + if (inType === DataTypeName.FloatNumber) { + return DataTypeName.FloatNumber; + } if (realType === DataTypeName.FloatNumber && inType === DataTypeName.WholeNumber) { return DataTypeName.FloatNumber; } - if (realType === DataTypeName.BigIntNumber) { return DataTypeName.BigIntNumber; } + if (realType === DataTypeName.BigIntNumber) { + return DataTypeName.BigIntNumber; + } if (inType === DataTypeName.BigIntNumber && realType === DataTypeName.WholeNumber) { return DataTypeName.BigIntNumber; } @@ -370,16 +507,21 @@ export function workoutDataType(value: ScalarType, inType: DataTypeName | undefi } /** - * generates a field descriptions (first level only) that can be used for relational table definition. - * if any properties are Objects, it would use JSON.stringify to calculate maxSize field. - * @param items + * generates a field descriptions (first level only) that can be used for relational table definition. + * if any properties are Objects, it would use JSON.stringify to calculate maxSize field. + * @param items */ -export function getFieldsInfo(items: Record[] | ScalarType[]): FieldDescription[] { +export function getFieldsInfo( + items: Record[] | ScalarType[] +): FieldDescription[] { const resultMap: Record = Object.create(null); const valuesMap: Record> = Object.create(null); let index = 0; - function processItem(name: string, value: string | number | bigint | boolean | Date | null): void { + function processItem( + name: string, + value: string | number | bigint | boolean | Date | null + ): void { let fDesc = resultMap[name]; let valuesSet = valuesMap[name]; @@ -401,8 +543,11 @@ export function getFieldsInfo(items: Record[] | ScalarType[] if (value === null || value === undefined) { fDesc.isNullable = true; } else { - strValue = value instanceof Date ? dateToString(value) - : typeof value === 'object' ? JSON.stringify(value) + strValue = + value instanceof Date + ? dateToString(value) + : typeof value === 'object' + ? JSON.stringify(value) : String(value); if (!fDesc.isObject && !(value instanceof Date)) { @@ -411,13 +556,19 @@ export function getFieldsInfo(items: Record[] | ScalarType[] const newType = workoutDataType(value, fDesc.dataTypeName); - if (newType !== fDesc.dataTypeName + if ( + newType !== fDesc.dataTypeName && // special case when datetime can't be date again - && !(fDesc.dataTypeName === DataTypeName.DateTime && newType === DataTypeName.Date)) { + !(fDesc.dataTypeName === DataTypeName.DateTime && newType === DataTypeName.Date) + ) { fDesc.dataTypeName = newType; } - if ((fDesc.dataTypeName == DataTypeName.String || fDesc.dataTypeName == DataTypeName.LargeString) && strValue.length > (fDesc.maxSize || 0)) { + if ( + (fDesc.dataTypeName == DataTypeName.String || + fDesc.dataTypeName == DataTypeName.LargeString) && + strValue.length > (fDesc.maxSize || 0) + ) { fDesc.maxSize = strValue.length; } } @@ -428,11 +579,12 @@ export function getFieldsInfo(items: Record[] | ScalarType[] } for (const item of items) { - if (item !== null - && item !== undefined - && typeof item === 'object' - && !(item instanceof Date) - && !Array.isArray(item) + if ( + item !== null && + item !== undefined && + typeof item === 'object' && + !(item instanceof Date) && + !Array.isArray(item) ) { for (const [name, value] of Object.entries(item as Record)) { processItem(name, value); @@ -442,12 +594,12 @@ export function getFieldsInfo(items: Record[] | ScalarType[] } } - const fields = Object.values(resultMap) - .sort((a: FieldDescription, b: FieldDescription) => (a.index > b.index) ? 1 : ((b.index > a.index) ? -1 : 0)) + const fields = Object.values(resultMap).sort((a: FieldDescription, b: FieldDescription) => + a.index > b.index ? 1 : b.index > a.index ? -1 : 0 + ); - return fields - .map(r => { - r.isUnique = valuesMap[r.fieldName].size === items.length; - return r; - }); + return fields.map(r => { + r.isUnique = valuesMap[r.fieldName].size === items.length; + return r; + }); } diff --git a/src/utils/index.ts b/src/utils/index.ts index c5e95a2..03c4cde 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,3 +1,3 @@ -export * from './dsv-parser' -export * from './helpers' -export * from './table' +export * from './dsv-parser'; +export * from './helpers'; +export * from './table'; diff --git a/src/utils/table.ts b/src/utils/table.ts index 9a048e4..10ed5ce 100644 --- a/src/utils/table.ts +++ b/src/utils/table.ts @@ -1,5 +1,5 @@ -import { ScalarObject, PrimitiveType, TableDto, DataTypeName } from "../types"; -import { parseDatetimeOrNull, dateToString } from "./helpers"; +import { ScalarObject, PrimitiveType, TableDto, DataTypeName } from '../types'; +import { parseDatetimeOrNull, dateToString } from './helpers'; /** * Get JSON type array for tabel type array. @@ -7,9 +7,11 @@ import { parseDatetimeOrNull, dateToString } from "./helpers"; * @param fieldNames Column names. If not provided then, it will be auto generated * @param fieldDataTypes Column names */ -export function fromTable(rowsOrTable: PrimitiveType[][] | TableDto, fieldNames?: string[], - fieldDataTypes?: DataTypeName[]): ScalarObject[] { - +export function fromTable( + rowsOrTable: PrimitiveType[][] | TableDto, + fieldNames?: string[], + fieldDataTypes?: DataTypeName[] +): ScalarObject[] { const table = rowsOrTable as TableDto; const rows = table?.rows || rowsOrTable || []; @@ -25,12 +27,12 @@ export function fromTable(rowsOrTable: PrimitiveType[][] | TableDto, fieldNames? for (const row of rows) { const value: ScalarObject = {}; for (let i = 0, len = fieldNames.length; i < len; i++) { - const fieldName = fieldNames[i]; const dataType = fieldDataTypes.length ? fieldDataTypes[i] : null; - value[fieldName] = ((dataType === DataTypeName.DateTime || dataType === DataTypeName.Date) && row[i]) ? - parseDatetimeOrNull(row[i] as string | Date) - : row[i]; + value[fieldName] = + (dataType === DataTypeName.DateTime || dataType === DataTypeName.Date) && row[i] + ? parseDatetimeOrNull(row[i] as string | Date) + : row[i]; } values.push(value); } @@ -60,14 +62,13 @@ export function toTable(values: ScalarObject[]): TableDto { }); tableDto.fieldNames = Array.from(fN.values()); - tableDto.rows = values - .map(rowValues => { - return tableDto.fieldNames.reduce((r, field) => { - const v = rowValues[field]; - const val = v instanceof Date ? dateToString(v) : v; - r.push(val as PrimitiveType); - return r; - }, [] as PrimitiveType[]); - }); + tableDto.rows = values.map(rowValues => { + return tableDto.fieldNames.reduce((r, field) => { + const v = rowValues[field]; + const val = v instanceof Date ? dateToString(v) : v; + r.push(val as PrimitiveType); + return r; + }, [] as PrimitiveType[]); + }); return tableDto; }