diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..f77a02d --- /dev/null +++ b/.eslintignore @@ -0,0 +1,4 @@ +node_modules +dist +**/*/*.d.ts +./*js diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..11643ef --- /dev/null +++ b/.eslintrc @@ -0,0 +1,12 @@ +{ + "root": true, + "parser": "@typescript-eslint/parser", + "plugins": [ + "@typescript-eslint" + ], + "extends": [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended" + ] +} diff --git a/jest.config.js b/jest.config.js index f498143..da2ee0d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -188,5 +188,5 @@ module.exports = { // watchPathIgnorePatterns: [], // Whether to use watchman for file crawling - // watchman: true, + // watchman: true }; diff --git a/package.json b/package.json index 7dd6de5..4147237 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "datapipe-js", - "version": "0.2.7", + "version": "0.2.8", "description": "dataPipe is a JavaScript library for data manipulations, data transformations and data wrangling library inspired by LINQ (C#) and Pandas (Python)", "main": "dist/data-pipe.min.js", "module": "dist/data-pipe.esm.js", @@ -20,7 +20,9 @@ "docs": "npx typedoc src --plugin none", "docs:md": "npx typedoc src --out md-docs --plugin typedoc-plugin-markdown", "deploy": "npm run docs && npx gh-pages -d docs", - "dev": "npx rollup --config rollup.config.dev.js --watch" + "dev": "npx rollup --config rollup.config.dev.js --watch", + "lint": "eslint . --ext .ts", + "lint-fix": "eslint . --ext .ts --fix" }, "repository": { "type": "git", @@ -48,6 +50,9 @@ ], "devDependencies": { "@types/jest": "^24.9.1", + "@typescript-eslint/eslint-plugin": "^2.24.0", + "@typescript-eslint/parser": "^2.24.0", + "eslint": "^6.8.0", "jest": "^24.9.0", "jest-fetch-mock": "^2.1.2", "rollup": "^1.31.1", @@ -57,8 +62,8 @@ "rollup-plugin-typescript2": "^0.25.3", "rollup-plugin-uglify": "^6.0.4", "ts-jest": "^24.3.0", - "typedoc": "^0.15.8", - "typedoc-plugin-markdown": "^2.2.16", - "typescript": "^3.7.5" + "typedoc": "^0.17.3", + "typedoc-plugin-markdown": "^2.2.17", + "typescript": "^3.8.3" } } diff --git a/src/array/joins.ts b/src/array/joins.ts index bf9ea81..c60f6ba 100644 --- a/src/array/joins.ts +++ b/src/array/joins.ts @@ -63,17 +63,17 @@ export function fullJoin( // build a lookup maps for both arrays. // so, both of them have to be unique, otherwise it will flattern result const leftArrayMap = Object.create(null); - for (let item of rightArray) { + for (const item of rightArray) { leftArrayMap[leftKeySelector(item)] = item; } const rightArrayMap = Object.create(null); - for (let item of rightArray) { + for (const item of rightArray) { rightArrayMap[rightKeySelector(item)] = item; } const result: any[] = []; - for (let leftItem of leftArray) { + for (const leftItem of leftArray) { const leftKey = leftKeySelector(leftItem); const rightItem = rightArrayMap[leftKey] || null; @@ -87,7 +87,7 @@ export function fullJoin( } // add remaining right items - for (let rightItemKey in rightArrayMap) { + for (const rightItemKey in rightArrayMap) { const rightItem = rightArrayMap[rightItemKey]; const resultItem = resultSelector(null, rightItem); @@ -119,16 +119,16 @@ export function merge( // build a lookup maps for both arrays. // so, both of them have to be unique, otherwise it will flattern result const targetArrayMap = Object.create(null); - for (let item of sourceArray) { + for (const item of sourceArray) { targetArrayMap[targetKeySelector(item)] = item; } const sourceArrayMap = Object.create(null); - for (let item of sourceArray) { + for (const item of sourceArray) { sourceArrayMap[sourceKeySelector(item)] = item; } - for (let sourceItemKey of Object.keys(sourceArrayMap)) { + for (const sourceItemKey of Object.keys(sourceArrayMap)) { const sourceItem = sourceArrayMap[sourceItemKey]; if (!targetArrayMap[sourceItemKey]) { targetArray.push(sourceItem); @@ -184,12 +184,12 @@ function leftOrInnerJoin( // build a lookup map const rightArrayMap = Object.create(null); - for (let item of rightArray) { + for (const item of rightArray) { rightArrayMap[rightKeySelector(item)] = item; } const result: any[] = []; - for (let leftItem of leftArray) { + for (const leftItem of leftArray) { const leftKey = leftKeySelector(leftItem); const rightItem = rightArrayMap[leftKey] || null; diff --git a/src/array/stats.ts b/src/array/stats.ts index d0feeed..5a1c12b 100644 --- a/src/array/stats.ts +++ b/src/array/stats.ts @@ -14,11 +14,11 @@ import { isArrayEmptyOrNull } from "./utils"; * sum([{ val: 1 }, { val: 5 }], i => i.val); // 6 */ export function sum(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } const elementSelector = fieldSelector(field); - let sum: number = 0; + let sum = 0; for (const item of array) { const numberVal = parseNumber(item, elementSelector); if (numberVal) { @@ -38,7 +38,7 @@ export function sum(array: any[], field?: Selector | string): number | null { * avg([1, 5, 3]); // 3 */ export function avg(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } const elementSelector = fieldSelector(field); @@ -73,7 +73,7 @@ export function min(array: any[], field?: Selector | string): number | Date | nu export function max(array: any[], field?: Selector | string): number | Date | null { const elementSelector = fieldSelector(field); const numberArray = getNumberValuesArray(array, elementSelector); - if (isArrayEmptyOrNull(numberArray)) { return null }; + if (isArrayEmptyOrNull(numberArray)) { return null } const max = Math.max(...numberArray); const item = elementSelector ? elementSelector(array[0]) : array[0]; if (item instanceof Date) { @@ -103,7 +103,7 @@ export function count(array: any[], predicate?: Predicate): number | null { * @param predicate Predicate function invoked per iteration. */ export function first(array: T[], predicate?: Predicate): T | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } if (!predicate) { return array[0]; @@ -122,7 +122,7 @@ export function first(array: T[], predicate?: Predicate): T | null { * @param predicate Predicate function invoked per iteration. */ export function last(array: T[], predicate?: Predicate): T | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } let lastIndex = array.length - 1; if (!predicate) { @@ -144,7 +144,7 @@ export function last(array: T[], predicate?: Predicate): T | null { * @param elementSelector Function invoked per iteration. */ export function countBy(array: any[], elementSelector: Selector): { [key: string]: number } { - if (!array || !Array.isArray(array)) { throw Error('No array provided') }; + if (!array || !Array.isArray(array)) { throw Error('No array provided') } const results: { [key: string]: number } = {}; const length = array.length; @@ -165,7 +165,7 @@ export function countBy(array: any[], elementSelector: Selector): { [key: string * @param field Property name or Selector function invoked per iteration. */ export function mean(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } let res = 0; for (let i = 0, c = 0, len = array.length; i < len; ++i) { @@ -185,7 +185,7 @@ export function mean(array: any[], field?: Selector | string): number | null { * @param p quantile. */ export function quantile(array: any[], p: number, field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } const len = (array.length - 1) * p + 1; const l = Math.floor(len); @@ -193,7 +193,7 @@ export function quantile(array: any[], p: number, field?: Selector | string): nu const val = elementSelector ? elementSelector(array[l - 1]) : array[l - 1]; const e = len - l; return e ? val + e * (array[l] - val) : val; -}; +} /** * Get sample variance of an array. @@ -201,7 +201,7 @@ export function quantile(array: any[], p: number, field?: Selector | string): nu * @param field Property name or Selector function invoked per iteration. */ export function variance(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } const elementSelector = fieldSelector(field); if (!Array.isArray(array) || array.length < 2) { @@ -218,7 +218,7 @@ export function variance(array: any[], field?: Selector | string): number | null } M2 = M2 / (c - 1); return M2; -}; +} /** * Get the sample standard deviation of an array. @@ -226,14 +226,14 @@ export function variance(array: any[], field?: Selector | string): number | null * @param field Property name or Selector function invoked per iteration. */ export function stdev(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } const varr = variance(array, field); if (varr === null) { return null; } return Math.sqrt(varr); -}; +} /** * Get median of an array. @@ -241,11 +241,11 @@ export function stdev(array: any[], field?: Selector | string): number | null { * @param field Property name or Selector function invoked per iteration. */ export function median(array: any[], field?: Selector | string): number | null { - if (isArrayEmptyOrNull(array)) { return null }; + if (isArrayEmptyOrNull(array)) { return null } array.sort(fieldComparator(field)); return quantile(getNumberValuesArray(array, field), 0.5); -}; +} function fieldComparator(field?: string | Selector): (a: any, b: any) => number { return (a: any, b: any) => { diff --git a/src/array/transform.ts b/src/array/transform.ts index e9fa6d7..345c75e 100644 --- a/src/array/transform.ts +++ b/src/array/transform.ts @@ -8,7 +8,7 @@ import { sum } from "./stats"; * @param elementSelector Function invoked per iteration. */ export function groupBy(array: any[], groupByFields: string | string[] | Selector): any[] { - if (!Array.isArray(array)) { throw Error('An array is not provided') }; + if (!Array.isArray(array)) { throw Error('An array is not provided') } if (!array.length) { return array; } @@ -33,7 +33,7 @@ export function groupBy(array: any[], groupByFields: string | string[] | Selecto * flatten([1, 4, [2, [5, 5, [9, 7]], 11], 0]); // length 9 */ export function flatten(array: any[]): any[] { - if (!Array.isArray(array)) { throw Error('An array is not provided') }; + if (!Array.isArray(array)) { throw Error('An array is not provided') } if (!array.length) { return array; } @@ -41,7 +41,7 @@ export function flatten(array: any[]): any[] { const length = array.length; for (let i = 0; i < length; i++) { - var value = array[i]; + const value = array[i]; if (Array.isArray(value)) { res = [...res, ...flatten(value)]; } else { @@ -63,7 +63,7 @@ export function flatten(array: any[]): any[] { export function pivot(array: any, rowFields: string | string[],columnField: string, dataField: string, aggFunction?: (array: any[]) => any | null, columnValues?: string[]): any[] { - if (!Array.isArray(array)) { throw Error('An array is not provided') }; + if (!Array.isArray(array)) { throw Error('An array is not provided') } if (!array.length) { return array; } @@ -90,7 +90,7 @@ export function pivot(array: any, rowFields: string | string[],columnField: stri } } - var result: any[] = []; + const result: any[] = []; for (const groupName of Object.keys(groups)) { const item = Object.create(null); @@ -118,7 +118,7 @@ export function pivot(array: any, rowFields: string | string[],columnField: stri * @param data */ export function transpose(data: any[]): any[] { - if (!Array.isArray(data)) { throw Error('An array is not provided') }; + if (!Array.isArray(data)) { throw Error('An array is not provided') } if (!data.length) { return data; } diff --git a/src/array/utils.ts b/src/array/utils.ts index 41b6321..c45e8e9 100644 --- a/src/array/utils.ts +++ b/src/array/utils.ts @@ -2,7 +2,7 @@ import { parseNumber, parseDatetimeOrNull } from "../utils"; /** * Checks if array is empty or null or array at all - * @param array + * @param array */ export function isArrayEmptyOrNull(array: any[]): boolean { return !array || !Array.isArray(array) || !array.length; @@ -30,13 +30,31 @@ export function sort(array: any[], ...fields: string[]) { return { asc, field: field.replace(asc ? /\sASC$/ : /\sDESC$/, '') - } + }; }); array.sort(comparator(sortFields)); return array; } +function compare(a: any, b: any, { field, asc }: any): number { + const valA = a[field]; + const valB = b[field]; + const order = asc ? 1 : -1; + + if (valA !== undefined && valB === undefined) { + return order; + } + + switch (typeof valA) { + case 'number': return order * compareNumbers(valA, valB); + case 'string': return order * compareStrings(valA, valB); + case 'object': return order * compareObjects(valA, valB); + case 'undefined': return valB === undefined ? 0 : (-1 * order); + } + return 0; +} + function comparator(sortFields: any[]) { if (sortFields.length) { return (a: any, b: any) => { @@ -48,22 +66,8 @@ function comparator(sortFields: any[]) { } } return 0; - } - } -} - -function compare(a: any, b: any, { field, asc }: any): number { - const valA = a[field]; - const valB = b[field]; - const order = asc ? 1 : -1; - - switch (typeof valA) { - case 'number': return order * compareNumbers(valA, valB); - case 'string': return order * compareStrings(valA, valB); - case 'object': return order * compareObjects(valA, valB); - case 'undefined': return -1 * order; + }; } - return 0; } function compareStrings(a: string, b: any): number { @@ -72,25 +76,28 @@ function compareStrings(a: string, b: any): number { function compareNumbers(a: number, b: any): number { const bNumVal = parseNumber(b); - if (bNumVal == undefined) { + if (bNumVal === undefined) { return 1; } return a - bNumVal; } -function compareObjects(a: any, b: any) { +function compareObjects(a: any, b: any): number { const aDate = parseDatetimeOrNull(a); + const bDate = parseDatetimeOrNull(b); + + if (!aDate && !bDate) { + return 0; + } + if (!aDate) { - return -1 + return -1; } - const bDate = parseDatetimeOrNull(b); + if (!bDate) { - return 1 - } - if (a instanceof Date) { - return a.getTime() - bDate.getTime(); + return 1; } - return -1; + return aDate.getTime() - bDate.getTime(); } diff --git a/src/string/stringUtils.ts b/src/string/stringUtils.ts index 9f05309..682a400 100644 --- a/src/string/stringUtils.ts +++ b/src/string/stringUtils.ts @@ -1,7 +1,7 @@ -export function formatCamelStr(str: string = ''): string { +export function formatCamelStr(str = ''): string { return str.replace(/^\w/, c => c.toUpperCase()).replace(/([a-z0-9])([A-Z])/g, '$1 $2').replace(/_/g, ' '); } -export function replaceAll(text: string, searchValue: string, replaceValue: string = ''): string { +export function replaceAll(text: string, searchValue: string, replaceValue = ''): string { return text.replace(new RegExp(searchValue, 'g'), replaceValue || ''); } diff --git a/src/tests/array.spec.ts b/src/tests/array.spec.ts index 21dbf5e..ad0f1c1 100644 --- a/src/tests/array.spec.ts +++ b/src/tests/array.spec.ts @@ -307,27 +307,27 @@ describe('Test array methods', () => { // expect(res.filter(r => r.product === 'P2')[0]['2018']).toBe('Data21'); // expect(res.filter(r => r.product === 'P3')[0]['2019']).toBe('Data33'); // expect(res.filter(r => r.product === 'P3')[0]['2018']).toBe(null); - }) + }); it('stats quantile for sorted array', () => { - const data = [3, 1, 2, 4, 0].sort(); - expect(quantile(data, 0)).toBe(0); - expect(quantile(data, 1 / 4)).toBe(1); - expect(quantile(data, 1.5 / 4)).toBe(1.5); - expect(quantile(data, 2 / 4)).toBe(2); - expect(quantile(data, 2.5 / 4)).toBe(2.5); - expect(quantile(data, 3 / 4)).toBe(3); - expect(quantile(data, 3.2 / 4)).toBe(3.2); - expect(quantile(data, 4 / 4)).toBe(4); - - var even = [3, 6, 7, 8, 8, 10, 13, 15, 16, 20]; + const numbersData = [3, 1, 2, 4, 0].sort(); + expect(quantile(numbersData, 0)).toBe(0); + expect(quantile(numbersData, 1 / 4)).toBe(1); + expect(quantile(numbersData, 1.5 / 4)).toBe(1.5); + expect(quantile(numbersData, 2 / 4)).toBe(2); + expect(quantile(numbersData, 2.5 / 4)).toBe(2.5); + expect(quantile(numbersData, 3 / 4)).toBe(3); + expect(quantile(numbersData, 3.2 / 4)).toBe(3.2); + expect(quantile(numbersData, 4 / 4)).toBe(4); + + const even = [3, 6, 7, 8, 8, 10, 13, 15, 16, 20]; expect(quantile(even, 0)).toBe(3); expect(quantile(even, 0.25)).toBe(7.25); expect(quantile(even, 0.5)).toBe(9); expect(quantile(even, 0.75)).toBe(14.5); expect(quantile(even, 1)).toBe(20); - var odd = [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]; + const odd = [3, 6, 7, 8, 8, 9, 10, 13, 15, 16, 20]; expect(quantile(odd, 0)).toBe(3); expect(quantile(odd, 0.25)).toBe(7.5); expect(quantile(odd, 0.5)).toBe(9); @@ -386,5 +386,15 @@ describe('Test array methods', () => { expect(arr[2].name).toBe('Marry'); arr = pipeFuncs.sort([5, 2, 9, 4]); expect(arr[2]).toBe(5); - }) -}) + + const arrWithUndefinedProps = [ + { name: 'Tom', age: 7 }, + { name: 'Bob', age: 10 }, + { age: 5 }, + { name: 'Jerry', age: 3 }]; + arr = pipeFuncs.sort(arrWithUndefinedProps, 'name ASC') || []; + expect(arr[0].age).toBe(5); + arr = pipeFuncs.sort(arrWithUndefinedProps, 'name DESC') || []; + expect(arr[0].age).toBe(7); + }); +}); diff --git a/src/tests/dsv-parser.spec.ts b/src/tests/dsv-parser.spec.ts index 4f8d1c3..195df15 100644 --- a/src/tests/dsv-parser.spec.ts +++ b/src/tests/dsv-parser.spec.ts @@ -67,7 +67,7 @@ describe('Dsv Parser specification', () => { it('skipUntil', () => { const csv = ["", " * not Empty *", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`].join('\n') - var options = new ParsingOptions(); + const options = new ParsingOptions(); options.delimiter = '\t'; options.skipUntil = t => t && t.length > 1; @@ -78,7 +78,7 @@ describe('Dsv Parser specification', () => { it('empty values', () => { const csv = ["", "", , "\t\t\t", "F1\tF2\tF3", `1\t1,000.32\t"Test, comma"`, "\t\t"].join('\n') - var options = new ParsingOptions(); + const options = new ParsingOptions(); options.delimiter = '\t'; const result = parseCsv(csv, options); @@ -88,7 +88,7 @@ describe('Dsv Parser specification', () => { it('Date Fields', () => { const csv = ["F1\tF2\tF3", `2020-02-11\t1,000.32\t"Test, comma"`].join('\n') - var options = new ParsingOptions(); + const options = new ParsingOptions(); options.delimiter = '\t'; options.dateFields = ['F1'] diff --git a/src/types.ts b/src/types.ts index 8750dd5..c1d23a0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,8 +1,8 @@ export type Predicate = (p: T) => boolean; export type Selector = (p: T) => V; export class ParsingOptions { - delimiter: string = ','; - skipRows: number = 0; + delimiter = ','; + skipRows = 0; dateFields: string[] = []; numberFields: string[] = []; booleanFields: string[] = []; diff --git a/src/utils/dsv-parser.ts b/src/utils/dsv-parser.ts index e66e831..0b6a2c1 100644 --- a/src/utils/dsv-parser.ts +++ b/src/utils/dsv-parser.ts @@ -12,10 +12,10 @@ export function parseCsv(content: string, options?: ParsingOptions): any[] { return getLineTokens(content, options || new ParsingOptions()); } -export function toCsv(array: any[], delimiter: string = ','): string { +export function toCsv(array: any[], delimiter = ','): string { array = array || []; - var result = ""; + const result = ""; const headers: string[] = []; // workout all headers @@ -47,8 +47,8 @@ export function toCsv(array: any[], delimiter: string = ','): string { } type ParsingContext = { - content: string, - currentIndex: number + content: string; + currentIndex: number; }; function getObjectElement(fieldNames: string[], tokens: string[], options: ParsingOptions): any { @@ -135,7 +135,7 @@ function getLineTokens(content: string, options: ParsingOptions): string[][] { return result; } -function nextLineTokens(context: ParsingContext, delimiter: string = ','): string[] { +function nextLineTokens(context: ParsingContext, delimiter = ','): string[] { const tokens: string[] = []; let token = ''; diff --git a/tsconfig.json b/tsconfig.json index c6ed90a..92d9b71 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -44,7 +44,7 @@ // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "typeRoots": [], /* List of folders to include type definitions from. */ - // "types": [], /* Type declaration files to be included in compilation. */ + "types": ["node", "jest"], /* Type declaration files to be included in compilation. */ // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ "esModuleInterop": true, /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */