From 0bfdaa7623a4cde36af4f75188d58523c7f40c8a Mon Sep 17 00:00:00 2001 From: zabkwak Date: Tue, 22 Mar 2022 14:10:37 +0100 Subject: [PATCH] parquest field --- package-lock.json | 125 +++++++++++++++++++++++++++++++++++++++++++ package.json | 9 +++- src/index.ts | 2 + src/parquet/field.ts | 71 ++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 src/parquet/field.ts diff --git a/package-lock.json b/package-lock.json index 7b571cc..a84aef3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -151,6 +151,24 @@ "integrity": "sha512-BzcaRsnFuznzOItW1WpQrDHM7plAa7GIDMZ6b5pnMbkqEtM/6WCOhvZar39oeMQP79gwvFUWjjptE7/KGcNqFg==", "dev": true }, + "@types/node-int64": { + "version": "0.4.29", + "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", + "integrity": "sha512-rHXvenLTj/CcsmNAebaBOhxQ2MqEGl3yXZZcZ21XYR+gzGTTcpOy2N4IxpvTCz48loyQNatHvfn6GhIbbZ1R3Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/parquetjs": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/@types/parquetjs/-/parquetjs-0.10.3.tgz", + "integrity": "sha512-n0xVEor3+3qHfCmFAf0pO4m/Pxc5JEmiVkEWWqJexN+p11/Nr+rqABKcIEj4X6tGKF1cnVIeBqy67mW2Yd+Kbg==", + "dev": true, + "requires": { + "@types/node-int64": "*" + } + }, "@ungap/promise-all-settled": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@ungap/promise-all-settled/-/promise-all-settled-1.1.2.tgz", @@ -227,12 +245,25 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "bindings": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.2.1.tgz", + "integrity": "sha1-FK1hE4EtLTfXLme0ystLtyZQXxE=", + "dev": true, + "optional": true + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -252,12 +283,27 @@ "fill-range": "^7.0.1" } }, + "brotli": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/brotli/-/brotli-1.3.2.tgz", + "integrity": "sha1-UlqcrU/LqWR119OI9q7LE+7VL0Y=", + "dev": true, + "requires": { + "base64-js": "^1.1.2" + } + }, "browser-stdout": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "bson": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/bson/-/bson-1.1.6.tgz", + "integrity": "sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg==", + "dev": true + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -561,6 +607,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "int53": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/int53/-/int53-0.2.4.tgz", + "integrity": "sha1-XtjXqtbFxlZ8rmmqf/xKEJ7oD4Y=", + "dev": true + }, "is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -671,6 +723,16 @@ "integrity": "sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==", "dev": true }, + "lzo": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/lzo/-/lzo-0.4.11.tgz", + "integrity": "sha512-apQHNoW2Alg72FMqaC/7pn03I7umdgSVFt2KRkCXXils4Z9u3QBh1uOtl2O5WmZIDLd9g6Lu4lIdOLmiSTFVCQ==", + "dev": true, + "optional": true, + "requires": { + "bindings": "~1.2.1" + } + }, "make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", @@ -807,12 +869,24 @@ "integrity": "sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw==", "dev": true }, + "node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=", + "dev": true + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "object-stream": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/object-stream/-/object-stream-0.0.1.tgz", + "integrity": "sha1-OgOibpT9ESyav/60ZR4HpeI8+EA=", + "dev": true + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -840,6 +914,22 @@ "p-limit": "^3.0.2" } }, + "parquetjs": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/parquetjs/-/parquetjs-0.11.2.tgz", + "integrity": "sha512-Y6FOc3Oi2AxY4TzJPz7fhICCR8tQNL3p+2xGQoUAMbmlJBR7+JJmMrwuyMjIpDiM7G8Wj/8oqOH4UDUmu4I5ZA==", + "dev": true, + "requires": { + "brotli": "^1.3.0", + "bson": "^1.0.4", + "int53": "^0.2.4", + "lzo": "^0.4.0", + "object-stream": "0.0.1", + "snappyjs": "^0.6.0", + "thrift": "^0.11.0", + "varint": "^5.0.0" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -870,6 +960,12 @@ "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", "dev": true }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", + "dev": true + }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -941,6 +1037,12 @@ "vscode-textmate": "5.2.0" } }, + "snappyjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/snappyjs/-/snappyjs-0.6.1.tgz", + "integrity": "sha512-YIK6I2lsH072UE0aOFxxY1dPDCS43I5ktqHpeAsuLNYWkE5pGxRGWfDM4/vSUfNzXjC1Ivzt3qx31PCLmc9yqg==", + "dev": true + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -998,6 +1100,17 @@ "has-flag": "^4.0.0" } }, + "thrift": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.11.0.tgz", + "integrity": "sha1-JWEV5P+Hhx4SU39LUQvStCXhOZA=", + "dev": true, + "requires": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": ">= 2.2.3" + } + }, "to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -1200,6 +1313,12 @@ "integrity": "sha512-mpSYqfsFvASnSn5qMiwrr4VKfumbPyONLCOPmsR3A6pTY/r0+tSaVbgPWSAIuzbk3lCTa+FForeTiO+wBQGkjA==", "dev": true }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==", + "dev": true + }, "vscode-oniguruma": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/vscode-oniguruma/-/vscode-oniguruma-1.6.2.tgz", @@ -1244,6 +1363,12 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, + "ws": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.5.0.tgz", + "integrity": "sha512-BWX0SWVgLPzYwF8lTzEy1egjhS4S4OEAHfsO8o65WOVsrnSRGaSiUaa9e0ggGlkMTtBlmOpEXiie9RUcBO86qg==", + "dev": true + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/package.json b/package.json index b2cfe98..85cf022 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mat-utils", - "version": "0.1.0", + "version": "0.2.0", "description": "Utils for Media analytics tool", "main": "index.js", "types": "./dist/index.d.ts", @@ -46,14 +46,19 @@ "@types/core-js": "^2.5.5", "@types/mocha": "^9.1.0", "@types/node": "^12.20.47", + "@types/parquetjs": "^0.10.3", "chai": "^4.3.6", "dotenv": "^16.0.0", "mocha": "^9.2.2", + "parquetjs": "^0.11.2", "run-script-os-fix": "^1.0.4", "source-map-support": "^0.5.21", "ts-node": "^10.7.0", "tslint": "^6.1.3", "typedoc": "^0.22.13", "typescript": "^4.6.2" + }, + "peerDependencies": { + "parquetjs": "^0.11.2" } -} +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index 691714f..839da4f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,6 @@ import CSV from './csv'; import Logger from './logger'; +import ParquetField from './parquet/field'; const sleep = (timeout: number) => { return new Promise((resolve) => { @@ -10,5 +11,6 @@ const sleep = (timeout: number) => { export { CSV, Logger, + ParquetField, sleep, }; diff --git a/src/parquet/field.ts b/src/parquet/field.ts new file mode 100644 index 0000000..59eb527 --- /dev/null +++ b/src/parquet/field.ts @@ -0,0 +1,71 @@ +interface IMapField { + fields: Record; + repeated: boolean; + optional: boolean; +} + +export default class Field { + + // #region Types + + public static get string() { + return new this('UTF8'); + } + + public static get integer() { + return new this('INT64'); + } + + public static get float() { + return new this('DOUBLE'); + } + + public static get timestamp() { + return new this('TIMESTAMP_MILLIS'); + } + + public static get boolean() { + return new this('BOOLEAN'); + } + + public static get date() { + return new this('DATE'); + } + + public static arrayOf(type: Field): Field { + const f = new this(type.type); + f.array(); + return f; + } + + public static map(fields: Record, array: boolean = false): IMapField { + return { + fields, + repeated: array, + optional: true, + }; + } + + // #endregion + + public type: string; + + public optional: boolean = true; + + public repeated: boolean = false; + + constructor(type: string) { + this.type = type; + this.optional = true; + } + + public required(): this { + this.optional = false; + return this; + } + + public array(): this { + this.repeated = true; + return this; + } +}