From 5a70fc675f719257c921df5e540b223054cd0a8e Mon Sep 17 00:00:00 2001 From: Connor Pearson Date: Fri, 3 Jan 2025 11:47:58 +0100 Subject: [PATCH 1/6] feat(redis): add `saveRawAsBinary` driver option (#559) --- src/drivers/redis.ts | 75 +++++++++++++++++++++++++++++++------- test/drivers/redis.test.ts | 68 ++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 13 deletions(-) diff --git a/src/drivers/redis.ts b/src/drivers/redis.ts index 47e7c03c..c10606b8 100644 --- a/src/drivers/redis.ts +++ b/src/drivers/redis.ts @@ -1,4 +1,5 @@ -import { defineDriver, joinKeys } from "./utils"; +import type { TransactionOptions } from "../types"; +import { defineDriver, joinKeys, createError } from "./utils"; // TODO: use named import in v2 import Redis, { Cluster, @@ -32,6 +33,12 @@ export interface RedisOptions extends _RedisOptions { * Default TTL for all items in seconds. */ ttl?: number; + + /** + * Whether to save Buffer/Uint8Arry as binary data or a base64-encoded string + * This option applies to the experimental getItemRaw/setItemRaw methods + */ + saveRawAsBinary?: boolean; } const DRIVER_NAME = "redis"; @@ -56,6 +63,52 @@ export default defineDriver((opts: RedisOptions) => { const p = (...keys: string[]) => joinKeys(base, ...keys); // Prefix a key. Uses base for backwards compatibility const d = (key: string) => (base ? key.replace(base, "") : key); // Deprefix a key + const getItem = async (key: string) => { + const value = await getRedisClient().get(p(key)); + return value ?? null; + }; + + const setItem = async ( + key: string, + value: any, + tOptions: TransactionOptions + ) => { + const ttl = tOptions?.ttl ?? opts.ttl; + if (ttl) { + await getRedisClient().set(p(key), value, "EX", ttl); + } else { + await getRedisClient().set(p(key), value); + } + }; + + const getItemRaw = async (key: string) => { + const value = await getRedisClient().getBuffer(p(key)); + return value ?? null; + }; + + const setItemRaw = async ( + key: string, + value: Uint8Array, + tOptions: TransactionOptions + ) => { + let valueToSave: Buffer; + if (value instanceof Uint8Array) { + if (value instanceof Buffer) { + valueToSave = value; + } else { + valueToSave = Buffer.copyBytesFrom( + value, + value.byteOffset, + value.byteLength + ); + } + } else { + throw createError(DRIVER_NAME, "Expected Buffer or Uint8Array"); + } + + await setItem(key, valueToSave, tOptions); + }; + return { name: DRIVER_NAME, options: opts, @@ -63,18 +116,14 @@ export default defineDriver((opts: RedisOptions) => { async hasItem(key) { return Boolean(await getRedisClient().exists(p(key))); }, - async getItem(key) { - const value = await getRedisClient().get(p(key)); - return value ?? null; - }, - async setItem(key, value, tOptions) { - const ttl = tOptions?.ttl ?? opts.ttl; - if (ttl) { - await getRedisClient().set(p(key), value, "EX", ttl); - } else { - await getRedisClient().set(p(key), value); - } - }, + getItem, + setItem, + ...(opts.saveRawAsBinary + ? { + getItemRaw, + setItemRaw, + } + : {}), async removeItem(key) { await getRedisClient().del(p(key)); }, diff --git a/test/drivers/redis.test.ts b/test/drivers/redis.test.ts index 54c6863f..0e4eee96 100644 --- a/test/drivers/redis.test.ts +++ b/test/drivers/redis.test.ts @@ -32,9 +32,77 @@ describe("drivers: redis", () => { await client.disconnect(); }); + it("saves raw data as a base64 string", async () => { + const helloBuffer = Buffer.from("Hello, world!", "utf8"); + const byteArray = new Uint8Array(4); + byteArray[0] = 2; + byteArray[1] = 0; + byteArray[2] = 2; + byteArray[3] = 5; + + await ctx.storage.setItemRaw("s4:a", helloBuffer); + await ctx.storage.setItemRaw("s5:a", byteArray); + + const client = new ioredis.default("ioredis://localhost:6379/0"); + + const bufferValue = await client.get("test:s4:a"); + expect(bufferValue).toEqual("base64:SGVsbG8sIHdvcmxkIQ=="); + + const byteArrayValue = await client.get("test:s5:a"); + expect(byteArrayValue).toEqual("base64:AgACBQ=="); + + await client.disconnect(); + }); + it("exposes instance", () => { expect(driver.getInstance?.()).toBeInstanceOf(ioredis.default); }); }, }); + + const binaryDriver = redisDriver({ + base: "test:", + url: "ioredis://localhost:6379/0", + lazyConnect: false, + saveRawAsBinary: true, + }); + + testDriver({ + driver: binaryDriver, + additionalTests(ctx) { + it("saves raw data as binary", async () => { + const helloBuffer = Buffer.from("Hello, world!", "utf8"); + const byteArray = new Uint8Array(4); + byteArray[0] = 2; + byteArray[1] = 0; + byteArray[2] = 2; + byteArray[3] = 5; + + await ctx.storage.setItemRaw("s4:a", helloBuffer); + await ctx.storage.setItemRaw("s5:a", byteArray); + + const client = new ioredis.default("ioredis://localhost:6379/0"); + + const bufferValue = await client.getBuffer("test:s4:a"); + expect(bufferValue).toEqual(helloBuffer); + + const byteArrayValue = await client.getBuffer("test:s5:a"); + expect(byteArrayValue).toEqual(Buffer.from([2, 0, 2, 5])); + + await client.disconnect(); + }); + + it("expects binary data to be sent to setItemRaw", async () => { + expect(() => + ctx.storage.setItemRaw("s4:a", "Hello, world!") + ).rejects.toThrow("Expected Buffer or Uint8Array"); + expect(() => ctx.storage.setItemRaw("s5:a", 100)).rejects.toThrow( + "Expected Buffer or Uint8Array" + ); + expect(() => + ctx.storage.setItemRaw("s5:a", { foo: "bar" }) + ).rejects.toThrow("Expected Buffer or Uint8Array"); + }); + }, + }); }); From ca6ed01d23f301868a058e62430b79338ec3b64d Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 3 Jan 2025 17:32:25 +0100 Subject: [PATCH 2/6] update --- docs/2.drivers/redis.md | 3 + src/drivers/redis.ts | 109 ++++++++++++++++++------------------- test/drivers/redis.test.ts | 6 +- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/docs/2.drivers/redis.md b/docs/2.drivers/redis.md index b9cf70ea..9b94fe41 100644 --- a/docs/2.drivers/redis.md +++ b/docs/2.drivers/redis.md @@ -30,6 +30,7 @@ import redisDriver from "unstorage/drivers/redis"; const storage = createStorage({ driver: redisDriver({ + raw: true, base: "unstorage", host: 'HOSTNAME', tls: true as any, @@ -46,6 +47,7 @@ Usage with Redis cluster (e.g. AWS ElastiCache or Azure Redis Cache): ```js const storage = createStorage({ driver: redisDriver({ + raw: true, base: "{unstorage}", cluster: [ { @@ -70,6 +72,7 @@ const storage = createStorage({ - `cluster`: List of redis nodes to use for cluster mode. Takes precedence over `url` and `host` options. - `clusterOptions`: Options to use for cluster mode. - `ttl`: Default TTL for all items in **seconds**. +- `useRaw`: If enabled, `getItemRaw` and `setItemRaw` will use binary data instead of base64 encoded strings. (this option will be enabled by default in the next major version.) See [ioredis](https://github.com/luin/ioredis/blob/master/API.md#new-redisport-host-options) for all available options. diff --git a/src/drivers/redis.ts b/src/drivers/redis.ts index c10606b8..bbde680e 100644 --- a/src/drivers/redis.ts +++ b/src/drivers/redis.ts @@ -35,10 +35,10 @@ export interface RedisOptions extends _RedisOptions { ttl?: number; /** - * Whether to save Buffer/Uint8Arry as binary data or a base64-encoded string - * This option applies to the experimental getItemRaw/setItemRaw methods + * If enabled, `getItemRaw` and `setItemRaw` will use binary data instead of base64 encoded strings. + * This option will be enabled by default in the next major version. */ - saveRawAsBinary?: boolean; + raw?: boolean; } const DRIVER_NAME = "redis"; @@ -63,52 +63,6 @@ export default defineDriver((opts: RedisOptions) => { const p = (...keys: string[]) => joinKeys(base, ...keys); // Prefix a key. Uses base for backwards compatibility const d = (key: string) => (base ? key.replace(base, "") : key); // Deprefix a key - const getItem = async (key: string) => { - const value = await getRedisClient().get(p(key)); - return value ?? null; - }; - - const setItem = async ( - key: string, - value: any, - tOptions: TransactionOptions - ) => { - const ttl = tOptions?.ttl ?? opts.ttl; - if (ttl) { - await getRedisClient().set(p(key), value, "EX", ttl); - } else { - await getRedisClient().set(p(key), value); - } - }; - - const getItemRaw = async (key: string) => { - const value = await getRedisClient().getBuffer(p(key)); - return value ?? null; - }; - - const setItemRaw = async ( - key: string, - value: Uint8Array, - tOptions: TransactionOptions - ) => { - let valueToSave: Buffer; - if (value instanceof Uint8Array) { - if (value instanceof Buffer) { - valueToSave = value; - } else { - valueToSave = Buffer.copyBytesFrom( - value, - value.byteOffset, - value.byteLength - ); - } - } else { - throw createError(DRIVER_NAME, "Expected Buffer or Uint8Array"); - } - - await setItem(key, valueToSave, tOptions); - }; - return { name: DRIVER_NAME, options: opts, @@ -116,14 +70,55 @@ export default defineDriver((opts: RedisOptions) => { async hasItem(key) { return Boolean(await getRedisClient().exists(p(key))); }, - getItem, - setItem, - ...(opts.saveRawAsBinary - ? { - getItemRaw, - setItemRaw, - } - : {}), + async getItem(key) { + const value = await getRedisClient().get(p(key)); + return value ?? null; + }, + getItemRaw: + opts.raw === true + ? async (key: string) => { + const value = await getRedisClient().getBuffer(p(key)); + return value ?? null; + } + : undefined, + async setItem(key, value, tOptions) { + const ttl = tOptions?.ttl ?? opts.ttl; + if (ttl) { + await getRedisClient().set(p(key), value, "EX", ttl); + } else { + await getRedisClient().set(p(key), value); + } + }, + setItemRaw: + opts.raw === true + ? async ( + key: string, + value: Uint8Array, + tOptions: TransactionOptions + ) => { + let valueToSave: Buffer; + if (value instanceof Uint8Array) { + if (value instanceof Buffer) { + valueToSave = value; + } else { + valueToSave = Buffer.copyBytesFrom( + value, + value.byteOffset, + value.byteLength + ); + } + } else { + throw createError(DRIVER_NAME, "Expected Buffer or Uint8Array"); + } + + const ttl = tOptions?.ttl ?? opts.ttl; + if (ttl) { + await getRedisClient().set(p(key), valueToSave, "EX", ttl); + } else { + await getRedisClient().set(p(key), valueToSave); + } + } + : undefined, async removeItem(key) { await getRedisClient().del(p(key)); }, diff --git a/test/drivers/redis.test.ts b/test/drivers/redis.test.ts index 0e4eee96..5f0bb6ad 100644 --- a/test/drivers/redis.test.ts +++ b/test/drivers/redis.test.ts @@ -5,7 +5,7 @@ import { testDriver } from "./utils"; vi.mock("ioredis", () => ioredis); -describe("drivers: redis", () => { +describe("drivers: redis (raw: false)", () => { const driver = redisDriver({ base: "test:", url: "ioredis://localhost:6379/0", @@ -59,12 +59,14 @@ describe("drivers: redis", () => { }); }, }); +}); +describe("drivers: redis (raw: true)", () => { const binaryDriver = redisDriver({ base: "test:", url: "ioredis://localhost:6379/0", lazyConnect: false, - saveRawAsBinary: true, + raw: true, }); testDriver({ From 74712ec79381b3e76c4a2c3e09a5a02a8a85ba1c Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 3 Jan 2025 17:38:07 +0100 Subject: [PATCH 3/6] update docs --- docs/2.drivers/redis.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/2.drivers/redis.md b/docs/2.drivers/redis.md index 9b94fe41..721f34b5 100644 --- a/docs/2.drivers/redis.md +++ b/docs/2.drivers/redis.md @@ -72,7 +72,7 @@ const storage = createStorage({ - `cluster`: List of redis nodes to use for cluster mode. Takes precedence over `url` and `host` options. - `clusterOptions`: Options to use for cluster mode. - `ttl`: Default TTL for all items in **seconds**. -- `useRaw`: If enabled, `getItemRaw` and `setItemRaw` will use binary data instead of base64 encoded strings. (this option will be enabled by default in the next major version.) +- `raw`: If enabled, `getItemRaw` and `setItemRaw` will use binary data instead of base64 encoded strings. (this option will be enabled by default in the next major version.) See [ioredis](https://github.com/luin/ioredis/blob/master/API.md#new-redisport-host-options) for all available options. From fb2977a9855bad1637798d87eb82ad850bc87e70 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 3 Jan 2025 18:06:13 +0100 Subject: [PATCH 4/6] improve normalize support --- src/drivers/redis.ts | 63 +++++++++++++++++++++++++------------- test/drivers/redis.test.ts | 12 -------- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/drivers/redis.ts b/src/drivers/redis.ts index bbde680e..b45c7f27 100644 --- a/src/drivers/redis.ts +++ b/src/drivers/redis.ts @@ -91,31 +91,13 @@ export default defineDriver((opts: RedisOptions) => { }, setItemRaw: opts.raw === true - ? async ( - key: string, - value: Uint8Array, - tOptions: TransactionOptions - ) => { - let valueToSave: Buffer; - if (value instanceof Uint8Array) { - if (value instanceof Buffer) { - valueToSave = value; - } else { - valueToSave = Buffer.copyBytesFrom( - value, - value.byteOffset, - value.byteLength - ); - } - } else { - throw createError(DRIVER_NAME, "Expected Buffer or Uint8Array"); - } - + ? async (key: string, value: unknown, tOptions: TransactionOptions) => { + const _value = normalizeValue(value); const ttl = tOptions?.ttl ?? opts.ttl; if (ttl) { - await getRedisClient().set(p(key), valueToSave, "EX", ttl); + await getRedisClient().set(p(key), _value, "EX", ttl); } else { - await getRedisClient().set(p(key), valueToSave); + await getRedisClient().set(p(key), _value); } } : undefined, @@ -140,3 +122,40 @@ export default defineDriver((opts: RedisOptions) => { }, }; }); + +function normalizeValue(value: unknown): Buffer | string | number { + const type = typeof value; + if (type === "string" || type === "number") { + return value as string | number; + } + if (Buffer.isBuffer(value)) { + return value; + } + if (isTypedArray(value)) { + if (Buffer.copyBytesFrom) { + return Buffer.copyBytesFrom(value, value.byteOffset, value.byteLength); + } else { + return Buffer.from(value.buffer, value.byteOffset, value.byteLength); + } + } + if (value instanceof ArrayBuffer) { + return Buffer.from(value); + } + return JSON.stringify(value); +} + +function isTypedArray(value: unknown): value is TypedArray { + return ( + value instanceof Int8Array || + value instanceof Uint8Array || + value instanceof Uint8ClampedArray || + value instanceof Int16Array || + value instanceof Uint16Array || + value instanceof Int32Array || + value instanceof Uint32Array || + value instanceof Float32Array || + value instanceof Float64Array || + value instanceof BigInt64Array || + value instanceof BigUint64Array + ); +} diff --git a/test/drivers/redis.test.ts b/test/drivers/redis.test.ts index 5f0bb6ad..86e9ce29 100644 --- a/test/drivers/redis.test.ts +++ b/test/drivers/redis.test.ts @@ -93,18 +93,6 @@ describe("drivers: redis (raw: true)", () => { await client.disconnect(); }); - - it("expects binary data to be sent to setItemRaw", async () => { - expect(() => - ctx.storage.setItemRaw("s4:a", "Hello, world!") - ).rejects.toThrow("Expected Buffer or Uint8Array"); - expect(() => ctx.storage.setItemRaw("s5:a", 100)).rejects.toThrow( - "Expected Buffer or Uint8Array" - ); - expect(() => - ctx.storage.setItemRaw("s5:a", { foo: "bar" }) - ).rejects.toThrow("Expected Buffer or Uint8Array"); - }); }, }); }); From 79d470f236bacc6d6e7317ab9f258387dea52de1 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 3 Jan 2025 18:09:41 +0100 Subject: [PATCH 5/6] lint --- src/drivers/redis.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/redis.ts b/src/drivers/redis.ts index b45c7f27..6681ab92 100644 --- a/src/drivers/redis.ts +++ b/src/drivers/redis.ts @@ -1,5 +1,5 @@ import type { TransactionOptions } from "../types"; -import { defineDriver, joinKeys, createError } from "./utils"; +import { defineDriver, joinKeys } from "./utils"; // TODO: use named import in v2 import Redis, { Cluster, From b92591eb6833821283357c56f4678a809bc08032 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 3 Jan 2025 18:10:28 +0100 Subject: [PATCH 6/6] use inherited type --- src/drivers/redis.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/drivers/redis.ts b/src/drivers/redis.ts index 6681ab92..e19b374b 100644 --- a/src/drivers/redis.ts +++ b/src/drivers/redis.ts @@ -1,4 +1,3 @@ -import type { TransactionOptions } from "../types"; import { defineDriver, joinKeys } from "./utils"; // TODO: use named import in v2 import Redis, { @@ -91,7 +90,7 @@ export default defineDriver((opts: RedisOptions) => { }, setItemRaw: opts.raw === true - ? async (key: string, value: unknown, tOptions: TransactionOptions) => { + ? async (key, value, tOptions) => { const _value = normalizeValue(value); const ttl = tOptions?.ttl ?? opts.ttl; if (ttl) {