From 5c3f4be13fef8bf64c98f8e4f3cd25ef7f4f4606 Mon Sep 17 00:00:00 2001 From: Karsten Schmidt Date: Mon, 22 Jul 2024 13:15:32 +0200 Subject: [PATCH] feat(pixel-convolve): import as new pkg (#486) - migrate convolve, normalMap and imagePyramid functions from thi.ng/pixel pkg --- packages/pixel-convolve/LICENSE | 201 ++++++++ packages/pixel-convolve/README.md | 201 ++++++++ packages/pixel-convolve/api-extractor.json | 3 + packages/pixel-convolve/package.json | 103 ++++ packages/pixel-convolve/src/api.ts | 97 ++++ packages/pixel-convolve/src/convolve.ts | 561 +++++++++++++++++++++ packages/pixel-convolve/src/index.ts | 4 + packages/pixel-convolve/src/normal-map.ts | 53 ++ packages/pixel-convolve/src/pyramid.ts | 31 ++ packages/pixel-convolve/test/main.test.ts | 4 + packages/pixel-convolve/test/tsconfig.json | 8 + packages/pixel-convolve/tpl.readme.md | 138 +++++ packages/pixel-convolve/tsconfig.json | 9 + yarn.lock | 12 + 14 files changed, 1425 insertions(+) create mode 100644 packages/pixel-convolve/LICENSE create mode 100644 packages/pixel-convolve/README.md create mode 100644 packages/pixel-convolve/api-extractor.json create mode 100644 packages/pixel-convolve/package.json create mode 100644 packages/pixel-convolve/src/api.ts create mode 100644 packages/pixel-convolve/src/convolve.ts create mode 100644 packages/pixel-convolve/src/index.ts create mode 100644 packages/pixel-convolve/src/normal-map.ts create mode 100644 packages/pixel-convolve/src/pyramid.ts create mode 100644 packages/pixel-convolve/test/main.test.ts create mode 100644 packages/pixel-convolve/test/tsconfig.json create mode 100644 packages/pixel-convolve/tpl.readme.md create mode 100644 packages/pixel-convolve/tsconfig.json diff --git a/packages/pixel-convolve/LICENSE b/packages/pixel-convolve/LICENSE new file mode 100644 index 0000000000..8dada3edaf --- /dev/null +++ b/packages/pixel-convolve/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/pixel-convolve/README.md b/packages/pixel-convolve/README.md new file mode 100644 index 0000000000..1aa56b0d67 --- /dev/null +++ b/packages/pixel-convolve/README.md @@ -0,0 +1,201 @@ + + +# ![@thi.ng/pixel-convolve](https://media.thi.ng/umbrella/banners-20230807/thing-pixel-convolve.svg?68dd5cd4) + +[![npm version](https://img.shields.io/npm/v/@thi.ng/pixel-convolve.svg)](https://www.npmjs.com/package/@thi.ng/pixel-convolve) +![npm downloads](https://img.shields.io/npm/dm/@thi.ng/pixel-convolve.svg) +[![Mastodon Follow](https://img.shields.io/mastodon/follow/109331703950160316?domain=https%3A%2F%2Fmastodon.thi.ng&style=social)](https://mastodon.thi.ng/@toxi) + +> [!NOTE] +> This is one of 198 standalone projects, maintained as part +> of the [@thi.ng/umbrella](https://github.com/thi-ng/umbrella/) monorepo +> and anti-framework. +> +> 🚀 Please help me to work full-time on these projects by [sponsoring me on +> GitHub](https://github.com/sponsors/postspectacular). Thank you! ❤️ + +- [About](#about) + - [Strided convolution & pooling](#strided-convolution--pooling) + - [Normal map generation](#normal-map-generation) +- [Status](#status) +- [Installation](#installation) +- [Dependencies](#dependencies) +- [API](#api) +- [Authors](#authors) +- [License](#license) + +## About + +Extensible bitmap image convolution, kernel presets, normal map & image pyramid generation. This is a support package for [@thi.ng/pixel](https://github.com/thi-ng/umbrella/tree/develop/packages/pixel). + +This package contains functionality which was previously part of and has been +extracted from the [@thi.ng/pixel](https://thi.ng/pixel) package. + +- Convolution w/ arbitrary shaped/sized kernels, pooling, striding +- Convolution kernel & pooling kernels presets + - Higher order kernel generators (Gaussian, Lanczos) +- Image pooling filters (min/max, mean, adaptive threshold, custom) +- Image pyramid generation (w/ customizable kernels) +- Customizable normal map generation (i.e. X/Y gradients plus static Z component) + +### Strided convolution & pooling + +Floating point buffers can be processed using arbitrary convolution kernels. The +following convolution kernel presets are provided for convenience: + +| Kernel | Size | +|------------------|-------------| +| `BOX_BLUR3` | 3x3 | +| `BOX_BLUR5` | 5x5 | +| `GAUSSIAN_BLUR3` | 3x3 | +| `GAUSSIAN_BLUR5` | 5x5 | +| `GAUSSIAN(n)` | 2n+1 x 2n+1 | +| `HIGHPASS3` | 3x3 | +| `LANCZOS(a,s)` | as+1 x as+1 | +| `SHARPEN3` | 3x3 | +| `SOBEL_X` | 3x3 | +| `SOBEL_Y` | 3x3 | +| `UNSHARP_MASK5` | 5x5 | + +Custom kernels can be defined (and code generated) using an array of +coefficients and a given kernel size. See above presets and +[`defKernel()`](https://docs.thi.ng/umbrella/pixel/functions/defKernel.html) for +reference. + +Furthermore, convolution supports striding (i.e. only processing & keeping every +nth pixel column/row, aka downscaling) and pixel pooling (e.g. for ML +applications). Available pooling kernel presets (kernel sizes must be configured +independently): + +| Kernel | Description | +|------------------------|--------------------| +| `POOL_MEAN` | Moving average | +| `POOL_MAX` | Local maximum | +| `POOL_MIN` | Local minimum | +| `POOL_NEAREST` | Nearest neighbor | +| `POOL_THRESHOLD(bias)` | Adaptive threshold | + +Convolution can be applied to single, multiple or all channels of a +`FloatBuffer`. See +[`convolveChannel()`](https://docs.thi.ng/umbrella/pixel/functions/convolveChannel.html) +and +[`convolveImage()`](https://docs.thi.ng/umbrella/pixel/functions/convolveImage.html). + +See +[ConvolveOpts](https://docs.thi.ng/umbrella/pixel/interfaces/ConvolveOpts.html) +for config options. + +```js tangle:export/readme-convolve.ts +import { floatBufferFromImage, FLOAT_RGB, imageFromURL } from "@thi.ng/pixel"; +import { convolveImage, SOBEL_X } from "@thi.ng/pixel-convolve"; + +// convolutions are only available for float buffers (for now) +const src = floatBufferFromImage(await imageFromURL("test.jpg"), FLOAT_RGB); + +// apply horizontal Sobel kernel preset to all channels +// downscale image by factor 2 (must be integer) +// scale kernel result values by factor 4 +const dest = convolveImage(src, { kernel: SOBEL_X, stride: 2, scale: 4 }); +``` + +### Normal map generation + +Normal maps can be created via `normalMap()`. This function uses an adjustable +convolution kernel size to control gradient smoothness & details. Result X/Y +gradients can also be scaled (uniform or anisotropic) and the Z component can be +customized to (default: 1.0). The resulting image is in `FLOAT_NORMAL` format, +using signed channel values. This channel format is auto-translating these into +unsigned values when the image is converted into an integer format. + +| Step | Scale = 1 | Scale = 2 | Scale = 4 | Scale = 8 | +|------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------| +| 0 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-8.jpg) | +| 1 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-8.jpg) | +| 2 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-8.jpg) | +| 3 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-8.jpg) | + +```ts tangle:export/readme-normalmap.ts +import { ARGB8888, FLOAT_GRAY, floatBufferFromImage, imageFromURL } from "@thi.ng/pixel"; +import { normalMap } from "@thi.ng/pixel-convolve"; + +// read source image into a single channel floating point buffer +const src = floatBufferFromImage(await imageFromURL("noise.png"), FLOAT_GRAY); + +// create normal map (w/ default options) +// this results in a new float pixel buffer with FLOAT_RGB format +const nmap = normalMap(src, { step: 0, scale: 1 }); + +// pixel lookup (vectors are stored _un_normalized) +nmap.getAt(10, 10); +// Float32Array(3) [ -0.019607841968536377, -0.04313725233078003, 1 ] + +// convert to 32bit packed int format +const nmapARGB = nmap.as(ARGB8888); +``` + +## Status + +**STABLE** - used in production + +[Search or submit any issues for this package](https://github.com/thi-ng/umbrella/issues?q=%5Bpixel-convolve%5D+in%3Atitle) + +## Installation + +```bash +yarn add @thi.ng/pixel-convolve +``` + +ESM import: + +```ts +import * as pc from "@thi.ng/pixel-convolve"; +``` + +Browser ESM import: + +```html + +``` + +[JSDelivr documentation](https://www.jsdelivr.com/) + +For Node.js REPL: + +```js +const pc = await import("@thi.ng/pixel-convolve"); +``` + +Package sizes (brotli'd, pre-treeshake): ESM: 2.27 KB + +## Dependencies + +- [@thi.ng/api](https://github.com/thi-ng/umbrella/tree/develop/packages/api) +- [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks) +- [@thi.ng/errors](https://github.com/thi-ng/umbrella/tree/develop/packages/errors) +- [@thi.ng/math](https://github.com/thi-ng/umbrella/tree/develop/packages/math) +- [@thi.ng/pixel](https://github.com/thi-ng/umbrella/tree/develop/packages/pixel) + +Note: @thi.ng/api is in _most_ cases a type-only import (not used at runtime) + +## API + +[Generated API docs](https://docs.thi.ng/umbrella/pixel-convolve/) + +## Authors + +- [Karsten Schmidt](https://thi.ng) + +If this project contributes to an academic publication, please cite it as: + +```bibtex +@misc{thing-pixel-convolve, + title = "@thi.ng/pixel-convolve", + author = "Karsten Schmidt", + note = "https://thi.ng/pixel-convolve", + year = 2021 +} +``` + +## License + +© 2021 - 2024 Karsten Schmidt // Apache License 2.0 diff --git a/packages/pixel-convolve/api-extractor.json b/packages/pixel-convolve/api-extractor.json new file mode 100644 index 0000000000..bc73f2cc02 --- /dev/null +++ b/packages/pixel-convolve/api-extractor.json @@ -0,0 +1,3 @@ +{ + "extends": "../../api-extractor.json" +} diff --git a/packages/pixel-convolve/package.json b/packages/pixel-convolve/package.json new file mode 100644 index 0000000000..dc750b1acf --- /dev/null +++ b/packages/pixel-convolve/package.json @@ -0,0 +1,103 @@ +{ + "name": "@thi.ng/pixel-convolve", + "version": "0.0.1", + "description": "Extensible bitmap image convolution, kernel presets, normal map & image pyramid generation", + "type": "module", + "module": "./index.js", + "typings": "./index.d.ts", + "sideEffects": false, + "repository": { + "type": "git", + "url": "https://github.com/thi-ng/umbrella.git" + }, + "homepage": "https://thi.ng/pixel-convolve", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/postspectacular" + }, + { + "type": "patreon", + "url": "https://patreon.com/thing_umbrella" + } + ], + "author": "Karsten Schmidt (https://thi.ng)", + "license": "Apache-2.0", + "scripts": { + "build": "yarn build:esbuild && yarn build:decl", + "build:decl": "tsc --declaration --emitDeclarationOnly", + "build:esbuild": "esbuild --format=esm --platform=neutral --target=es2022 --tsconfig=tsconfig.json --outdir=. src/**/*.ts", + "clean": "bun ../../tools/src/clean-package.ts", + "doc": "typedoc --excludePrivate --excludeInternal --out doc src/index.ts", + "doc:ae": "mkdir -p .ae/doc .ae/temp && api-extractor run --local --verbose", + "doc:readme": "bun ../../tools/src/module-stats.ts && bun ../../tools/src/readme.ts", + "pub": "yarn npm publish --access public", + "test": "bun test", + "tool:tangle": "../../node_modules/.bin/tangle src/**/*.ts" + }, + "dependencies": { + "@thi.ng/api": "^8.11.6", + "@thi.ng/checks": "^3.6.8", + "@thi.ng/errors": "^2.5.12", + "@thi.ng/math": "^5.11.4", + "@thi.ng/pixel": "^6.1.37" + }, + "devDependencies": { + "@microsoft/api-extractor": "^7.47.0", + "esbuild": "^0.23.0", + "typedoc": "^0.26.3", + "typescript": "^5.5.3" + }, + "keywords": [ + "blur", + "channel", + "convolution", + "edge", + "float", + "gaussian", + "mean", + "normal", + "pixel", + "pool", + "pyramid", + "resize", + "sample", + "sharpen", + "typescript" + ], + "publishConfig": { + "access": "public" + }, + "browser": { + "process": false, + "setTimeout": false + }, + "engines": { + "node": ">=18" + }, + "files": [ + "./*.js", + "./*.d.ts" + ], + "exports": { + ".": { + "default": "./index.js" + }, + "./api": { + "default": "./api.js" + }, + "./convolve": { + "default": "./convolve.js" + }, + "./normal-map": { + "default": "./normal-map.js" + }, + "./pyramid": { + "default": "./pyramid.js" + } + }, + "thi.ng": { + "parent": "@thi.ng/pixel", + "year": 2021 + } +} diff --git a/packages/pixel-convolve/src/api.ts b/packages/pixel-convolve/src/api.ts new file mode 100644 index 0000000000..22821763f0 --- /dev/null +++ b/packages/pixel-convolve/src/api.ts @@ -0,0 +1,97 @@ +import type { FloatArray, Fn, Fn3, FnN3, NumericArray } from "@thi.ng/api"; +import type { IPixelBuffer } from "@thi.ng/pixel"; + +export type PoolTemplate = Fn3; + +export interface ConvolutionKernelSpec { + /** + * Kernel coefficients. + */ + spec: NumericArray; + /** + * Kernel size. If given as number, expands to `[size, size]`. + */ + size: number | [number, number]; +} + +export interface PoolKernelSpec { + /** + * Code template function for {@link defKernel}. + */ + pool: PoolTemplate; + /** + * Kernel size. If given as number, expands to `[size, size]`. + */ + size: number | [number, number]; +} + +export interface KernelFnSpec { + /** + * Kernel factory. + */ + fn: Fn, FnN3>; + /** + * Kernel size. If given as number, expands to `[size, size]`. + */ + size: number | [number, number]; +} + +export type KernelSpec = ConvolutionKernelSpec | PoolKernelSpec | KernelFnSpec; + +export interface ConvolveOpts { + /** + * Convolution kernel details/implementation. + */ + kernel: KernelSpec; + /** + * Channel ID to convolve. + * + * @defaultValue 0 + */ + channel?: number; + /** + * Result scale factor + * + * @defaultValue 1 + */ + scale?: number; + /** + * Step size to process only every nth pixel. + * + * @defaultValue 1 + */ + stride?: number | [number, number]; + /** + * Pixel read offset, only to be used for pooling operations. Should be set + * to `kernelSize/2`, and for the X-axis MUST be in `[0,stride)` interval. + */ + offset?: number | [number, number]; +} + +export interface NormalMapOpts { + /** + * Channel ID to use for gradient extraction in source image. + * + * @defaultValue 0 + */ + channel: number; + /** + * Step size (aka number of pixels) between left/right, top/bottom + * neighbors. + * + * @defaultValue 0 + */ + step: number; + /** + * Result gradient scale factor(s). + * + * @defaultValue 1 + */ + scale: number | [number, number]; + /** + * Z-axis value to use in blue channel of normal map. + * + * @defaultValue 1 + */ + z: number; +} diff --git a/packages/pixel-convolve/src/convolve.ts b/packages/pixel-convolve/src/convolve.ts new file mode 100644 index 0000000000..ba06d7fb3b --- /dev/null +++ b/packages/pixel-convolve/src/convolve.ts @@ -0,0 +1,561 @@ +import type { Fn, FnN3, NumericArray } from "@thi.ng/api"; +import { isFunction } from "@thi.ng/checks/is-function"; +import { assert } from "@thi.ng/errors/assert"; +import { clamp } from "@thi.ng/math/interval"; +import { lanczos } from "@thi.ng/math/mix"; +import { ensureChannel } from "@thi.ng/pixel/checks"; +import { FloatBuffer } from "@thi.ng/pixel/float"; +import { FLOAT_GRAY } from "@thi.ng/pixel/format/float-gray"; +import { __range } from "@thi.ng/pixel/internal/range"; +import { __asIntVec } from "@thi.ng/pixel/internal/utils"; +import type { + ConvolutionKernelSpec, + ConvolveOpts, + KernelFnSpec, + KernelSpec, + PoolKernelSpec, + PoolTemplate, +} from "./api.js"; + +/** + * Convolves a single channel from given `src` float buffer with provided + * convolution or pooling kernel with support for strided sampling (resulting in + * smaller dimensions). Returns result as single channel buffer (in + * {@link FLOAT_GRAY} format). + * + * @remarks + * Use {@link convolveImage} to process multiple or all channels in a buffer. + * + * References: + * - https://en.wikipedia.org/wiki/Kernel_(image_processing) + * + * @param src - + * @param opts - + */ +export const convolveChannel = (src: FloatBuffer, opts: ConvolveOpts) => + __convolve(__initConvolve(src, opts)); + +/** + * Similar to {@link convolveChannel}, but processes multiple or all channels + * (default) in a buffer and returns a new buffer in same format as original. + * + * @remarks + * This function re-uses as much as internal state & memory as possible, so will + * be faster than individual applications of {@link convolveChannel}. + * + * @param src - + * @param opts - + */ +export const convolveImage = ( + src: FloatBuffer, + opts: Exclude & { channels?: number[] } +) => { + const state = __initConvolve(src, opts); + const dest = new FloatBuffer(state.dwidth, state.dheight, src.format); + for (let channel of opts.channels || __range(src.format.channels.length)) { + dest.setChannel(channel, __convolve({ ...state, channel })); + } + return dest; +}; + +/** @internal */ +const __convolve = ({ + channel, + dest, + dwidth, + dheight, + kernel, + offsetX, + offsetY, + rowStride, + scale, + src, + srcStride, + strideX, + strideY, +}: ReturnType) => { + ensureChannel(src.format, channel); + const dpix = dest.data; + const stepX = strideX * srcStride; + const stepY = strideY * rowStride; + for ( + let sy = offsetY * rowStride, dy = 0, i = 0; + dy < dheight; + sy += stepY, dy++ + ) { + for ( + let sx = offsetX * srcStride + channel, dx = 0; + dx < dwidth; + sx += stepX, dx++, i++ + ) { + dpix[i] = kernel(sx, sy, channel) * scale; + } + } + return dest; +}; + +/** @internal */ +const __initKernel = ( + src: FloatBuffer, + kernel: KernelSpec, + kw: number, + kh: number +) => + (isFunction((kernel).fn) + ? (kernel).fn + : defKernel( + (kernel).spec || + (kernel).pool, + kw, + kh + ))(src); + +/** @internal */ +const __initConvolve = (src: FloatBuffer, opts: ConvolveOpts) => { + const { + channel = 0, + offset = 0, + scale = 1, + stride: sampleStride = 1, + kernel, + } = opts; + const size = kernel.size; + const [kw, kh] = __asIntVec(size); + const [strideX, strideY] = __asIntVec(sampleStride); + const [offsetX, offsetY] = __asIntVec(offset); + assert(strideX >= 1 && strideY >= 1, `illegal stride: ${sampleStride}`); + const { + size: [width, height], + stride: [srcStride, rowStride], + } = src; + const dwidth = Math.floor(width / strideX); + const dheight = Math.floor(height / strideY); + assert(dwidth > 0 && dheight > 0, `too large stride(s) for given image`); + const dest = new FloatBuffer(dwidth, dheight, FLOAT_GRAY); + return { + channel, + dest, + dheight, + dwidth, + kernel: __initKernel(src, kernel, kw, kh), + offsetX, + offsetY, + rowStride, + scale, + src, + srcStride, + strideX, + strideY, + }; +}; + +/** @internal */ +const __declOffset = ( + idx: number, + i: number, + pre: string, + stride: string, + min: string, + max: string +) => + idx < 0 + ? `const ${pre}${i} = max(${pre}${ + idx < -1 ? idx + "*" : "-" + }${stride},${min});` + : `const ${pre}${i} = min(${pre}+${ + idx > 1 ? idx + "*" : "" + }${stride},${max});`; + +/** + * HOF convolution or pooling kernel code generator. Takes either a + * {@link PoolTemplate} function or array of kernel coefficients and kernel + * width/height. Returns optimized kernel function for use with + * {@link __convolve}. If `normalize` is true (default: false), the given + * coefficients are divided by their sum (only used if provided as array). + * + * @remarks + * If total kernel size (width * height) is < 512, the result function will use + * unrolled loops to access pixels and hence kernel sizes shouldn't be larger + * than ~22x22 to avoid excessive function bodies. For dynamically generated + * kernel functions, only non-zero weighted pixels will be included in the + * result function to avoid extraneous lookups. Row & column offsets are + * pre-calculated too. Larger kernel sizes are handled via + * {@link defLargeKernel}. + * + * @param tpl - + * @param w - + * @param h - + * @param normalize - + */ +export const defKernel = ( + tpl: NumericArray | PoolTemplate, + w: number, + h: number, + normalize = false +) => { + if (w * h > 512 && !isFunction(tpl)) + return defLargeKernel(tpl, w, h, normalize); + const isPool = isFunction(tpl); + const prefix: string[] = []; + const body: string[] = []; + const kvars: string[] = []; + const h2 = h >> 1; + const w2 = w >> 1; + if (normalize) tpl = __normalize(tpl); + for (let y = 0, i = 0; y < h; y++) { + const yy = y - h2; + const row: string[] = []; + for (let x = 0; x < w; x++, i++) { + const kv = `k${y}_${x}`; + kvars.push(kv); + const xx = x - w2; + const idx = + (yy !== 0 ? `y${y}` : `y`) + (xx !== 0 ? `+x${x}` : "+x"); + isPool + ? row.push(`pix[${idx}]`) + : (tpl)[i] !== 0 && row.push(`${kv}*pix[${idx}]`); + if (y === 0 && xx !== 0) { + prefix.push( + __declOffset( + xx, + x, + "x", + "stride", + "channel", + "maxX+channel" + ) + ); + } + } + row.length && body.push(...row); + if (yy !== 0) { + prefix.push(__declOffset(yy, y, "y", "rowStride", "0", "maxY")); + } + } + const decls = isPool + ? "" + : `const [${kvars.join(", ")}] = [${(tpl).join(", ")}];`; + const inner = isPool ? (tpl)(body, w, h) : body.join(" + "); + const fnBody = [ + decls, + "const { min, max } = Math;", + "const { data: pix, stride: [stride, rowStride] } = src;", + "const maxX = (src.width - 1) * stride;", + "const maxY = (src.height - 1) * rowStride;", + "return (x, y, channel) => {", + ...prefix, + `return ${inner};`, + "}", + ].join("\n"); + // console.log(fnBody); + return >new Function("src", fnBody); +}; + +/** + * Loop based fallback for {@link defKernel}, intended for larger kernel sizes + * for which loop-unrolled approach is prohibitive. If `normalize` is true + * (default: false), the given coefficients are divided by their sum. + * + * @param kernel - + * @param w - + * @param h - + * @param normalize - + */ +export const defLargeKernel = ( + kernel: NumericArray, + w: number, + h: number, + normalize = false +): Fn => { + if (normalize) kernel = __normalize(kernel); + return (src) => { + const { + data, + stride: [stride, rowStride], + } = src; + const x0 = -(w >> 1) * stride; + const x1 = -x0 + (w & 1 ? stride : 0); + const y0 = -(h >> 1) * rowStride; + const y1 = -y0 + (h & 1 ? rowStride : 0); + const maxX = (src.width - 1) * stride; + const maxY = (src.height - 1) * rowStride; + return (xx, yy, channel) => { + const $maxX = maxX + channel; + let sum = 0, + y: number, + x: number, + k: number, + row: number; + for (y = y0, k = 0; y < y1; y += rowStride) { + for ( + x = x0, row = clamp(yy + y, 0, maxY); + x < x1; + x += stride, k++ + ) { + sum += + kernel[k] * data[row + clamp(xx + x, channel, $maxX)]; + } + } + return sum; + }; + }; +}; + +/** @internal */ +const __normalize = (kernel: NumericArray) => { + const scale = 1 / (kernel).reduce((acc, x) => acc + x, 0); + return (kernel).map((x) => x * scale); +}; + +export const POOL_NEAREST: PoolTemplate = (body, w, h) => + body[(h >> 1) * w + (w >> 1)]; + +export const POOL_MEAN: PoolTemplate = (body, w, h) => + `(${body.join("+")})*${1 / (w * h)}`; + +export const POOL_MIN: PoolTemplate = (body) => `Math.min(${body.join(",")})`; + +export const POOL_MAX: PoolTemplate = (body) => `Math.max(${body.join(",")})`; + +/** + * Higher order adaptive threshold {@link PoolTemplate}. Computes: `step(C - + * mean(K) + B)`, where `C` is the center pixel, `K` the entire set of pixels in + * the kernel and `B` an arbitrary bias/offset value. + * + * @example + * ```ts + * import { convolveChannel, POOL_THRESHOLD } from "@thi.ng/pixel"; + * + * // 3x3 adaptive threshold w/ bias = 1 + * convolveChannel(src, { kernel: { pool: POOL_THRESHOLD(1), size: 3 }}); + * ``` + * + * @param bias - + */ +export const POOL_THRESHOLD = + (bias = 0): PoolTemplate => + (body, w, h) => { + const center = POOL_NEAREST(body, w, h); + const mean = `(${body.join("+")})/${w * h}`; + return `(${center} - ${mean} + ${bias}) < 0 ? 0 : 1`; + }; + +export const SOBEL_X: KernelSpec = { + spec: [-1, -2, -1, 0, 0, 0, 1, 2, 1], + size: 3, +}; + +export const SOBEL_Y: KernelSpec = { + spec: [-1, 0, 1, -2, 0, 2, -1, 0, 1], + size: 3, +}; + +export const SHARPEN3: KernelSpec = { + spec: [0, -1, 0, -1, 5, -1, 0, -1, 0], + size: 3, +}; + +export const HIGHPASS3: KernelSpec = { + spec: [-1, -1, -1, -1, 9, -1, -1, -1, -1], + size: 3, +}; + +export const BOX_BLUR3: KernelSpec = { + pool: POOL_MEAN, + size: 3, +}; + +export const BOX_BLUR5: KernelSpec = { + pool: POOL_MEAN, + size: 5, +}; + +export const GAUSSIAN_BLUR3: KernelSpec = { + spec: [1 / 16, 1 / 8, 1 / 16, 1 / 8, 1 / 4, 1 / 8, 1 / 16, 1 / 8, 1 / 16], + size: 3, +}; + +export const GAUSSIAN_BLUR5: KernelSpec = { + // prettier-ignore + spec: [ + 1 / 256, 1 / 64, 3 / 128, 1 / 64, 1 / 256, + 1 / 64, 1 / 16, 3 / 32, 1 / 16, 1 / 64, + 3 / 128, 3 / 32, 9 / 64, 3 / 32, 3 / 128, + 1 / 64, 1 / 16, 3 / 32, 1 / 16, 1 / 64, + 1 / 256, 1 / 64, 3 / 128, 1 / 64, 1 / 256, + ], + size: 5, +}; + +/** + * Higher order Gaussian blur kernel for given pixel radius `r` (integer). + * Returns {@link ConvolutionKernelSpec} with resulting kernel size of `2r+1`. + * + * @param r - + */ +export const GAUSSIAN = (r: number): ConvolutionKernelSpec => { + r |= 0; + assert(r > 0, `invalid kernel radius: ${r}`); + const sigma = -1 / (2 * (Math.hypot(r, r) / 3) ** 2); + const res: number[] = []; + let sum = 0; + for (let y = -r; y <= r; y++) { + for (let x = -r; x <= r; x++) { + const g = Math.exp((x * x + y * y) * sigma); + res.push(g); + sum += g; + } + } + return { spec: res.map((x) => x / sum), size: r * 2 + 1 }; +}; + +/** + * Higher-order Lanczos filter kernel generator for given `a` value (recommended + * 2 or 3) and `scale` (num pixels per `a`). + * + * @remarks + * https://en.wikipedia.org/wiki/Lanczos_resampling#Lanczos_kernel + * + * @param a - + * @param scale - + */ +export const LANCZOS = (a: number, scale = 2): ConvolutionKernelSpec => { + assert(a > 0, `invalid coefficient: ${a}`); + const r = Math.ceil(a * scale); + const res: number[] = []; + let sum = 0; + for (let y = -r; y <= r; y++) { + const yy = y / scale; + const ly = lanczos(a, yy); + for (let x = -r; x <= r; x++) { + const m = Math.hypot(x / scale, yy); + const l = m < a ? ly * lanczos(a, x / scale) : 0; + res.push(l); + sum += l; + } + } + return { spec: res.map((x) => x / sum), size: r * 2 + 1 }; +}; + +export const UNSHARP_MASK5: KernelSpec = { + // prettier-ignore + spec: [ + -1 / 256, -1 / 64, -3 / 128, -1 / 64, -1 / 256, + -1 / 64, -1 / 16, -3 / 32, -1 / 16, -1 / 64, + -3 / 128, -3 / 32, 119 / 64, -3 / 32, -3 / 128, + -1 / 64, -1 / 16, -3 / 32, -1 / 16, -1 / 64, + -1 / 256, -1 / 64, -3 / 128, -1 / 64, -1 / 256, + ], + size: 5, +}; + +const { min, max } = Math; + +/** + * 3x3 convolution kernel to detect local maxima in a Von Neumann neighborhood. + * Returns in 1.0 if the center pixel is either higher valued than A & D or B & C, + * otherwise return zero. + * + * @remarks + * ```text + * |---|---|---| + * | | A | | + * |---|---|---| + * | B | X | C | + * |---|---|---| + * | | D | | + * |---|---|---| + * ``` + * + * Also see {@link MAXIMA4_DIAG} for alternative. + */ +export const MAXIMA4_CROSS: KernelFnSpec = { + fn: (src) => { + const { + data: pix, + stride: [stride, rowStride], + } = src; + const maxX = (src.width - 1) * stride; + const maxY = (src.height - 1) * rowStride; + return (x, y, channel) => { + const x0 = max(x - stride, channel); + const x2 = min(x + stride, maxX + channel); + const y0 = max(y - rowStride, 0); + const y2 = min(y + rowStride, maxY); + const c = pix[x + y]; + return (c > pix[y + x0] && c > pix[y + x2]) || + (c > pix[y0 + x] && c > pix[y2 + x]) + ? 1 + : 0; + }; + }, + size: 3, +}; + +/** + * Similar to {@link MAXIMA4_CROSS}, a 3x3 convolution kernel to detect local + * maxima in a 45 degree rotated Von Neumann neighborhood. Returns in 1.0 if the + * center pixel is either higher valued than A & D or B & C, otherwise return + * zero. + * + * @remarks + * ```text + * |---|---|---| + * | A | | B | + * |---|---|---| + * | | X | | + * |---|---|---| + * | C | | D | + * |---|---|---| + * ``` + */ +export const MAXIMA4_DIAG: KernelFnSpec = { + fn: (src) => { + const { + data: pix, + stride: [stride, rowStride], + } = src; + const maxX = (src.width - 1) * stride; + const maxY = (src.height - 1) * rowStride; + return (x, y, channel) => { + const x0 = max(x - stride, channel); + const x2 = min(x + stride, maxX + channel); + const y0 = max(y - rowStride, 0); + const y2 = min(y + rowStride, maxY); + const c = pix[x + y]; + return (c > pix[y0 + x0] && c > pix[y2 + x2]) || + (c > pix[y0 + x2] && c > pix[y2 + x0]) + ? 1 + : 0; + }; + }, + size: 3, +}; + +/** + * Union kernel of {@link MAXIMA4_CROSS} and {@link MAXIMA4_DIAG}. + */ +export const MAXIMA8: KernelFnSpec = { + fn: (src) => { + const { + data: pix, + stride: [stride, rowStride], + } = src; + const maxX = (src.width - 1) * stride; + const maxY = (src.height - 1) * rowStride; + return (x, y, channel) => { + const x0 = max(x - stride, channel); + const x2 = min(x + stride, maxX + channel); + const y0 = max(y - rowStride, 0); + const y2 = min(y + rowStride, maxY); + const c = pix[x + y]; + return (c > pix[y + x0] && c > pix[y + x2]) || + (c > pix[y0 + x] && c > pix[y2 + x]) || + (c > pix[y0 + x0] && c > pix[y2 + x2]) || + (c > pix[y0 + x2] && c > pix[y2 + x0]) + ? 1 + : 0; + }; + }, + size: 3, +}; diff --git a/packages/pixel-convolve/src/index.ts b/packages/pixel-convolve/src/index.ts new file mode 100644 index 0000000000..34a7696d6c --- /dev/null +++ b/packages/pixel-convolve/src/index.ts @@ -0,0 +1,4 @@ +export * from "./api.js"; +export * from "./convolve.js"; +export * from "./normal-map.js"; +export * from "./pyramid.js"; diff --git a/packages/pixel-convolve/src/normal-map.ts b/packages/pixel-convolve/src/normal-map.ts new file mode 100644 index 0000000000..80f9ef567c --- /dev/null +++ b/packages/pixel-convolve/src/normal-map.ts @@ -0,0 +1,53 @@ +import { ensureChannel } from "@thi.ng/pixel/checks"; +import { FloatBuffer } from "@thi.ng/pixel/float"; +import { FLOAT_NORMAL } from "@thi.ng/pixel/format/float-norm"; +import { __asVec } from "@thi.ng/pixel/internal/utils"; +import type { NormalMapOpts } from "./api.js"; +import { convolveChannel } from "./convolve.js"; + +/** + * Computes normal map image (aka gradient in X & Y directions and a static Z + * value) for a single channel in given {@link FloatBuffer}. The resulting + * buffer will use the {@link FLOAT_NORMAL} format, storing the horizontal + * gradient in the 1st channel (red), vertical gradient in the 2nd channel + * (green) and sets last channel to given `z` value (blue). + * + * @remarks + * The gradient values will be scaled with `scale` (default: 1, but supports + * individual X/Y factors). Gradient values will be signed. + * + * The partial gradients of the last column/row will be set to zero + * (respectively). I.e. the right most pixel column will have `red = 0` and last + * row will have `green = 0`. + * + * @param src - + * @param opts - + */ +export const normalMap = ( + src: FloatBuffer, + opts: Partial = {} +) => { + const { channel = 0, step = 0, scale = 1, z = 1 } = opts; + ensureChannel(src.format, channel); + const spec = [-1, ...new Array(step).fill(0), 1]; + const [sx, sy] = __asVec(scale); + const dest = new FloatBuffer(src.width, src.height, FLOAT_NORMAL); + dest.setChannel( + 0, + convolveChannel(src, { + kernel: { spec, size: [step + 2, 1] }, + scale: sx, + channel, + }) + ); + dest.setChannel( + 1, + convolveChannel(src, { + kernel: { spec, size: [1, step + 2] }, + scale: sy, + channel, + }) + ); + dest.setChannel(2, z); + return dest; +}; diff --git a/packages/pixel-convolve/src/pyramid.ts b/packages/pixel-convolve/src/pyramid.ts new file mode 100644 index 0000000000..2d59c10034 --- /dev/null +++ b/packages/pixel-convolve/src/pyramid.ts @@ -0,0 +1,31 @@ +import { assert } from "@thi.ng/errors/assert"; +import type { FloatBuffer } from "@thi.ng/pixel/float"; +import type { KernelSpec } from "./api.js"; +import { convolveImage, LANCZOS } from "./convolve.js"; + +/** + * Yields an iterator of progressively downsampled versions of `src` (using + * `kernel` for filtering, default: {@link LANCZOS}(2)). Each image will be half + * size of the previous result, stopping only once either width or height + * becomes less than `minSize` (default: 1). If `includeOrig` is enabled + * (default), the first emitted image will be the original `src`. + * + * @param src - + * @param kernel - + * @param minSize - + * @param includeOrig - + */ +export function* imagePyramid( + src: FloatBuffer, + kernel: KernelSpec = LANCZOS(2), + minSize = 1, + includeOrig = true +) { + assert(minSize > 0, `invalid min size`); + minSize <<= 1; + if (includeOrig) yield src; + while (src.width >= minSize && src.height >= minSize) { + src = convolveImage(src, { kernel, stride: 2 }); + yield src; + } +} diff --git a/packages/pixel-convolve/test/main.test.ts b/packages/pixel-convolve/test/main.test.ts new file mode 100644 index 0000000000..a31ee4cad2 --- /dev/null +++ b/packages/pixel-convolve/test/main.test.ts @@ -0,0 +1,4 @@ +import { expect, test } from "bun:test"; +// import { } from "../src/index.js" + +test.todo("pixel-convolve", () => {}); diff --git a/packages/pixel-convolve/test/tsconfig.json b/packages/pixel-convolve/test/tsconfig.json new file mode 100644 index 0000000000..10a781ee02 --- /dev/null +++ b/packages/pixel-convolve/test/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "types": ["bun-types"], + "noEmit": true + }, + "include": ["./**/*.ts"] +} diff --git a/packages/pixel-convolve/tpl.readme.md b/packages/pixel-convolve/tpl.readme.md new file mode 100644 index 0000000000..1c07f5495b --- /dev/null +++ b/packages/pixel-convolve/tpl.readme.md @@ -0,0 +1,138 @@ + + + + +## About + +{{pkg.description}} + +This package contains functionality which was previously part of and has been +extracted from the [@thi.ng/pixel](https://thi.ng/pixel) package. + +- Convolution w/ arbitrary shaped/sized kernels, pooling, striding +- Convolution kernel & pooling kernels presets + - Higher order kernel generators (Gaussian, Lanczos) +- Image pooling filters (min/max, mean, adaptive threshold, custom) +- Image pyramid generation (w/ customizable kernels) +- Customizable normal map generation (i.e. X/Y gradients plus static Z component) + +### Strided convolution & pooling + +Floating point buffers can be processed using arbitrary convolution kernels. The +following convolution kernel presets are provided for convenience: + +| Kernel | Size | +|------------------|-------------| +| `BOX_BLUR3` | 3x3 | +| `BOX_BLUR5` | 5x5 | +| `GAUSSIAN_BLUR3` | 3x3 | +| `GAUSSIAN_BLUR5` | 5x5 | +| `GAUSSIAN(n)` | 2n+1 x 2n+1 | +| `HIGHPASS3` | 3x3 | +| `LANCZOS(a,s)` | as+1 x as+1 | +| `SHARPEN3` | 3x3 | +| `SOBEL_X` | 3x3 | +| `SOBEL_Y` | 3x3 | +| `UNSHARP_MASK5` | 5x5 | + +Custom kernels can be defined (and code generated) using an array of +coefficients and a given kernel size. See above presets and +[`defKernel()`](https://docs.thi.ng/umbrella/pixel/functions/defKernel.html) for +reference. + +Furthermore, convolution supports striding (i.e. only processing & keeping every +nth pixel column/row, aka downscaling) and pixel pooling (e.g. for ML +applications). Available pooling kernel presets (kernel sizes must be configured +independently): + +| Kernel | Description | +|------------------------|--------------------| +| `POOL_MEAN` | Moving average | +| `POOL_MAX` | Local maximum | +| `POOL_MIN` | Local minimum | +| `POOL_NEAREST` | Nearest neighbor | +| `POOL_THRESHOLD(bias)` | Adaptive threshold | + +Convolution can be applied to single, multiple or all channels of a +`FloatBuffer`. See +[`convolveChannel()`](https://docs.thi.ng/umbrella/pixel/functions/convolveChannel.html) +and +[`convolveImage()`](https://docs.thi.ng/umbrella/pixel/functions/convolveImage.html). + +See +[ConvolveOpts](https://docs.thi.ng/umbrella/pixel/interfaces/ConvolveOpts.html) +for config options. + +```js tangle:export/readme-convolve.ts +import { floatBufferFromImage, FLOAT_RGB, imageFromURL } from "@thi.ng/pixel"; +import { convolveImage, SOBEL_X } from "@thi.ng/pixel-convolve"; + +// convolutions are only available for float buffers (for now) +const src = floatBufferFromImage(await imageFromURL("test.jpg"), FLOAT_RGB); + +// apply horizontal Sobel kernel preset to all channels +// downscale image by factor 2 (must be integer) +// scale kernel result values by factor 4 +const dest = convolveImage(src, { kernel: SOBEL_X, stride: 2, scale: 4 }); +``` + +### Normal map generation + +Normal maps can be created via `normalMap()`. This function uses an adjustable +convolution kernel size to control gradient smoothness & details. Result X/Y +gradients can also be scaled (uniform or anisotropic) and the Z component can be +customized to (default: 1.0). The resulting image is in `FLOAT_NORMAL` format, +using signed channel values. This channel format is auto-translating these into +unsigned values when the image is converted into an integer format. + +| Step | Scale = 1 | Scale = 2 | Scale = 4 | Scale = 8 | +|------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------| +| 0 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-0-8.jpg) | +| 1 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-1-8.jpg) | +| 2 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-2-8.jpg) | +| 3 | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-1.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-2.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-4.jpg) | ![](https://raw.githubusercontent.com/thi-ng/umbrella/develop/assets/pixel/nmap-3-8.jpg) | + +```ts tangle:export/readme-normalmap.ts +import { ARGB8888, FLOAT_GRAY, floatBufferFromImage, imageFromURL } from "@thi.ng/pixel"; +import { normalMap } from "@thi.ng/pixel-convolve"; + +// read source image into a single channel floating point buffer +const src = floatBufferFromImage(await imageFromURL("noise.png"), FLOAT_GRAY); + +// create normal map (w/ default options) +// this results in a new float pixel buffer with FLOAT_RGB format +const nmap = normalMap(src, { step: 0, scale: 1 }); + +// pixel lookup (vectors are stored _un_normalized) +nmap.getAt(10, 10); +// Float32Array(3) [ -0.019607841968536377, -0.04313725233078003, 1 ] + +// convert to 32bit packed int format +const nmapARGB = nmap.as(ARGB8888); +``` + +{{meta.status}} + +{{repo.supportPackages}} + +{{repo.relatedPackages}} + +{{meta.blogPosts}} + +## Installation + +{{pkg.install}} + +{{pkg.size}} + +## Dependencies + +{{pkg.deps}} + +{{repo.examples}} + +## API + +{{pkg.docs}} + + diff --git a/packages/pixel-convolve/tsconfig.json b/packages/pixel-convolve/tsconfig.json new file mode 100644 index 0000000000..1cd5465cf2 --- /dev/null +++ b/packages/pixel-convolve/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "." + }, + "include": [ + "./src/**/*.ts" + ] +} diff --git a/yarn.lock b/yarn.lock index c1c34f61a0..109fc8269e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5691,6 +5691,18 @@ __metadata: languageName: unknown linkType: soft +"@thi.ng/pixel-convolve@workspace:packages/pixel-convolve": + version: 0.0.0-use.local + resolution: "@thi.ng/pixel-convolve@workspace:packages/pixel-convolve" + dependencies: + "@microsoft/api-extractor": "npm:^7.47.0" + "@thi.ng/api": "npm:^8.11.6" + esbuild: "npm:^0.23.0" + typedoc: "npm:^0.26.3" + typescript: "npm:^5.5.3" + languageName: unknown + linkType: soft + "@thi.ng/pixel-dither@npm:^1.1.135, @thi.ng/pixel-dither@workspace:^, @thi.ng/pixel-dither@workspace:packages/pixel-dither": version: 0.0.0-use.local resolution: "@thi.ng/pixel-dither@workspace:packages/pixel-dither"