Add KDTree normalization helpers and TypeScript demo with Vite

This commit is contained in:
Snider 2025-11-04 02:15:04 +00:00
parent 3c83fc38e4
commit 38a6c6aad3
19 changed files with 552 additions and 5 deletions

View file

@ -42,6 +42,21 @@ jobs:
- name: Benchmarks (linear) - name: Benchmarks (linear)
run: go test -bench . -benchmem -run=^$ ./... | tee bench-linear.txt run: go test -bench . -benchmem -run=^$ ./... | tee bench-linear.txt
- name: Coverage summary (linear)
run: |
if [ -f coverage.out ]; then
go tool cover -func=coverage.out > coverage-summary.md;
else
echo "coverage.out not found" > coverage-summary.md;
fi
- name: Upload coverage summary (linear)
uses: actions/upload-artifact@v4
with:
name: coverage-summary
path: coverage-summary.md
if-no-files-found: error
- name: Upload benchmarks (linear) - name: Upload benchmarks (linear)
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:
@ -55,6 +70,9 @@ jobs:
- name: Prepare npm package folder - name: Prepare npm package folder
run: make npm-pack run: make npm-pack
- name: WASM smoke test (Node)
run: node npm/poindexter-wasm/smoke.mjs
- name: Create npm tarball - name: Create npm tarball
id: npm_pack id: npm_pack
run: | run: |
@ -137,6 +155,21 @@ jobs:
path: coverage-gonum.out path: coverage-gonum.out
if-no-files-found: error if-no-files-found: error
- name: Coverage summary (gonum)
run: |
if [ -f coverage-gonum.out ]; then
go tool cover -func=coverage-gonum.out > coverage-summary-gonum.md;
else
echo "coverage-gonum.out not found" > coverage-summary-gonum.md;
fi
- name: Upload coverage summary (gonum)
uses: actions/upload-artifact@v4
with:
name: coverage-summary-gonum
path: coverage-summary-gonum.md
if-no-files-found: error
- name: Upload benchmarks (gonum) - name: Upload benchmarks (gonum)
uses: actions/upload-artifact@v4 uses: actions/upload-artifact@v4
with: with:

View file

@ -117,8 +117,26 @@ fuzz: ## Run Go fuzz tests for $(FUZZTIME)
done done
.PHONY: bench .PHONY: bench
bench: ## Run benchmarks and write $(BENCHOUT) # Benchmark configuration variables
$(GO) test -bench . -benchmem -run=^$$ ./... | tee $(BENCHOUT) BENCHPKG ?= ./...
BENCHFILTER ?= .
BENCHTAGS ?=
BENCHMEMFLAG ?= -benchmem
bench: ## Run benchmarks (configurable: BENCHPKG, BENCHFILTER, BENCHTAGS, BENCHOUT)
$(GO) test $(if $(BENCHTAGS),-tags=$(BENCHTAGS),) -bench $(BENCHFILTER) $(BENCHMEMFLAG) -run=^$$ $(BENCHPKG) | tee $(BENCHOUT)
.PHONY: bench-linear
bench-linear: ## Run linear-backend benchmarks and write bench-linear.txt
$(MAKE) bench BENCHTAGS= BENCHOUT=bench-linear.txt
.PHONY: bench-gonum
bench-gonum: ## Run gonum-backend benchmarks (includes 100k benches) and write bench-gonum.txt
$(MAKE) bench BENCHTAGS=gonum BENCHOUT=bench-gonum.txt
.PHONY: bench-list
bench-list: ## List available benchmark names for BENCHPKG (use with BENCHPKG=./pkg)
$(GO) test $(if $(BENCHTAGS),-tags=$(BENCHTAGS),) -run=^$$ -bench ^$$ -list '^Benchmark' $(BENCHPKG)
.PHONY: lint .PHONY: lint
lint: ## Run golangci-lint (requires it installed) lint: ## Run golangci-lint (requires it installed)

View file

@ -71,7 +71,9 @@ Explore runnable examples in the repository:
- examples/kdtree_2d_ping_hop - examples/kdtree_2d_ping_hop
- examples/kdtree_3d_ping_hop_geo - examples/kdtree_3d_ping_hop_geo
- examples/kdtree_4d_ping_hop_geo_score - examples/kdtree_4d_ping_hop_geo_score
- examples/dht_helpers (convenience wrappers for common DHT schemas)
- examples/wasm-browser (browser demo using the ESM loader) - examples/wasm-browser (browser demo using the ESM loader)
- examples/wasm-browser-ts (TypeScript + Vite local demo)
### KDTree performance and notes ### KDTree performance and notes
- Dual backend support: Linear (always available) and an optimized KD backend enabled when building with `-tags=gonum`. Linear is the default; with the `gonum` tag, the optimized backend becomes the default. - Dual backend support: Linear (always available) and an optimized KD backend enabled when building with `-tags=gonum`. Linear is the default; with the `gonum` tag, the optimized backend becomes the default.
@ -216,4 +218,21 @@ This project is licensed under the European Union Public Licence v1.2 (EUPL-1.2)
## Contributing ## Contributing
Contributions are welcome! Please feel free to submit a Pull Request. Contributions are welcome! Please feel free to submit a Pull Request.
## Coverage
- CI produces coverage summaries as artifacts on every push/PR:
- Default job: `coverage-summary.md` (from `coverage.out`)
- Gonum-tag job: `coverage-summary-gonum.md` (from `coverage-gonum.out`)
- Locally, you can generate and inspect coverage with the Makefile:
```bash
make cover # runs tests with race + coverage and prints the total
make coverfunc # prints per-function coverage
make cover-kdtree # filters coverage to kdtree.go
make coverhtml # writes coverage.html for visual inspection
```
Note: CI also uploads raw coverage profiles as artifacts (`coverage.out`, `coverage-gonum.out`).

View file

@ -0,0 +1,35 @@
//go:build gonum
package poindexter
import "testing"
// 100k-size benchmarks run only in the gonum-tag job to keep CI time reasonable.
func BenchmarkNearest_Linear_Uniform_100k_2D(b *testing.B) {
benchNearestBackend(b, 100_000, 2, BackendLinear, true, 0)
}
func BenchmarkNearest_Gonum_Uniform_100k_2D(b *testing.B) {
benchNearestBackend(b, 100_000, 2, BackendGonum, true, 0)
}
func BenchmarkNearest_Linear_Uniform_100k_4D(b *testing.B) {
benchNearestBackend(b, 100_000, 4, BackendLinear, true, 0)
}
func BenchmarkNearest_Gonum_Uniform_100k_4D(b *testing.B) {
benchNearestBackend(b, 100_000, 4, BackendGonum, true, 0)
}
func BenchmarkNearest_Linear_Clustered_100k_2D(b *testing.B) {
benchNearestBackend(b, 100_000, 2, BackendLinear, false, 3)
}
func BenchmarkNearest_Gonum_Clustered_100k_2D(b *testing.B) {
benchNearestBackend(b, 100_000, 2, BackendGonum, false, 3)
}
func BenchmarkNearest_Linear_Clustered_100k_4D(b *testing.B) {
benchNearestBackend(b, 100_000, 4, BackendLinear, false, 3)
}
func BenchmarkNearest_Gonum_Clustered_100k_4D(b *testing.B) {
benchNearestBackend(b, 100_000, 4, BackendGonum, false, 3)
}

View file

@ -496,6 +496,53 @@ Notes:
- These helpers mirror `Build2D/3D/4D`, but use your provided `NormStats` instead of recomputing from the items slice. - These helpers mirror `Build2D/3D/4D`, but use your provided `NormStats` instead of recomputing from the items slice.
---
## KDTree Normalization Helpers (ND)
Poindexter includes helpers to build KD points from arbitrary dimensions.
```go
func BuildND[T any](
items []T,
id func(T) string,
features []func(T) float64,
weights []float64,
invert []bool,
) ([]KDPoint[T], error)
// Like BuildND but never returns an error. It performs no validation beyond
// basic length checks and propagates NaN/Inf values from feature extractors.
func BuildNDNoErr[T any](
items []T,
id func(T) string,
features []func(T) float64,
weights []float64,
invert []bool,
) []KDPoint[T]
```
- `features`: extract raw values per axis.
- `weights`: per-axis weights, same length as `features`.
- `invert`: if true for an axis, uses `1 - normalized` before weighting (turns “higher is better” into lower cost).
- Use `ComputeNormStatsND` + `BuildNDWithStats` to reuse normalization between updates.
Example:
```go
pts := poindexter.BuildNDNoErr(records,
func(r Rec) string { return r.ID },
[]func(Rec) float64{
func(r Rec) float64 { return r.PingMS },
func(r Rec) float64 { return r.Hops },
func(r Rec) float64 { return r.GeoKM },
func(r Rec) float64 { return r.Score },
},
[]float64{1.0, 0.7, 0.2, 1.2},
[]bool{false, false, false, true},
)
```
--- ---
## KDTree Backend selection ## KDTree Backend selection

View file

@ -4,8 +4,9 @@ This page summarizes how to measure KDTree performance in this repository and ho
## How benchmarks are organized ## How benchmarks are organized
- Micro-benchmarks live in `bench_kdtree_test.go` and `bench_kdtree_dual_test.go` and cover: - Micro-benchmarks live in `bench_kdtree_test.go`, `bench_kdtree_dual_test.go`, and `bench_kdtree_dual_100k_test.go` and cover:
- `Nearest` in 2D and 4D with N = 1k, 10k - `Nearest` in 2D and 4D with N = 1k, 10k (both backends)
- `Nearest` in 2D and 4D with N = 100k (gonum-tag job; linear also measured there)
- `KNearest(k=10)` in 2D/4D with N = 1k, 10k - `KNearest(k=10)` in 2D/4D with N = 1k, 10k
- `Radius` (mid radius r≈0.5 after normalization) in 2D/4D with N = 1k, 10k - `Radius` (mid radius r≈0.5 after normalization) in 2D/4D with N = 1k, 10k
- Datasets: Uniform and 3-cluster synthetic generators in normalized [0,1] spaces. - Datasets: Uniform and 3-cluster synthetic generators in normalized [0,1] spaces.

View file

@ -150,3 +150,22 @@ Then open:
- http://127.0.0.1:8000/examples/wasm-browser/ - http://127.0.0.1:8000/examples/wasm-browser/
Open the browser console to see outputs from `nearest`, `kNearest`, and `radius` queries. Open the browser console to see outputs from `nearest`, `kNearest`, and `radius` queries.
### TypeScript + Vite demo (local-only)
A minimal TypeScript demo using Vite is also included:
- Path: `examples/wasm-browser-ts/`
- Prerequisites: run `make wasm-build` at the repo root first.
- From the example folder:
```bash
npm install
npm run dev
```
Then open the URL printed by Vite (usually http://127.0.0.1:5173/) and check the browser console.
Notes:
- The dev script copies `dist/poindexter.wasm`, `dist/wasm_exec.js`, and the ESM loader into the example's `public/` folder before serving.
- This example is intentionally excluded from CI to keep the pipeline lean.

View file

@ -0,0 +1,69 @@
package main
import (
"fmt"
po "github.com/Snider/Poindexter"
)
// BuildPingHop2D wraps poindexter.Build2D to construct 2D points from (ping_ms, hop_count).
func BuildPingHop2D[T any](
items []T,
id func(T) string,
ping func(T) float64,
hops func(T) float64,
weights [2]float64,
invert [2]bool,
) ([]po.KDPoint[T], error) {
return po.Build2D(items, id, ping, hops, weights, invert)
}
// BuildPingHopGeo3D wraps poindexter.Build3D for (ping_ms, hop_count, geo_km).
func BuildPingHopGeo3D[T any](
items []T,
id func(T) string,
ping func(T) float64,
hops func(T) float64,
geoKM func(T) float64,
weights [3]float64,
invert [3]bool,
) ([]po.KDPoint[T], error) {
return po.Build3D(items, id, ping, hops, geoKM, weights, invert)
}
// BuildPingHopGeoScore4D wraps poindexter.Build4D for (ping_ms, hop_count, geo_km, score).
// Typical usage sets invert for score=true so higher score => lower cost.
func BuildPingHopGeoScore4D[T any](
items []T,
id func(T) string,
ping func(T) float64,
hops func(T) float64,
geoKM func(T) float64,
score func(T) float64,
weights [4]float64,
invert [4]bool,
) ([]po.KDPoint[T], error) {
return po.Build4D(items, id, ping, hops, geoKM, score, weights, invert)
}
// Demo program that builds a small tree using the 2D helper and performs a query.
func main() {
type Peer struct {
ID string
PingMS, Hops float64
}
peers := []Peer{{"A", 20, 1}, {"B", 50, 2}, {"C", 10, 3}}
pts, err := BuildPingHop2D(peers,
func(p Peer) string { return p.ID },
func(p Peer) float64 { return p.PingMS },
func(p Peer) float64 { return p.Hops },
[2]float64{1.0, 0.7},
[2]bool{false, false},
)
if err != nil {
panic(err)
}
kdt, _ := po.NewKDTree(pts, po.WithMetric(po.EuclideanDistance{}))
best, dist, _ := kdt.Nearest([]float64{0, 0})
fmt.Println(best.ID, dist)
}

View file

@ -0,0 +1,8 @@
package main
import "testing"
func TestMain_Run(t *testing.T) {
// Just ensure the example main runs without panic to contribute to coverage
main()
}

View file

@ -0,0 +1,46 @@
# WASM Browser Example (TypeScript + Vite)
This is a minimal TypeScript example that runs Poindexters WebAssembly build in the browser.
It bundles a tiny page with Vite and demonstrates creating a KDTree and running `Nearest`,
`KNearest`, and `Radius` queries.
## Prerequisites
- Go toolchain installed
- Node.js 18+ (tested with Node 20)
## Quick start
1) Build the WASM artifacts at the repo root:
```bash
make wasm-build
```
This creates `dist/poindexter.wasm` and `dist/wasm_exec.js`.
2) From this example directory, install deps and start the dev server (the script copies the required files into `public/` before starting Vite):
```bash
npm install
npm run dev
```
3) Open the URL printed by Vite (usually http://127.0.0.1:5173/). Open the browser console to see outputs.
## What the dev script does
- Copies `../../dist/poindexter.wasm` and `../../dist/wasm_exec.js` into `public/`
- Copies `../../npm/poindexter-wasm/loader.js` into `public/`
- Starts Vite with `public/` as the static root for those assets
The TypeScript code imports the loader from `/loader.js` and initializes with:
```ts
const px = await init({
wasmURL: '/poindexter.wasm',
wasmExecURL: '/wasm_exec.js',
});
```
## Notes
- This example is local-only and not built in CI to keep jobs light.
- You can adapt the same structure inside your own web projects; alternatively, install the published npm package when available and serve `dist/` as static assets.

View file

@ -0,0 +1,25 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Poindexter WASM TS Demo</title>
<style>
body { font-family: system-ui, -apple-system, Segoe UI, Roboto, sans-serif; margin: 2rem; }
pre { background: #f6f8fa; padding: 1rem; overflow-x: auto; }
code { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; }
</style>
</head>
<body>
<h1>Poindexter WASM (TypeScript + Vite)</h1>
<p>
This demo initializes the WebAssembly build and performs KDTree queries. Open your browser console to see results.
</p>
<p>
Before running, build the WASM artifacts at the repo root:
</p>
<pre><code>make wasm-build</code></pre>
<script type="module" src="/src/main.ts"></script>
</body>
</html>

View file

@ -0,0 +1,16 @@
{
"name": "poindexter-wasm-browser-ts",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"predev": "node scripts/copy-assets.mjs",
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
"devDependencies": {
"typescript": "^5.4.0",
"vite": "^5.0.0"
}
}

View file

@ -0,0 +1,40 @@
// Copies WASM artifacts and loader into the public/ folder before Vite dev/build.
// Run as an npm script (predev) from this example directory.
import { cp, mkdir } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
async function main() {
const root = resolve(__dirname, '../../..');
const exampleDir = resolve(__dirname, '..');
const publicDir = resolve(exampleDir, 'public');
await mkdir(publicDir, { recursive: true });
const sources = [
// WASM artifacts built by `make wasm-build`
resolve(root, 'dist/poindexter.wasm'),
resolve(root, 'dist/wasm_exec.js'),
// ESM loader shipped with the repo's npm folder
resolve(root, 'npm/poindexter-wasm/loader.js'),
];
const targets = [
resolve(publicDir, 'poindexter.wasm'),
resolve(publicDir, 'wasm_exec.js'),
resolve(publicDir, 'loader.js'),
];
for (let i = 0; i < sources.length; i++) {
await cp(sources[i], targets[i]);
console.log(`Copied ${sources[i]} -> ${targets[i]}`);
}
}
main().catch((err) => {
console.error('copy-assets failed:', err);
process.exit(1);
});

View file

@ -0,0 +1,33 @@
// Minimal TypeScript demo that uses the Poindexter WASM ESM loader.
// Precondition: run `make wasm-build` at repo root, then `npm run dev` in this folder.
// We copy the loader and wasm artifacts to /public via scripts/copy-assets.mjs before dev starts.
// @ts-ignore
import { init } from '/loader.js';
async function run() {
const px = await init({
wasmURL: '/poindexter.wasm',
wasmExecURL: '/wasm_exec.js',
});
console.log('Poindexter (WASM) version:', await px.version());
const tree = await px.newTree(2);
await tree.insert({ id: 'a', coords: [0, 0], value: 'A' });
await tree.insert({ id: 'b', coords: [1, 0], value: 'B' });
await tree.insert({ id: 'c', coords: [0, 1], value: 'C' });
const nn = await tree.nearest([0.9, 0.1]);
console.log('Nearest [0.9,0.1]:', nn);
const kn = await tree.kNearest([0.9, 0.9], 2);
console.log('kNN k=2 [0.9,0.9]:', kn);
const rad = await tree.radius([0, 0], 1.1);
console.log('Radius r=1.1 [0,0]:', rad);
}
run().catch((err) => {
console.error('WASM demo error:', err);
});

View file

@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2020",
"module": "ESNext",
"moduleResolution": "Bundler",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"jsx": "react-jsx",
"allowJs": false,
"types": []
},
"include": ["src/**/*"]
}

View file

@ -0,0 +1,17 @@
import { defineConfig } from 'vite';
// Minimal Vite config for the WASM TS example.
// Serves files from project root; our dev script copies required artifacts to public/.
export default defineConfig({
root: '.',
server: {
host: '127.0.0.1',
port: 5173,
open: false,
},
preview: {
host: '127.0.0.1',
port: 5173,
open: false,
},
});

View file

@ -88,6 +88,23 @@ func BuildND[T any](items []T, id func(T) string, features []func(T) float64, we
return BuildNDWithStats(items, id, features, weights, invert, stats) return BuildNDWithStats(items, id, features, weights, invert, stats)
} }
// BuildNDNoErr constructs normalized-and-weighted KD points like BuildND but never returns an error.
// It performs no input validation beyond basic length checks and will propagate NaN/Inf values
// from feature extractors into the resulting coordinates. Use when you control inputs and want a
// simpler call signature.
func BuildNDNoErr[T any](items []T, id func(T) string, features []func(T) float64, weights []float64, invert []bool) []KDPoint[T] {
if len(items) == 0 || len(features) == 0 {
return nil
}
// If lengths are inconsistent, return empty (no panic); this function is intentionally lenient.
if len(weights) != len(features) || len(invert) != len(features) {
return nil
}
stats, _ := ComputeNormStatsND(items, features)
pts, _ := BuildNDWithStats(items, id, features, weights, invert, stats)
return pts
}
// BuildNDWithStats builds points using provided normalisation stats. // BuildNDWithStats builds points using provided normalisation stats.
func BuildNDWithStats[T any](items []T, id func(T) string, features []func(T) float64, weights []float64, invert []bool, stats NormStats) ([]KDPoint[T], error) { func BuildNDWithStats[T any](items []T, id func(T) string, features []func(T) float64, weights []float64, invert []bool, stats NormStats) ([]KDPoint[T], error) {
if len(items) == 0 { if len(items) == 0 {

63
kdtree_nd_noerr_test.go Normal file
View file

@ -0,0 +1,63 @@
package poindexter
import (
"fmt"
"math"
"testing"
)
// TestBuildNDNoErr_Parity checks that BuildNDNoErr matches BuildND on valid inputs.
func TestBuildNDNoErr_Parity(t *testing.T) {
type rec struct {
A, B, C float64
ID string
}
items := []rec{
{A: 10, B: 100, C: 1, ID: "x"},
{A: 20, B: 200, C: 2, ID: "y"},
{A: 30, B: 300, C: 3, ID: "z"},
}
features := []func(rec) float64{
func(r rec) float64 { return r.A },
func(r rec) float64 { return r.B },
func(r rec) float64 { return r.C },
}
weights := []float64{1, 0.5, 2}
invert := []bool{false, true, false}
idfn := func(r rec) string { return r.ID }
ptsStrict, err := BuildND(items, idfn, features, weights, invert)
if err != nil {
t.Fatalf("BuildND returned error: %v", err)
}
ptsLoose := BuildNDNoErr(items, idfn, features, weights, invert)
if len(ptsStrict) != len(ptsLoose) {
t.Fatalf("length mismatch: strict %d loose %d", len(ptsStrict), len(ptsLoose))
}
for i := range ptsStrict {
if ptsStrict[i].ID != ptsLoose[i].ID {
t.Fatalf("ID mismatch at %d: %s vs %s", i, ptsStrict[i].ID, ptsLoose[i].ID)
}
if len(ptsStrict[i].Coords) != len(ptsLoose[i].Coords) {
t.Fatalf("dim mismatch at %d: %d vs %d", i, len(ptsStrict[i].Coords), len(ptsLoose[i].Coords))
}
for d := range ptsStrict[i].Coords {
if math.Abs(ptsStrict[i].Coords[d]-ptsLoose[i].Coords[d]) > 1e-12 {
t.Fatalf("coord mismatch at %d dim %d: %v vs %v", i, d, ptsStrict[i].Coords[d], ptsLoose[i].Coords[d])
}
}
}
}
// TestBuildNDNoErr_Lenient ensures the no-error builder is lenient and returns nil on bad lengths.
func TestBuildNDNoErr_Lenient(t *testing.T) {
type rec struct{ A float64 }
items := []rec{{A: 1}, {A: 2}}
features := []func(rec) float64{func(r rec) float64 { return r.A }}
weightsBad := []float64{} // wrong length
invert := []bool{false}
pts := BuildNDNoErr(items, func(r rec) string { return fmt.Sprint(r.A) }, features, weightsBad, invert)
if pts != nil {
t.Fatalf("expected nil result on bad weights length, got %v", pts)
}
}

View file

@ -0,0 +1,26 @@
// Minimal Node smoke test for the WASM loader.
// Assumes npm-pack has prepared npm/poindexter-wasm with loader and dist assets.
import { init } from './loader.js';
(async function () {
try {
const px = await init({
// In CI, dist/ is placed at repo root via make wasm-build && make npm-pack
wasmURL: new URL('./dist/poindexter.wasm', import.meta.url).pathname,
wasmExecURL: new URL('./dist/wasm_exec.js', import.meta.url).pathname,
});
const ver = await px.version();
if (!ver || typeof ver !== 'string') throw new Error('version not string');
const tree = await px.newTree(2);
await tree.insert({ id: 'a', coords: [0, 0], value: 'A' });
await tree.insert({ id: 'b', coords: [1, 0], value: 'B' });
const nn = await tree.nearest([0.9, 0.1]);
if (!nn || !nn.id) throw new Error('nearest failed');
console.log('WASM smoke ok:', ver, 'nearest.id=', nn.id);
} catch (err) {
console.error('WASM smoke failed:', err);
process.exit(1);
}
})();