From 9331f5067caa772377ffd1b8ab549bdfa80a18c7 Mon Sep 17 00:00:00 2001 From: Snider Date: Wed, 18 Mar 2026 00:21:08 +0000 Subject: [PATCH] feat: add Slicer[T] generics + Pack (asset packing without go:embed) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slicer[T] — generic typed slice operations (leaanthony/slicer rewrite): s := core.NewSlicer("a", "b", "c") s.AddUnique("d") s.Contains("a") // true s.Filter(fn) // new filtered slicer s.Deduplicate() // remove dupes s.Each(fn) // iterate Pack — build-time asset packing (leaanthony/mewn pattern): Build tool: core.ScanAssets(files) → core.GeneratePack(pkg) Runtime: core.AddAsset(group, name, data) / core.GetAsset(group, name) Scans Go AST for core.GetAsset() calls, reads referenced files, gzip+base64 compresses, generates Go source with init(). Works without go:embed — language-agnostic pattern for CoreTS bridge. Both zero external dependencies. Co-Authored-By: Virgil --- pkg/core/pack.go | 291 +++++++++++++++++++++++++++++++++++++++++++++ pkg/core/slicer.go | 96 +++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 pkg/core/pack.go create mode 100644 pkg/core/slicer.go diff --git a/pkg/core/pack.go b/pkg/core/pack.go new file mode 100644 index 0000000..33d5a89 --- /dev/null +++ b/pkg/core/pack.go @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: EUPL-1.2 + +// Build-time asset packing for the Core framework. +// Based on leaanthony/mewn — scans Go source AST for asset references, +// reads files, compresses, and generates Go source with embedded data. +// +// This enables asset embedding WITHOUT go:embed — the packer runs at +// build time and generates a .go file with init() that registers assets. +// This pattern works cross-language (Go, TypeScript, etc). +// +// Usage (build tool): +// +// refs, _ := core.ScanAssets([]string{"main.go", "app.go"}) +// source, _ := core.GeneratePack(refs) +// os.WriteFile("pack.go", []byte(source), 0644) +// +// Usage (runtime): +// +// core.AddAsset(".", "template.html", compressedData) +// content := core.GetAsset(".", "template.html") +package core + +import ( + "compress/gzip" + "encoding/base64" + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "path/filepath" + "strings" + "sync" +) + +// --- Runtime: Asset Registry --- + +// AssetGroup holds a named collection of packed assets. +type AssetGroup struct { + name string + assets map[string]string // name → compressed data +} + +var ( + assetGroups = make(map[string]*AssetGroup) + assetGroupsMu sync.RWMutex +) + +// AddAsset registers a packed asset at runtime (called from generated init()). +func AddAsset(group, name, data string) { + assetGroupsMu.Lock() + defer assetGroupsMu.Unlock() + + g, ok := assetGroups[group] + if !ok { + g = &AssetGroup{name: group, assets: make(map[string]string)} + assetGroups[group] = g + } + g.assets[name] = data +} + +// GetAsset retrieves and decompresses a packed asset. +func GetAsset(group, name string) (string, error) { + assetGroupsMu.RLock() + g, ok := assetGroups[group] + assetGroupsMu.RUnlock() + if !ok { + return "", fmt.Errorf("asset group %q not found", group) + } + data, ok := g.assets[name] + if !ok { + return "", fmt.Errorf("asset %q not found in group %q", name, group) + } + return decompress(data) +} + +// GetAssetBytes retrieves a packed asset as bytes. +func GetAssetBytes(group, name string) ([]byte, error) { + s, err := GetAsset(group, name) + return []byte(s), err +} + +// --- Build-time: AST Scanner --- + +// AssetRef is a reference to an asset found in source code. +type AssetRef struct { + Name string + Path string + Group string + FullPath string +} + +// ScannedPackage holds all asset references from a set of source files. +type ScannedPackage struct { + PackageName string + BaseDir string + Groups []string + Assets []AssetRef +} + +// ScanAssets parses Go source files and finds asset references. +// Looks for calls to: core.GetAsset("group", "name"), core.AddAsset, etc. +func ScanAssets(filenames []string) ([]ScannedPackage, error) { + packageMap := make(map[string]*ScannedPackage) + groupPaths := make(map[string]string) // variable name → path + + for _, filename := range filenames { + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, filename, nil, parser.AllErrors) + if err != nil { + return nil, err + } + + baseDir := filepath.Dir(filename) + pkg, ok := packageMap[baseDir] + if !ok { + pkg = &ScannedPackage{BaseDir: baseDir} + packageMap[baseDir] = pkg + } + pkg.PackageName = node.Name.Name + + ast.Inspect(node, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + + ident, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + + // Look for core.GetAsset or mewn.String patterns + if ident.Name == "core" || ident.Name == "mewn" { + switch sel.Sel.Name { + case "GetAsset", "GetAssetBytes", "String", "MustString", "Bytes", "MustBytes": + if len(call.Args) >= 1 { + if lit, ok := call.Args[len(call.Args)-1].(*ast.BasicLit); ok { + path := strings.Trim(lit.Value, "\"") + group := "." + if len(call.Args) >= 2 { + if glit, ok := call.Args[0].(*ast.BasicLit); ok { + group = strings.Trim(glit.Value, "\"") + } + } + fullPath, _ := filepath.Abs(filepath.Join(baseDir, group, path)) + pkg.Assets = append(pkg.Assets, AssetRef{ + Name: path, + Path: path, + Group: group, + FullPath: fullPath, + }) + } + } + case "Group": + // Variable assignment: g := core.Group("./assets") + if len(call.Args) == 1 { + if lit, ok := call.Args[0].(*ast.BasicLit); ok { + path := strings.Trim(lit.Value, "\"") + fullPath, _ := filepath.Abs(filepath.Join(baseDir, path)) + pkg.Groups = append(pkg.Groups, fullPath) + // Track for variable resolution + groupPaths[path] = fullPath + } + } + } + } + + return true + }) + } + + var result []ScannedPackage + for _, pkg := range packageMap { + result = append(result, *pkg) + } + return result, nil +} + +// GeneratePack creates Go source code that embeds the scanned assets. +func GeneratePack(pkg ScannedPackage) (string, error) { + var b strings.Builder + + b.WriteString(fmt.Sprintf("package %s\n\n", pkg.PackageName)) + b.WriteString("// Code generated by core pack. DO NOT EDIT.\n\n") + + if len(pkg.Assets) == 0 && len(pkg.Groups) == 0 { + return b.String(), nil + } + + b.WriteString("import \"forge.lthn.ai/core/go/pkg/core\"\n\n") + b.WriteString("func init() {\n") + + // Pack groups (entire directories) + packed := make(map[string]bool) + for _, groupPath := range pkg.Groups { + files, err := getAllFiles(groupPath) + if err != nil { + continue + } + for _, file := range files { + if packed[file] { + continue + } + data, err := compressFile(file) + if err != nil { + continue + } + localPath := strings.TrimPrefix(file, groupPath+"/") + relGroup, _ := filepath.Rel(pkg.BaseDir, groupPath) + b.WriteString(fmt.Sprintf("\tcore.AddAsset(%q, %q, %q)\n", relGroup, localPath, data)) + packed[file] = true + } + } + + // Pack individual assets + for _, asset := range pkg.Assets { + if packed[asset.FullPath] { + continue + } + data, err := compressFile(asset.FullPath) + if err != nil { + continue + } + b.WriteString(fmt.Sprintf("\tcore.AddAsset(%q, %q, %q)\n", asset.Group, asset.Name, data)) + packed[asset.FullPath] = true + } + + b.WriteString("}\n") + return b.String(), nil +} + +// --- Compression --- + +func compressFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + return compress(string(data)) +} + +func compress(input string) (string, error) { + var buf bytes.Buffer + b64 := base64.NewEncoder(base64.StdEncoding, &buf) + gz, err := gzip.NewWriterLevel(b64, gzip.BestCompression) + if err != nil { + return "", err + } + if _, err := gz.Write([]byte(input)); err != nil { + return "", err + } + gz.Close() + b64.Close() + return buf.String(), nil +} + +func decompress(input string) (string, error) { + b64 := base64.NewDecoder(base64.StdEncoding, strings.NewReader(input)) + gz, err := gzip.NewReader(b64) + if err != nil { + return "", err + } + defer gz.Close() + data, err := io.ReadAll(gz) + if err != nil { + return "", err + } + return string(data), nil +} + +func getAllFiles(dir string) ([]string, error) { + var result []string + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsRegular() { + result = append(result, path) + } + return nil + }) + return result, err +} diff --git a/pkg/core/slicer.go b/pkg/core/slicer.go new file mode 100644 index 0000000..a690ea4 --- /dev/null +++ b/pkg/core/slicer.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: EUPL-1.2 + +// Generic slice operations for the Core framework. +// Based on leaanthony/slicer, rewritten with Go 1.18+ generics. + +package core + +// Slicer is a typed slice with common operations. +type Slicer[T comparable] struct { + items []T +} + +// NewSlicer creates an empty Slicer. +func NewSlicer[T comparable](items ...T) *Slicer[T] { + return &Slicer[T]{items: items} +} + +// Add appends values. +func (s *Slicer[T]) Add(values ...T) { + s.items = append(s.items, values...) +} + +// AddUnique appends values only if not already present. +func (s *Slicer[T]) AddUnique(values ...T) { + for _, v := range values { + if !s.Contains(v) { + s.items = append(s.items, v) + } + } +} + +// Contains returns true if the value is in the slice. +func (s *Slicer[T]) Contains(val T) bool { + for _, v := range s.items { + if v == val { + return true + } + } + return false +} + +// Filter returns a new Slicer with elements matching the predicate. +func (s *Slicer[T]) Filter(fn func(T) bool) *Slicer[T] { + result := &Slicer[T]{} + for _, v := range s.items { + if fn(v) { + result.items = append(result.items, v) + } + } + return result +} + +// Each runs a function on every element. +func (s *Slicer[T]) Each(fn func(T)) { + for _, v := range s.items { + fn(v) + } +} + +// Remove removes the first occurrence of a value. +func (s *Slicer[T]) Remove(val T) { + for i, v := range s.items { + if v == val { + s.items = append(s.items[:i], s.items[i+1:]...) + return + } + } +} + +// Deduplicate removes duplicate values, preserving order. +func (s *Slicer[T]) Deduplicate() { + seen := make(map[T]struct{}) + result := make([]T, 0, len(s.items)) + for _, v := range s.items { + if _, exists := seen[v]; !exists { + seen[v] = struct{}{} + result = append(result, v) + } + } + s.items = result +} + +// Len returns the number of elements. +func (s *Slicer[T]) Len() int { + return len(s.items) +} + +// Clear removes all elements. +func (s *Slicer[T]) Clear() { + s.items = nil +} + +// AsSlice returns the underlying slice. +func (s *Slicer[T]) AsSlice() []T { + return s.items +}