a43cc099cd
Virgil split go-ai into standalone modules (go-agentic, go-ml, go-mlx, go-rag). This migrates all CLI imports to the new module paths and fixes API mismatches from the split. Key changes: - go-ai/agentic → go-agentic (cmd/ai, cmd/dev) - go-ai/ml → go-ml (31 files in cmd/ml) - go-ai/rag → go-rag (3 files in cmd/rag) - go-ai/mlx → go-mlx (1 file) - Fix go.work path (../core → ../go) - Add all split repos to go.work and go.mod - Simplify daemon to goroutine-based MCP (remove missing supervisor) - Wire go-agentic SQLiteRegistry into dispatch watch (--agent-id flag) - Add `core ai agent fleet` command for local registry status - Fix rag collections API (PointCount, Status string) - Fix ml live/expand-status to use available go-ml API Co-Authored-By: Charon <charon@lethean.io>
66 lines
1.5 KiB
Go
66 lines
1.5 KiB
Go
package ml
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
|
|
"forge.lthn.ai/core/go/pkg/cli"
|
|
"forge.lthn.ai/core/go-ml"
|
|
)
|
|
|
|
var (
|
|
probeOutput string
|
|
)
|
|
|
|
var probeCmd = &cli.Command{
|
|
Use: "probe",
|
|
Short: "Run capability and content probes against a model",
|
|
Long: "Runs 23 capability probes and 6 content probes against an OpenAI-compatible API.",
|
|
RunE: runProbe,
|
|
}
|
|
|
|
func init() {
|
|
probeCmd.Flags().StringVar(&probeOutput, "output", "", "Output JSON file for probe results")
|
|
}
|
|
|
|
func runProbe(cmd *cli.Command, args []string) error {
|
|
if apiURL == "" {
|
|
return fmt.Errorf("--api-url is required")
|
|
}
|
|
|
|
model := modelName
|
|
if model == "" {
|
|
model = "default"
|
|
}
|
|
|
|
ctx := context.Background()
|
|
backend := ml.NewHTTPBackend(apiURL, model)
|
|
|
|
fmt.Printf("Running %d capability probes against %s...\n", len(ml.CapabilityProbes), apiURL)
|
|
results := ml.RunCapabilityProbes(ctx, backend)
|
|
|
|
fmt.Printf("\nResults: %.1f%% (%d/%d)\n", results.Accuracy, results.Correct, results.Total)
|
|
|
|
for cat, data := range results.ByCategory {
|
|
catAcc := 0.0
|
|
if data.Total > 0 {
|
|
catAcc = float64(data.Correct) / float64(data.Total) * 100
|
|
}
|
|
fmt.Printf(" %-20s %d/%d (%.0f%%)\n", cat, data.Correct, data.Total, catAcc)
|
|
}
|
|
|
|
if probeOutput != "" {
|
|
data, err := json.MarshalIndent(results, "", " ")
|
|
if err != nil {
|
|
return fmt.Errorf("marshal results: %w", err)
|
|
}
|
|
if err := os.WriteFile(probeOutput, data, 0644); err != nil {
|
|
return fmt.Errorf("write output: %w", err)
|
|
}
|
|
fmt.Printf("\nResults written to %s\n", probeOutput)
|
|
}
|
|
|
|
return nil
|
|
}
|