a43cc099cd
Virgil split go-ai into standalone modules (go-agentic, go-ml, go-mlx, go-rag). This migrates all CLI imports to the new module paths and fixes API mismatches from the split. Key changes: - go-ai/agentic → go-agentic (cmd/ai, cmd/dev) - go-ai/ml → go-ml (31 files in cmd/ml) - go-ai/rag → go-rag (3 files in cmd/rag) - go-ai/mlx → go-mlx (1 file) - Fix go.work path (../core → ../go) - Add all split repos to go.work and go.mod - Simplify daemon to goroutine-based MCP (remove missing supervisor) - Wire go-agentic SQLiteRegistry into dispatch watch (--agent-id flag) - Add `core ai agent fleet` command for local registry status - Fix rag collections API (PointCount, Status string) - Fix ml live/expand-status to use available go-ml API Co-Authored-By: Charon <charon@lethean.io>
77 lines
2 KiB
Go
77 lines
2 KiB
Go
package ml
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
"forge.lthn.ai/core/go/pkg/cli"
|
|
"forge.lthn.ai/core/go-ml"
|
|
)
|
|
|
|
var (
|
|
scoreInput string
|
|
scoreSuites string
|
|
scoreOutput string
|
|
scoreConcur int
|
|
)
|
|
|
|
var scoreCmd = &cli.Command{
|
|
Use: "score",
|
|
Short: "Score responses with heuristic and LLM judges",
|
|
Long: "Reads a JSONL file of prompt/response pairs and scores them across configured suites.",
|
|
RunE: runScore,
|
|
}
|
|
|
|
func init() {
|
|
scoreCmd.Flags().StringVar(&scoreInput, "input", "", "Input JSONL file with prompt/response pairs (required)")
|
|
scoreCmd.Flags().StringVar(&scoreSuites, "suites", "all", "Comma-separated scoring suites (heuristic,semantic,content,exact,truthfulqa,donotanswer,toxigen)")
|
|
scoreCmd.Flags().StringVar(&scoreOutput, "output", "", "Output JSON file for scores")
|
|
scoreCmd.Flags().IntVar(&scoreConcur, "concurrency", 4, "Number of concurrent scoring workers")
|
|
scoreCmd.MarkFlagRequired("input")
|
|
}
|
|
|
|
func runScore(cmd *cli.Command, args []string) error {
|
|
responses, err := ml.ReadResponses(scoreInput)
|
|
if err != nil {
|
|
return fmt.Errorf("read input: %w", err)
|
|
}
|
|
|
|
var judge *ml.Judge
|
|
if judgeURL != "" {
|
|
backend := ml.NewHTTPBackend(judgeURL, judgeModel)
|
|
judge = ml.NewJudge(backend)
|
|
}
|
|
|
|
engine := ml.NewEngine(judge, scoreConcur, scoreSuites)
|
|
|
|
ctx := context.Background()
|
|
perPrompt := engine.ScoreAll(ctx, responses)
|
|
averages := ml.ComputeAverages(perPrompt)
|
|
|
|
if scoreOutput != "" {
|
|
output := &ml.ScorerOutput{
|
|
Metadata: ml.Metadata{
|
|
JudgeModel: judgeModel,
|
|
JudgeURL: judgeURL,
|
|
ScoredAt: time.Now(),
|
|
Suites: ml.SplitComma(scoreSuites),
|
|
},
|
|
ModelAverages: averages,
|
|
PerPrompt: perPrompt,
|
|
}
|
|
if err := ml.WriteScores(scoreOutput, output); err != nil {
|
|
return fmt.Errorf("write output: %w", err)
|
|
}
|
|
fmt.Printf("Scores written to %s\n", scoreOutput)
|
|
} else {
|
|
for model, avgs := range averages {
|
|
fmt.Printf("%s:\n", model)
|
|
for field, val := range avgs {
|
|
fmt.Printf(" %-25s %.3f\n", field, val)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|