1
0
Fork 0
forked from lthn/LEM

feat(cli): add cmd/lemcmd command registration package

6 command groups (score, gen, data, export, mon, infra) with 25
commands. All pass through to existing lem.Run* functions via
the Core framework's cli package.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Snider 2026-02-22 18:55:57 +00:00
parent a0a0118155
commit cf1d8156dd
7 changed files with 200 additions and 0 deletions

28
cmd/lemcmd/data.go Normal file
View file

@ -0,0 +1,28 @@
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addDataCommands(root *cli.Command) {
dataGroup := cli.NewGroup("data", "Data management commands", "Import, consolidate, normalise, and approve training data.")
dataGroup.AddCommand(cli.NewRun("import-all", "Import ALL LEM data into DuckDB from M3", "", func(cmd *cli.Command, args []string) {
lem.RunImport(args)
}))
dataGroup.AddCommand(cli.NewRun("consolidate", "Pull worker JSONLs from M3, merge, deduplicate", "", func(cmd *cli.Command, args []string) {
lem.RunConsolidate(args)
}))
dataGroup.AddCommand(cli.NewRun("normalize", "Normalise seeds to deduplicated expansion prompts", "", func(cmd *cli.Command, args []string) {
lem.RunNormalize(args)
}))
dataGroup.AddCommand(cli.NewRun("approve", "Filter scored expansions to training JSONL", "", func(cmd *cli.Command, args []string) {
lem.RunApprove(args)
}))
root.AddCommand(dataGroup)
}

28
cmd/lemcmd/export.go Normal file
View file

@ -0,0 +1,28 @@
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addExportCommands(root *cli.Command) {
exportGroup := cli.NewGroup("export", "Export and publish commands", "Export training data to JSONL, Parquet, HuggingFace, and PEFT formats.")
exportGroup.AddCommand(cli.NewRun("jsonl", "Export golden set to training-format JSONL splits", "", func(cmd *cli.Command, args []string) {
lem.RunExport(args)
}))
exportGroup.AddCommand(cli.NewRun("parquet", "Export JSONL training splits to Parquet", "", func(cmd *cli.Command, args []string) {
lem.RunParquet(args)
}))
exportGroup.AddCommand(cli.NewRun("publish", "Push Parquet files to HuggingFace dataset repo", "", func(cmd *cli.Command, args []string) {
lem.RunPublish(args)
}))
exportGroup.AddCommand(cli.NewRun("convert", "Convert MLX LoRA adapter to PEFT format", "", func(cmd *cli.Command, args []string) {
lem.RunConvert(args)
}))
root.AddCommand(exportGroup)
}

24
cmd/lemcmd/gen.go Normal file
View file

@ -0,0 +1,24 @@
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addGenCommands(root *cli.Command) {
genGroup := cli.NewGroup("gen", "Generation commands", "Distill, expand, and generate training data.")
genGroup.AddCommand(cli.NewRun("distill", "Native Metal distillation (go-mlx + grammar scoring)", "", func(cmd *cli.Command, args []string) {
lem.RunDistill(args)
}))
genGroup.AddCommand(cli.NewRun("expand", "Generate expansion responses via trained LEM model", "", func(cmd *cli.Command, args []string) {
lem.RunExpand(args)
}))
genGroup.AddCommand(cli.NewRun("conv", "Generate conversational training data (calm phase)", "", func(cmd *cli.Command, args []string) {
lem.RunConv(args)
}))
root.AddCommand(genGroup)
}

28
cmd/lemcmd/infra.go Normal file
View file

@ -0,0 +1,28 @@
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addInfraCommands(root *cli.Command) {
infraGroup := cli.NewGroup("infra", "Infrastructure commands", "InfluxDB ingestion, DuckDB queries, and distributed workers.")
infraGroup.AddCommand(cli.NewRun("ingest", "Ingest benchmark data into InfluxDB", "", func(cmd *cli.Command, args []string) {
lem.RunIngest(args)
}))
infraGroup.AddCommand(cli.NewRun("seed-influx", "Seed InfluxDB golden_gen from DuckDB", "", func(cmd *cli.Command, args []string) {
lem.RunSeedInflux(args)
}))
infraGroup.AddCommand(cli.NewRun("query", "Run ad-hoc SQL against DuckDB", "", func(cmd *cli.Command, args []string) {
lem.RunQuery(args)
}))
infraGroup.AddCommand(cli.NewRun("worker", "Run as distributed inference worker node", "", func(cmd *cli.Command, args []string) {
lem.RunWorker(args)
}))
root.AddCommand(infraGroup)
}

17
cmd/lemcmd/lem.go Normal file
View file

@ -0,0 +1,17 @@
// Package lemcmd provides CLI commands for the LEM binary.
// Commands register through the Core framework's cli.WithCommands lifecycle.
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
)
// AddLEMCommands registers all LEM command groups on the root command.
func AddLEMCommands(root *cli.Command) {
addScoreCommands(root)
addGenCommands(root)
addDataCommands(root)
addExportCommands(root)
addMonCommands(root)
addInfraCommands(root)
}

32
cmd/lemcmd/mon.go Normal file
View file

@ -0,0 +1,32 @@
package lemcmd
import (
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addMonCommands(root *cli.Command) {
monGroup := cli.NewGroup("mon", "Monitoring commands", "Training progress, pipeline status, inventory, coverage, and metrics.")
monGroup.AddCommand(cli.NewRun("status", "Show training and generation progress (InfluxDB)", "", func(cmd *cli.Command, args []string) {
lem.RunStatus(args)
}))
monGroup.AddCommand(cli.NewRun("expand-status", "Show expansion pipeline status (DuckDB)", "", func(cmd *cli.Command, args []string) {
lem.RunExpandStatus(args)
}))
monGroup.AddCommand(cli.NewRun("inventory", "Show DuckDB table inventory", "", func(cmd *cli.Command, args []string) {
lem.RunInventory(args)
}))
monGroup.AddCommand(cli.NewRun("coverage", "Analyse seed coverage gaps", "", func(cmd *cli.Command, args []string) {
lem.RunCoverage(args)
}))
monGroup.AddCommand(cli.NewRun("metrics", "Push DuckDB golden set stats to InfluxDB", "", func(cmd *cli.Command, args []string) {
lem.RunMetrics(args)
}))
root.AddCommand(monGroup)
}

43
cmd/lemcmd/score.go Normal file
View file

@ -0,0 +1,43 @@
package lemcmd
import (
"fmt"
"forge.lthn.ai/core/go/pkg/cli"
"forge.lthn.ai/lthn/lem/pkg/lem"
)
func addScoreCommands(root *cli.Command) {
scoreGroup := cli.NewGroup("score", "Scoring commands", "Score responses, probe models, compare results.")
scoreGroup.AddCommand(cli.NewRun("run", "Score existing response files", "", func(cmd *cli.Command, args []string) {
lem.RunScore(args)
}))
scoreGroup.AddCommand(cli.NewRun("probe", "Generate responses and score them", "", func(cmd *cli.Command, args []string) {
lem.RunProbe(args)
}))
// compare has a different signature — it takes two named args, not []string.
compareCmd := cli.NewCommand("compare", "Compare two score files", "", nil)
var compareOld, compareNew string
cli.StringFlag(compareCmd, &compareOld, "old", "", "", "Old score file (required)")
cli.StringFlag(compareCmd, &compareNew, "new", "", "", "New score file (required)")
compareCmd.RunE = func(cmd *cli.Command, args []string) error {
if compareOld == "" || compareNew == "" {
return fmt.Errorf("--old and --new are required")
}
return lem.RunCompare(compareOld, compareNew)
}
scoreGroup.AddCommand(compareCmd)
scoreGroup.AddCommand(cli.NewRun("tier", "Score expansion responses (heuristic/judge tiers)", "", func(cmd *cli.Command, args []string) {
lem.RunTierScore(args)
}))
scoreGroup.AddCommand(cli.NewRun("agent", "ROCm scoring daemon (polls M3, scores checkpoints)", "", func(cmd *cli.Command, args []string) {
lem.RunAgent(args)
}))
root.AddCommand(scoreGroup)
}