From cf1d8156ddbcd1c91731660ccd9915f9e1c03493 Mon Sep 17 00:00:00 2001 From: Snider Date: Sun, 22 Feb 2026 18:55:57 +0000 Subject: [PATCH] feat(cli): add cmd/lemcmd command registration package 6 command groups (score, gen, data, export, mon, infra) with 25 commands. All pass through to existing lem.Run* functions via the Core framework's cli package. Co-Authored-By: Claude Opus 4.6 --- cmd/lemcmd/data.go | 28 ++++++++++++++++++++++++++++ cmd/lemcmd/export.go | 28 ++++++++++++++++++++++++++++ cmd/lemcmd/gen.go | 24 ++++++++++++++++++++++++ cmd/lemcmd/infra.go | 28 ++++++++++++++++++++++++++++ cmd/lemcmd/lem.go | 17 +++++++++++++++++ cmd/lemcmd/mon.go | 32 ++++++++++++++++++++++++++++++++ cmd/lemcmd/score.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 200 insertions(+) create mode 100644 cmd/lemcmd/data.go create mode 100644 cmd/lemcmd/export.go create mode 100644 cmd/lemcmd/gen.go create mode 100644 cmd/lemcmd/infra.go create mode 100644 cmd/lemcmd/lem.go create mode 100644 cmd/lemcmd/mon.go create mode 100644 cmd/lemcmd/score.go diff --git a/cmd/lemcmd/data.go b/cmd/lemcmd/data.go new file mode 100644 index 0000000..5971208 --- /dev/null +++ b/cmd/lemcmd/data.go @@ -0,0 +1,28 @@ +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addDataCommands(root *cli.Command) { + dataGroup := cli.NewGroup("data", "Data management commands", "Import, consolidate, normalise, and approve training data.") + + dataGroup.AddCommand(cli.NewRun("import-all", "Import ALL LEM data into DuckDB from M3", "", func(cmd *cli.Command, args []string) { + lem.RunImport(args) + })) + + dataGroup.AddCommand(cli.NewRun("consolidate", "Pull worker JSONLs from M3, merge, deduplicate", "", func(cmd *cli.Command, args []string) { + lem.RunConsolidate(args) + })) + + dataGroup.AddCommand(cli.NewRun("normalize", "Normalise seeds to deduplicated expansion prompts", "", func(cmd *cli.Command, args []string) { + lem.RunNormalize(args) + })) + + dataGroup.AddCommand(cli.NewRun("approve", "Filter scored expansions to training JSONL", "", func(cmd *cli.Command, args []string) { + lem.RunApprove(args) + })) + + root.AddCommand(dataGroup) +} diff --git a/cmd/lemcmd/export.go b/cmd/lemcmd/export.go new file mode 100644 index 0000000..3dc3288 --- /dev/null +++ b/cmd/lemcmd/export.go @@ -0,0 +1,28 @@ +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addExportCommands(root *cli.Command) { + exportGroup := cli.NewGroup("export", "Export and publish commands", "Export training data to JSONL, Parquet, HuggingFace, and PEFT formats.") + + exportGroup.AddCommand(cli.NewRun("jsonl", "Export golden set to training-format JSONL splits", "", func(cmd *cli.Command, args []string) { + lem.RunExport(args) + })) + + exportGroup.AddCommand(cli.NewRun("parquet", "Export JSONL training splits to Parquet", "", func(cmd *cli.Command, args []string) { + lem.RunParquet(args) + })) + + exportGroup.AddCommand(cli.NewRun("publish", "Push Parquet files to HuggingFace dataset repo", "", func(cmd *cli.Command, args []string) { + lem.RunPublish(args) + })) + + exportGroup.AddCommand(cli.NewRun("convert", "Convert MLX LoRA adapter to PEFT format", "", func(cmd *cli.Command, args []string) { + lem.RunConvert(args) + })) + + root.AddCommand(exportGroup) +} diff --git a/cmd/lemcmd/gen.go b/cmd/lemcmd/gen.go new file mode 100644 index 0000000..7828ab0 --- /dev/null +++ b/cmd/lemcmd/gen.go @@ -0,0 +1,24 @@ +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addGenCommands(root *cli.Command) { + genGroup := cli.NewGroup("gen", "Generation commands", "Distill, expand, and generate training data.") + + genGroup.AddCommand(cli.NewRun("distill", "Native Metal distillation (go-mlx + grammar scoring)", "", func(cmd *cli.Command, args []string) { + lem.RunDistill(args) + })) + + genGroup.AddCommand(cli.NewRun("expand", "Generate expansion responses via trained LEM model", "", func(cmd *cli.Command, args []string) { + lem.RunExpand(args) + })) + + genGroup.AddCommand(cli.NewRun("conv", "Generate conversational training data (calm phase)", "", func(cmd *cli.Command, args []string) { + lem.RunConv(args) + })) + + root.AddCommand(genGroup) +} diff --git a/cmd/lemcmd/infra.go b/cmd/lemcmd/infra.go new file mode 100644 index 0000000..16e3dea --- /dev/null +++ b/cmd/lemcmd/infra.go @@ -0,0 +1,28 @@ +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addInfraCommands(root *cli.Command) { + infraGroup := cli.NewGroup("infra", "Infrastructure commands", "InfluxDB ingestion, DuckDB queries, and distributed workers.") + + infraGroup.AddCommand(cli.NewRun("ingest", "Ingest benchmark data into InfluxDB", "", func(cmd *cli.Command, args []string) { + lem.RunIngest(args) + })) + + infraGroup.AddCommand(cli.NewRun("seed-influx", "Seed InfluxDB golden_gen from DuckDB", "", func(cmd *cli.Command, args []string) { + lem.RunSeedInflux(args) + })) + + infraGroup.AddCommand(cli.NewRun("query", "Run ad-hoc SQL against DuckDB", "", func(cmd *cli.Command, args []string) { + lem.RunQuery(args) + })) + + infraGroup.AddCommand(cli.NewRun("worker", "Run as distributed inference worker node", "", func(cmd *cli.Command, args []string) { + lem.RunWorker(args) + })) + + root.AddCommand(infraGroup) +} diff --git a/cmd/lemcmd/lem.go b/cmd/lemcmd/lem.go new file mode 100644 index 0000000..e1f3967 --- /dev/null +++ b/cmd/lemcmd/lem.go @@ -0,0 +1,17 @@ +// Package lemcmd provides CLI commands for the LEM binary. +// Commands register through the Core framework's cli.WithCommands lifecycle. +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" +) + +// AddLEMCommands registers all LEM command groups on the root command. +func AddLEMCommands(root *cli.Command) { + addScoreCommands(root) + addGenCommands(root) + addDataCommands(root) + addExportCommands(root) + addMonCommands(root) + addInfraCommands(root) +} diff --git a/cmd/lemcmd/mon.go b/cmd/lemcmd/mon.go new file mode 100644 index 0000000..bc8925f --- /dev/null +++ b/cmd/lemcmd/mon.go @@ -0,0 +1,32 @@ +package lemcmd + +import ( + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addMonCommands(root *cli.Command) { + monGroup := cli.NewGroup("mon", "Monitoring commands", "Training progress, pipeline status, inventory, coverage, and metrics.") + + monGroup.AddCommand(cli.NewRun("status", "Show training and generation progress (InfluxDB)", "", func(cmd *cli.Command, args []string) { + lem.RunStatus(args) + })) + + monGroup.AddCommand(cli.NewRun("expand-status", "Show expansion pipeline status (DuckDB)", "", func(cmd *cli.Command, args []string) { + lem.RunExpandStatus(args) + })) + + monGroup.AddCommand(cli.NewRun("inventory", "Show DuckDB table inventory", "", func(cmd *cli.Command, args []string) { + lem.RunInventory(args) + })) + + monGroup.AddCommand(cli.NewRun("coverage", "Analyse seed coverage gaps", "", func(cmd *cli.Command, args []string) { + lem.RunCoverage(args) + })) + + monGroup.AddCommand(cli.NewRun("metrics", "Push DuckDB golden set stats to InfluxDB", "", func(cmd *cli.Command, args []string) { + lem.RunMetrics(args) + })) + + root.AddCommand(monGroup) +} diff --git a/cmd/lemcmd/score.go b/cmd/lemcmd/score.go new file mode 100644 index 0000000..f162fc7 --- /dev/null +++ b/cmd/lemcmd/score.go @@ -0,0 +1,43 @@ +package lemcmd + +import ( + "fmt" + + "forge.lthn.ai/core/go/pkg/cli" + "forge.lthn.ai/lthn/lem/pkg/lem" +) + +func addScoreCommands(root *cli.Command) { + scoreGroup := cli.NewGroup("score", "Scoring commands", "Score responses, probe models, compare results.") + + scoreGroup.AddCommand(cli.NewRun("run", "Score existing response files", "", func(cmd *cli.Command, args []string) { + lem.RunScore(args) + })) + + scoreGroup.AddCommand(cli.NewRun("probe", "Generate responses and score them", "", func(cmd *cli.Command, args []string) { + lem.RunProbe(args) + })) + + // compare has a different signature — it takes two named args, not []string. + compareCmd := cli.NewCommand("compare", "Compare two score files", "", nil) + var compareOld, compareNew string + cli.StringFlag(compareCmd, &compareOld, "old", "", "", "Old score file (required)") + cli.StringFlag(compareCmd, &compareNew, "new", "", "", "New score file (required)") + compareCmd.RunE = func(cmd *cli.Command, args []string) error { + if compareOld == "" || compareNew == "" { + return fmt.Errorf("--old and --new are required") + } + return lem.RunCompare(compareOld, compareNew) + } + scoreGroup.AddCommand(compareCmd) + + scoreGroup.AddCommand(cli.NewRun("tier", "Score expansion responses (heuristic/judge tiers)", "", func(cmd *cli.Command, args []string) { + lem.RunTierScore(args) + })) + + scoreGroup.AddCommand(cli.NewRun("agent", "ROCm scoring daemon (polls M3, scores checkpoints)", "", func(cmd *cli.Command, args []string) { + lem.RunAgent(args) + })) + + root.AddCommand(scoreGroup) +}