1b570b8229
Export distill_results from DuckDB back to compressed JSONL.zst files, completing the cold -> warm -> cold round-trip data pipeline. Co-Authored-By: Virgil <virgil@lethean.io>
47 lines
1 KiB
Go
47 lines
1 KiB
Go
package lem
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func TestExportCold(t *testing.T) {
|
|
dir := t.TempDir()
|
|
dbPath := filepath.Join(dir, "test.duckdb")
|
|
|
|
db, err := OpenDBReadWrite(dbPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// Setup tables and insert test data
|
|
ensureSetupTables(db)
|
|
db.conn.Exec(`INSERT INTO distill_results VALUES ('P01', 'gemma3/12b', 0, 'What is X?', 'X is...', 'lesson-0.jsonl', CURRENT_TIMESTAMP)`)
|
|
db.conn.Exec(`INSERT INTO distill_results VALUES ('P02', 'gemma3/12b', 0, 'What is Y?', 'Y is...', 'lesson-0.jsonl', CURRENT_TIMESTAMP)`)
|
|
db.Close()
|
|
|
|
outDir := filepath.Join(dir, "export")
|
|
os.MkdirAll(outDir, 0755)
|
|
|
|
cfg := ExportColdOpts{
|
|
DB: dbPath,
|
|
Model: "gemma3/12b",
|
|
OutputDir: outDir,
|
|
Compress: true,
|
|
}
|
|
|
|
n, err := RunExportCold(cfg)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 2 {
|
|
t.Fatalf("expected 2 exported, got %d", n)
|
|
}
|
|
|
|
// Verify .zst file exists
|
|
matches, _ := filepath.Glob(filepath.Join(outDir, "*.jsonl.zst"))
|
|
if len(matches) == 0 {
|
|
t.Fatal("expected .jsonl.zst output file")
|
|
}
|
|
}
|