fix: address PR review comments from CodeRabbit, Copilot, and Gemini
Fixes across 25 files addressing 46+ review comments: - pkg/ai/metrics.go: handle error from Close() on writable file handle - pkg/ansible: restore loop vars after loop, restore become settings, fix Upload with become=true and no password (use sudo -n), honour SSH timeout config, use E() helper for contextual errors, quote git refs in checkout commands - pkg/rag: validate chunk config, guard negative-to-uint64 conversion, use E() helper for errors, add context timeout to Ollama HTTP calls - pkg/deploy/python: fix exec.ExitError type assertion (was os.PathError), handle os.UserHomeDir() error - pkg/build/buildcmd: use cmd.Context() instead of context.Background() for proper Ctrl+C cancellation - install.bat: add curl timeouts, CRLF line endings, use --connect-timeout for archive downloads - install.sh: use absolute path for version check in CI mode - tools/rag: fix broken ingest.py function def, escape HTML in query.py, pin qdrant-client version, add markdown code block languages - internal/cmd/rag: add chunk size validation, env override handling Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Snider
parent
8d1aca4053
commit
f4ba17b9f5
25 changed files with 374 additions and 150 deletions
11
install.bat
11
install.bat
|
|
@ -33,7 +33,7 @@ set "INSTALL_DIR=%LOCALAPPDATA%\Programs\core"
|
|||
|
||||
REM === Resolve Version ===
|
||||
if "%VERSION%"=="latest" (
|
||||
for /f "tokens=2 delims=:" %%a in ('curl -fsSL "https://api.github.com/repos/%REPO%/releases/latest" ^| findstr "tag_name"') do (
|
||||
for /f "tokens=2 delims=:" %%a in ('curl -fsSL --max-time 10 "https://api.github.com/repos/%REPO%/releases/latest" ^| findstr "tag_name"') do (
|
||||
set "VERSION=%%a"
|
||||
set "VERSION=!VERSION:"=!"
|
||||
set "VERSION=!VERSION: =!"
|
||||
|
|
@ -79,7 +79,8 @@ if errorlevel 1 exit /b 1
|
|||
call :install_binary
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
%BINARY% --version || exit /b 1
|
||||
%BINARY% --version
|
||||
if errorlevel 1 exit /b 1
|
||||
goto :done
|
||||
|
||||
:install_dev
|
||||
|
|
@ -120,7 +121,7 @@ set "_result=%~2"
|
|||
REM Try variant-specific first, then full
|
||||
if not "%_variant%"=="" (
|
||||
set "_try=%BINARY%-%_variant%-windows-amd64.zip"
|
||||
curl -fsSLI "https://github.com/%REPO%/releases/download/!VERSION!/!_try!" 2>nul | findstr /r "HTTP/[12].* [23][0-9][0-9]" >nul
|
||||
curl -fsSLI --max-time 10 "https://github.com/%REPO%/releases/download/!VERSION!/!_try!" 2>nul | findstr /r "HTTP/[12].* [23][0-9][0-9]" >nul
|
||||
if not errorlevel 1 (
|
||||
set "%_result%=!_try!"
|
||||
exit /b 0
|
||||
|
|
@ -132,7 +133,7 @@ set "%_result%=%BINARY%-windows-amd64.zip"
|
|||
exit /b 0
|
||||
|
||||
:download_and_extract
|
||||
curl -fsSL "https://github.com/%REPO%/releases/download/!VERSION!/!ARCHIVE!" -o "%TEMP%\!ARCHIVE!"
|
||||
curl -fsSL --connect-timeout 10 "https://github.com/%REPO%/releases/download/!VERSION!/!ARCHIVE!" -o "%TEMP%\!ARCHIVE!"
|
||||
if errorlevel 1 (
|
||||
echo ERROR: Failed to download !ARCHIVE!
|
||||
exit /b 1
|
||||
|
|
@ -166,4 +167,4 @@ if errorlevel 1 exit /b 1
|
|||
exit /b 0
|
||||
|
||||
:done
|
||||
endlocal
|
||||
endlocal
|
||||
|
|
@ -171,7 +171,7 @@ install_ci() {
|
|||
sudo mv "$WORK_DIR/${BINARY}" /usr/local/bin/
|
||||
fi
|
||||
|
||||
${BINARY} --version
|
||||
/usr/local/bin/${BINARY} --version
|
||||
}
|
||||
|
||||
install_dev() {
|
||||
|
|
|
|||
|
|
@ -291,8 +291,8 @@ func runAnsibleTest(cmd *cobra.Command, args []string) error {
|
|||
fmt.Printf(" Disk: %s\n", strings.TrimSpace(stdout))
|
||||
|
||||
// Docker
|
||||
stdout, _, rc, _ := client.Run(ctx, "docker --version 2>/dev/null")
|
||||
if rc == 0 {
|
||||
stdout, _, _, err = client.Run(ctx, "docker --version 2>/dev/null")
|
||||
if err == nil {
|
||||
fmt.Printf(" Docker: %s\n", cli.SuccessStyle.Render(strings.TrimSpace(stdout)))
|
||||
} else {
|
||||
fmt.Printf(" Docker: %s\n", cli.DimStyle.Render("not installed"))
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ func runTeam(cmd *cobra.Command, args []string) error {
|
|||
func runCall(cmd *cobra.Command, args []string) error {
|
||||
client, err := getClient()
|
||||
if err != nil {
|
||||
return err
|
||||
return cli.WrapVerb(err, "initialize", "client")
|
||||
}
|
||||
|
||||
operation := args[0]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package rag
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/host-uk/core/pkg/cli"
|
||||
"github.com/host-uk/core/pkg/i18n"
|
||||
|
|
@ -66,6 +65,13 @@ func runIngest(cmd *cobra.Command, args []string) error {
|
|||
}
|
||||
|
||||
// Configure ingestion
|
||||
if chunkSize <= 0 {
|
||||
return fmt.Errorf("chunk-size must be > 0")
|
||||
}
|
||||
if chunkOverlap < 0 || chunkOverlap >= chunkSize {
|
||||
return fmt.Errorf("chunk-overlap must be >= 0 and < chunk-size")
|
||||
}
|
||||
|
||||
cfg := rag.IngestConfig{
|
||||
Directory: directory,
|
||||
Collection: collection,
|
||||
|
|
@ -164,15 +170,4 @@ func IngestFile(ctx context.Context, filePath, collectionName string) (int, erro
|
|||
return rag.IngestFile(ctx, qdrantClient, ollamaClient, collectionName, filePath, rag.DefaultChunkConfig())
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Check for environment variable overrides
|
||||
if host := os.Getenv("QDRANT_HOST"); host != "" {
|
||||
qdrantHost = host
|
||||
}
|
||||
if host := os.Getenv("OLLAMA_HOST"); host != "" {
|
||||
ollamaHost = host
|
||||
}
|
||||
if m := os.Getenv("EMBEDDING_MODEL"); m != "" {
|
||||
model = m
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -51,6 +51,9 @@ func runQuery(cmd *cobra.Command, args []string) error {
|
|||
}
|
||||
|
||||
// Configure query
|
||||
if limit < 0 {
|
||||
limit = 0
|
||||
}
|
||||
cfg := rag.QueryConfig{
|
||||
Collection: queryCollection,
|
||||
Limit: uint64(limit),
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package rag
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/host-uk/core/pkg/i18n"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
|
@ -23,13 +26,40 @@ var ragCmd = &cobra.Command{
|
|||
|
||||
func initFlags() {
|
||||
// Qdrant connection flags (persistent) - defaults to localhost for local development
|
||||
ragCmd.PersistentFlags().StringVar(&qdrantHost, "qdrant-host", "localhost", i18n.T("cmd.rag.flag.qdrant_host"))
|
||||
ragCmd.PersistentFlags().IntVar(&qdrantPort, "qdrant-port", 6334, i18n.T("cmd.rag.flag.qdrant_port"))
|
||||
qHost := "localhost"
|
||||
if v := os.Getenv("QDRANT_HOST"); v != "" {
|
||||
qHost = v
|
||||
}
|
||||
ragCmd.PersistentFlags().StringVar(&qdrantHost, "qdrant-host", qHost, i18n.T("cmd.rag.flag.qdrant_host"))
|
||||
|
||||
qPort := 6334
|
||||
if v := os.Getenv("QDRANT_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil {
|
||||
qPort = p
|
||||
}
|
||||
}
|
||||
ragCmd.PersistentFlags().IntVar(&qdrantPort, "qdrant-port", qPort, i18n.T("cmd.rag.flag.qdrant_port"))
|
||||
|
||||
// Ollama connection flags (persistent) - defaults to localhost for local development
|
||||
ragCmd.PersistentFlags().StringVar(&ollamaHost, "ollama-host", "localhost", i18n.T("cmd.rag.flag.ollama_host"))
|
||||
ragCmd.PersistentFlags().IntVar(&ollamaPort, "ollama-port", 11434, i18n.T("cmd.rag.flag.ollama_port"))
|
||||
ragCmd.PersistentFlags().StringVar(&model, "model", "nomic-embed-text", i18n.T("cmd.rag.flag.model"))
|
||||
oHost := "localhost"
|
||||
if v := os.Getenv("OLLAMA_HOST"); v != "" {
|
||||
oHost = v
|
||||
}
|
||||
ragCmd.PersistentFlags().StringVar(&ollamaHost, "ollama-host", oHost, i18n.T("cmd.rag.flag.ollama_host"))
|
||||
|
||||
oPort := 11434
|
||||
if v := os.Getenv("OLLAMA_PORT"); v != "" {
|
||||
if p, err := strconv.Atoi(v); err == nil {
|
||||
oPort = p
|
||||
}
|
||||
}
|
||||
ragCmd.PersistentFlags().IntVar(&ollamaPort, "ollama-port", oPort, i18n.T("cmd.rag.flag.ollama_port"))
|
||||
|
||||
m := "nomic-embed-text"
|
||||
if v := os.Getenv("EMBEDDING_MODEL"); v != "" {
|
||||
m = v
|
||||
}
|
||||
ragCmd.PersistentFlags().StringVar(&model, "model", m, i18n.T("cmd.rag.flag.model"))
|
||||
|
||||
// Verbose flag (persistent)
|
||||
ragCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, i18n.T("common.flag.verbose"))
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ func metricsFilePath(dir string, t time.Time) string {
|
|||
|
||||
// Record appends an event to the daily JSONL file at
|
||||
// ~/.core/ai/metrics/YYYY-MM-DD.jsonl.
|
||||
func Record(event Event) error {
|
||||
func Record(event Event) (err error) {
|
||||
if event.Timestamp.IsZero() {
|
||||
event.Timestamp = time.Now()
|
||||
}
|
||||
|
|
@ -56,7 +56,11 @@ func Record(event Event) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("open metrics file: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
defer func() {
|
||||
if cerr := f.Close(); cerr != nil && err == nil {
|
||||
err = fmt.Errorf("close metrics file: %w", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
data, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ import (
|
|||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
)
|
||||
|
||||
// Executor runs Ansible playbooks.
|
||||
|
|
@ -176,7 +178,7 @@ func (e *Executor) runRole(ctx context.Context, hosts []string, roleRef *RoleRef
|
|||
// Parse role tasks
|
||||
tasks, err := e.parser.ParseRole(roleRef.Role, roleRef.TasksFrom)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse role %s: %w", roleRef.Role, err)
|
||||
return log.E("executor.runRole", fmt.Sprintf("parse role %s", roleRef.Role), err)
|
||||
}
|
||||
|
||||
// Merge role vars
|
||||
|
|
@ -308,12 +310,25 @@ func (e *Executor) runLoop(ctx context.Context, host string, client *SSHClient,
|
|||
loopVar = task.LoopControl.LoopVar
|
||||
}
|
||||
|
||||
// Save loop state to restore after loop
|
||||
savedVars := make(map[string]any)
|
||||
if v, ok := e.vars[loopVar]; ok {
|
||||
savedVars[loopVar] = v
|
||||
}
|
||||
indexVar := ""
|
||||
if task.LoopControl != nil && task.LoopControl.IndexVar != "" {
|
||||
indexVar = task.LoopControl.IndexVar
|
||||
if v, ok := e.vars[indexVar]; ok {
|
||||
savedVars[indexVar] = v
|
||||
}
|
||||
}
|
||||
|
||||
var results []TaskResult
|
||||
for i, item := range items {
|
||||
// Set loop variables
|
||||
e.vars[loopVar] = item
|
||||
if task.LoopControl != nil && task.LoopControl.IndexVar != "" {
|
||||
e.vars[task.LoopControl.IndexVar] = i
|
||||
if indexVar != "" {
|
||||
e.vars[indexVar] = i
|
||||
}
|
||||
|
||||
result, err := e.executeModule(ctx, host, client, task, play)
|
||||
|
|
@ -327,6 +342,20 @@ func (e *Executor) runLoop(ctx context.Context, host string, client *SSHClient,
|
|||
}
|
||||
}
|
||||
|
||||
// Restore loop variables
|
||||
if v, ok := savedVars[loopVar]; ok {
|
||||
e.vars[loopVar] = v
|
||||
} else {
|
||||
delete(e.vars, loopVar)
|
||||
}
|
||||
if indexVar != "" {
|
||||
if v, ok := savedVars[indexVar]; ok {
|
||||
e.vars[indexVar] = v
|
||||
} else {
|
||||
delete(e.vars, indexVar)
|
||||
}
|
||||
}
|
||||
|
||||
// Store combined result
|
||||
if task.Register != "" {
|
||||
combined := &TaskResult{
|
||||
|
|
@ -371,7 +400,11 @@ func (e *Executor) runBlock(ctx context.Context, hosts []string, task *Task, pla
|
|||
|
||||
// Always run always block
|
||||
for _, t := range task.Always {
|
||||
e.runTaskOnHosts(ctx, hosts, &t, play) //nolint:errcheck
|
||||
if err := e.runTaskOnHosts(ctx, hosts, &t, play); err != nil {
|
||||
if blockErr == nil {
|
||||
blockErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if blockErr != nil && len(task.Rescue) == 0 {
|
||||
|
|
@ -578,14 +611,6 @@ func (e *Executor) gatherFacts(ctx context.Context, host string, play *Play) err
|
|||
e.facts[host] = facts
|
||||
e.mu.Unlock()
|
||||
|
||||
// Also set as vars
|
||||
e.SetVar("ansible_hostname", facts.Hostname)
|
||||
e.SetVar("ansible_fqdn", facts.FQDN)
|
||||
e.SetVar("ansible_distribution", facts.Distribution)
|
||||
e.SetVar("ansible_distribution_version", facts.Version)
|
||||
e.SetVar("ansible_architecture", facts.Architecture)
|
||||
e.SetVar("ansible_kernel", facts.Kernel)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -788,6 +813,12 @@ func (e *Executor) resolveExpr(expr string, host string, task *Task) string {
|
|||
return facts.FQDN
|
||||
case "ansible_distribution":
|
||||
return facts.Distribution
|
||||
case "ansible_distribution_version":
|
||||
return facts.Version
|
||||
case "ansible_architecture":
|
||||
return facts.Architecture
|
||||
case "ansible_kernel":
|
||||
return facts.Kernel
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -959,8 +990,30 @@ func (e *Executor) TemplateFile(src, host string, task *Task) (string, error) {
|
|||
return e.templateString(string(content), host, task), nil
|
||||
}
|
||||
|
||||
// Build context map
|
||||
context := make(map[string]any)
|
||||
for k, v := range e.vars {
|
||||
context[k] = v
|
||||
}
|
||||
// Add host vars
|
||||
if e.inventory != nil {
|
||||
hostVars := GetHostVars(e.inventory, host)
|
||||
for k, v := range hostVars {
|
||||
context[k] = v
|
||||
}
|
||||
}
|
||||
// Add facts
|
||||
if facts, ok := e.facts[host]; ok {
|
||||
context["ansible_hostname"] = facts.Hostname
|
||||
context["ansible_fqdn"] = facts.FQDN
|
||||
context["ansible_distribution"] = facts.Distribution
|
||||
context["ansible_distribution_version"] = facts.Version
|
||||
context["ansible_architecture"] = facts.Architecture
|
||||
context["ansible_kernel"] = facts.Kernel
|
||||
}
|
||||
|
||||
var buf strings.Builder
|
||||
if err := tmpl.Execute(&buf, e.vars); err != nil {
|
||||
if err := tmpl.Execute(&buf, context); err != nil {
|
||||
return e.templateString(string(content), host, task), nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,14 @@ func (e *Executor) executeModule(ctx context.Context, host string, client *SSHCl
|
|||
|
||||
// Apply task-level become
|
||||
if task.Become != nil && *task.Become {
|
||||
// Save old state to restore
|
||||
oldBecome := client.become
|
||||
oldUser := client.becomeUser
|
||||
oldPass := client.becomePass
|
||||
|
||||
client.SetBecome(true, task.BecomeUser, "")
|
||||
|
||||
defer client.SetBecome(oldBecome, oldUser, oldPass)
|
||||
}
|
||||
|
||||
// Template the args
|
||||
|
|
@ -770,8 +777,14 @@ func (e *Executor) moduleUser(ctx context.Context, client *SSHClient, args map[s
|
|||
}
|
||||
|
||||
// Try usermod first, then useradd
|
||||
cmd := fmt.Sprintf("id %s >/dev/null 2>&1 && usermod %s %s || useradd %s %s",
|
||||
name, strings.Join(opts, " "), name, strings.Join(opts, " "), name)
|
||||
optsStr := strings.Join(opts, " ")
|
||||
var cmd string
|
||||
if optsStr == "" {
|
||||
cmd = fmt.Sprintf("id %s >/dev/null 2>&1 || useradd %s", name, name)
|
||||
} else {
|
||||
cmd = fmt.Sprintf("id %s >/dev/null 2>&1 && usermod %s %s || useradd %s %s",
|
||||
name, optsStr, name, optsStr, name)
|
||||
}
|
||||
|
||||
stdout, stderr, rc, err := client.Run(ctx, cmd)
|
||||
if err != nil || rc != 0 {
|
||||
|
|
@ -1008,10 +1021,10 @@ func (e *Executor) moduleGit(ctx context.Context, client *SSHClient, args map[st
|
|||
|
||||
var cmd string
|
||||
if exists {
|
||||
cmd = fmt.Sprintf("cd %q && git fetch --all && git checkout %s && git pull origin %s 2>/dev/null || true",
|
||||
dest, version, version)
|
||||
// Fetch and checkout (force to ensure clean state)
|
||||
cmd = fmt.Sprintf("cd %q && git fetch --all && git checkout --force %q", dest, version)
|
||||
} else {
|
||||
cmd = fmt.Sprintf("git clone %q %q && cd %q && git checkout %s",
|
||||
cmd = fmt.Sprintf("git clone %q %q && cd %q && git checkout %q",
|
||||
repo, dest, dest, version)
|
||||
}
|
||||
|
||||
|
|
@ -1414,5 +1427,11 @@ func (e *Executor) moduleDockerCompose(ctx context.Context, client *SSHClient, a
|
|||
return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil
|
||||
}
|
||||
|
||||
return &TaskResult{Changed: true, Stdout: stdout}, nil
|
||||
// Heuristic for changed
|
||||
changed := true
|
||||
if strings.Contains(stdout, "Up to date") || strings.Contains(stderr, "Up to date") {
|
||||
changed = false
|
||||
}
|
||||
|
||||
return &TaskResult{Changed: changed, Stdout: stdout}, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
|
|
@ -112,7 +113,7 @@ func (p *Parser) ParseRole(name string, tasksFrom string) ([]Task, error) {
|
|||
}
|
||||
|
||||
if tasksPath == "" {
|
||||
return nil, fmt.Errorf("role %s not found in search paths: %v", name, searchPaths)
|
||||
return nil, log.E("parser.ParseRole", fmt.Sprintf("role %s not found in search paths: %v", name, searchPaths), nil)
|
||||
}
|
||||
|
||||
// Load role defaults
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
"golang.org/x/crypto/ssh"
|
||||
"golang.org/x/crypto/ssh/knownhosts"
|
||||
)
|
||||
|
|
@ -28,6 +29,8 @@ type SSHClient struct {
|
|||
become bool
|
||||
becomeUser string
|
||||
becomePass string
|
||||
timeout time.Duration
|
||||
insecure bool
|
||||
}
|
||||
|
||||
// SSHConfig holds SSH connection configuration.
|
||||
|
|
@ -41,6 +44,7 @@ type SSHConfig struct {
|
|||
BecomeUser string
|
||||
BecomePass string
|
||||
Timeout time.Duration
|
||||
Insecure bool
|
||||
}
|
||||
|
||||
// NewSSHClient creates a new SSH client.
|
||||
|
|
@ -64,6 +68,8 @@ func NewSSHClient(cfg SSHConfig) (*SSHClient, error) {
|
|||
become: cfg.Become,
|
||||
becomeUser: cfg.BecomeUser,
|
||||
becomePass: cfg.BecomePass,
|
||||
timeout: cfg.Timeout,
|
||||
insecure: cfg.Insecure,
|
||||
}
|
||||
|
||||
return client, nil
|
||||
|
|
@ -125,25 +131,33 @@ func (c *SSHClient) Connect(ctx context.Context) error {
|
|||
}
|
||||
|
||||
if len(authMethods) == 0 {
|
||||
return fmt.Errorf("no authentication method available")
|
||||
return log.E("ssh.Connect", "no authentication method available", nil)
|
||||
}
|
||||
|
||||
// Use known_hosts file for host key verification, fall back to accepting any key
|
||||
// if known_hosts doesn't exist (common in containerized/ephemeral environments)
|
||||
hostKeyCallback := ssh.InsecureIgnoreHostKey()
|
||||
home, _ := os.UserHomeDir()
|
||||
knownHostsPath := filepath.Join(home, ".ssh", "known_hosts")
|
||||
if _, err := os.Stat(knownHostsPath); err == nil {
|
||||
if cb, err := knownhosts.New(knownHostsPath); err == nil {
|
||||
hostKeyCallback = cb
|
||||
// Host key verification
|
||||
var hostKeyCallback ssh.HostKeyCallback
|
||||
|
||||
if c.insecure {
|
||||
hostKeyCallback = ssh.InsecureIgnoreHostKey()
|
||||
} else {
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return log.E("ssh.Connect", "failed to get user home dir", err)
|
||||
}
|
||||
knownHostsPath := filepath.Join(home, ".ssh", "known_hosts")
|
||||
|
||||
cb, err := knownhosts.New(knownHostsPath)
|
||||
if err != nil {
|
||||
return log.E("ssh.Connect", "failed to load known_hosts (use Insecure=true to bypass)", err)
|
||||
}
|
||||
hostKeyCallback = cb
|
||||
}
|
||||
|
||||
config := &ssh.ClientConfig{
|
||||
User: c.user,
|
||||
Auth: authMethods,
|
||||
HostKeyCallback: hostKeyCallback,
|
||||
Timeout: 30 * time.Second,
|
||||
Timeout: c.timeout,
|
||||
}
|
||||
|
||||
addr := fmt.Sprintf("%s:%d", c.host, c.port)
|
||||
|
|
@ -152,15 +166,16 @@ func (c *SSHClient) Connect(ctx context.Context) error {
|
|||
var d net.Dialer
|
||||
conn, err := d.DialContext(ctx, "tcp", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial %s: %w", addr, err)
|
||||
return log.E("ssh.Connect", fmt.Sprintf("dial %s", addr), err)
|
||||
}
|
||||
|
||||
sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return fmt.Errorf("ssh connect %s: %w", addr, err)
|
||||
// conn is closed by NewClientConn on error
|
||||
return log.E("ssh.Connect", fmt.Sprintf("ssh connect %s", addr), err)
|
||||
}
|
||||
|
||||
|
||||
c.client = ssh.NewClient(sshConn, chans, reqs)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -186,7 +201,7 @@ func (c *SSHClient) Run(ctx context.Context, cmd string) (stdout, stderr string,
|
|||
|
||||
session, err := c.client.NewSession()
|
||||
if err != nil {
|
||||
return "", "", -1, fmt.Errorf("new session: %w", err)
|
||||
return "", "", -1, log.E("ssh.Run", "new session", err)
|
||||
}
|
||||
defer func() { _ = session.Close() }()
|
||||
|
||||
|
|
@ -204,10 +219,27 @@ func (c *SSHClient) Run(ctx context.Context, cmd string) (stdout, stderr string,
|
|||
escapedCmd := strings.ReplaceAll(cmd, "'", "'\\''")
|
||||
if c.becomePass != "" {
|
||||
// Use sudo with password via stdin (-S flag)
|
||||
cmd = fmt.Sprintf("echo '%s' | sudo -S -u %s bash -c '%s'", c.becomePass, becomeUser, escapedCmd)
|
||||
// We launch a goroutine to write the password to stdin
|
||||
cmd = fmt.Sprintf("sudo -S -u %s bash -c '%s'", becomeUser, escapedCmd)
|
||||
stdin, err := session.StdinPipe()
|
||||
if err != nil {
|
||||
return "", "", -1, log.E("ssh.Run", "stdin pipe", err)
|
||||
}
|
||||
go func() {
|
||||
defer stdin.Close()
|
||||
_, _ = io.WriteString(stdin, c.becomePass+"\n")
|
||||
}()
|
||||
} else if c.password != "" {
|
||||
// Try using connection password for sudo
|
||||
cmd = fmt.Sprintf("echo '%s' | sudo -S -u %s bash -c '%s'", c.password, becomeUser, escapedCmd)
|
||||
cmd = fmt.Sprintf("sudo -S -u %s bash -c '%s'", becomeUser, escapedCmd)
|
||||
stdin, err := session.StdinPipe()
|
||||
if err != nil {
|
||||
return "", "", -1, log.E("ssh.Run", "stdin pipe", err)
|
||||
}
|
||||
go func() {
|
||||
defer stdin.Close()
|
||||
_, _ = io.WriteString(stdin, c.password+"\n")
|
||||
}()
|
||||
} else {
|
||||
// Try passwordless sudo
|
||||
cmd = fmt.Sprintf("sudo -n -u %s bash -c '%s'", becomeUser, escapedCmd)
|
||||
|
|
@ -250,16 +282,10 @@ func (c *SSHClient) Upload(ctx context.Context, local io.Reader, remote string,
|
|||
return err
|
||||
}
|
||||
|
||||
session, err := c.client.NewSession()
|
||||
if err != nil {
|
||||
return fmt.Errorf("new session: %w", err)
|
||||
}
|
||||
defer func() { _ = session.Close() }()
|
||||
|
||||
// Read content
|
||||
content, err := io.ReadAll(local)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read content: %w", err)
|
||||
return log.E("ssh.Upload", "read content", err)
|
||||
}
|
||||
|
||||
// Create parent directory
|
||||
|
|
@ -269,40 +295,76 @@ func (c *SSHClient) Upload(ctx context.Context, local io.Reader, remote string,
|
|||
dirCmd = fmt.Sprintf("sudo mkdir -p %q", dir)
|
||||
}
|
||||
if _, _, _, err := c.Run(ctx, dirCmd); err != nil {
|
||||
return fmt.Errorf("create parent dir: %w", err)
|
||||
return log.E("ssh.Upload", "create parent dir", err)
|
||||
}
|
||||
|
||||
// Use cat to write the file (simpler than SCP)
|
||||
writeCmd := fmt.Sprintf("cat > %q && chmod %o %q", remote, mode, remote)
|
||||
if c.become {
|
||||
writeCmd = fmt.Sprintf("sudo bash -c 'cat > %q && chmod %o %q'", remote, mode, remote)
|
||||
}
|
||||
|
||||
|
||||
// If become is needed, we construct a command that reads password then content from stdin
|
||||
// But we need to be careful with handling stdin for sudo + cat.
|
||||
// We'll use a session with piped stdin.
|
||||
|
||||
session2, err := c.client.NewSession()
|
||||
if err != nil {
|
||||
return fmt.Errorf("new session for write: %w", err)
|
||||
return log.E("ssh.Upload", "new session for write", err)
|
||||
}
|
||||
defer func() { _ = session2.Close() }()
|
||||
|
||||
stdin, err := session2.StdinPipe()
|
||||
if err != nil {
|
||||
return fmt.Errorf("stdin pipe: %w", err)
|
||||
return log.E("ssh.Upload", "stdin pipe", err)
|
||||
}
|
||||
|
||||
var stderrBuf bytes.Buffer
|
||||
session2.Stderr = &stderrBuf
|
||||
|
||||
if err := session2.Start(writeCmd); err != nil {
|
||||
return fmt.Errorf("start write: %w", err)
|
||||
}
|
||||
if c.become {
|
||||
becomeUser := c.becomeUser
|
||||
if becomeUser == "" {
|
||||
becomeUser = "root"
|
||||
}
|
||||
|
||||
if _, err := stdin.Write(content); err != nil {
|
||||
return fmt.Errorf("write content: %w", err)
|
||||
pass := c.becomePass
|
||||
if pass == "" {
|
||||
pass = c.password
|
||||
}
|
||||
|
||||
if pass != "" {
|
||||
// Use sudo -S with password from stdin
|
||||
writeCmd = fmt.Sprintf("sudo -S -u %s bash -c 'cat > %q && chmod %o %q'",
|
||||
becomeUser, remote, mode, remote)
|
||||
} else {
|
||||
// Use passwordless sudo (sudo -n) to avoid consuming file content as password
|
||||
writeCmd = fmt.Sprintf("sudo -n -u %s bash -c 'cat > %q && chmod %o %q'",
|
||||
becomeUser, remote, mode, remote)
|
||||
}
|
||||
|
||||
if err := session2.Start(writeCmd); err != nil {
|
||||
return log.E("ssh.Upload", "start write", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer stdin.Close()
|
||||
if pass != "" {
|
||||
_, _ = io.WriteString(stdin, pass+"\n")
|
||||
}
|
||||
_, _ = stdin.Write(content)
|
||||
}()
|
||||
} else {
|
||||
// Normal write
|
||||
if err := session2.Start(writeCmd); err != nil {
|
||||
return log.E("ssh.Upload", "start write", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer stdin.Close()
|
||||
_, _ = stdin.Write(content)
|
||||
}()
|
||||
}
|
||||
_ = stdin.Close()
|
||||
|
||||
if err := session2.Wait(); err != nil {
|
||||
return fmt.Errorf("write failed: %w (stderr: %s)", err, stderrBuf.String())
|
||||
return log.E("ssh.Upload", fmt.Sprintf("write failed (stderr: %s)", stderrBuf.String()), err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -315,16 +377,13 @@ func (c *SSHClient) Download(ctx context.Context, remote string) ([]byte, error)
|
|||
}
|
||||
|
||||
cmd := fmt.Sprintf("cat %q", remote)
|
||||
if c.become {
|
||||
cmd = fmt.Sprintf("sudo cat %q", remote)
|
||||
}
|
||||
|
||||
stdout, stderr, exitCode, err := c.Run(ctx, cmd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exitCode != 0 {
|
||||
return nil, fmt.Errorf("cat failed: %s", stderr)
|
||||
return nil, log.E("ssh.Download", fmt.Sprintf("cat failed: %s", stderr), nil)
|
||||
}
|
||||
|
||||
return []byte(stdout), nil
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"os"
|
||||
|
||||
"github.com/host-uk/core/pkg/cli"
|
||||
"github.com/host-uk/core/pkg/framework/core"
|
||||
"github.com/host-uk/core/pkg/i18n"
|
||||
"github.com/host-uk/core/pkg/release"
|
||||
)
|
||||
|
|
@ -24,7 +25,7 @@ var releaseCmd = &cli.Command{
|
|||
Short: i18n.T("cmd.build.release.short"),
|
||||
Long: i18n.T("cmd.build.release.long"),
|
||||
RunE: func(cmd *cli.Command, args []string) error {
|
||||
return runRelease(releaseDryRun, releaseVersion, releaseDraft, releasePrerelease)
|
||||
return runRelease(cmd.Context(), releaseDryRun, releaseVersion, releaseDraft, releasePrerelease)
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -41,13 +42,11 @@ func AddReleaseCommand(buildCmd *cli.Command) {
|
|||
}
|
||||
|
||||
// runRelease executes the full release workflow: build + archive + checksum + publish.
|
||||
func runRelease(dryRun bool, version string, draft, prerelease bool) error {
|
||||
ctx := context.Background()
|
||||
|
||||
func runRelease(ctx context.Context, dryRun bool, version string, draft, prerelease bool) error {
|
||||
// Get current directory
|
||||
projectDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return cli.WrapVerb(err, "get", "working directory")
|
||||
return core.E("release", "get working directory", err)
|
||||
}
|
||||
|
||||
// Check for release config
|
||||
|
|
@ -57,13 +56,13 @@ func runRelease(dryRun bool, version string, draft, prerelease bool) error {
|
|||
i18n.T("cmd.build.release.error.no_config"),
|
||||
)
|
||||
cli.Print(" %s\n", buildDimStyle.Render(i18n.T("cmd.build.release.hint.create_config")))
|
||||
return cli.Err("release config not found")
|
||||
return core.E("release", "config not found", nil)
|
||||
}
|
||||
|
||||
// Load configuration
|
||||
cfg, err := release.LoadConfig(projectDir)
|
||||
if err != nil {
|
||||
return cli.WrapVerb(err, "load", "config")
|
||||
return core.E("release", "load config", err)
|
||||
}
|
||||
|
||||
// Apply CLI overrides
|
||||
|
|
|
|||
|
|
@ -70,7 +70,10 @@ func (c *Client) Call(ctx context.Context, operationID string, params map[string
|
|||
}
|
||||
|
||||
// Generate and run Python script
|
||||
script := python.CoolifyScript(c.baseURL, c.apiToken, operationID, params)
|
||||
script, err := python.CoolifyScript(c.baseURL, c.apiToken, operationID, params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate script: %w", err)
|
||||
}
|
||||
output, err := python.RunScript(ctx, script)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("API call %s failed: %w", operationID, err)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,11 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/host-uk/core/pkg/framework/core"
|
||||
"github.com/kluctl/go-embed-python/python"
|
||||
)
|
||||
|
||||
|
|
@ -39,13 +41,13 @@ func RunScript(ctx context.Context, code string, args ...string) (string, error)
|
|||
// Write code to temp file
|
||||
tmpFile, err := os.CreateTemp("", "core-*.py")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temp file: %w", err)
|
||||
return "", core.E("python", "create temp file", err)
|
||||
}
|
||||
defer func() { _ = os.Remove(tmpFile.Name()) }()
|
||||
|
||||
if _, err := tmpFile.WriteString(code); err != nil {
|
||||
_ = tmpFile.Close()
|
||||
return "", fmt.Errorf("failed to write script: %w", err)
|
||||
return "", core.E("python", "write script", err)
|
||||
}
|
||||
_ = tmpFile.Close()
|
||||
|
||||
|
|
@ -55,17 +57,17 @@ func RunScript(ctx context.Context, code string, args ...string) (string, error)
|
|||
// Get the command
|
||||
cmd, err := ep.PythonCmd(cmdArgs...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create Python command: %w", err)
|
||||
return "", core.E("python", "create command", err)
|
||||
}
|
||||
|
||||
// Run with context
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
// Try to get stderr for better error message
|
||||
if exitErr, ok := err.(*os.PathError); ok {
|
||||
return "", fmt.Errorf("script failed: %v", exitErr)
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
return "", core.E("python", "run script", fmt.Errorf("%w: %s", err, string(exitErr.Stderr)))
|
||||
}
|
||||
return "", fmt.Errorf("script failed: %w", err)
|
||||
return "", core.E("python", "run script", err)
|
||||
}
|
||||
|
||||
return string(output), nil
|
||||
|
|
@ -80,34 +82,49 @@ func RunModule(ctx context.Context, module string, args ...string) (string, erro
|
|||
cmdArgs := append([]string{"-m", module}, args...)
|
||||
cmd, err := ep.PythonCmd(cmdArgs...)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create Python command: %w", err)
|
||||
return "", core.E("python", "create command", err)
|
||||
}
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("module %s failed: %w", module, err)
|
||||
return "", core.E("python", fmt.Sprintf("run module %s", module), err)
|
||||
}
|
||||
|
||||
return string(output), nil
|
||||
}
|
||||
|
||||
// DevOpsPath returns the path to the DevOps repo.
|
||||
func DevOpsPath() string {
|
||||
func DevOpsPath() (string, error) {
|
||||
if path := os.Getenv("DEVOPS_PATH"); path != "" {
|
||||
return path
|
||||
return path, nil
|
||||
}
|
||||
home, _ := os.UserHomeDir()
|
||||
return filepath.Join(home, "Code", "DevOps")
|
||||
home, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
return "", core.E("python", "get user home", err)
|
||||
}
|
||||
return filepath.Join(home, "Code", "DevOps"), nil
|
||||
}
|
||||
|
||||
// CoolifyModulePath returns the path to the Coolify module_utils.
|
||||
func CoolifyModulePath() string {
|
||||
return filepath.Join(DevOpsPath(), "playbooks", "roles", "coolify", "module_utils")
|
||||
func CoolifyModulePath() (string, error) {
|
||||
path, err := DevOpsPath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filepath.Join(path, "playbooks", "roles", "coolify", "module_utils"), nil
|
||||
}
|
||||
|
||||
// CoolifyScript generates Python code to call the Coolify API.
|
||||
func CoolifyScript(baseURL, apiToken, operation string, params map[string]any) string {
|
||||
paramsJSON, _ := json.Marshal(params)
|
||||
func CoolifyScript(baseURL, apiToken, operation string, params map[string]any) (string, error) {
|
||||
paramsJSON, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return "", core.E("python", "marshal params", err)
|
||||
}
|
||||
|
||||
modulePath, err := CoolifyModulePath()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf(`
|
||||
import sys
|
||||
|
|
@ -126,5 +143,5 @@ client = CoolifyClient(
|
|||
params = json.loads(%q)
|
||||
result = client._call(%q, params, check_response=False)
|
||||
print(json.dumps(result))
|
||||
`, CoolifyModulePath(), baseURL, apiToken, string(paramsJSON), operation)
|
||||
`, modulePath, baseURL, apiToken, string(paramsJSON), operation), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -428,9 +428,33 @@
|
|||
"qa.flag.full": "Run all stages including slow checks",
|
||||
"build.short": "Build Docker or LinuxKit image",
|
||||
"deploy.short": "Deploy to Coolify",
|
||||
"deploy.long": "Deploy the PHP application to Coolify",
|
||||
"deploy.deploying": "Deploying to {{.Environment}}",
|
||||
"deploy.warning_status": "Deployment finished with status: {{.Status}}",
|
||||
"deploy.triggered": "Deployment triggered successfully",
|
||||
"deploy.flag.staging": "Deploy to staging environment",
|
||||
"deploy.flag.force": "Force deployment even if no changes detected",
|
||||
"deploy.flag.wait": "Wait for deployment to complete",
|
||||
"deploy_list.short": "List deployments",
|
||||
"deploy_list.long": "List recent deployments",
|
||||
"deploy_list.recent": "Recent deployments for {{.Environment}}",
|
||||
"deploy_list.none_found": "No deployments found",
|
||||
"deploy_list.flag.staging": "List staging deployments",
|
||||
"deploy_list.flag.limit": "Number of deployments to list",
|
||||
"deploy_rollback.short": "Rollback to previous deployment",
|
||||
"deploy_rollback.long": "Rollback to a previous deployment",
|
||||
"deploy_rollback.rolling_back": "Rolling back {{.Environment}}",
|
||||
"deploy_rollback.warning_status": "Rollback finished with status: {{.Status}}",
|
||||
"deploy_rollback.triggered": "Rollback triggered successfully",
|
||||
"deploy_rollback.flag.staging": "Rollback staging environment",
|
||||
"deploy_rollback.flag.id": "Specific deployment ID to rollback to",
|
||||
"deploy_rollback.flag.wait": "Wait for rollback to complete",
|
||||
"deploy_status.short": "Show deployment status",
|
||||
"deploy_status.long": "Show the status of a deployment",
|
||||
"deploy_status.flag.staging": "Check staging deployment",
|
||||
"deploy_status.flag.id": "Specific deployment ID",
|
||||
"label.deploy": "Deploy",
|
||||
"error.deploy_failed": "Deployment failed",
|
||||
"serve.short": "Run production container",
|
||||
"ssl.short": "Setup SSL certificates with mkcert",
|
||||
"packages.short": "Manage local PHP packages",
|
||||
|
|
|
|||
|
|
@ -32,6 +32,13 @@ type Chunk struct {
|
|||
// ChunkMarkdown splits markdown text into chunks by sections and paragraphs.
|
||||
// Preserves context with configurable overlap.
|
||||
func ChunkMarkdown(text string, cfg ChunkConfig) []Chunk {
|
||||
if cfg.Size <= 0 {
|
||||
cfg.Size = 500
|
||||
}
|
||||
if cfg.Overlap < 0 || cfg.Overlap >= cfg.Size {
|
||||
cfg.Overlap = 0
|
||||
}
|
||||
|
||||
var chunks []Chunk
|
||||
|
||||
// Split by ## headers
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ import (
|
|||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
)
|
||||
|
||||
// IngestConfig holds ingestion configuration.
|
||||
|
|
@ -50,26 +52,26 @@ func Ingest(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, cfg
|
|||
// Resolve directory
|
||||
absDir, err := filepath.Abs(cfg.Directory)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error resolving directory: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error resolving directory", err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(absDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error accessing directory: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error accessing directory", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("not a directory: %s", absDir)
|
||||
return nil, log.E("rag.Ingest", fmt.Sprintf("not a directory: %s", absDir), nil)
|
||||
}
|
||||
|
||||
// Check/create collection
|
||||
exists, err := qdrant.CollectionExists(ctx, cfg.Collection)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error checking collection: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error checking collection", err)
|
||||
}
|
||||
|
||||
if cfg.Recreate && exists {
|
||||
if err := qdrant.DeleteCollection(ctx, cfg.Collection); err != nil {
|
||||
return nil, fmt.Errorf("error deleting collection: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error deleting collection", err)
|
||||
}
|
||||
exists = false
|
||||
}
|
||||
|
|
@ -77,7 +79,7 @@ func Ingest(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, cfg
|
|||
if !exists {
|
||||
vectorDim := ollama.EmbedDimension()
|
||||
if err := qdrant.CreateCollection(ctx, cfg.Collection, vectorDim); err != nil {
|
||||
return nil, fmt.Errorf("error creating collection: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error creating collection", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -93,11 +95,11 @@ func Ingest(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, cfg
|
|||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error walking directory: %w", err)
|
||||
return nil, log.E("rag.Ingest", "error walking directory", err)
|
||||
}
|
||||
|
||||
if len(files) == 0 {
|
||||
return nil, fmt.Errorf("no markdown files found in %s", absDir)
|
||||
return nil, log.E("rag.Ingest", fmt.Sprintf("no markdown files found in %s", absDir), nil)
|
||||
}
|
||||
|
||||
// Process files
|
||||
|
|
@ -164,7 +166,7 @@ func Ingest(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, cfg
|
|||
}
|
||||
batch := points[i:end]
|
||||
if err := qdrant.UpsertPoints(ctx, cfg.Collection, batch); err != nil {
|
||||
return stats, fmt.Errorf("error upserting batch %d: %w", i/cfg.BatchSize+1, err)
|
||||
return stats, log.E("rag.Ingest", fmt.Sprintf("error upserting batch %d", i/cfg.BatchSize+1), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -176,7 +178,7 @@ func Ingest(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, cfg
|
|||
func IngestFile(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, collection string, filePath string, chunkCfg ChunkConfig) (int, error) {
|
||||
content, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error reading file: %w", err)
|
||||
return 0, log.E("rag.IngestFile", "error reading file", err)
|
||||
}
|
||||
|
||||
if len(strings.TrimSpace(string(content))) == 0 {
|
||||
|
|
@ -190,7 +192,7 @@ func IngestFile(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient,
|
|||
for _, chunk := range chunks {
|
||||
embedding, err := ollama.Embed(ctx, chunk.Text)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("error embedding chunk %d: %w", chunk.Index, err)
|
||||
return 0, log.E("rag.IngestFile", fmt.Sprintf("error embedding chunk %d", chunk.Index), err)
|
||||
}
|
||||
|
||||
points = append(points, Point{
|
||||
|
|
@ -207,8 +209,8 @@ func IngestFile(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient,
|
|||
}
|
||||
|
||||
if err := qdrant.UpsertPoints(ctx, collection, points); err != nil {
|
||||
return 0, fmt.Errorf("error upserting points: %w", err)
|
||||
return 0, log.E("rag.IngestFile", "error upserting points", err)
|
||||
}
|
||||
|
||||
return len(points), nil
|
||||
}
|
||||
}
|
||||
|
|
@ -5,7 +5,9 @@ import (
|
|||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
"github.com/ollama/ollama/api"
|
||||
)
|
||||
|
||||
|
|
@ -39,7 +41,9 @@ func NewOllamaClient(cfg OllamaConfig) (*OllamaClient, error) {
|
|||
Host: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
}
|
||||
|
||||
client := api.NewClient(baseURL, http.DefaultClient)
|
||||
client := api.NewClient(baseURL, &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
|
||||
return &OllamaClient{
|
||||
client: client,
|
||||
|
|
@ -71,11 +75,11 @@ func (o *OllamaClient) Embed(ctx context.Context, text string) ([]float32, error
|
|||
|
||||
resp, err := o.client.Embed(ctx, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate embedding: %w", err)
|
||||
return nil, log.E("rag.Ollama.Embed", "failed to generate embedding", err)
|
||||
}
|
||||
|
||||
if len(resp.Embeddings) == 0 || len(resp.Embeddings[0]) == 0 {
|
||||
return nil, fmt.Errorf("empty embedding response")
|
||||
return nil, log.E("rag.Ollama.Embed", "empty embedding response", nil)
|
||||
}
|
||||
|
||||
// Convert float64 to float32 for Qdrant
|
||||
|
|
@ -94,7 +98,7 @@ func (o *OllamaClient) EmbedBatch(ctx context.Context, texts []string) ([][]floa
|
|||
for i, text := range texts {
|
||||
embedding, err := o.Embed(ctx, text)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to embed text %d: %w", i, err)
|
||||
return nil, log.E("rag.Ollama.EmbedBatch", fmt.Sprintf("failed to embed text %d", i), err)
|
||||
}
|
||||
results[i] = embedding
|
||||
}
|
||||
|
|
@ -105,7 +109,7 @@ func (o *OllamaClient) EmbedBatch(ctx context.Context, texts []string) ([][]floa
|
|||
func (o *OllamaClient) VerifyModel(ctx context.Context) error {
|
||||
_, err := o.Embed(ctx, "test")
|
||||
if err != nil {
|
||||
return fmt.Errorf("model %s not available: %w (run: ollama pull %s)", o.config.Model, err, o.config.Model)
|
||||
return log.E("rag.Ollama.VerifyModel", fmt.Sprintf("model %s not available (run: ollama pull %s)", o.config.Model, o.config.Model), err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -113,4 +117,4 @@ func (o *OllamaClient) VerifyModel(ctx context.Context) error {
|
|||
// Model returns the configured embedding model name.
|
||||
func (o *OllamaClient) Model() string {
|
||||
return o.config.Model
|
||||
}
|
||||
}
|
||||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
"github.com/qdrant/go-client/qdrant"
|
||||
)
|
||||
|
||||
|
|
@ -44,7 +45,7 @@ func NewQdrantClient(cfg QdrantConfig) (*QdrantClient, error) {
|
|||
UseTLS: cfg.UseTLS,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to Qdrant at %s: %w", addr, err)
|
||||
return nil, log.E("rag.Qdrant", fmt.Sprintf("failed to connect to Qdrant at %s", addr), err)
|
||||
}
|
||||
|
||||
return &QdrantClient{
|
||||
|
|
@ -221,4 +222,4 @@ func valueToGo(v *qdrant.Value) any {
|
|||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,8 @@ import (
|
|||
"fmt"
|
||||
"html"
|
||||
"strings"
|
||||
|
||||
"github.com/host-uk/core/pkg/log"
|
||||
)
|
||||
|
||||
// QueryConfig holds query configuration.
|
||||
|
|
@ -39,7 +41,7 @@ func Query(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, quer
|
|||
// Generate embedding for query
|
||||
embedding, err := ollama.Embed(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error generating query embedding: %w", err)
|
||||
return nil, log.E("rag.Query", "error generating query embedding", err)
|
||||
}
|
||||
|
||||
// Build filter
|
||||
|
|
@ -51,7 +53,7 @@ func Query(ctx context.Context, qdrant *QdrantClient, ollama *OllamaClient, quer
|
|||
// Search Qdrant
|
||||
results, err := qdrant.Search(ctx, cfg.Collection, embedding, cfg.Limit, filter)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error searching: %w", err)
|
||||
return nil, log.E("rag.Query", "error searching", err)
|
||||
}
|
||||
|
||||
// Convert and filter by threshold
|
||||
|
|
@ -158,4 +160,4 @@ func FormatResultsJSON(results []QueryResult) string {
|
|||
}
|
||||
sb.WriteString("]")
|
||||
return sb.String()
|
||||
}
|
||||
}
|
||||
|
|
@ -158,7 +158,7 @@ The ingestion automatically categorizes files:
|
|||
|
||||
## Vector Math (For Understanding)
|
||||
|
||||
```
|
||||
```text
|
||||
"How do I make a Flux button?"
|
||||
↓ Embedding
|
||||
[0.12, -0.45, 0.78, ...768 floats...]
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ except ImportError:
|
|||
|
||||
|
||||
# Configuration
|
||||
QDRANT_HOST = os.getenv("QDRANT_HOST", "linux.snider.dev")
|
||||
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
|
||||
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text")
|
||||
CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "500")) # chars
|
||||
|
|
@ -117,7 +117,7 @@ def get_file_category(path: str) -> str:
|
|||
return "documentation"
|
||||
|
||||
|
||||
def ingest_directory(do we
|
||||
def ingest_directory(
|
||||
directory: Path,
|
||||
client: QdrantClient,
|
||||
collection: str,
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ Requirements:
|
|||
"""
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
|
@ -29,7 +30,7 @@ except ImportError:
|
|||
|
||||
|
||||
# Configuration
|
||||
QDRANT_HOST = os.getenv("QDRANT_HOST", "linux.snider.dev")
|
||||
QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost")
|
||||
QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333"))
|
||||
EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text")
|
||||
|
||||
|
|
@ -120,18 +121,17 @@ def format_for_context(results: list[dict], query: str) -> str:
|
|||
return ""
|
||||
|
||||
output = []
|
||||
output.append(f"<retrieved_context query=\"{query}\">")
|
||||
output.append(f'<retrieved_context query="{html.escape(query)}">')
|
||||
|
||||
for r in results:
|
||||
output.append(f"\n<document source=\"{r['source']}\" category=\"{r['category']}\">")
|
||||
output.append(r['text'])
|
||||
output.append(f'\n<document source="{html.escape(r["source"])}" category="{html.escape(r["category"])}">')
|
||||
output.append(html.escape(r['text']))
|
||||
output.append("</document>")
|
||||
|
||||
output.append("\n</retrieved_context>")
|
||||
|
||||
return "\n".join(output)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Query RAG documentation")
|
||||
parser.add_argument("query", nargs="?", help="Search query")
|
||||
|
|
@ -193,4 +193,4 @@ def main():
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
main()
|
||||
|
|
@ -1,2 +1,2 @@
|
|||
qdrant-client>=1.7.0
|
||||
ollama>=0.1.0
|
||||
qdrant-client>=1.12.0,<2.0.0
|
||||
ollama>=0.1.0
|
||||
Loading…
Add table
Reference in a new issue