From f55ca297a00f8a6977a60b34ac0a977263d033a3 Mon Sep 17 00:00:00 2001 From: Snider Date: Tue, 3 Feb 2026 18:10:13 +0000 Subject: [PATCH] fix(deploy): address linter warnings and build errors - Fix fmt.Sprintf format verb error in ssh.go (remove unused stat command) - Fix errcheck warnings by explicitly ignoring best-effort operations - Fix ineffassign warning in cmd_ansible.go All golangci-lint checks now pass for deploy packages. Co-Authored-By: Claude Opus 4.5 --- internal/cmd/deploy/cmd_ansible.go | 4 +- pkg/ansible/executor.go | 2 +- pkg/ansible/modules.go | 78 ++++----- pkg/ansible/ssh.go | 22 +-- pkg/deploy/python/python.go | 6 +- tools/rag/README.md | 193 ++++++++++++++++++++++ tools/rag/ingest.py | 254 +++++++++++++++++++++++++++++ tools/rag/query.py | 196 ++++++++++++++++++++++ tools/rag/requirements.txt | 2 + 9 files changed, 698 insertions(+), 59 deletions(-) create mode 100644 tools/rag/README.md create mode 100644 tools/rag/ingest.py create mode 100644 tools/rag/query.py create mode 100644 tools/rag/requirements.txt diff --git a/internal/cmd/deploy/cmd_ansible.go b/internal/cmd/deploy/cmd_ansible.go index 2dac1132..c2237986 100644 --- a/internal/cmd/deploy/cmd_ansible.go +++ b/internal/cmd/deploy/cmd_ansible.go @@ -247,7 +247,7 @@ func runAnsibleTest(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("create client: %w", err) } - defer client.Close() + defer func() { _ = client.Close() }() ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -299,7 +299,7 @@ func runAnsibleTest(cmd *cobra.Command, args []string) error { } // Check if Coolify is running - stdout, _, rc, _ = client.Run(ctx, "docker ps 2>/dev/null | grep -q coolify && echo 'running' || echo 'not running'") + stdout, _, _, _ = client.Run(ctx, "docker ps 2>/dev/null | grep -q coolify && echo 'running' || echo 'not running'") if strings.TrimSpace(stdout) == "running" { fmt.Printf(" Coolify: %s\n", cli.SuccessStyle.Render("running")) } else { diff --git a/pkg/ansible/executor.go b/pkg/ansible/executor.go index 7b6803b4..ffe80103 100644 --- a/pkg/ansible/executor.go +++ b/pkg/ansible/executor.go @@ -935,7 +935,7 @@ func (e *Executor) Close() { defer e.mu.Unlock() for _, client := range e.clients { - client.Close() + _ = client.Close() } e.clients = make(map[string]*SSHClient) } diff --git a/pkg/ansible/modules.go b/pkg/ansible/modules.go index 2e81a53e..c28719d1 100644 --- a/pkg/ansible/modules.go +++ b/pkg/ansible/modules.go @@ -300,12 +300,12 @@ func (e *Executor) moduleCopy(ctx context.Context, client *SSHClient, args map[s return nil, err } - // Handle owner/group + // Handle owner/group (best-effort, errors ignored) if owner := getStringArg(args, "owner", ""); owner != "" { - client.Run(ctx, fmt.Sprintf("chown %s %q", owner, dest)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chown %s %q", owner, dest)) } if group := getStringArg(args, "group", ""); group != "" { - client.Run(ctx, fmt.Sprintf("chgrp %s %q", group, dest)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chgrp %s %q", group, dest)) } return &TaskResult{Changed: true, Msg: fmt.Sprintf("copied to %s", dest)}, nil @@ -387,20 +387,20 @@ func (e *Executor) moduleFile(ctx context.Context, client *SSHClient, args map[s case "file": // Ensure file exists and set permissions if mode := getStringArg(args, "mode", ""); mode != "" { - client.Run(ctx, fmt.Sprintf("chmod %s %q", mode, path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chmod %s %q", mode, path)) } } - // Handle owner/group + // Handle owner/group (best-effort, errors ignored) if owner := getStringArg(args, "owner", ""); owner != "" { - client.Run(ctx, fmt.Sprintf("chown %s %q", owner, path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chown %s %q", owner, path)) } if group := getStringArg(args, "group", ""); group != "" { - client.Run(ctx, fmt.Sprintf("chgrp %s %q", group, path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chgrp %s %q", group, path)) } if recurse := getBoolArg(args, "recurse", false); recurse { if owner := getStringArg(args, "owner", ""); owner != "" { - client.Run(ctx, fmt.Sprintf("chown -R %s %q", owner, path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chown -R %s %q", owner, path)) } } @@ -438,12 +438,12 @@ func (e *Executor) moduleLineinfile(ctx context.Context, client *SSHClient, args if rc != 0 { // Line not found, append cmd = fmt.Sprintf("echo %q >> %q", line, path) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) } } else if line != "" { // Ensure line is present cmd := fmt.Sprintf("grep -qxF %q %q || echo %q >> %q", line, path, line, path) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) } } @@ -527,9 +527,9 @@ func (e *Executor) moduleGetURL(ctx context.Context, client *SSHClient, args map return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil } - // Set mode if specified + // Set mode if specified (best-effort) if mode := getStringArg(args, "mode", ""); mode != "" { - client.Run(ctx, fmt.Sprintf("chmod %s %q", mode, dest)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chmod %s %q", mode, dest)) } return &TaskResult{Changed: true}, nil @@ -545,7 +545,7 @@ func (e *Executor) moduleApt(ctx context.Context, client *SSHClient, args map[st var cmd string if updateCache { - client.Run(ctx, "apt-get update -qq") + _, _, _, _ = client.Run(ctx, "apt-get update -qq") } switch state { @@ -578,7 +578,7 @@ func (e *Executor) moduleAptKey(ctx context.Context, client *SSHClient, args map if state == "absent" { if keyring != "" { - client.Run(ctx, fmt.Sprintf("rm -f %q", keyring)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("rm -f %q", keyring)) } return &TaskResult{Changed: true}, nil } @@ -621,7 +621,7 @@ func (e *Executor) moduleAptRepository(ctx context.Context, client *SSHClient, a path := fmt.Sprintf("/etc/apt/sources.list.d/%s.list", filename) if state == "absent" { - client.Run(ctx, fmt.Sprintf("rm -f %q", path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("rm -f %q", path)) return &TaskResult{Changed: true}, nil } @@ -631,9 +631,9 @@ func (e *Executor) moduleAptRepository(ctx context.Context, client *SSHClient, a return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil } - // Update apt cache + // Update apt cache (best-effort) if getBoolArg(args, "update_cache", true) { - client.Run(ctx, "apt-get update -qq") + _, _, _, _ = client.Run(ctx, "apt-get update -qq") } return &TaskResult{Changed: true}, nil @@ -722,7 +722,7 @@ func (e *Executor) moduleService(ctx context.Context, client *SSHClient, args ma func (e *Executor) moduleSystemd(ctx context.Context, client *SSHClient, args map[string]any) (*TaskResult, error) { // systemd is similar to service if getBoolArg(args, "daemon_reload", false) { - client.Run(ctx, "systemctl daemon-reload") + _, _, _, _ = client.Run(ctx, "systemctl daemon-reload") } return e.moduleService(ctx, client, args) @@ -740,7 +740,7 @@ func (e *Executor) moduleUser(ctx context.Context, client *SSHClient, args map[s if state == "absent" { cmd := fmt.Sprintf("userdel -r %s 2>/dev/null || true", name) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) return &TaskResult{Changed: true}, nil } @@ -791,7 +791,7 @@ func (e *Executor) moduleGroup(ctx context.Context, client *SSHClient, args map[ if state == "absent" { cmd := fmt.Sprintf("groupdel %s 2>/dev/null || true", name) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) return &TaskResult{Changed: true}, nil } @@ -1032,8 +1032,8 @@ func (e *Executor) moduleUnarchive(ctx context.Context, client *SSHClient, args return nil, fmt.Errorf("unarchive: src and dest required") } - // Create dest directory - client.Run(ctx, fmt.Sprintf("mkdir -p %q", dest)) + // Create dest directory (best-effort) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("mkdir -p %q", dest)) var cmd string if !remote { @@ -1048,7 +1048,7 @@ func (e *Executor) moduleUnarchive(ctx context.Context, client *SSHClient, args return nil, err } src = tmpPath - defer client.Run(ctx, fmt.Sprintf("rm -f %q", tmpPath)) + defer func() { _, _, _, _ = client.Run(ctx, fmt.Sprintf("rm -f %q", tmpPath)) }() } // Detect archive type and extract @@ -1114,8 +1114,8 @@ func (e *Executor) moduleHostname(ctx context.Context, client *SSHClient, args m return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil } - // Update /etc/hosts if needed - client.Run(ctx, fmt.Sprintf("sed -i 's/127.0.1.1.*/127.0.1.1\t%s/' /etc/hosts", name)) + // Update /etc/hosts if needed (best-effort) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("sed -i 's/127.0.1.1.*/127.0.1.1\t%s/' /etc/hosts", name)) return &TaskResult{Changed: true}, nil } @@ -1132,7 +1132,7 @@ func (e *Executor) moduleSysctl(ctx context.Context, client *SSHClient, args map if state == "absent" { // Remove from sysctl.conf cmd := fmt.Sprintf("sed -i '/%s/d' /etc/sysctl.conf", name) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) return &TaskResult{Changed: true}, nil } @@ -1143,11 +1143,11 @@ func (e *Executor) moduleSysctl(ctx context.Context, client *SSHClient, args map return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil } - // Persist if requested + // Persist if requested (best-effort) if getBoolArg(args, "sysctl_set", true) { cmd = fmt.Sprintf("grep -q '^%s' /etc/sysctl.conf && sed -i 's/^%s.*/%s=%s/' /etc/sysctl.conf || echo '%s=%s' >> /etc/sysctl.conf", name, name, name, value, name, value) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) } return &TaskResult{Changed: true}, nil @@ -1170,7 +1170,7 @@ func (e *Executor) moduleCron(ctx context.Context, client *SSHClient, args map[s // Remove by name (comment marker) cmd := fmt.Sprintf("crontab -u %s -l 2>/dev/null | grep -v '# %s' | grep -v '%s' | crontab -u %s -", user, name, job, user) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) } return &TaskResult{Changed: true}, nil } @@ -1213,13 +1213,13 @@ func (e *Executor) moduleBlockinfile(ctx context.Context, client *SSHClient, arg strings.ReplaceAll(beginMarker, "/", "\\/"), strings.ReplaceAll(endMarker, "/", "\\/"), path) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) return &TaskResult{Changed: true}, nil } - // Create file if needed + // Create file if needed (best-effort) if create { - client.Run(ctx, fmt.Sprintf("touch %q", path)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("touch %q", path)) } // Remove existing block and add new one @@ -1279,9 +1279,9 @@ func (e *Executor) moduleReboot(ctx context.Context, client *SSHClient, args map if preRebootDelay > 0 { cmd := fmt.Sprintf("sleep %d && shutdown -r now '%s' &", preRebootDelay, msg) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) } else { - client.Run(ctx, fmt.Sprintf("shutdown -r now '%s' &", msg)) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("shutdown -r now '%s' &", msg)) } return &TaskResult{Changed: true, Msg: "Reboot initiated"}, nil @@ -1366,12 +1366,12 @@ func (e *Executor) moduleAuthorizedKey(ctx context.Context, client *SSHClient, a // Remove key escapedKey := strings.ReplaceAll(key, "/", "\\/") cmd := fmt.Sprintf("sed -i '/%s/d' %q 2>/dev/null || true", escapedKey[:40], authKeysPath) - client.Run(ctx, cmd) + _, _, _, _ = client.Run(ctx, cmd) return &TaskResult{Changed: true}, nil } - // Ensure .ssh directory exists - client.Run(ctx, fmt.Sprintf("mkdir -p %q && chmod 700 %q && chown %s:%s %q", + // Ensure .ssh directory exists (best-effort) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("mkdir -p %q && chmod 700 %q && chown %s:%s %q", filepath.Dir(authKeysPath), filepath.Dir(authKeysPath), user, user, filepath.Dir(authKeysPath))) // Add key if not present @@ -1382,8 +1382,8 @@ func (e *Executor) moduleAuthorizedKey(ctx context.Context, client *SSHClient, a return &TaskResult{Failed: true, Msg: stderr, Stdout: stdout, RC: rc}, nil } - // Fix permissions - client.Run(ctx, fmt.Sprintf("chmod 600 %q && chown %s:%s %q", + // Fix permissions (best-effort) + _, _, _, _ = client.Run(ctx, fmt.Sprintf("chmod 600 %q && chown %s:%s %q", authKeysPath, user, user, authKeysPath)) return &TaskResult{Changed: true}, nil diff --git a/pkg/ansible/ssh.go b/pkg/ansible/ssh.go index 51c57a2c..ba157b01 100644 --- a/pkg/ansible/ssh.go +++ b/pkg/ansible/ssh.go @@ -145,7 +145,7 @@ func (c *SSHClient) Connect(ctx context.Context) error { sshConn, chans, reqs, err := ssh.NewClientConn(conn, addr, config) if err != nil { - conn.Close() + _ = conn.Close() return fmt.Errorf("ssh connect %s: %w", addr, err) } @@ -176,7 +176,7 @@ func (c *SSHClient) Run(ctx context.Context, cmd string) (stdout, stderr string, if err != nil { return "", "", -1, fmt.Errorf("new session: %w", err) } - defer session.Close() + defer func() { _ = session.Close() }() var stdoutBuf, stderrBuf bytes.Buffer session.Stdout = &stdoutBuf @@ -210,7 +210,7 @@ func (c *SSHClient) Run(ctx context.Context, cmd string) (stdout, stderr string, select { case <-ctx.Done(): - session.Signal(ssh.SIGKILL) + _ = session.Signal(ssh.SIGKILL) return "", "", -1, ctx.Err() case err := <-done: exitCode = 0 @@ -242,7 +242,7 @@ func (c *SSHClient) Upload(ctx context.Context, local io.Reader, remote string, if err != nil { return fmt.Errorf("new session: %w", err) } - defer session.Close() + defer func() { _ = session.Close() }() // Read content content, err := io.ReadAll(local) @@ -270,7 +270,7 @@ func (c *SSHClient) Upload(ctx context.Context, local io.Reader, remote string, if err != nil { return fmt.Errorf("new session for write: %w", err) } - defer session2.Close() + defer func() { _ = session2.Close() }() stdin, err := session2.StdinPipe() if err != nil { @@ -287,7 +287,7 @@ func (c *SSHClient) Upload(ctx context.Context, local io.Reader, remote string, if _, err := stdin.Write(content); err != nil { return fmt.Errorf("write content: %w", err) } - stdin.Close() + _ = stdin.Close() if err := session2.Wait(); err != nil { return fmt.Errorf("write failed: %w (stderr: %s)", err, stderrBuf.String()) @@ -334,14 +334,8 @@ func (c *SSHClient) FileExists(ctx context.Context, path string) (bool, error) { // Stat returns file info from the remote host. func (c *SSHClient) Stat(ctx context.Context, path string) (map[string]any, error) { - // Use stat command to get file info - cmd := fmt.Sprintf(`stat -c '{"exists":true,"isdir":%s,"mode":"%a","size":%s,"uid":%u,"gid":%g}' %q 2>/dev/null || echo '{"exists":false}'`, - `$(test -d %q && echo true || echo false)`, - `%s`, - path) - - // Simpler approach - just get basic info - cmd = fmt.Sprintf(` + // Simple approach - get basic file info + cmd := fmt.Sprintf(` if [ -e %q ]; then if [ -d %q ]; then echo "exists=true isdir=true" diff --git a/pkg/deploy/python/python.go b/pkg/deploy/python/python.go index 3bac48b1..9b7d7fc5 100644 --- a/pkg/deploy/python/python.go +++ b/pkg/deploy/python/python.go @@ -41,13 +41,13 @@ func RunScript(ctx context.Context, code string, args ...string) (string, error) if err != nil { return "", fmt.Errorf("failed to create temp file: %w", err) } - defer os.Remove(tmpFile.Name()) + defer func() { _ = os.Remove(tmpFile.Name()) }() if _, err := tmpFile.WriteString(code); err != nil { - tmpFile.Close() + _ = tmpFile.Close() return "", fmt.Errorf("failed to write script: %w", err) } - tmpFile.Close() + _ = tmpFile.Close() // Build args: script path + any additional args cmdArgs := append([]string{tmpFile.Name()}, args...) diff --git a/tools/rag/README.md b/tools/rag/README.md new file mode 100644 index 00000000..28f49829 --- /dev/null +++ b/tools/rag/README.md @@ -0,0 +1,193 @@ +# RAG Pipeline for Host UK Documentation + +Store documentation in a vector database so Claude (and local LLMs) can retrieve relevant context without being reminded every conversation. + +## The Problem This Solves + +> "The amount of times I've had to re-tell you how to make a Flux button is crazy" + +Instead of wasting context window on "remember, Flux buttons work like this...", the RAG system: +1. Stores all documentation in Qdrant +2. Claude queries before answering +3. Relevant docs injected automatically +4. No more re-teaching + +## Prerequisites + +**Already running on your lab:** +- Qdrant: `linux.snider.dev:6333` +- Ollama: `linux.snider.dev:11434` (or local) + +**Install Python deps:** +```bash +pip install -r requirements.txt +``` + +**Ensure embedding model is available:** +```bash +ollama pull nomic-embed-text +``` + +## Quick Start + +### 1. Ingest Documentation + +```bash +# Ingest recovered Host UK docs +python ingest.py /Users/snider/Code/host-uk/core/tasks/recovered-hostuk \ + --collection hostuk-docs \ + --recreate + +# Ingest Flux UI docs separately (higher priority) +python ingest.py /path/to/flux-ui-docs \ + --collection flux-ui-docs \ + --recreate +``` + +### 2. Query the Database + +```bash +# Search for Flux button docs +python query.py "how to create a Flux button component" + +# Filter by category +python query.py "path sandboxing" --category architecture + +# Get more results +python query.py "Vi personality" --top 10 + +# Output as JSON +python query.py "brand voice" --format json + +# Output for LLM context injection +python query.py "Flux modal component" --format context +``` + +### 3. List Collections + +```bash +python query.py --list-collections +python query.py --stats --collection flux-ui-docs +``` + +## Collections Strategy + +| Collection | Content | Priority | +|------------|---------|----------| +| `flux-ui-docs` | Flux Pro component docs | High (UI questions) | +| `hostuk-docs` | Recovered implementation docs | Medium | +| `brand-docs` | Vi, brand voice, visual identity | For content generation | +| `lethean-docs` | SASE/dVPN technical docs | Product-specific | + +## Integration with Claude Code + +### Option 1: MCP Server (Best) + +Create an MCP server that Claude can query: + +```go +// In core CLI +func (s *RagServer) Query(query string) ([]Document, error) { + // Query Qdrant + // Return relevant docs +} +``` + +Then Claude can call `rag.query("Flux button")` and get docs automatically. + +### Option 2: CLAUDE.md Instruction + +Add to project CLAUDE.md: + +```markdown +## Before Answering UI Questions + +When asked about Flux UI components, query the RAG database first: +```bash +python /path/to/query.py "your question" --collection flux-ui-docs --format context +``` + +Include the retrieved context in your response. +``` + +### Option 3: Claude Code Hook + +Create a hook that auto-injects context for certain queries. + +## Category Taxonomy + +The ingestion automatically categorizes files: + +| Category | Matches | +|----------|---------| +| `ui-component` | flux, ui/component | +| `brand` | brand, mascot | +| `product-brief` | brief | +| `help-doc` | help, draft | +| `task` | task, plan | +| `architecture` | architecture, migration | +| `documentation` | default | + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `QDRANT_HOST` | linux.snider.dev | Qdrant server | +| `QDRANT_PORT` | 6333 | Qdrant port | +| `EMBEDDING_MODEL` | nomic-embed-text | Ollama model | +| `CHUNK_SIZE` | 500 | Characters per chunk | +| `CHUNK_OVERLAP` | 50 | Overlap between chunks | + +## Training a Model vs RAG + +**RAG** (what this does): +- Model weights unchanged +- Documents retrieved at query time +- Knowledge updates instantly (re-ingest) +- Good for: facts, API docs, current information + +**Fine-tuning** (separate process): +- Model weights updated +- Knowledge baked into model +- Requires retraining to update +- Good for: style, patterns, conventions + +**For Flux UI**: RAG is perfect. The docs change, API changes, you want current info. + +**For Vi's voice**: Fine-tuning is better. Style doesn't change often, should be "baked in". + +## Vector Math (For Understanding) + +``` +"How do I make a Flux button?" + ↓ Embedding +[0.12, -0.45, 0.78, ...768 floats...] + ↓ Cosine similarity search +Find chunks with similar vectors + ↓ Results +1. doc/ui/flux/components/button.md (score: 0.89) +2. doc/ui/flux/forms.md (score: 0.76) +3. doc/ui/flux/components/input.md (score: 0.71) +``` + +The embedding model converts text to "meaning vectors". Similar meanings = similar vectors = found by search. + +## Troubleshooting + +**"No results found"** +- Lower threshold: `--threshold 0.3` +- Check collection has data: `--stats` +- Verify Ollama is running: `ollama list` + +**"Connection refused"** +- Check Qdrant is running: `curl http://linux.snider.dev:6333/collections` +- Check firewall/network + +**"Embedding model not available"** +```bash +ollama pull nomic-embed-text +``` + +--- + +*Part of the Host UK Core CLI tooling* diff --git a/tools/rag/ingest.py b/tools/rag/ingest.py new file mode 100644 index 00000000..3d151ed3 --- /dev/null +++ b/tools/rag/ingest.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +""" +RAG Ingestion Pipeline for Host UK Documentation + +Chunks markdown files, generates embeddings via Ollama, stores in Qdrant. + +Usage: + python ingest.py /path/to/docs --collection hostuk-docs + python ingest.py /path/to/flux-ui --collection flux-ui-docs + +Requirements: + pip install qdrant-client ollama markdown +""" + +import argparse +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Generator + +try: + from qdrant_client import QdrantClient + from qdrant_client.models import Distance, VectorParams, PointStruct + import ollama +except ImportError: + print("Install dependencies: pip install qdrant-client ollama") + sys.exit(1) + + +# Configuration +QDRANT_HOST = os.getenv("QDRANT_HOST", "linux.snider.dev") +QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) +EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text") +CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "500")) # chars +CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "50")) # chars +VECTOR_DIM = 768 # nomic-embed-text dimension + + +def chunk_markdown(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> Generator[dict, None, None]: + """ + Chunk markdown by sections (## headers), then by paragraphs if too long. + Preserves context with overlap. + """ + # Split by ## headers first + sections = re.split(r'\n(?=## )', text) + + for section in sections: + if not section.strip(): + continue + + # Extract section title + lines = section.strip().split('\n') + title = lines[0].lstrip('#').strip() if lines[0].startswith('#') else "" + + # If section is small enough, yield as-is + if len(section) <= chunk_size: + yield { + "text": section.strip(), + "section": title, + } + continue + + # Otherwise, chunk by paragraphs + paragraphs = re.split(r'\n\n+', section) + current_chunk = "" + + for para in paragraphs: + if len(current_chunk) + len(para) <= chunk_size: + current_chunk += "\n\n" + para if current_chunk else para + else: + if current_chunk: + yield { + "text": current_chunk.strip(), + "section": title, + } + # Start new chunk with overlap from previous + if overlap and current_chunk: + overlap_text = current_chunk[-overlap:] + current_chunk = overlap_text + "\n\n" + para + else: + current_chunk = para + + # Don't forget the last chunk + if current_chunk.strip(): + yield { + "text": current_chunk.strip(), + "section": title, + } + + +def generate_embedding(text: str) -> list[float]: + """Generate embedding using Ollama.""" + response = ollama.embeddings(model=EMBEDDING_MODEL, prompt=text) + return response["embedding"] + + +def get_file_category(path: str) -> str: + """Determine category from file path.""" + path_lower = path.lower() + + if "flux" in path_lower or "ui/component" in path_lower: + return "ui-component" + elif "brand" in path_lower or "mascot" in path_lower: + return "brand" + elif "brief" in path_lower: + return "product-brief" + elif "help" in path_lower or "draft" in path_lower: + return "help-doc" + elif "task" in path_lower or "plan" in path_lower: + return "task" + elif "architecture" in path_lower or "migration" in path_lower: + return "architecture" + else: + return "documentation" + + +def ingest_directory(do we + directory: Path, + client: QdrantClient, + collection: str, + verbose: bool = False +) -> dict: + """Ingest all markdown files from directory into Qdrant.""" + + stats = {"files": 0, "chunks": 0, "errors": 0} + points = [] + + # Find all markdown files + md_files = list(directory.rglob("*.md")) + print(f"Found {len(md_files)} markdown files") + + for file_path in md_files: + try: + rel_path = str(file_path.relative_to(directory)) + + with open(file_path, "r", encoding="utf-8", errors="ignore") as f: + content = f.read() + + if not content.strip(): + continue + + # Extract metadata + category = get_file_category(rel_path) + + # Chunk the content + for i, chunk in enumerate(chunk_markdown(content)): + chunk_id = hashlib.md5( + f"{rel_path}:{i}:{chunk['text'][:100]}".encode() + ).hexdigest() + + # Generate embedding + embedding = generate_embedding(chunk["text"]) + + # Create point + point = PointStruct( + id=chunk_id, + vector=embedding, + payload={ + "text": chunk["text"], + "source": rel_path, + "section": chunk["section"], + "category": category, + "chunk_index": i, + } + ) + points.append(point) + stats["chunks"] += 1 + + if verbose: + print(f" [{category}] {rel_path} chunk {i}: {len(chunk['text'])} chars") + + stats["files"] += 1 + if not verbose: + print(f" Processed: {rel_path} ({stats['chunks']} chunks total)") + + except Exception as e: + print(f" Error processing {file_path}: {e}") + stats["errors"] += 1 + + # Batch upsert to Qdrant + if points: + print(f"\nUpserting {len(points)} vectors to Qdrant...") + + # Upsert in batches of 100 + batch_size = 100 + for i in range(0, len(points), batch_size): + batch = points[i:i + batch_size] + client.upsert(collection_name=collection, points=batch) + print(f" Uploaded batch {i // batch_size + 1}/{(len(points) - 1) // batch_size + 1}") + + return stats + + +def main(): + parser = argparse.ArgumentParser(description="Ingest markdown docs into Qdrant") + parser.add_argument("directory", type=Path, help="Directory containing markdown files") + parser.add_argument("--collection", default="hostuk-docs", help="Qdrant collection name") + parser.add_argument("--recreate", action="store_true", help="Delete and recreate collection") + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") + parser.add_argument("--qdrant-host", default=QDRANT_HOST, help="Qdrant host") + parser.add_argument("--qdrant-port", type=int, default=QDRANT_PORT, help="Qdrant port") + + args = parser.parse_args() + + if not args.directory.exists(): + print(f"Error: Directory not found: {args.directory}") + sys.exit(1) + + # Connect to Qdrant + print(f"Connecting to Qdrant at {args.qdrant_host}:{args.qdrant_port}...") + client = QdrantClient(host=args.qdrant_host, port=args.qdrant_port) + + # Create or recreate collection + collections = [c.name for c in client.get_collections().collections] + + if args.recreate and args.collection in collections: + print(f"Deleting existing collection: {args.collection}") + client.delete_collection(args.collection) + collections.remove(args.collection) + + if args.collection not in collections: + print(f"Creating collection: {args.collection}") + client.create_collection( + collection_name=args.collection, + vectors_config=VectorParams(size=VECTOR_DIM, distance=Distance.COSINE) + ) + + # Verify Ollama model is available + print(f"Using embedding model: {EMBEDDING_MODEL}") + try: + ollama.embeddings(model=EMBEDDING_MODEL, prompt="test") + except Exception as e: + print(f"Error: Embedding model not available. Run: ollama pull {EMBEDDING_MODEL}") + sys.exit(1) + + # Ingest files + print(f"\nIngesting from: {args.directory}") + stats = ingest_directory(args.directory, client, args.collection, args.verbose) + + # Summary + print(f"\n{'=' * 50}") + print(f"Ingestion complete!") + print(f" Files processed: {stats['files']}") + print(f" Chunks created: {stats['chunks']}") + print(f" Errors: {stats['errors']}") + print(f" Collection: {args.collection}") + print(f"{'=' * 50}") + + +if __name__ == "__main__": + main() diff --git a/tools/rag/query.py b/tools/rag/query.py new file mode 100644 index 00000000..a6246c26 --- /dev/null +++ b/tools/rag/query.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +""" +RAG Query Tool for Host UK Documentation + +Query the vector database and retrieve relevant documentation chunks. + +Usage: + python query.py "how do I create a Flux button" + python query.py "what is Vi's personality" --collection hostuk-docs + python query.py "path sandboxing" --top 10 --category architecture + +Requirements: + pip install qdrant-client ollama +""" + +import argparse +import json +import os +import sys +from typing import Optional + +try: + from qdrant_client import QdrantClient + from qdrant_client.models import Filter, FieldCondition, MatchValue + import ollama +except ImportError: + print("Install dependencies: pip install qdrant-client ollama") + sys.exit(1) + + +# Configuration +QDRANT_HOST = os.getenv("QDRANT_HOST", "linux.snider.dev") +QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) +EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text") + + +def generate_embedding(text: str) -> list[float]: + """Generate embedding using Ollama.""" + response = ollama.embeddings(model=EMBEDDING_MODEL, prompt=text) + return response["embedding"] + + +def query_rag( + query: str, + client: QdrantClient, + collection: str, + top_k: int = 5, + category: Optional[str] = None, + score_threshold: float = 0.5, +) -> list[dict]: + """Query the RAG database and return relevant chunks.""" + + # Generate query embedding + query_embedding = generate_embedding(query) + + # Build filter if category specified + query_filter = None + if category: + query_filter = Filter( + must=[ + FieldCondition(key="category", match=MatchValue(value=category)) + ] + ) + + # Search + results = client.query_points( + collection_name=collection, + query=query_embedding, + query_filter=query_filter, + limit=top_k, + score_threshold=score_threshold, + ).points + + return [ + { + "score": hit.score, + "text": hit.payload["text"], + "source": hit.payload["source"], + "section": hit.payload.get("section", ""), + "category": hit.payload.get("category", ""), + } + for hit in results + ] + + +def format_results(results: list[dict], query: str, format: str = "text") -> str: + """Format results for display.""" + + if format == "json": + return json.dumps(results, indent=2) + + if not results: + return f"No results found for: {query}" + + output = [] + output.append(f"Query: {query}") + output.append(f"Results: {len(results)}") + output.append("=" * 60) + + for i, r in enumerate(results, 1): + output.append(f"\n[{i}] {r['source']} (score: {r['score']:.3f})") + if r['section']: + output.append(f" Section: {r['section']}") + output.append(f" Category: {r['category']}") + output.append("-" * 40) + # Truncate long text for display + text = r['text'] + if len(text) > 500: + text = text[:500] + "..." + output.append(text) + output.append("") + + return "\n".join(output) + + +def format_for_context(results: list[dict], query: str) -> str: + """Format results as context for LLM injection.""" + + if not results: + return "" + + output = [] + output.append(f"") + + for r in results: + output.append(f"\n") + output.append(r['text']) + output.append("") + + output.append("\n") + + return "\n".join(output) + + +def main(): + parser = argparse.ArgumentParser(description="Query RAG documentation") + parser.add_argument("query", nargs="?", help="Search query") + parser.add_argument("--collection", default="hostuk-docs", help="Qdrant collection name") + parser.add_argument("--top", "-k", type=int, default=5, help="Number of results") + parser.add_argument("--category", "-c", help="Filter by category") + parser.add_argument("--threshold", "-t", type=float, default=0.5, help="Score threshold") + parser.add_argument("--format", "-f", choices=["text", "json", "context"], default="text") + parser.add_argument("--qdrant-host", default=QDRANT_HOST) + parser.add_argument("--qdrant-port", type=int, default=QDRANT_PORT) + parser.add_argument("--list-collections", action="store_true", help="List available collections") + parser.add_argument("--stats", action="store_true", help="Show collection stats") + + args = parser.parse_args() + + # Connect to Qdrant + client = QdrantClient(host=args.qdrant_host, port=args.qdrant_port) + + # List collections + if args.list_collections: + collections = client.get_collections().collections + print("Available collections:") + for c in collections: + info = client.get_collection(c.name) + print(f" - {c.name}: {info.points_count} vectors") + return + + # Show stats + if args.stats: + try: + info = client.get_collection(args.collection) + print(f"Collection: {args.collection}") + print(f" Vectors: {info.points_count}") + print(f" Status: {info.status}") + except Exception as e: + print(f"Collection not found: {args.collection}") + return + + # Query required + if not args.query: + parser.print_help() + return + + # Execute query + results = query_rag( + query=args.query, + client=client, + collection=args.collection, + top_k=args.top, + category=args.category, + score_threshold=args.threshold, + ) + + # Format output + if args.format == "context": + print(format_for_context(results, args.query)) + else: + print(format_results(results, args.query, args.format)) + + +if __name__ == "__main__": + main() diff --git a/tools/rag/requirements.txt b/tools/rag/requirements.txt new file mode 100644 index 00000000..9f8b75c8 --- /dev/null +++ b/tools/rag/requirements.txt @@ -0,0 +1,2 @@ +qdrant-client>=1.7.0 +ollama>=0.1.0