diff --git a/playbooks/galera-backup.yml b/playbooks/galera-backup.yml deleted file mode 100644 index 0109d5a9..00000000 --- a/playbooks/galera-backup.yml +++ /dev/null @@ -1,63 +0,0 @@ -# Galera Database Backup -# Dumps the database and uploads to Hetzner S3 -# -# Usage: -# core deploy ansible playbooks/galera-backup.yml -i playbooks/inventory.yml -l de ---- -- name: Backup Galera Database to S3 - hosts: app_servers - become: true - vars: - db_root_password: "{{ lookup('env', 'DB_ROOT_PASSWORD') }}" - s3_endpoint: "{{ lookup('env', 'HETZNER_S3_ENDPOINT') | default('fsn1.your-objectstorage.com', true) }}" - s3_bucket: "{{ lookup('env', 'HETZNER_S3_BUCKET') | default('hostuk', true) }}" - s3_access_key: "{{ lookup('env', 'HETZNER_S3_ACCESS_KEY') }}" - s3_secret_key: "{{ lookup('env', 'HETZNER_S3_SECRET_KEY') }}" - backup_prefix: backup/galera - backup_retain_days: 30 - - tasks: - - name: Create backup directory - file: - path: /opt/backup - state: directory - mode: "0700" - - - name: Dump database - shell: | - TIMESTAMP=$(date +%Y%m%d-%H%M%S) - DUMP_FILE="/opt/backup/hostuk-${TIMESTAMP}-{{ galera_node_name }}.sql.gz" - docker exec galera mariadb-dump \ - -u root -p{{ db_root_password }} \ - --all-databases \ - --single-transaction \ - --routines \ - --triggers \ - --events \ - | gzip > "${DUMP_FILE}" - echo "${DUMP_FILE}" - register: dump_result - - - name: Install s3cmd if missing - shell: | - which s3cmd 2>/dev/null || pip3 install s3cmd - changed_when: false - - - name: Upload to S3 - shell: | - s3cmd put {{ dump_result.stdout | trim }} \ - s3://{{ s3_bucket }}/{{ backup_prefix }}/$(basename {{ dump_result.stdout | trim }}) \ - --host={{ s3_endpoint }} \ - --host-bucket='%(bucket)s.{{ s3_endpoint }}' \ - --access_key={{ s3_access_key }} \ - --secret_key={{ s3_secret_key }} - when: s3_access_key != "" - - - name: Clean old local backups - shell: | - find /opt/backup -name "hostuk-*.sql.gz" -mtime +{{ backup_retain_days }} -delete - changed_when: false - - - name: Show backup result - debug: - msg: "Backup completed: {{ dump_result.stdout | trim }}" diff --git a/playbooks/galera-deploy.yml b/playbooks/galera-deploy.yml deleted file mode 100644 index 58594fb4..00000000 --- a/playbooks/galera-deploy.yml +++ /dev/null @@ -1,96 +0,0 @@ -# MariaDB Galera Cluster Deployment -# Deploys a 2-node Galera cluster on de + de2 -# -# Usage: -# core deploy ansible playbooks/galera-deploy.yml -i playbooks/inventory.yml -# core deploy ansible playbooks/galera-deploy.yml -i playbooks/inventory.yml -l de # Single node -# -# First-time bootstrap: -# Set galera_bootstrap=true for the first node: -# core deploy ansible playbooks/galera-deploy.yml -i playbooks/inventory.yml -l de -e galera_bootstrap=true ---- -- name: Deploy MariaDB Galera Cluster - hosts: app_servers - become: true - vars: - mariadb_version: "11" - galera_cluster_address: "gcomm://116.202.82.115,88.99.195.41" - galera_bootstrap: false - db_root_password: "{{ lookup('env', 'DB_ROOT_PASSWORD') }}" - db_password: "{{ lookup('env', 'DB_PASSWORD') }}" - - tasks: - - name: Create MariaDB data directory - file: - path: /opt/galera/data - state: directory - mode: "0755" - - - name: Create MariaDB config directory - file: - path: /opt/galera/conf.d - state: directory - mode: "0755" - - - name: Write Galera configuration - copy: - dest: /opt/galera/conf.d/galera.cnf - content: | - [mysqld] - wsrep_on=ON - wsrep_provider=/usr/lib/galera/libgalera_smm.so - wsrep_cluster_name={{ galera_cluster_name }} - wsrep_cluster_address={{ 'gcomm://' if galera_bootstrap else galera_cluster_address }} - wsrep_node_address={{ galera_node_address }} - wsrep_node_name={{ galera_node_name }} - wsrep_sst_method={{ galera_sst_method }} - binlog_format=ROW - default_storage_engine=InnoDB - innodb_autoinc_lock_mode=2 - innodb_buffer_pool_size=1G - innodb_log_file_size=256M - character_set_server=utf8mb4 - collation_server=utf8mb4_unicode_ci - - - name: Stop existing MariaDB container - shell: docker stop galera 2>/dev/null || true - changed_when: false - - - name: Remove existing MariaDB container - shell: docker rm galera 2>/dev/null || true - changed_when: false - - - name: Start MariaDB Galera container - shell: | - docker run -d \ - --name galera \ - --restart unless-stopped \ - --network host \ - -v /opt/galera/data:/var/lib/mysql \ - -v /opt/galera/conf.d:/etc/mysql/conf.d \ - -e MARIADB_ROOT_PASSWORD={{ db_root_password }} \ - -e MARIADB_DATABASE={{ db_name }} \ - -e MARIADB_USER={{ db_user }} \ - -e MARIADB_PASSWORD={{ db_password }} \ - mariadb:{{ mariadb_version }} - - - name: Wait for MariaDB to be ready - shell: | - for i in $(seq 1 60); do - docker exec galera mariadb -u root -p{{ db_root_password }} -e "SELECT 1" 2>/dev/null && exit 0 - sleep 2 - done - exit 1 - changed_when: false - - - name: Check Galera cluster status - shell: | - docker exec galera mariadb -u root -p{{ db_root_password }} \ - -e "SHOW STATUS WHERE Variable_name IN ('wsrep_cluster_size','wsrep_ready','wsrep_cluster_status')" \ - --skip-column-names - register: galera_status - changed_when: false - - - name: Display cluster status - debug: - var: galera_status.stdout_lines diff --git a/playbooks/inventory.yml b/playbooks/inventory.yml deleted file mode 100644 index 3e24226e..00000000 --- a/playbooks/inventory.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Ansible inventory for Host UK production -# Used by: core deploy ansible -i playbooks/inventory.yml -all: - vars: - ansible_user: root - ansible_ssh_private_key_file: ~/.ssh/hostuk - - children: - bastion: - hosts: - noc: - ansible_host: 77.42.42.205 - private_ip: 10.0.0.4 - - app_servers: - hosts: - de: - ansible_host: 116.202.82.115 - galera_node_name: de - galera_node_address: 116.202.82.115 - de2: - ansible_host: 88.99.195.41 - galera_node_name: de2 - galera_node_address: 88.99.195.41 - vars: - galera_cluster_name: hostuk-galera - galera_sst_method: mariabackup - db_name: hostuk - db_user: hostuk - redis_maxmemory: 512mb - - builders: - hosts: - build: - ansible_host: 46.224.93.62 - private_ip: 10.0.0.5 diff --git a/playbooks/redis-deploy.yml b/playbooks/redis-deploy.yml deleted file mode 100644 index ed3b86e7..00000000 --- a/playbooks/redis-deploy.yml +++ /dev/null @@ -1,98 +0,0 @@ -# Redis Sentinel Deployment -# Deploys Redis with Sentinel on de + de2 -# -# Usage: -# core deploy ansible playbooks/redis-deploy.yml -i playbooks/inventory.yml ---- -- name: Deploy Redis with Sentinel - hosts: app_servers - become: true - vars: - redis_version: "7" - redis_password: "{{ lookup('env', 'REDIS_PASSWORD') | default('', true) }}" - - tasks: - - name: Create Redis data directory - file: - path: /opt/redis/data - state: directory - mode: "0755" - - - name: Create Redis config directory - file: - path: /opt/redis/conf - state: directory - mode: "0755" - - - name: Write Redis configuration - copy: - dest: /opt/redis/conf/redis.conf - content: | - maxmemory {{ redis_maxmemory }} - maxmemory-policy allkeys-lru - appendonly yes - appendfsync everysec - tcp-keepalive 300 - timeout 0 - {% if redis_password %} - requirepass {{ redis_password }} - masterauth {{ redis_password }} - {% endif %} - - - name: Write Sentinel configuration - copy: - dest: /opt/redis/conf/sentinel.conf - content: | - port 26379 - sentinel monitor hostuk-redis 116.202.82.115 6379 2 - sentinel down-after-milliseconds hostuk-redis 5000 - sentinel failover-timeout hostuk-redis 60000 - sentinel parallel-syncs hostuk-redis 1 - {% if redis_password %} - sentinel auth-pass hostuk-redis {{ redis_password }} - {% endif %} - - - name: Stop existing Redis containers - shell: | - docker stop redis redis-sentinel 2>/dev/null || true - docker rm redis redis-sentinel 2>/dev/null || true - changed_when: false - - - name: Start Redis container - shell: | - docker run -d \ - --name redis \ - --restart unless-stopped \ - --network host \ - -v /opt/redis/data:/data \ - -v /opt/redis/conf/redis.conf:/usr/local/etc/redis/redis.conf \ - redis:{{ redis_version }}-alpine \ - redis-server /usr/local/etc/redis/redis.conf - - - name: Start Redis Sentinel container - shell: | - docker run -d \ - --name redis-sentinel \ - --restart unless-stopped \ - --network host \ - -v /opt/redis/conf/sentinel.conf:/usr/local/etc/redis/sentinel.conf \ - redis:{{ redis_version }}-alpine \ - redis-sentinel /usr/local/etc/redis/sentinel.conf - - - name: Wait for Redis to be ready - shell: | - for i in $(seq 1 30); do - docker exec redis redis-cli ping 2>/dev/null | grep -q PONG && exit 0 - sleep 1 - done - exit 1 - changed_when: false - - - name: Check Redis info - shell: docker exec redis redis-cli info replication | head -10 - register: redis_info - changed_when: false - - - name: Display Redis info - debug: - var: redis_info.stdout_lines diff --git a/tasks/plans/2026-01-29-code-signing-design.md b/tasks/plans/2026-01-29-code-signing-design.md deleted file mode 100644 index cedf7380..00000000 --- a/tasks/plans/2026-01-29-code-signing-design.md +++ /dev/null @@ -1,236 +0,0 @@ -# Code Signing Design (S3.3) - -## Summary - -Integrate standard code signing tools into the build pipeline. GPG signs checksums by default. macOS codesign + notarization for Apple binaries. Windows signtool deferred. - -## Design Decisions - -- **Sign during build**: Signing happens in `pkg/build/signing/` after compilation, before archiving -- **Config location**: `.core/build.yaml` with environment variable fallbacks for secrets -- **GPG scope**: Signs `checksums.txt` only (standard pattern like Go, Terraform) -- **macOS flow**: Codesign always when identity configured, notarize optional with flag/config -- **Windows**: Placeholder for later implementation - -## Package Structure - -``` -pkg/build/signing/ -├── signer.go # Signer interface + SignConfig -├── gpg.go # GPG checksums signing -├── codesign.go # macOS codesign + notarize -└── signtool.go # Windows placeholder -``` - -## Signer Interface - -```go -// pkg/build/signing/signer.go -type Signer interface { - Name() string - Available() bool - Sign(ctx context.Context, artifact string) error -} - -type SignConfig struct { - Enabled bool `yaml:"enabled"` - GPG GPGConfig `yaml:"gpg,omitempty"` - MacOS MacOSConfig `yaml:"macos,omitempty"` - Windows WindowsConfig `yaml:"windows,omitempty"` -} - -type GPGConfig struct { - Key string `yaml:"key"` // Key ID or fingerprint, supports $ENV -} - -type MacOSConfig struct { - Identity string `yaml:"identity"` // Developer ID Application: ... - Notarize bool `yaml:"notarize"` // Submit to Apple - AppleID string `yaml:"apple_id"` // Apple account email - TeamID string `yaml:"team_id"` // Team ID - AppPassword string `yaml:"app_password"` // App-specific password -} - -type WindowsConfig struct { - Certificate string `yaml:"certificate"` // Path to .pfx - Password string `yaml:"password"` // Certificate password -} -``` - -## Config Schema - -In `.core/build.yaml`: - -```yaml -sign: - enabled: true - - gpg: - key: $GPG_KEY_ID - - macos: - identity: "Developer ID Application: Your Name (TEAM_ID)" - notarize: false - apple_id: $APPLE_ID - team_id: $APPLE_TEAM_ID - app_password: $APPLE_APP_PASSWORD - - # windows: (deferred) - # certificate: $WINDOWS_CERT_PATH - # password: $WINDOWS_CERT_PASSWORD -``` - -## Build Pipeline Integration - -``` -Build() in pkg/build/builders/go.go - ↓ -compile binaries - ↓ -Sign macOS binaries (codesign) ← NEW - ↓ -Notarize if enabled (wait) ← NEW - ↓ -Create archives (tar.gz, zip) - ↓ -Generate checksums.txt - ↓ -GPG sign checksums.txt ← NEW - ↓ -Return artifacts -``` - -## GPG Signer - -```go -// pkg/build/signing/gpg.go -type GPGSigner struct { - KeyID string -} - -func (s *GPGSigner) Name() string { return "gpg" } - -func (s *GPGSigner) Available() bool { - _, err := exec.LookPath("gpg") - return err == nil && s.KeyID != "" -} - -func (s *GPGSigner) Sign(ctx context.Context, file string) error { - cmd := exec.CommandContext(ctx, "gpg", - "--detach-sign", - "--armor", - "--local-user", s.KeyID, - "--output", file+".asc", - file, - ) - return cmd.Run() -} -``` - -**Output:** `checksums.txt.asc` (ASCII armored detached signature) - -**User verification:** -```bash -gpg --verify checksums.txt.asc checksums.txt -sha256sum -c checksums.txt -``` - -## macOS Codesign - -```go -// pkg/build/signing/codesign.go -type MacOSSigner struct { - Identity string - Notarize bool - AppleID string - TeamID string - AppPassword string -} - -func (s *MacOSSigner) Name() string { return "codesign" } - -func (s *MacOSSigner) Available() bool { - if runtime.GOOS != "darwin" { - return false - } - _, err := exec.LookPath("codesign") - return err == nil && s.Identity != "" -} - -func (s *MacOSSigner) Sign(ctx context.Context, binary string) error { - cmd := exec.CommandContext(ctx, "codesign", - "--sign", s.Identity, - "--timestamp", - "--options", "runtime", - "--force", - binary, - ) - return cmd.Run() -} - -func (s *MacOSSigner) NotarizeAndStaple(ctx context.Context, binary string) error { - // 1. Create ZIP for submission - zipPath := binary + ".zip" - exec.CommandContext(ctx, "zip", "-j", zipPath, binary).Run() - defer os.Remove(zipPath) - - // 2. Submit and wait - cmd := exec.CommandContext(ctx, "xcrun", "notarytool", "submit", - zipPath, - "--apple-id", s.AppleID, - "--team-id", s.TeamID, - "--password", s.AppPassword, - "--wait", - ) - if err := cmd.Run(); err != nil { - return fmt.Errorf("notarization failed: %w", err) - } - - // 3. Staple ticket - return exec.CommandContext(ctx, "xcrun", "stapler", "staple", binary).Run() -} -``` - -## CLI Flags - -```bash -core build # Sign with defaults (GPG + codesign if configured) -core build --no-sign # Skip all signing -core build --notarize # Enable macOS notarization (overrides config) -``` - -## Environment Variables - -| Variable | Purpose | -|----------|---------| -| `GPG_KEY_ID` | GPG key ID or fingerprint | -| `CODESIGN_IDENTITY` | macOS Developer ID (fallback) | -| `APPLE_ID` | Apple account email | -| `APPLE_TEAM_ID` | Apple Developer Team ID | -| `APPLE_APP_PASSWORD` | App-specific password for notarization | - -## Deferred - -- **Windows signtool**: Placeholder implementation returning nil -- **Sigstore/keyless signing**: Future consideration -- **Binary-level GPG signatures**: Only checksums.txt signed - -## Implementation Steps - -1. Create `pkg/build/signing/` package structure -2. Implement Signer interface and SignConfig -3. Implement GPGSigner -4. Implement MacOSSigner with codesign -5. Add notarization support to MacOSSigner -6. Add SignConfig to build.Config -7. Integrate signing into build pipeline -8. Add CLI flags (--no-sign, --notarize) -9. Add Windows placeholder -10. Tests with mocked exec - -## Dependencies - -- `gpg` CLI (system) -- `codesign` CLI (macOS Xcode Command Line Tools) -- `xcrun notarytool` (macOS Xcode Command Line Tools) -- `xcrun stapler` (macOS Xcode Command Line Tools) diff --git a/tasks/plans/2026-01-29-code-signing-impl.md b/tasks/plans/2026-01-29-code-signing-impl.md deleted file mode 100644 index 4345c348..00000000 --- a/tasks/plans/2026-01-29-code-signing-impl.md +++ /dev/null @@ -1,967 +0,0 @@ -# Code Signing Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Add GPG checksums signing and macOS codesign/notarization to the build pipeline. - -**Architecture:** `pkg/build/signing/` package with Signer interface. GPG signs CHECKSUMS.txt. macOS codesign runs after binary compilation, before archiving. Config in `.core/build.yaml` with env var fallbacks. - -**Tech Stack:** Go, os/exec for gpg/codesign/xcrun CLI tools - ---- - -### Task 1: Create Signing Package Structure - -**Files:** -- Create: `pkg/build/signing/signer.go` - -**Step 1: Create signer.go with interface and config types** - -```go -// Package signing provides code signing for build artifacts. -package signing - -import ( - "context" - "os" - "strings" -) - -// Signer defines the interface for code signing implementations. -type Signer interface { - // Name returns the signer's identifier. - Name() string - // Available checks if this signer can be used. - Available() bool - // Sign signs the artifact at the given path. - Sign(ctx context.Context, path string) error -} - -// SignConfig holds signing configuration from .core/build.yaml. -type SignConfig struct { - Enabled bool `yaml:"enabled"` - GPG GPGConfig `yaml:"gpg,omitempty"` - MacOS MacOSConfig `yaml:"macos,omitempty"` - Windows WindowsConfig `yaml:"windows,omitempty"` -} - -// GPGConfig holds GPG signing configuration. -type GPGConfig struct { - Key string `yaml:"key"` // Key ID or fingerprint, supports $ENV -} - -// MacOSConfig holds macOS codesign configuration. -type MacOSConfig struct { - Identity string `yaml:"identity"` // Developer ID Application: ... - Notarize bool `yaml:"notarize"` // Submit to Apple for notarization - AppleID string `yaml:"apple_id"` // Apple account email - TeamID string `yaml:"team_id"` // Team ID - AppPassword string `yaml:"app_password"` // App-specific password -} - -// WindowsConfig holds Windows signtool configuration (placeholder). -type WindowsConfig struct { - Certificate string `yaml:"certificate"` // Path to .pfx - Password string `yaml:"password"` // Certificate password -} - -// DefaultSignConfig returns sensible defaults. -func DefaultSignConfig() SignConfig { - return SignConfig{ - Enabled: true, - GPG: GPGConfig{ - Key: os.Getenv("GPG_KEY_ID"), - }, - MacOS: MacOSConfig{ - Identity: os.Getenv("CODESIGN_IDENTITY"), - AppleID: os.Getenv("APPLE_ID"), - TeamID: os.Getenv("APPLE_TEAM_ID"), - AppPassword: os.Getenv("APPLE_APP_PASSWORD"), - }, - } -} - -// ExpandEnv expands environment variables in config values. -func (c *SignConfig) ExpandEnv() { - c.GPG.Key = expandEnv(c.GPG.Key) - c.MacOS.Identity = expandEnv(c.MacOS.Identity) - c.MacOS.AppleID = expandEnv(c.MacOS.AppleID) - c.MacOS.TeamID = expandEnv(c.MacOS.TeamID) - c.MacOS.AppPassword = expandEnv(c.MacOS.AppPassword) - c.Windows.Certificate = expandEnv(c.Windows.Certificate) - c.Windows.Password = expandEnv(c.Windows.Password) -} - -// expandEnv expands $VAR or ${VAR} in a string. -func expandEnv(s string) string { - if strings.HasPrefix(s, "$") { - return os.ExpandEnv(s) - } - return s -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/build/signing/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/build/signing/signer.go -git commit -m "feat(signing): add Signer interface and config types - -Defines interface for GPG, macOS, and Windows signing. -Config supports env var expansion for secrets. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 2: Implement GPG Signer - -**Files:** -- Create: `pkg/build/signing/gpg.go` -- Create: `pkg/build/signing/gpg_test.go` - -**Step 1: Write the failing test** - -```go -package signing - -import ( - "testing" -) - -func TestGPGSigner_Good_Name(t *testing.T) { - s := NewGPGSigner("ABCD1234") - if s.Name() != "gpg" { - t.Errorf("expected name 'gpg', got %q", s.Name()) - } -} - -func TestGPGSigner_Good_Available(t *testing.T) { - s := NewGPGSigner("ABCD1234") - // Available depends on gpg being installed - _ = s.Available() -} - -func TestGPGSigner_Bad_NoKey(t *testing.T) { - s := NewGPGSigner("") - if s.Available() { - t.Error("expected Available() to be false when key is empty") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/signing/... -run TestGPGSigner -v` -Expected: FAIL (NewGPGSigner not defined) - -**Step 3: Write implementation** - -```go -package signing - -import ( - "context" - "fmt" - "os/exec" -) - -// GPGSigner signs files using GPG. -type GPGSigner struct { - KeyID string -} - -// Compile-time interface check. -var _ Signer = (*GPGSigner)(nil) - -// NewGPGSigner creates a new GPG signer. -func NewGPGSigner(keyID string) *GPGSigner { - return &GPGSigner{KeyID: keyID} -} - -// Name returns "gpg". -func (s *GPGSigner) Name() string { - return "gpg" -} - -// Available checks if gpg is installed and key is configured. -func (s *GPGSigner) Available() bool { - if s.KeyID == "" { - return false - } - _, err := exec.LookPath("gpg") - return err == nil -} - -// Sign creates a detached ASCII-armored signature. -// For file.txt, creates file.txt.asc -func (s *GPGSigner) Sign(ctx context.Context, file string) error { - if !s.Available() { - return fmt.Errorf("gpg.Sign: gpg not available or key not configured") - } - - cmd := exec.CommandContext(ctx, "gpg", - "--detach-sign", - "--armor", - "--local-user", s.KeyID, - "--output", file+".asc", - file, - ) - - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("gpg.Sign: %w\nOutput: %s", err, string(output)) - } - - return nil -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/signing/... -run TestGPGSigner -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/build/signing/gpg.go pkg/build/signing/gpg_test.go -git commit -m "feat(signing): add GPG signer - -Signs files with detached ASCII-armored signatures (.asc). - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 3: Implement macOS Codesign - -**Files:** -- Create: `pkg/build/signing/codesign.go` -- Create: `pkg/build/signing/codesign_test.go` - -**Step 1: Write the failing test** - -```go -package signing - -import ( - "runtime" - "testing" -) - -func TestMacOSSigner_Good_Name(t *testing.T) { - s := NewMacOSSigner(MacOSConfig{Identity: "Developer ID Application: Test"}) - if s.Name() != "codesign" { - t.Errorf("expected name 'codesign', got %q", s.Name()) - } -} - -func TestMacOSSigner_Good_Available(t *testing.T) { - s := NewMacOSSigner(MacOSConfig{Identity: "Developer ID Application: Test"}) - - // Only available on macOS with identity set - if runtime.GOOS == "darwin" { - // May or may not be available depending on Xcode - _ = s.Available() - } else { - if s.Available() { - t.Error("expected Available() to be false on non-macOS") - } - } -} - -func TestMacOSSigner_Bad_NoIdentity(t *testing.T) { - s := NewMacOSSigner(MacOSConfig{}) - if s.Available() { - t.Error("expected Available() to be false when identity is empty") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/signing/... -run TestMacOSSigner -v` -Expected: FAIL (NewMacOSSigner not defined) - -**Step 3: Write implementation** - -```go -package signing - -import ( - "context" - "fmt" - "os" - "os/exec" - "runtime" -) - -// MacOSSigner signs binaries using macOS codesign. -type MacOSSigner struct { - config MacOSConfig -} - -// Compile-time interface check. -var _ Signer = (*MacOSSigner)(nil) - -// NewMacOSSigner creates a new macOS signer. -func NewMacOSSigner(cfg MacOSConfig) *MacOSSigner { - return &MacOSSigner{config: cfg} -} - -// Name returns "codesign". -func (s *MacOSSigner) Name() string { - return "codesign" -} - -// Available checks if running on macOS with codesign and identity configured. -func (s *MacOSSigner) Available() bool { - if runtime.GOOS != "darwin" { - return false - } - if s.config.Identity == "" { - return false - } - _, err := exec.LookPath("codesign") - return err == nil -} - -// Sign codesigns a binary with hardened runtime. -func (s *MacOSSigner) Sign(ctx context.Context, binary string) error { - if !s.Available() { - return fmt.Errorf("codesign.Sign: codesign not available") - } - - cmd := exec.CommandContext(ctx, "codesign", - "--sign", s.config.Identity, - "--timestamp", - "--options", "runtime", // Hardened runtime for notarization - "--force", - binary, - ) - - output, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("codesign.Sign: %w\nOutput: %s", err, string(output)) - } - - return nil -} - -// Notarize submits binary to Apple for notarization and staples the ticket. -// This blocks until Apple responds (typically 1-5 minutes). -func (s *MacOSSigner) Notarize(ctx context.Context, binary string) error { - if s.config.AppleID == "" || s.config.TeamID == "" || s.config.AppPassword == "" { - return fmt.Errorf("codesign.Notarize: missing Apple credentials (apple_id, team_id, app_password)") - } - - // Create ZIP for submission - zipPath := binary + ".zip" - zipCmd := exec.CommandContext(ctx, "zip", "-j", zipPath, binary) - if output, err := zipCmd.CombinedOutput(); err != nil { - return fmt.Errorf("codesign.Notarize: failed to create zip: %w\nOutput: %s", err, string(output)) - } - defer os.Remove(zipPath) - - // Submit to Apple and wait - submitCmd := exec.CommandContext(ctx, "xcrun", "notarytool", "submit", - zipPath, - "--apple-id", s.config.AppleID, - "--team-id", s.config.TeamID, - "--password", s.config.AppPassword, - "--wait", - ) - if output, err := submitCmd.CombinedOutput(); err != nil { - return fmt.Errorf("codesign.Notarize: notarization failed: %w\nOutput: %s", err, string(output)) - } - - // Staple the ticket - stapleCmd := exec.CommandContext(ctx, "xcrun", "stapler", "staple", binary) - if output, err := stapleCmd.CombinedOutput(); err != nil { - return fmt.Errorf("codesign.Notarize: failed to staple: %w\nOutput: %s", err, string(output)) - } - - return nil -} - -// ShouldNotarize returns true if notarization is enabled. -func (s *MacOSSigner) ShouldNotarize() bool { - return s.config.Notarize -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/signing/... -run TestMacOSSigner -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/build/signing/codesign.go pkg/build/signing/codesign_test.go -git commit -m "feat(signing): add macOS codesign + notarization - -Signs binaries with Developer ID and hardened runtime. -Notarization submits to Apple and staples ticket. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 4: Add Windows Placeholder - -**Files:** -- Create: `pkg/build/signing/signtool.go` - -**Step 1: Create placeholder implementation** - -```go -package signing - -import ( - "context" -) - -// WindowsSigner signs binaries using Windows signtool (placeholder). -type WindowsSigner struct { - config WindowsConfig -} - -// Compile-time interface check. -var _ Signer = (*WindowsSigner)(nil) - -// NewWindowsSigner creates a new Windows signer. -func NewWindowsSigner(cfg WindowsConfig) *WindowsSigner { - return &WindowsSigner{config: cfg} -} - -// Name returns "signtool". -func (s *WindowsSigner) Name() string { - return "signtool" -} - -// Available returns false (not yet implemented). -func (s *WindowsSigner) Available() bool { - return false -} - -// Sign is a placeholder that does nothing. -func (s *WindowsSigner) Sign(ctx context.Context, binary string) error { - // TODO: Implement Windows signing - return nil -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/build/signing/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/build/signing/signtool.go -git commit -m "feat(signing): add Windows signtool placeholder - -Placeholder for future Windows code signing support. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 5: Add SignConfig to BuildConfig - -**Files:** -- Modify: `pkg/build/config.go` -- Modify: `pkg/build/config_test.go` - -**Step 1: Add Sign field to BuildConfig** - -In `pkg/build/config.go`, add to the `BuildConfig` struct: - -```go -// Add import -import "forge.lthn.ai/core/cli/pkg/build/signing" - -// Add to BuildConfig struct after Targets field: - // Sign contains code signing configuration. - Sign signing.SignConfig `yaml:"sign,omitempty"` -``` - -**Step 2: Update DefaultConfig** - -In `DefaultConfig()`, add: - -```go - Sign: signing.DefaultSignConfig(), -``` - -**Step 3: Update applyDefaults** - -In `applyDefaults()`, add: - -```go - // Expand environment variables in sign config - cfg.Sign.ExpandEnv() -``` - -**Step 4: Add test for sign config loading** - -Add to `pkg/build/config_test.go`: - -```go -func TestLoadConfig_Good_SignConfig(t *testing.T) { - tmpDir := t.TempDir() - coreDir := filepath.Join(tmpDir, ".core") - os.MkdirAll(coreDir, 0755) - - configContent := `version: 1 -sign: - enabled: true - gpg: - key: "ABCD1234" - macos: - identity: "Developer ID Application: Test" - notarize: true -` - os.WriteFile(filepath.Join(coreDir, "build.yaml"), []byte(configContent), 0644) - - cfg, err := LoadConfig(tmpDir) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if !cfg.Sign.Enabled { - t.Error("expected Sign.Enabled to be true") - } - if cfg.Sign.GPG.Key != "ABCD1234" { - t.Errorf("expected GPG.Key 'ABCD1234', got %q", cfg.Sign.GPG.Key) - } - if cfg.Sign.MacOS.Identity != "Developer ID Application: Test" { - t.Errorf("expected MacOS.Identity, got %q", cfg.Sign.MacOS.Identity) - } - if !cfg.Sign.MacOS.Notarize { - t.Error("expected MacOS.Notarize to be true") - } -} -``` - -**Step 5: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/... -run TestLoadConfig -v` -Expected: PASS - -**Step 6: Commit** - -```bash -git add pkg/build/config.go pkg/build/config_test.go -git commit -m "feat(build): add SignConfig to BuildConfig - -Loads signing configuration from .core/build.yaml. -Expands environment variables for secrets. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 6: Create Sign Helper Functions - -**Files:** -- Create: `pkg/build/signing/sign.go` - -**Step 1: Create orchestration helpers** - -```go -package signing - -import ( - "context" - "fmt" - "runtime" - - "forge.lthn.ai/core/cli/pkg/build" -) - -// SignBinaries signs macOS binaries in the artifacts list. -// Only signs darwin binaries when running on macOS with a configured identity. -func SignBinaries(ctx context.Context, cfg SignConfig, artifacts []build.Artifact) error { - if !cfg.Enabled { - return nil - } - - // Only sign on macOS - if runtime.GOOS != "darwin" { - return nil - } - - signer := NewMacOSSigner(cfg.MacOS) - if !signer.Available() { - return nil // Silently skip if not configured - } - - for _, artifact := range artifacts { - if artifact.OS != "darwin" { - continue - } - - fmt.Printf(" Signing %s...\n", artifact.Path) - if err := signer.Sign(ctx, artifact.Path); err != nil { - return fmt.Errorf("failed to sign %s: %w", artifact.Path, err) - } - } - - return nil -} - -// NotarizeBinaries notarizes macOS binaries if enabled. -func NotarizeBinaries(ctx context.Context, cfg SignConfig, artifacts []build.Artifact) error { - if !cfg.Enabled || !cfg.MacOS.Notarize { - return nil - } - - if runtime.GOOS != "darwin" { - return nil - } - - signer := NewMacOSSigner(cfg.MacOS) - if !signer.Available() { - return fmt.Errorf("notarization requested but codesign not available") - } - - for _, artifact := range artifacts { - if artifact.OS != "darwin" { - continue - } - - fmt.Printf(" Notarizing %s (this may take a few minutes)...\n", artifact.Path) - if err := signer.Notarize(ctx, artifact.Path); err != nil { - return fmt.Errorf("failed to notarize %s: %w", artifact.Path, err) - } - } - - return nil -} - -// SignChecksums signs the checksums file with GPG. -func SignChecksums(ctx context.Context, cfg SignConfig, checksumFile string) error { - if !cfg.Enabled { - return nil - } - - signer := NewGPGSigner(cfg.GPG.Key) - if !signer.Available() { - return nil // Silently skip if not configured - } - - fmt.Printf(" Signing %s with GPG...\n", checksumFile) - if err := signer.Sign(ctx, checksumFile); err != nil { - return fmt.Errorf("failed to sign checksums: %w", err) - } - - return nil -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/build/signing/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/build/signing/sign.go -git commit -m "feat(signing): add orchestration helpers - -SignBinaries, NotarizeBinaries, SignChecksums for pipeline integration. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 7: Integrate Signing into CLI - -**Files:** -- Modify: `cmd/core/cmd/build.go` - -**Step 1: Add --no-sign and --notarize flags** - -After the existing flag declarations (around line 74), add: - -```go - var noSign bool - var notarize bool - - buildCmd.BoolFlag("no-sign", "Skip all code signing", &noSign) - buildCmd.BoolFlag("notarize", "Enable macOS notarization (requires Apple credentials)", ¬arize) -``` - -**Step 2: Update runProjectBuild signature** - -Update the function signature and call: - -```go -// Update function signature: -func runProjectBuild(buildType string, ciMode bool, targetsFlag string, outputDir string, doArchive bool, doChecksum bool, configPath string, format string, push bool, imageName string, noSign bool, notarize bool) error { - -// Update the Action call: -buildCmd.Action(func() error { - return runProjectBuild(buildType, ciMode, targets, outputDir, doArchive, doChecksum, configPath, format, push, imageName, noSign, notarize) -}) -``` - -**Step 3: Add signing import** - -Add to imports: - -```go - "forge.lthn.ai/core/cli/pkg/build/signing" -``` - -**Step 4: Add signing after build, before archive** - -After the build succeeds (around line 228), add: - -```go - // Sign macOS binaries if enabled - signCfg := buildCfg.Sign - if notarize { - signCfg.MacOS.Notarize = true - } - if noSign { - signCfg.Enabled = false - } - - if signCfg.Enabled && runtime.GOOS == "darwin" { - if !ciMode { - fmt.Println() - fmt.Printf("%s Signing binaries...\n", buildHeaderStyle.Render("Sign:")) - } - - if err := signing.SignBinaries(ctx, signCfg, artifacts); err != nil { - if !ciMode { - fmt.Printf("%s Signing failed: %v\n", buildErrorStyle.Render("Error:"), err) - } - return err - } - - if signCfg.MacOS.Notarize { - if err := signing.NotarizeBinaries(ctx, signCfg, artifacts); err != nil { - if !ciMode { - fmt.Printf("%s Notarization failed: %v\n", buildErrorStyle.Render("Error:"), err) - } - return err - } - } - } -``` - -**Step 5: Add GPG signing after checksums** - -After WriteChecksumFile (around line 297), add: - -```go - // Sign checksums with GPG - if signCfg.Enabled { - if err := signing.SignChecksums(ctx, signCfg, checksumPath); err != nil { - if !ciMode { - fmt.Printf("%s GPG signing failed: %v\n", buildErrorStyle.Render("Error:"), err) - } - return err - } - } -``` - -**Step 6: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./cmd/core/...` -Expected: No errors - -**Step 7: Commit** - -```bash -git add cmd/core/cmd/build.go -git commit -m "feat(cli): integrate signing into build command - -Adds --no-sign and --notarize flags. -Signs macOS binaries after build, GPG signs checksums. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 8: Add Integration Test - -**Files:** -- Create: `pkg/build/signing/signing_test.go` - -**Step 1: Create integration test** - -```go -package signing - -import ( - "context" - "os" - "path/filepath" - "runtime" - "testing" - - "forge.lthn.ai/core/cli/pkg/build" -) - -func TestSignBinaries_Good_SkipsNonDarwin(t *testing.T) { - ctx := context.Background() - cfg := SignConfig{ - Enabled: true, - MacOS: MacOSConfig{ - Identity: "Developer ID Application: Test", - }, - } - - // Create fake artifact for linux - artifacts := []build.Artifact{ - {Path: "/tmp/test-binary", OS: "linux", Arch: "amd64"}, - } - - // Should not error even though binary doesn't exist (skips non-darwin) - err := SignBinaries(ctx, cfg, artifacts) - if err != nil { - t.Errorf("unexpected error: %v", err) - } -} - -func TestSignBinaries_Good_DisabledConfig(t *testing.T) { - ctx := context.Background() - cfg := SignConfig{ - Enabled: false, - } - - artifacts := []build.Artifact{ - {Path: "/tmp/test-binary", OS: "darwin", Arch: "arm64"}, - } - - err := SignBinaries(ctx, cfg, artifacts) - if err != nil { - t.Errorf("unexpected error: %v", err) - } -} - -func TestSignChecksums_Good_SkipsNoKey(t *testing.T) { - ctx := context.Background() - cfg := SignConfig{ - Enabled: true, - GPG: GPGConfig{ - Key: "", // No key configured - }, - } - - // Should silently skip when no key - err := SignChecksums(ctx, cfg, "/tmp/CHECKSUMS.txt") - if err != nil { - t.Errorf("unexpected error: %v", err) - } -} - -func TestSignChecksums_Good_Disabled(t *testing.T) { - ctx := context.Background() - cfg := SignConfig{ - Enabled: false, - } - - err := SignChecksums(ctx, cfg, "/tmp/CHECKSUMS.txt") - if err != nil { - t.Errorf("unexpected error: %v", err) - } -} -``` - -**Step 2: Run all signing tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/signing/... -v` -Expected: All tests pass - -**Step 3: Commit** - -```bash -git add pkg/build/signing/signing_test.go -git commit -m "test(signing): add integration tests - -Tests for skip conditions and disabled configs. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 9: Update TODO.md and Final Verification - -**Step 1: Build CLI** - -Run: `cd /Users/snider/Code/Core && go build -o bin/core ./cmd/core` -Expected: No errors - -**Step 2: Test help output** - -Run: `./bin/core build --help` -Expected: Shows --no-sign and --notarize flags - -**Step 3: Run all tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/build/... -v` -Expected: All tests pass - -**Step 4: Update TODO.md** - -Mark S3.3 tasks as complete in `tasks/TODO.md`: - -```markdown -### S3.3 Code Signing (Standard) ✅ -- [x] macOS codesign integration -- [x] macOS notarization -- [ ] Windows signtool integration (placeholder added) -- [x] GPG signing (standard tools) -``` - -**Step 5: Final commit** - -```bash -git add tasks/TODO.md -git commit -m "chore(signing): finalize S3.3 code signing - -Implemented: -- GPG signing of CHECKSUMS.txt -- macOS codesign with hardened runtime -- macOS notarization via notarytool -- Windows signtool placeholder - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -## Summary - -9 tasks covering: -1. Signing package structure (Signer interface, SignConfig) -2. GPG signer implementation -3. macOS codesign + notarization -4. Windows signtool placeholder -5. Add SignConfig to BuildConfig -6. Orchestration helpers (SignBinaries, SignChecksums) -7. CLI integration (--no-sign, --notarize) -8. Integration tests -9. Final verification and TODO update diff --git a/tasks/plans/2026-01-29-core-devops-design.md b/tasks/plans/2026-01-29-core-devops-design.md deleted file mode 100644 index 1b66e671..00000000 --- a/tasks/plans/2026-01-29-core-devops-design.md +++ /dev/null @@ -1,306 +0,0 @@ -# Core DevOps CLI Design (S4.6) - -## Summary - -Portable development environment CLI commands for the core-devops LinuxKit image. Provides a sandboxed, immutable environment with 100+ embedded tools. - -## Design Decisions - -- **Image sources**: GitHub Releases + Container Registry + CDN (try in order, configurable) -- **Local storage**: `~/.core/images/` with `CORE_IMAGES_DIR` env override -- **Shell connection**: SSH by default, `--console` for serial fallback -- **Serve**: Mount PWD into VM via 9P/SSHFS, run auto-detected dev server -- **Test**: Auto-detect framework + `.core/test.yaml` config + `--` override -- **Update**: Simple hash/version check, `--force` to always download -- **Claude sandbox**: SSH in with forwarded auth, safe experimentation in immutable image - -## Package Structure - -``` -pkg/devops/ -├── devops.go # DevOps struct, Boot/Stop/Status -├── images.go # ImageManager, manifest handling -├── mount.go # Directory mounting (9P, SSHFS) -├── serve.go # Project detection, serve command -├── test.go # Test detection, .core/test.yaml parsing -├── config.go # ~/.core/config.yaml handling -└── sources/ - ├── source.go # ImageSource interface - ├── github.go # GitHub Releases - ├── registry.go # Container registry - └── cdn.go # CDN/S3 - -cmd/core/cmd/dev.go # CLI commands -``` - -## Image Storage - -``` -~/.core/ -├── config.yaml # Global config (image source preference, etc.) -└── images/ - ├── core-devops-darwin-arm64.qcow2 - ├── core-devops-darwin-amd64.qcow2 - ├── core-devops-linux-amd64.qcow2 - └── manifest.json # Tracks versions, hashes, last-updated -``` - -## ImageSource Interface - -```go -type ImageSource interface { - Name() string - Available() bool - LatestVersion() (string, error) - Download(ctx context.Context, dest string) error -} -``` - -Sources tried in order: GitHub → Registry → CDN, or respect user preference in config. - -## CLI Commands - -```go -// cmd/core/cmd/dev.go - -func AddDevCommand(app *clir.Cli) { - devCmd := app.NewSubCommand("dev", "Portable development environment") - - // core dev install [--source github|registry|cdn] - // Downloads core-devops image for current platform - - // core dev boot [--memory 4096] [--cpus 4] [--name mydev] - // Boots the dev environment (detached by default) - - // core dev shell [--console] - // SSH into running dev env (or serial console with --console) - - // core dev serve [--port 8000] - // Mount PWD → /app, run FrankenPHP, forward port - - // core dev test [-- custom command] - // Auto-detect tests or use .core/test.yaml or pass custom - - // core dev claude [--auth] [--model opus|sonnet] - // SSH in with forwarded auth, start Claude in sandbox - - // core dev update [--force] - // Check for newer image, download if available - - // core dev status - // Show if dev env is running, resource usage, ports - - // core dev stop - // Stop the running dev environment -} -``` - -## Command Flow - -``` -First time: - core dev install → Downloads ~/.core/images/core-devops-{os}-{arch}.qcow2 - core dev boot → Starts VM in background - core dev shell → SSH in - -Daily use: - core dev boot → Start (if not running) - core dev serve → Mount project, start server - core dev test → Run tests inside VM - core dev shell → Interactive work - -AI sandbox: - core dev claude → SSH + forward auth + start Claude CLI - -Maintenance: - core dev update → Get latest image - core dev status → Check what's running -``` - -## `core dev claude` - Sandboxed AI Session - -```bash -core dev claude # Forward all auth by default -core dev claude --no-auth # Clean session, no host credentials -core dev claude --auth=gh,anthropic # Selective forwarding -``` - -**What it does:** -1. Ensures dev VM is running (auto-boots if not) -2. Forwards auth credentials from host: - - `~/.anthropic/` or `ANTHROPIC_API_KEY` - - `~/.config/gh/` (GitHub CLI auth) - - SSH agent forwarding - - Git config (name, email) -3. SSHs into VM with agent forwarding (`ssh -A`) -4. Starts `claude` CLI inside with forwarded context -5. Current project mounted at `/app` - -**Why this is powerful:** -- Immutable base = reset anytime with `core dev boot --fresh` -- Claude can experiment freely, install packages, make mistakes -- Host system untouched -- Still has real credentials to push code, create PRs -- Full 100+ tools available in core-devops image - -## Test Configuration - -**`.core/test.yaml` format:** -```yaml -version: 1 - -# Commands to run (in order) -commands: - - name: unit - run: vendor/bin/pest --parallel - - name: types - run: vendor/bin/phpstan analyse - - name: lint - run: vendor/bin/pint --test - -# Or simple single command -command: npm test - -# Environment variables -env: - APP_ENV: testing - DB_CONNECTION: sqlite -``` - -**Auto-Detection Priority:** -1. `.core/test.yaml` -2. `composer.json` scripts.test → `composer test` -3. `package.json` scripts.test → `npm test` -4. `go.mod` → `go test ./...` -5. `pytest.ini` or `pyproject.toml` → `pytest` -6. `Taskfile.yaml` → `task test` - -**CLI Usage:** -```bash -core dev test # Auto-detect and run -core dev test --unit # Run only "unit" from .core/test.yaml -core dev test -- go test -v ./pkg/... # Override with custom -``` - -## `core dev serve` - Mount & Serve - -**How it works:** -1. Ensure VM is running -2. Mount current directory into VM via 9P virtio-fs (or SSHFS fallback) -3. Start auto-detected dev server on /app inside VM -4. Forward port to host - -**Mount Strategy:** -```go -type MountMethod int -const ( - Mount9P MountMethod = iota // QEMU virtio-9p (faster) - MountSSHFS // sshfs reverse mount - MountRSync // Fallback: rsync on change -) -``` - -**CLI Usage:** -```bash -core dev serve # Mount PWD, serve on :8000 -core dev serve --port 3000 # Custom port -core dev serve --path ./backend # Serve subdirectory -``` - -**Project Detection:** -```go -func detectServeCommand(projectDir string) string { - if exists("artisan") { - return "php artisan octane:start --host=0.0.0.0 --port=8000" - } - if exists("package.json") && hasScript("dev") { - return "npm run dev -- --host 0.0.0.0" - } - if exists("composer.json") { - return "frankenphp php-server" - } - return "python -m http.server 8000" // Fallback -} -``` - -## Image Sources & Updates - -**~/.core/config.yaml:** -```yaml -version: 1 - -images: - source: auto # auto | github | registry | cdn - - cdn: - url: https://images.example.com/core-devops - - github: - repo: host-uk/core-images - - registry: - image: ghcr.io/host-uk/core-devops -``` - -**Manifest for Update Checking:** -```json -// ~/.core/images/manifest.json -{ - "core-devops-darwin-arm64.qcow2": { - "version": "v1.2.0", - "sha256": "abc123...", - "downloaded": "2026-01-29T10:00:00Z", - "source": "github" - } -} -``` - -**Update Flow:** -```go -func (d *DevOps) Update(force bool) error { - local := d.manifest.Get(imageName) - remote, _ := d.source.LatestVersion() - - if force || local.Version != remote { - fmt.Printf("Updating %s → %s\n", local.Version, remote) - return d.source.Download(ctx, imagePath) - } - fmt.Println("Already up to date") - return nil -} -``` - -## Commands Summary - -| Command | Description | -|---------|-------------| -| `core dev install` | Download image for platform | -| `core dev boot` | Start VM (auto-installs if needed) | -| `core dev shell` | SSH in (--console for serial) | -| `core dev serve` | Mount PWD, run dev server | -| `core dev test` | Run tests inside VM | -| `core dev claude` | Start Claude session in sandbox | -| `core dev update` | Check/download newer image | -| `core dev status` | Show VM state, ports, resources | -| `core dev stop` | Stop the VM | - -## Dependencies - -- Reuse existing `pkg/container` for VM management (LinuxKitManager) -- SSH client for shell/exec (golang.org/x/crypto/ssh) -- Progress bar for downloads (charmbracelet/bubbles or similar) - -## Implementation Steps - -1. Create `pkg/devops/` package structure -2. Implement ImageSource interface and sources (GitHub, Registry, CDN) -3. Implement image download with manifest tracking -4. Implement config loading (`~/.core/config.yaml`) -5. Add CLI commands to `cmd/core/cmd/dev.go` -6. Implement boot/stop using existing LinuxKitManager -7. Implement shell (SSH + serial console) -8. Implement serve (mount + project detection) -9. Implement test (detection + .core/test.yaml) -10. Implement claude (auth forwarding + sandbox) -11. Implement update (version check + download) -12. Implement status diff --git a/tasks/plans/2026-01-29-core-devops-impl.md b/tasks/plans/2026-01-29-core-devops-impl.md deleted file mode 100644 index e368bf93..00000000 --- a/tasks/plans/2026-01-29-core-devops-impl.md +++ /dev/null @@ -1,2183 +0,0 @@ -# Core DevOps CLI Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Implement `core dev` commands for portable development environment using core-devops LinuxKit images. - -**Architecture:** `pkg/devops` package handles image management, config, and orchestration. Reuses `pkg/container.LinuxKitManager` for VM lifecycle. Image sources (GitHub, Registry, CDN) implement common interface. CLI in `cmd/core/cmd/dev.go`. - -**Tech Stack:** Go, pkg/container, golang.org/x/crypto/ssh, os/exec for gh CLI, YAML config - ---- - -### Task 1: Create DevOps Package Structure - -**Files:** -- Create: `pkg/devops/devops.go` -- Create: `pkg/devops/go.mod` - -**Step 1: Create go.mod** - -```go -module forge.lthn.ai/core/cli/pkg/devops - -go 1.25 - -require ( - forge.lthn.ai/core/cli/pkg/container v0.0.0 - golang.org/x/crypto v0.32.0 - gopkg.in/yaml.v3 v3.0.1 -) - -replace forge.lthn.ai/core/cli/pkg/container => ../container -``` - -**Step 2: Create devops.go with core types** - -```go -// Package devops provides a portable development environment using LinuxKit images. -package devops - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime" - - "forge.lthn.ai/core/cli/pkg/container" -) - -// DevOps manages the portable development environment. -type DevOps struct { - config *Config - images *ImageManager - container *container.LinuxKitManager -} - -// New creates a new DevOps instance. -func New() (*DevOps, error) { - cfg, err := LoadConfig() - if err != nil { - return nil, fmt.Errorf("devops.New: failed to load config: %w", err) - } - - images, err := NewImageManager(cfg) - if err != nil { - return nil, fmt.Errorf("devops.New: failed to create image manager: %w", err) - } - - mgr, err := container.NewLinuxKitManager() - if err != nil { - return nil, fmt.Errorf("devops.New: failed to create container manager: %w", err) - } - - return &DevOps{ - config: cfg, - images: images, - container: mgr, - }, nil -} - -// ImageName returns the platform-specific image name. -func ImageName() string { - return fmt.Sprintf("core-devops-%s-%s.qcow2", runtime.GOOS, runtime.GOARCH) -} - -// ImagesDir returns the path to the images directory. -func ImagesDir() (string, error) { - if dir := os.Getenv("CORE_IMAGES_DIR"); dir != "" { - return dir, nil - } - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, ".core", "images"), nil -} - -// ImagePath returns the full path to the platform-specific image. -func ImagePath() (string, error) { - dir, err := ImagesDir() - if err != nil { - return "", err - } - return filepath.Join(dir, ImageName()), nil -} - -// IsInstalled checks if the dev image is installed. -func (d *DevOps) IsInstalled() bool { - path, err := ImagePath() - if err != nil { - return false - } - _, err = os.Stat(path) - return err == nil -} -``` - -**Step 3: Add to go.work** - -Run: `cd /Users/snider/Code/Core && echo " ./pkg/devops" >> go.work && go work sync` - -**Step 4: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: Error (missing Config, ImageManager) - that's OK for now - -**Step 5: Commit** - -```bash -git add pkg/devops/ -git add go.work go.work.sum -git commit -m "feat(devops): add package structure - -Initial pkg/devops setup with DevOps type and path helpers. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 2: Implement Config Loading - -**Files:** -- Create: `pkg/devops/config.go` -- Create: `pkg/devops/config_test.go` - -**Step 1: Write the failing test** - -```go -package devops - -import ( - "os" - "path/filepath" - "testing" -) - -func TestLoadConfig_Good_Default(t *testing.T) { - // Use temp home dir - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - cfg, err := LoadConfig() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if cfg.Images.Source != "auto" { - t.Errorf("expected source 'auto', got %q", cfg.Images.Source) - } -} - -func TestLoadConfig_Good_FromFile(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("HOME", tmpDir) - - configDir := filepath.Join(tmpDir, ".core") - os.MkdirAll(configDir, 0755) - - configContent := `version: 1 -images: - source: github - github: - repo: myorg/images -` - os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte(configContent), 0644) - - cfg, err := LoadConfig() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if cfg.Images.Source != "github" { - t.Errorf("expected source 'github', got %q", cfg.Images.Source) - } - if cfg.Images.GitHub.Repo != "myorg/images" { - t.Errorf("expected repo 'myorg/images', got %q", cfg.Images.GitHub.Repo) - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestLoadConfig -v` -Expected: FAIL (LoadConfig not defined) - -**Step 3: Write implementation** - -```go -package devops - -import ( - "os" - "path/filepath" - - "gopkg.in/yaml.v3" -) - -// Config holds global devops configuration from ~/.core/config.yaml. -type Config struct { - Version int `yaml:"version"` - Images ImagesConfig `yaml:"images"` -} - -// ImagesConfig holds image source configuration. -type ImagesConfig struct { - Source string `yaml:"source"` // auto, github, registry, cdn - GitHub GitHubConfig `yaml:"github,omitempty"` - Registry RegistryConfig `yaml:"registry,omitempty"` - CDN CDNConfig `yaml:"cdn,omitempty"` -} - -// GitHubConfig holds GitHub Releases configuration. -type GitHubConfig struct { - Repo string `yaml:"repo"` // owner/repo format -} - -// RegistryConfig holds container registry configuration. -type RegistryConfig struct { - Image string `yaml:"image"` // e.g., ghcr.io/host-uk/core-devops -} - -// CDNConfig holds CDN/S3 configuration. -type CDNConfig struct { - URL string `yaml:"url"` // base URL for downloads -} - -// DefaultConfig returns sensible defaults. -func DefaultConfig() *Config { - return &Config{ - Version: 1, - Images: ImagesConfig{ - Source: "auto", - GitHub: GitHubConfig{ - Repo: "host-uk/core-images", - }, - Registry: RegistryConfig{ - Image: "ghcr.io/host-uk/core-devops", - }, - }, - } -} - -// ConfigPath returns the path to the config file. -func ConfigPath() (string, error) { - home, err := os.UserHomeDir() - if err != nil { - return "", err - } - return filepath.Join(home, ".core", "config.yaml"), nil -} - -// LoadConfig loads configuration from ~/.core/config.yaml. -// Returns default config if file doesn't exist. -func LoadConfig() (*Config, error) { - configPath, err := ConfigPath() - if err != nil { - return DefaultConfig(), nil - } - - data, err := os.ReadFile(configPath) - if err != nil { - if os.IsNotExist(err) { - return DefaultConfig(), nil - } - return nil, err - } - - cfg := DefaultConfig() - if err := yaml.Unmarshal(data, cfg); err != nil { - return nil, err - } - - return cfg, nil -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestLoadConfig -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/devops/config.go pkg/devops/config_test.go -git commit -m "feat(devops): add config loading - -Loads ~/.core/config.yaml with image source preferences. -Defaults to auto-detection with host-uk/core-images. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 3: Implement ImageSource Interface - -**Files:** -- Create: `pkg/devops/sources/source.go` - -**Step 1: Create source interface** - -```go -// Package sources provides image download sources for core-devops. -package sources - -import ( - "context" -) - -// ImageSource defines the interface for downloading dev images. -type ImageSource interface { - // Name returns the source identifier. - Name() string - // Available checks if this source can be used. - Available() bool - // LatestVersion returns the latest available version. - LatestVersion(ctx context.Context) (string, error) - // Download downloads the image to the destination path. - // Reports progress via the callback if provided. - Download(ctx context.Context, dest string, progress func(downloaded, total int64)) error -} - -// SourceConfig holds configuration for a source. -type SourceConfig struct { - // GitHub configuration - GitHubRepo string - // Registry configuration - RegistryImage string - // CDN configuration - CDNURL string - // Image name (e.g., core-devops-darwin-arm64.qcow2) - ImageName string -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/devops/sources/source.go -git commit -m "feat(devops): add ImageSource interface - -Defines common interface for GitHub, Registry, and CDN sources. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 4: Implement GitHub Source - -**Files:** -- Create: `pkg/devops/sources/github.go` -- Create: `pkg/devops/sources/github_test.go` - -**Step 1: Write the failing test** - -```go -package sources - -import ( - "testing" -) - -func TestGitHubSource_Good_Available(t *testing.T) { - src := NewGitHubSource(SourceConfig{ - GitHubRepo: "host-uk/core-images", - ImageName: "core-devops-darwin-arm64.qcow2", - }) - - if src.Name() != "github" { - t.Errorf("expected name 'github', got %q", src.Name()) - } - - // Available depends on gh CLI being installed - _ = src.Available() -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/sources/... -run TestGitHubSource -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package sources - -import ( - "context" - "encoding/json" - "fmt" - "os" - "os/exec" - "strings" -) - -// GitHubSource downloads images from GitHub Releases. -type GitHubSource struct { - config SourceConfig -} - -// NewGitHubSource creates a new GitHub source. -func NewGitHubSource(cfg SourceConfig) *GitHubSource { - return &GitHubSource{config: cfg} -} - -// Name returns "github". -func (s *GitHubSource) Name() string { - return "github" -} - -// Available checks if gh CLI is installed and authenticated. -func (s *GitHubSource) Available() bool { - _, err := exec.LookPath("gh") - if err != nil { - return false - } - // Check if authenticated - cmd := exec.Command("gh", "auth", "status") - return cmd.Run() == nil -} - -// LatestVersion returns the latest release tag. -func (s *GitHubSource) LatestVersion(ctx context.Context) (string, error) { - cmd := exec.CommandContext(ctx, "gh", "release", "view", - "-R", s.config.GitHubRepo, - "--json", "tagName", - "-q", ".tagName", - ) - out, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("github.LatestVersion: %w", err) - } - return strings.TrimSpace(string(out)), nil -} - -// Download downloads the image from the latest release. -func (s *GitHubSource) Download(ctx context.Context, dest string, progress func(downloaded, total int64)) error { - // Get release assets to find our image - cmd := exec.CommandContext(ctx, "gh", "release", "download", - "-R", s.config.GitHubRepo, - "-p", s.config.ImageName, - "-D", dest, - "--clobber", - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("github.Download: %w", err) - } - return nil -} - -// releaseAsset represents a GitHub release asset. -type releaseAsset struct { - Name string `json:"name"` - Size int64 `json:"size"` - URL string `json:"url"` -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/sources/... -run TestGitHubSource -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/devops/sources/github.go pkg/devops/sources/github_test.go -git commit -m "feat(devops): add GitHub Releases source - -Downloads core-devops images from GitHub Releases using gh CLI. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 5: Implement CDN Source - -**Files:** -- Create: `pkg/devops/sources/cdn.go` -- Create: `pkg/devops/sources/cdn_test.go` - -**Step 1: Write the failing test** - -```go -package sources - -import ( - "testing" -) - -func TestCDNSource_Good_Available(t *testing.T) { - src := NewCDNSource(SourceConfig{ - CDNURL: "https://images.example.com", - ImageName: "core-devops-darwin-arm64.qcow2", - }) - - if src.Name() != "cdn" { - t.Errorf("expected name 'cdn', got %q", src.Name()) - } - - // CDN is available if URL is configured - if !src.Available() { - t.Error("expected Available() to be true when URL is set") - } -} - -func TestCDNSource_Bad_NoURL(t *testing.T) { - src := NewCDNSource(SourceConfig{ - ImageName: "core-devops-darwin-arm64.qcow2", - }) - - if src.Available() { - t.Error("expected Available() to be false when URL is empty") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/sources/... -run TestCDNSource -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package sources - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" -) - -// CDNSource downloads images from a CDN or S3 bucket. -type CDNSource struct { - config SourceConfig -} - -// NewCDNSource creates a new CDN source. -func NewCDNSource(cfg SourceConfig) *CDNSource { - return &CDNSource{config: cfg} -} - -// Name returns "cdn". -func (s *CDNSource) Name() string { - return "cdn" -} - -// Available checks if CDN URL is configured. -func (s *CDNSource) Available() bool { - return s.config.CDNURL != "" -} - -// LatestVersion fetches version from manifest or returns "latest". -func (s *CDNSource) LatestVersion(ctx context.Context) (string, error) { - // Try to fetch manifest.json for version info - url := fmt.Sprintf("%s/manifest.json", s.config.CDNURL) - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return "latest", nil - } - - resp, err := http.DefaultClient.Do(req) - if err != nil || resp.StatusCode != 200 { - return "latest", nil - } - defer resp.Body.Close() - - // For now, just return latest - could parse manifest for version - return "latest", nil -} - -// Download downloads the image from CDN. -func (s *CDNSource) Download(ctx context.Context, dest string, progress func(downloaded, total int64)) error { - url := fmt.Sprintf("%s/%s", s.config.CDNURL, s.config.ImageName) - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return fmt.Errorf("cdn.Download: %w", err) - } - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return fmt.Errorf("cdn.Download: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("cdn.Download: HTTP %d", resp.StatusCode) - } - - // Ensure dest directory exists - if err := os.MkdirAll(filepath.Dir(dest), 0755); err != nil { - return fmt.Errorf("cdn.Download: %w", err) - } - - // Create destination file - destPath := filepath.Join(dest, s.config.ImageName) - f, err := os.Create(destPath) - if err != nil { - return fmt.Errorf("cdn.Download: %w", err) - } - defer f.Close() - - // Copy with progress - total := resp.ContentLength - var downloaded int64 - - buf := make([]byte, 32*1024) - for { - n, err := resp.Body.Read(buf) - if n > 0 { - if _, werr := f.Write(buf[:n]); werr != nil { - return fmt.Errorf("cdn.Download: %w", werr) - } - downloaded += int64(n) - if progress != nil { - progress(downloaded, total) - } - } - if err == io.EOF { - break - } - if err != nil { - return fmt.Errorf("cdn.Download: %w", err) - } - } - - return nil -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/sources/... -run TestCDNSource -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/devops/sources/cdn.go pkg/devops/sources/cdn_test.go -git commit -m "feat(devops): add CDN/S3 source - -Downloads core-devops images from custom CDN with progress reporting. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 6: Implement ImageManager - -**Files:** -- Create: `pkg/devops/images.go` -- Create: `pkg/devops/images_test.go` - -**Step 1: Write the failing test** - -```go -package devops - -import ( - "os" - "path/filepath" - "testing" -) - -func TestImageManager_Good_IsInstalled(t *testing.T) { - tmpDir := t.TempDir() - t.Setenv("CORE_IMAGES_DIR", tmpDir) - - cfg := DefaultConfig() - mgr, err := NewImageManager(cfg) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - // Not installed yet - if mgr.IsInstalled() { - t.Error("expected IsInstalled() to be false") - } - - // Create fake image - imagePath := filepath.Join(tmpDir, ImageName()) - os.WriteFile(imagePath, []byte("fake"), 0644) - - // Now installed - if !mgr.IsInstalled() { - t.Error("expected IsInstalled() to be true") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestImageManager -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package devops - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "forge.lthn.ai/core/cli/pkg/devops/sources" -) - -// ImageManager handles image downloads and updates. -type ImageManager struct { - config *Config - manifest *Manifest - sources []sources.ImageSource -} - -// Manifest tracks installed images. -type Manifest struct { - Images map[string]ImageInfo `json:"images"` - path string -} - -// ImageInfo holds metadata about an installed image. -type ImageInfo struct { - Version string `json:"version"` - SHA256 string `json:"sha256,omitempty"` - Downloaded time.Time `json:"downloaded"` - Source string `json:"source"` -} - -// NewImageManager creates a new image manager. -func NewImageManager(cfg *Config) (*ImageManager, error) { - imagesDir, err := ImagesDir() - if err != nil { - return nil, err - } - - // Ensure images directory exists - if err := os.MkdirAll(imagesDir, 0755); err != nil { - return nil, err - } - - // Load or create manifest - manifestPath := filepath.Join(imagesDir, "manifest.json") - manifest, err := loadManifest(manifestPath) - if err != nil { - return nil, err - } - - // Build source list based on config - imageName := ImageName() - sourceCfg := sources.SourceConfig{ - GitHubRepo: cfg.Images.GitHub.Repo, - RegistryImage: cfg.Images.Registry.Image, - CDNURL: cfg.Images.CDN.URL, - ImageName: imageName, - } - - var srcs []sources.ImageSource - switch cfg.Images.Source { - case "github": - srcs = []sources.ImageSource{sources.NewGitHubSource(sourceCfg)} - case "cdn": - srcs = []sources.ImageSource{sources.NewCDNSource(sourceCfg)} - default: // "auto" - srcs = []sources.ImageSource{ - sources.NewGitHubSource(sourceCfg), - sources.NewCDNSource(sourceCfg), - } - } - - return &ImageManager{ - config: cfg, - manifest: manifest, - sources: srcs, - }, nil -} - -// IsInstalled checks if the dev image is installed. -func (m *ImageManager) IsInstalled() bool { - path, err := ImagePath() - if err != nil { - return false - } - _, err = os.Stat(path) - return err == nil -} - -// Install downloads and installs the dev image. -func (m *ImageManager) Install(ctx context.Context, progress func(downloaded, total int64)) error { - imagesDir, err := ImagesDir() - if err != nil { - return err - } - - // Find first available source - var src sources.ImageSource - for _, s := range m.sources { - if s.Available() { - src = s - break - } - } - if src == nil { - return fmt.Errorf("no image source available") - } - - // Get version - version, err := src.LatestVersion(ctx) - if err != nil { - return fmt.Errorf("failed to get latest version: %w", err) - } - - fmt.Printf("Downloading %s from %s...\n", ImageName(), src.Name()) - - // Download - if err := src.Download(ctx, imagesDir, progress); err != nil { - return err - } - - // Update manifest - m.manifest.Images[ImageName()] = ImageInfo{ - Version: version, - Downloaded: time.Now(), - Source: src.Name(), - } - - return m.manifest.Save() -} - -// CheckUpdate checks if an update is available. -func (m *ImageManager) CheckUpdate(ctx context.Context) (current, latest string, hasUpdate bool, err error) { - info, ok := m.manifest.Images[ImageName()] - if !ok { - return "", "", false, fmt.Errorf("image not installed") - } - current = info.Version - - // Find first available source - var src sources.ImageSource - for _, s := range m.sources { - if s.Available() { - src = s - break - } - } - if src == nil { - return current, "", false, fmt.Errorf("no image source available") - } - - latest, err = src.LatestVersion(ctx) - if err != nil { - return current, "", false, err - } - - hasUpdate = current != latest - return current, latest, hasUpdate, nil -} - -func loadManifest(path string) (*Manifest, error) { - m := &Manifest{ - Images: make(map[string]ImageInfo), - path: path, - } - - data, err := os.ReadFile(path) - if err != nil { - if os.IsNotExist(err) { - return m, nil - } - return nil, err - } - - if err := json.Unmarshal(data, m); err != nil { - return nil, err - } - m.path = path - - return m, nil -} - -// Save writes the manifest to disk. -func (m *Manifest) Save() error { - data, err := json.MarshalIndent(m, "", " ") - if err != nil { - return err - } - return os.WriteFile(m.path, data, 0644) -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestImageManager -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/devops/images.go pkg/devops/images_test.go -git commit -m "feat(devops): add ImageManager - -Manages image downloads, manifest tracking, and update checking. -Tries sources in priority order (GitHub, CDN). - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 7: Implement Boot/Stop/Status - -**Files:** -- Modify: `pkg/devops/devops.go` -- Create: `pkg/devops/devops_test.go` - -**Step 1: Add boot/stop/status methods to devops.go** - -```go -// Add to devops.go - -// BootOptions configures how to boot the dev environment. -type BootOptions struct { - Memory int // MB, default 4096 - CPUs int // default 2 - Name string // container name - Fresh bool // destroy existing and start fresh -} - -// DefaultBootOptions returns sensible defaults. -func DefaultBootOptions() BootOptions { - return BootOptions{ - Memory: 4096, - CPUs: 2, - Name: "core-dev", - } -} - -// Boot starts the dev environment. -func (d *DevOps) Boot(ctx context.Context, opts BootOptions) error { - if !d.images.IsInstalled() { - return fmt.Errorf("dev image not installed (run 'core dev install' first)") - } - - // Check if already running - if !opts.Fresh { - running, err := d.IsRunning(ctx) - if err == nil && running { - return fmt.Errorf("dev environment already running (use 'core dev stop' first or --fresh)") - } - } - - // Stop existing if fresh - if opts.Fresh { - _ = d.Stop(ctx) - } - - imagePath, err := ImagePath() - if err != nil { - return err - } - - runOpts := container.RunOptions{ - Name: opts.Name, - Detach: true, - Memory: opts.Memory, - CPUs: opts.CPUs, - SSHPort: 2222, - } - - _, err = d.container.Run(ctx, imagePath, runOpts) - return err -} - -// Stop stops the dev environment. -func (d *DevOps) Stop(ctx context.Context) error { - containers, err := d.container.List(ctx) - if err != nil { - return err - } - - for _, c := range containers { - if c.Name == "core-dev" && c.Status == container.StatusRunning { - return d.container.Stop(ctx, c.ID) - } - } - - return nil -} - -// IsRunning checks if the dev environment is running. -func (d *DevOps) IsRunning(ctx context.Context) (bool, error) { - containers, err := d.container.List(ctx) - if err != nil { - return false, err - } - - for _, c := range containers { - if c.Name == "core-dev" && c.Status == container.StatusRunning { - return true, nil - } - } - - return false, nil -} - -// Status returns information about the dev environment. -type DevStatus struct { - Installed bool - Running bool - ImageVersion string - ContainerID string - Memory int - CPUs int - SSHPort int - Uptime time.Duration -} - -// Status returns the current dev environment status. -func (d *DevOps) Status(ctx context.Context) (*DevStatus, error) { - status := &DevStatus{ - Installed: d.images.IsInstalled(), - } - - if info, ok := d.images.manifest.Images[ImageName()]; ok { - status.ImageVersion = info.Version - } - - containers, err := d.container.List(ctx) - if err != nil { - return status, nil - } - - for _, c := range containers { - if c.Name == "core-dev" && c.Status == container.StatusRunning { - status.Running = true - status.ContainerID = c.ID - status.Memory = c.Memory - status.CPUs = c.CPUs - status.SSHPort = 2222 - status.Uptime = time.Since(c.StartedAt) - break - } - } - - return status, nil -} -``` - -**Step 2: Add missing import to devops.go** - -```go -import ( - "time" - // ... other imports -) -``` - -**Step 3: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: No errors - -**Step 4: Commit** - -```bash -git add pkg/devops/devops.go -git commit -m "feat(devops): add Boot/Stop/Status methods - -Manages dev VM lifecycle using LinuxKitManager. -Supports fresh boot, status checking, graceful stop. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 8: Implement Shell Command - -**Files:** -- Create: `pkg/devops/shell.go` - -**Step 1: Create shell.go** - -```go -package devops - -import ( - "context" - "fmt" - "os" - "os/exec" -) - -// ShellOptions configures the shell connection. -type ShellOptions struct { - Console bool // Use serial console instead of SSH - Command []string // Command to run (empty = interactive shell) -} - -// Shell connects to the dev environment. -func (d *DevOps) Shell(ctx context.Context, opts ShellOptions) error { - running, err := d.IsRunning(ctx) - if err != nil { - return err - } - if !running { - return fmt.Errorf("dev environment not running (run 'core dev boot' first)") - } - - if opts.Console { - return d.serialConsole(ctx) - } - - return d.sshShell(ctx, opts.Command) -} - -// sshShell connects via SSH. -func (d *DevOps) sshShell(ctx context.Context, command []string) error { - args := []string{ - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - "-A", // Agent forwarding - "-p", "2222", - "root@localhost", - } - - if len(command) > 0 { - args = append(args, command...) - } - - cmd := exec.CommandContext(ctx, "ssh", args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - return cmd.Run() -} - -// serialConsole attaches to the QEMU serial console. -func (d *DevOps) serialConsole(ctx context.Context) error { - // Find the container to get its console socket - containers, err := d.container.List(ctx) - if err != nil { - return err - } - - for _, c := range containers { - if c.Name == "core-dev" { - // Use socat to connect to the console socket - socketPath := fmt.Sprintf("/tmp/core-%s-console.sock", c.ID) - cmd := exec.CommandContext(ctx, "socat", "-,raw,echo=0", "unix-connect:"+socketPath) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() - } - } - - return fmt.Errorf("console not available") -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/devops/shell.go -git commit -m "feat(devops): add Shell for SSH and console access - -Connects to dev VM via SSH (default) or serial console (--console). -Supports SSH agent forwarding for credential access. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 9: Implement Test Detection - -**Files:** -- Create: `pkg/devops/test.go` -- Create: `pkg/devops/test_test.go` - -**Step 1: Write the failing test** - -```go -package devops - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDetectTestCommand_Good_ComposerJSON(t *testing.T) { - tmpDir := t.TempDir() - os.WriteFile(filepath.Join(tmpDir, "composer.json"), []byte(`{"scripts":{"test":"pest"}}`), 0644) - - cmd := DetectTestCommand(tmpDir) - if cmd != "composer test" { - t.Errorf("expected 'composer test', got %q", cmd) - } -} - -func TestDetectTestCommand_Good_PackageJSON(t *testing.T) { - tmpDir := t.TempDir() - os.WriteFile(filepath.Join(tmpDir, "package.json"), []byte(`{"scripts":{"test":"vitest"}}`), 0644) - - cmd := DetectTestCommand(tmpDir) - if cmd != "npm test" { - t.Errorf("expected 'npm test', got %q", cmd) - } -} - -func TestDetectTestCommand_Good_GoMod(t *testing.T) { - tmpDir := t.TempDir() - os.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte("module example"), 0644) - - cmd := DetectTestCommand(tmpDir) - if cmd != "go test ./..." { - t.Errorf("expected 'go test ./...', got %q", cmd) - } -} - -func TestDetectTestCommand_Good_CoreTestYaml(t *testing.T) { - tmpDir := t.TempDir() - coreDir := filepath.Join(tmpDir, ".core") - os.MkdirAll(coreDir, 0755) - os.WriteFile(filepath.Join(coreDir, "test.yaml"), []byte("command: custom-test"), 0644) - - cmd := DetectTestCommand(tmpDir) - if cmd != "custom-test" { - t.Errorf("expected 'custom-test', got %q", cmd) - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestDetectTestCommand -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package devops - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "gopkg.in/yaml.v3" -) - -// TestConfig holds test configuration from .core/test.yaml. -type TestConfig struct { - Version int `yaml:"version"` - Command string `yaml:"command,omitempty"` - Commands []TestCommand `yaml:"commands,omitempty"` - Env map[string]string `yaml:"env,omitempty"` -} - -// TestCommand is a named test command. -type TestCommand struct { - Name string `yaml:"name"` - Run string `yaml:"run"` -} - -// TestOptions configures test execution. -type TestOptions struct { - Name string // Run specific named command from .core/test.yaml - Command []string // Override command (from -- args) -} - -// Test runs tests in the dev environment. -func (d *DevOps) Test(ctx context.Context, projectDir string, opts TestOptions) error { - running, err := d.IsRunning(ctx) - if err != nil { - return err - } - if !running { - return fmt.Errorf("dev environment not running (run 'core dev boot' first)") - } - - var cmd string - - // Priority: explicit command > named command > auto-detect - if len(opts.Command) > 0 { - cmd = joinCommand(opts.Command) - } else if opts.Name != "" { - cfg, err := LoadTestConfig(projectDir) - if err != nil { - return err - } - for _, c := range cfg.Commands { - if c.Name == opts.Name { - cmd = c.Run - break - } - } - if cmd == "" { - return fmt.Errorf("test command %q not found in .core/test.yaml", opts.Name) - } - } else { - cmd = DetectTestCommand(projectDir) - if cmd == "" { - return fmt.Errorf("could not detect test command (create .core/test.yaml)") - } - } - - // Run via SSH - return d.sshShell(ctx, []string{"cd", "/app", "&&", cmd}) -} - -// DetectTestCommand auto-detects the test command for a project. -func DetectTestCommand(projectDir string) string { - // 1. Check .core/test.yaml - cfg, err := LoadTestConfig(projectDir) - if err == nil && cfg.Command != "" { - return cfg.Command - } - - // 2. Check composer.json - if hasFile(projectDir, "composer.json") { - return "composer test" - } - - // 3. Check package.json - if hasFile(projectDir, "package.json") { - return "npm test" - } - - // 4. Check go.mod - if hasFile(projectDir, "go.mod") { - return "go test ./..." - } - - // 5. Check pytest - if hasFile(projectDir, "pytest.ini") || hasFile(projectDir, "pyproject.toml") { - return "pytest" - } - - // 6. Check Taskfile - if hasFile(projectDir, "Taskfile.yaml") || hasFile(projectDir, "Taskfile.yml") { - return "task test" - } - - return "" -} - -// LoadTestConfig loads .core/test.yaml. -func LoadTestConfig(projectDir string) (*TestConfig, error) { - path := filepath.Join(projectDir, ".core", "test.yaml") - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - var cfg TestConfig - if err := yaml.Unmarshal(data, &cfg); err != nil { - return nil, err - } - - return &cfg, nil -} - -func hasFile(dir, name string) bool { - _, err := os.Stat(filepath.Join(dir, name)) - return err == nil -} - -func joinCommand(parts []string) string { - result := "" - for i, p := range parts { - if i > 0 { - result += " " - } - result += p - } - return result -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -run TestDetectTestCommand -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/devops/test.go pkg/devops/test_test.go -git commit -m "feat(devops): add test detection and execution - -Auto-detects test framework from project files. -Supports .core/test.yaml for custom configuration. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 10: Implement Serve with Mount - -**Files:** -- Create: `pkg/devops/serve.go` - -**Step 1: Create serve.go** - -```go -package devops - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// ServeOptions configures the dev server. -type ServeOptions struct { - Port int // Port to serve on (default 8000) - Path string // Subdirectory to serve (default: current dir) -} - -// Serve mounts the project and starts a dev server. -func (d *DevOps) Serve(ctx context.Context, projectDir string, opts ServeOptions) error { - running, err := d.IsRunning(ctx) - if err != nil { - return err - } - if !running { - return fmt.Errorf("dev environment not running (run 'core dev boot' first)") - } - - if opts.Port == 0 { - opts.Port = 8000 - } - - servePath := projectDir - if opts.Path != "" { - servePath = filepath.Join(projectDir, opts.Path) - } - - // Mount project directory via SSHFS - if err := d.mountProject(ctx, servePath); err != nil { - return fmt.Errorf("failed to mount project: %w", err) - } - - // Detect and run serve command - serveCmd := DetectServeCommand(servePath) - fmt.Printf("Starting server: %s\n", serveCmd) - fmt.Printf("Listening on http://localhost:%d\n", opts.Port) - - // Run serve command via SSH - return d.sshShell(ctx, []string{"cd", "/app", "&&", serveCmd}) -} - -// mountProject mounts a directory into the VM via SSHFS. -func (d *DevOps) mountProject(ctx context.Context, path string) error { - absPath, err := filepath.Abs(path) - if err != nil { - return err - } - - // Use reverse SSHFS mount - // The VM connects back to host to mount the directory - cmd := exec.CommandContext(ctx, "ssh", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-R", "10000:localhost:22", // Reverse tunnel for SSHFS - "-p", "2222", - "root@localhost", - "mkdir -p /app && sshfs -p 10000 "+os.Getenv("USER")+"@localhost:"+absPath+" /app -o allow_other", - ) - return cmd.Run() -} - -// DetectServeCommand auto-detects the serve command for a project. -func DetectServeCommand(projectDir string) string { - // Laravel/Octane - if hasFile(projectDir, "artisan") { - return "php artisan octane:start --host=0.0.0.0 --port=8000" - } - - // Node.js with dev script - if hasFile(projectDir, "package.json") { - if hasPackageScript(projectDir, "dev") { - return "npm run dev -- --host 0.0.0.0" - } - if hasPackageScript(projectDir, "start") { - return "npm start" - } - } - - // PHP with composer - if hasFile(projectDir, "composer.json") { - return "frankenphp php-server -l :8000" - } - - // Go - if hasFile(projectDir, "go.mod") { - if hasFile(projectDir, "main.go") { - return "go run ." - } - } - - // Python - if hasFile(projectDir, "manage.py") { - return "python manage.py runserver 0.0.0.0:8000" - } - - // Fallback: simple HTTP server - return "python3 -m http.server 8000" -} - -func hasPackageScript(projectDir, script string) bool { - data, err := os.ReadFile(filepath.Join(projectDir, "package.json")) - if err != nil { - return false - } - - var pkg struct { - Scripts map[string]string `json:"scripts"` - } - if err := json.Unmarshal(data, &pkg); err != nil { - return false - } - - _, ok := pkg.Scripts[script] - return ok -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/devops/serve.go -git commit -m "feat(devops): add Serve with project mounting - -Mounts project via SSHFS and runs auto-detected dev server. -Supports Laravel, Node.js, PHP, Go, Python projects. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 11: Implement Claude Sandbox - -**Files:** -- Create: `pkg/devops/claude.go` - -**Step 1: Create claude.go** - -```go -package devops - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" -) - -// ClaudeOptions configures the Claude sandbox session. -type ClaudeOptions struct { - NoAuth bool // Don't forward any auth - Auth []string // Selective auth: "gh", "anthropic", "ssh", "git" - Model string // Model to use: opus, sonnet -} - -// Claude starts a sandboxed Claude session in the dev environment. -func (d *DevOps) Claude(ctx context.Context, projectDir string, opts ClaudeOptions) error { - // Auto-boot if not running - running, err := d.IsRunning(ctx) - if err != nil { - return err - } - if !running { - fmt.Println("Dev environment not running, booting...") - if err := d.Boot(ctx, DefaultBootOptions()); err != nil { - return fmt.Errorf("failed to boot: %w", err) - } - } - - // Mount project - if err := d.mountProject(ctx, projectDir); err != nil { - return fmt.Errorf("failed to mount project: %w", err) - } - - // Prepare environment variables to forward - envVars := []string{} - - if !opts.NoAuth { - authTypes := opts.Auth - if len(authTypes) == 0 { - authTypes = []string{"gh", "anthropic", "ssh", "git"} - } - - for _, auth := range authTypes { - switch auth { - case "anthropic": - if key := os.Getenv("ANTHROPIC_API_KEY"); key != "" { - envVars = append(envVars, "ANTHROPIC_API_KEY="+key) - } - case "git": - // Forward git config - name, _ := exec.Command("git", "config", "user.name").Output() - email, _ := exec.Command("git", "config", "user.email").Output() - if len(name) > 0 { - envVars = append(envVars, "GIT_AUTHOR_NAME="+strings.TrimSpace(string(name))) - envVars = append(envVars, "GIT_COMMITTER_NAME="+strings.TrimSpace(string(name))) - } - if len(email) > 0 { - envVars = append(envVars, "GIT_AUTHOR_EMAIL="+strings.TrimSpace(string(email))) - envVars = append(envVars, "GIT_COMMITTER_EMAIL="+strings.TrimSpace(string(email))) - } - } - } - } - - // Build SSH command with agent forwarding - args := []string{ - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - "-A", // SSH agent forwarding - "-p", "2222", - } - - // Add environment variables - for _, env := range envVars { - args = append(args, "-o", "SendEnv="+strings.Split(env, "=")[0]) - } - - args = append(args, "root@localhost") - - // Build command to run inside - claudeCmd := "cd /app && claude" - if opts.Model != "" { - claudeCmd += " --model " + opts.Model - } - args = append(args, claudeCmd) - - // Set environment for SSH - cmd := exec.CommandContext(ctx, "ssh", args...) - cmd.Stdin = os.Stdin - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Env = append(os.Environ(), envVars...) - - fmt.Println("Starting Claude in sandboxed environment...") - fmt.Println("Project mounted at /app") - fmt.Println("Auth forwarded: SSH agent" + formatAuthList(opts)) - fmt.Println() - - return cmd.Run() -} - -func formatAuthList(opts ClaudeOptions) string { - if opts.NoAuth { - return " (none)" - } - if len(opts.Auth) == 0 { - return ", gh, anthropic, git" - } - return ", " + strings.Join(opts.Auth, ", ") -} - -// CopyGHAuth copies GitHub CLI auth to the VM. -func (d *DevOps) CopyGHAuth(ctx context.Context) error { - home, err := os.UserHomeDir() - if err != nil { - return err - } - - ghConfigDir := filepath.Join(home, ".config", "gh") - if _, err := os.Stat(ghConfigDir); os.IsNotExist(err) { - return nil // No gh config to copy - } - - // Use scp to copy gh config - cmd := exec.CommandContext(ctx, "scp", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-P", "2222", - "-r", ghConfigDir, - "root@localhost:/root/.config/", - ) - return cmd.Run() -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/devops/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/devops/claude.go -git commit -m "feat(devops): add Claude sandbox session - -Starts Claude in immutable dev environment with auth forwarding. -Auto-boots VM, mounts project, forwards credentials. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 12: Add CLI Commands - -**Files:** -- Create: `cmd/core/cmd/dev.go` -- Modify: `cmd/core/cmd/root.go` - -**Step 1: Create dev.go** - -```go -package cmd - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/charmbracelet/lipgloss" - "forge.lthn.ai/core/cli/pkg/devops" - "github.com/leaanthony/clir" -) - -var ( - devHeaderStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#3b82f6")) - - devSuccessStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#22c55e")) - - devErrorStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#ef4444")) - - devDimStyle = lipgloss.NewStyle(). - Foreground(lipgloss.Color("#6b7280")) -) - -// AddDevCommand adds the dev command group. -func AddDevCommand(app *clir.Cli) { - devCmd := app.NewSubCommand("dev", "Portable development environment") - devCmd.LongDescription("Manage the core-devops portable development environment.\n" + - "A sandboxed, immutable Linux VM with 100+ development tools.") - - addDevInstallCommand(devCmd) - addDevBootCommand(devCmd) - addDevStopCommand(devCmd) - addDevStatusCommand(devCmd) - addDevShellCommand(devCmd) - addDevServeCommand(devCmd) - addDevTestCommand(devCmd) - addDevClaudeCommand(devCmd) - addDevUpdateCommand(devCmd) -} - -func addDevInstallCommand(parent *clir.Cli) { - var source string - cmd := parent.NewSubCommand("install", "Download the dev environment image") - cmd.StringFlag("source", "Image source: auto, github, registry, cdn", &source) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - if d.IsInstalled() { - fmt.Printf("%s Dev image already installed\n", devSuccessStyle.Render("OK:")) - fmt.Println("Use 'core dev update' to check for updates") - return nil - } - - fmt.Printf("%s Downloading dev image...\n", devHeaderStyle.Render("Install:")) - - progress := func(downloaded, total int64) { - if total > 0 { - pct := float64(downloaded) / float64(total) * 100 - fmt.Printf("\r %.1f%% (%d / %d MB)", pct, downloaded/1024/1024, total/1024/1024) - } - } - - if err := d.Install(ctx, progress); err != nil { - return err - } - - fmt.Println() - fmt.Printf("%s Dev image installed\n", devSuccessStyle.Render("Success:")) - return nil - }) -} - -func addDevBootCommand(parent *clir.Cli) { - var memory, cpus int - var fresh bool - - cmd := parent.NewSubCommand("boot", "Start the dev environment") - cmd.IntFlag("memory", "Memory in MB (default: 4096)", &memory) - cmd.IntFlag("cpus", "Number of CPUs (default: 2)", &cpus) - cmd.BoolFlag("fresh", "Destroy existing and start fresh", &fresh) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - opts := devops.DefaultBootOptions() - if memory > 0 { - opts.Memory = memory - } - if cpus > 0 { - opts.CPUs = cpus - } - opts.Fresh = fresh - - fmt.Printf("%s Starting dev environment...\n", devHeaderStyle.Render("Boot:")) - - if err := d.Boot(ctx, opts); err != nil { - return err - } - - fmt.Printf("%s Dev environment running\n", devSuccessStyle.Render("Success:")) - fmt.Printf(" Memory: %d MB\n", opts.Memory) - fmt.Printf(" CPUs: %d\n", opts.CPUs) - fmt.Printf(" SSH: ssh -p 2222 root@localhost\n") - return nil - }) -} - -func addDevStopCommand(parent *clir.Cli) { - cmd := parent.NewSubCommand("stop", "Stop the dev environment") - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - fmt.Printf("%s Stopping dev environment...\n", devHeaderStyle.Render("Stop:")) - - if err := d.Stop(ctx); err != nil { - return err - } - - fmt.Printf("%s Dev environment stopped\n", devSuccessStyle.Render("Success:")) - return nil - }) -} - -func addDevStatusCommand(parent *clir.Cli) { - cmd := parent.NewSubCommand("status", "Show dev environment status") - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - status, err := d.Status(ctx) - if err != nil { - return err - } - - fmt.Printf("%s Dev Environment\n\n", devHeaderStyle.Render("Status:")) - - if status.Installed { - fmt.Printf(" Image: %s\n", devSuccessStyle.Render("installed")) - fmt.Printf(" Version: %s\n", status.ImageVersion) - } else { - fmt.Printf(" Image: %s\n", devDimStyle.Render("not installed")) - } - - if status.Running { - fmt.Printf(" Status: %s\n", devSuccessStyle.Render("running")) - fmt.Printf(" ID: %s\n", status.ContainerID[:8]) - fmt.Printf(" Memory: %d MB\n", status.Memory) - fmt.Printf(" CPUs: %d\n", status.CPUs) - fmt.Printf(" SSH: port %d\n", status.SSHPort) - fmt.Printf(" Uptime: %s\n", status.Uptime.Round(1000000000)) - } else { - fmt.Printf(" Status: %s\n", devDimStyle.Render("stopped")) - } - - return nil - }) -} - -func addDevShellCommand(parent *clir.Cli) { - var console bool - cmd := parent.NewSubCommand("shell", "Open a shell in the dev environment") - cmd.BoolFlag("console", "Use serial console instead of SSH", &console) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - return d.Shell(ctx, devops.ShellOptions{Console: console}) - }) -} - -func addDevServeCommand(parent *clir.Cli) { - var port int - var path string - - cmd := parent.NewSubCommand("serve", "Mount project and start dev server") - cmd.IntFlag("port", "Port to serve on (default: 8000)", &port) - cmd.StringFlag("path", "Subdirectory to serve", &path) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - projectDir, _ := os.Getwd() - return d.Serve(ctx, projectDir, devops.ServeOptions{Port: port, Path: path}) - }) -} - -func addDevTestCommand(parent *clir.Cli) { - var name string - - cmd := parent.NewSubCommand("test", "Run tests in dev environment") - cmd.StringFlag("name", "Run specific named test from .core/test.yaml", &name) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - projectDir, _ := os.Getwd() - args := cmd.OtherArgs() - - return d.Test(ctx, projectDir, devops.TestOptions{ - Name: name, - Command: args, - }) - }) -} - -func addDevClaudeCommand(parent *clir.Cli) { - var noAuth bool - var auth string - var model string - - cmd := parent.NewSubCommand("claude", "Start Claude in sandboxed dev environment") - cmd.BoolFlag("no-auth", "Don't forward any credentials", &noAuth) - cmd.StringFlag("auth", "Selective auth forwarding: gh,anthropic,ssh,git", &auth) - cmd.StringFlag("model", "Model to use: opus, sonnet", &model) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - projectDir, _ := os.Getwd() - - var authList []string - if auth != "" { - authList = strings.Split(auth, ",") - } - - return d.Claude(ctx, projectDir, devops.ClaudeOptions{ - NoAuth: noAuth, - Auth: authList, - Model: model, - }) - }) -} - -func addDevUpdateCommand(parent *clir.Cli) { - var force bool - cmd := parent.NewSubCommand("update", "Check for and download image updates") - cmd.BoolFlag("force", "Force download even if up to date", &force) - - cmd.Action(func() error { - ctx := context.Background() - d, err := devops.New() - if err != nil { - return err - } - - if !d.IsInstalled() { - return fmt.Errorf("dev image not installed (run 'core dev install' first)") - } - - fmt.Printf("%s Checking for updates...\n", devHeaderStyle.Render("Update:")) - - current, latest, hasUpdate, err := d.CheckUpdate(ctx) - if err != nil { - return err - } - - if !hasUpdate && !force { - fmt.Printf("%s Already up to date (%s)\n", devSuccessStyle.Render("OK:"), current) - return nil - } - - fmt.Printf(" Current: %s\n", current) - fmt.Printf(" Latest: %s\n", latest) - - progress := func(downloaded, total int64) { - if total > 0 { - pct := float64(downloaded) / float64(total) * 100 - fmt.Printf("\r Downloading: %.1f%%", pct) - } - } - - if err := d.Install(ctx, progress); err != nil { - return err - } - - fmt.Println() - fmt.Printf("%s Updated to %s\n", devSuccessStyle.Render("Success:"), latest) - return nil - }) -} -``` - -**Step 2: Add to root.go** - -Add after other command registrations: -```go -AddDevCommand(app) -``` - -**Step 3: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./cmd/core/...` -Expected: No errors - -**Step 4: Commit** - -```bash -git add cmd/core/cmd/dev.go cmd/core/cmd/root.go -git commit -m "feat(cli): add dev command group - -Commands: -- core dev install/boot/stop/status -- core dev shell/serve/test -- core dev claude (sandboxed AI session) -- core dev update - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 13: Final Integration Test - -**Step 1: Build CLI** - -Run: `cd /Users/snider/Code/Core && go build -o bin/core ./cmd/core` -Expected: No errors - -**Step 2: Test help output** - -Run: `./bin/core dev --help` -Expected: Shows all dev subcommands - -**Step 3: Run package tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/devops/... -v` -Expected: All tests pass - -**Step 4: Update TODO.md** - -Mark S4.6 tasks as complete in tasks/TODO.md - -**Step 5: Final commit** - -```bash -git add -A -git commit -m "chore(devops): finalize S4.6 core-devops CLI - -All dev commands implemented: -- install/boot/stop/status -- shell/serve/test -- claude (sandboxed AI session) -- update - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -## Summary - -13 tasks covering: -1. Package structure -2. Config loading -3. ImageSource interface -4. GitHub source -5. CDN source -6. ImageManager -7. Boot/Stop/Status -8. Shell command -9. Test detection -10. Serve with mount -11. Claude sandbox -12. CLI commands -13. Integration test diff --git a/tasks/plans/2026-01-29-sdk-generation-design.md b/tasks/plans/2026-01-29-sdk-generation-design.md deleted file mode 100644 index ee189fcf..00000000 --- a/tasks/plans/2026-01-29-sdk-generation-design.md +++ /dev/null @@ -1,291 +0,0 @@ -# SDK Generation Design - -## Summary - -Generate typed API clients from OpenAPI specs for TypeScript, Python, Go, and PHP. Includes breaking change detection via semantic diff. - -## Design Decisions - -- **Generator approach**: Hybrid - native generators where available, openapi-generator fallback -- **Languages**: TypeScript, Python, Go, PHP (Core 4) -- **Detection**: Config → common paths → Laravel Scramble -- **Output**: Local `sdk/` + optional monorepo publish -- **Diff**: Semantic with oasdiff, CI-friendly exit codes -- **Priority**: DX (developer experience) - -## Package Structure - -``` -pkg/sdk/ -├── sdk.go # Main SDK type, orchestration -├── detect.go # OpenAPI spec detection -├── diff.go # Breaking change detection (oasdiff) -├── generators/ -│ ├── generator.go # Generator interface -│ ├── typescript.go # openapi-typescript-codegen -│ ├── python.go # openapi-python-client -│ ├── go.go # oapi-codegen -│ └── php.go # openapi-generator (Docker) -└── templates/ # Package scaffolding templates - ├── typescript/ - │ └── package.json.tmpl - ├── python/ - │ └── setup.py.tmpl - ├── go/ - │ └── go.mod.tmpl - └── php/ - └── composer.json.tmpl -``` - -## OpenAPI Detection Flow - -``` -1. Check config: sdk.spec in .core/release.yaml - ↓ not found -2. Check common paths: - - api/openapi.yaml - - api/openapi.json - - openapi.yaml - - openapi.json - - docs/api.yaml - - swagger.yaml - ↓ not found -3. Laravel Scramble detection: - - Check for scramble/scramble in composer.json - - Run: php artisan scramble:export --path=api/openapi.json - - Use generated spec - ↓ not found -4. Error: No OpenAPI spec found -``` - -## Generator Interface - -```go -type Generator interface { - // Language returns the generator's target language - Language() string - - // Generate creates SDK from OpenAPI spec - Generate(ctx context.Context, opts GenerateOptions) error - - // Available checks if generator dependencies are installed - Available() bool - - // Install provides installation instructions - Install() string -} - -type GenerateOptions struct { - SpecPath string // OpenAPI spec file - OutputDir string // Where to write SDK - PackageName string // Package/module name - Version string // SDK version -} -``` - -### Native Generators - -| Language | Tool | Install | -|------------|----------------------------|--------------------------------| -| TypeScript | openapi-typescript-codegen | `npm i -g openapi-typescript-codegen` | -| Python | openapi-python-client | `pip install openapi-python-client` | -| Go | oapi-codegen | `go install github.com/deepmap/oapi-codegen/cmd/oapi-codegen@latest` | -| PHP | openapi-generator (Docker) | Requires Docker | - -### Fallback Strategy - -```go -func (g *TypeScriptGenerator) Generate(ctx context.Context, opts GenerateOptions) error { - if g.Available() { - return g.generateNative(ctx, opts) - } - return g.generateDocker(ctx, opts) // openapi-generator in Docker -} -``` - -## Breaking Change Detection - -Using [oasdiff](https://github.com/Tufin/oasdiff) for semantic OpenAPI comparison: - -```go -import "github.com/tufin/oasdiff/diff" -import "github.com/tufin/oasdiff/checker" - -func (s *SDK) Diff(base, revision string) (*DiffResult, error) { - // Load specs - baseSpec, _ := load.From(loader, base) - revSpec, _ := load.From(loader, revision) - - // Compute diff - d, _ := diff.Get(diff.NewConfig(), baseSpec, revSpec) - - // Check for breaking changes - breaks := checker.CheckBackwardCompatibility( - checker.GetDefaultChecks(), - d, - baseSpec, - revSpec, - ) - - return &DiffResult{ - Breaking: len(breaks) > 0, - Changes: breaks, - Summary: formatSummary(d), - }, nil -} -``` - -### Exit Codes for CI - -| Exit Code | Meaning | -|-----------|---------| -| 0 | No breaking changes | -| 1 | Breaking changes detected | -| 2 | Error (invalid spec, etc.) | - -### Breaking Change Categories - -- Removed endpoints -- Changed required parameters -- Modified response schemas -- Changed authentication requirements - -## CLI Commands - -```bash -# Generate SDKs from OpenAPI spec -core sdk generate # Uses .core/release.yaml config -core sdk generate --spec api.yaml # Explicit spec file -core sdk generate --lang typescript # Single language - -# Check for breaking changes -core sdk diff # Compare current vs last release -core sdk diff --spec api.yaml --base v1.0.0 - -# Validate spec before generation -core sdk validate -core sdk validate --spec api.yaml -``` - -## Config Schema - -In `.core/release.yaml`: - -```yaml -sdk: - # OpenAPI spec source (auto-detected if omitted) - spec: api/openapi.yaml - - # Languages to generate - languages: - - typescript - - python - - go - - php - - # Output directory (default: sdk/) - output: sdk/ - - # Package naming - package: - name: myapi # Base name - version: "{{.Version}}" - - # Breaking change detection - diff: - enabled: true - fail_on_breaking: true # CI fails on breaking changes - - # Optional: publish to monorepo - publish: - repo: myorg/sdks - path: packages/myapi -``` - -## Output Structure - -Each generator outputs to `sdk/{lang}/`: - -``` -sdk/ -├── typescript/ -│ ├── package.json -│ ├── src/ -│ │ ├── index.ts -│ │ ├── client.ts -│ │ └── models/ -│ └── tsconfig.json -├── python/ -│ ├── setup.py -│ ├── myapi/ -│ │ ├── __init__.py -│ │ ├── client.py -│ │ └── models/ -│ └── requirements.txt -├── go/ -│ ├── go.mod -│ ├── client.go -│ └── models.go -└── php/ - ├── composer.json - ├── src/ - │ ├── Client.php - │ └── Models/ - └── README.md -``` - -## Publishing Workflow - -SDK publishing integrates with the existing release pipeline: - -``` -core release - → build artifacts - → generate SDKs (if sdk: configured) - → run diff check (warns or fails on breaking) - → publish to GitHub release - → publish SDKs (optional) -``` - -### Monorepo Publishing - -For projects using a shared SDK monorepo: - -1. Clone target repo (shallow) -2. Update `packages/{name}/{lang}/` -3. Commit with version tag -4. Push (triggers downstream CI) - -The SDK tarball is also attached to GitHub releases for direct download. - -## Implementation Steps - -1. Create `pkg/sdk/` package structure -2. Implement OpenAPI detection (`detect.go`) -3. Define Generator interface (`generators/generator.go`) -4. Implement TypeScript generator (native + fallback) -5. Implement Python generator (native + fallback) -6. Implement Go generator (native) -7. Implement PHP generator (Docker-based) -8. Add package templates (`templates/`) -9. Implement diff with oasdiff (`diff.go`) -10. Add CLI commands (`cmd/core/sdk.go`) -11. Integrate with release pipeline -12. Add monorepo publish support - -## Dependencies - -```go -// go.mod additions -require ( - github.com/tufin/oasdiff v1.x.x - github.com/getkin/kin-openapi v0.x.x -) -``` - -## Testing - -- Unit tests for each generator -- Integration tests with sample OpenAPI specs -- Diff tests with known breaking/non-breaking changes -- E2E test generating SDKs for a real API diff --git a/tasks/plans/2026-01-29-sdk-generation-impl.md b/tasks/plans/2026-01-29-sdk-generation-impl.md deleted file mode 100644 index 734ed024..00000000 --- a/tasks/plans/2026-01-29-sdk-generation-impl.md +++ /dev/null @@ -1,1861 +0,0 @@ -# SDK Generation Implementation Plan - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Generate typed API clients from OpenAPI specs for TypeScript, Python, Go, and PHP with breaking change detection. - -**Architecture:** Hybrid generator approach - native tools where available (openapi-typescript-codegen, openapi-python-client, oapi-codegen), Docker fallback for others (openapi-generator). Detection flow: config → common paths → Laravel Scramble. Breaking changes via oasdiff library. - -**Tech Stack:** Go, oasdiff, kin-openapi, embedded templates, exec for native generators, Docker for fallback - ---- - -### Task 1: Create SDK Package Structure - -**Files:** -- Create: `pkg/sdk/sdk.go` -- Create: `pkg/sdk/go.mod` - -**Step 1: Create go.mod for sdk package** - -```go -module forge.lthn.ai/core/cli/pkg/sdk - -go 1.25 - -require ( - github.com/getkin/kin-openapi v0.128.0 - github.com/tufin/oasdiff v1.10.25 - gopkg.in/yaml.v3 v3.0.1 -) -``` - -**Step 2: Create sdk.go with types and config** - -```go -// Package sdk provides OpenAPI SDK generation and diff capabilities. -package sdk - -import ( - "context" - "fmt" -) - -// Config holds SDK generation configuration from .core/release.yaml. -type Config struct { - // Spec is the path to the OpenAPI spec file (auto-detected if empty). - Spec string `yaml:"spec,omitempty"` - // Languages to generate SDKs for. - Languages []string `yaml:"languages,omitempty"` - // Output directory (default: sdk/). - Output string `yaml:"output,omitempty"` - // Package naming configuration. - Package PackageConfig `yaml:"package,omitempty"` - // Diff configuration for breaking change detection. - Diff DiffConfig `yaml:"diff,omitempty"` - // Publish configuration for monorepo publishing. - Publish PublishConfig `yaml:"publish,omitempty"` -} - -// PackageConfig holds package naming configuration. -type PackageConfig struct { - // Name is the base package name. - Name string `yaml:"name,omitempty"` - // Version is the SDK version (supports templates like {{.Version}}). - Version string `yaml:"version,omitempty"` -} - -// DiffConfig holds breaking change detection configuration. -type DiffConfig struct { - // Enabled determines whether to run diff checks. - Enabled bool `yaml:"enabled,omitempty"` - // FailOnBreaking fails the release if breaking changes are detected. - FailOnBreaking bool `yaml:"fail_on_breaking,omitempty"` -} - -// PublishConfig holds monorepo publishing configuration. -type PublishConfig struct { - // Repo is the SDK monorepo (e.g., "myorg/sdks"). - Repo string `yaml:"repo,omitempty"` - // Path is the subdirectory for this SDK (e.g., "packages/myapi"). - Path string `yaml:"path,omitempty"` -} - -// SDK orchestrates OpenAPI SDK generation. -type SDK struct { - config *Config - projectDir string -} - -// New creates a new SDK instance. -func New(projectDir string, config *Config) *SDK { - if config == nil { - config = DefaultConfig() - } - return &SDK{ - config: config, - projectDir: projectDir, - } -} - -// DefaultConfig returns sensible defaults for SDK configuration. -func DefaultConfig() *Config { - return &Config{ - Languages: []string{"typescript", "python", "go", "php"}, - Output: "sdk", - Diff: DiffConfig{ - Enabled: true, - FailOnBreaking: false, - }, - } -} - -// Generate generates SDKs for all configured languages. -func (s *SDK) Generate(ctx context.Context) error { - return fmt.Errorf("sdk.Generate: not implemented") -} - -// GenerateLanguage generates SDK for a specific language. -func (s *SDK) GenerateLanguage(ctx context.Context, lang string) error { - return fmt.Errorf("sdk.GenerateLanguage: not implemented") -} -``` - -**Step 3: Add to go.work** - -Run: `cd /Users/snider/Code/Core && echo " ./pkg/sdk" >> go.work && go work sync` - -**Step 4: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/sdk/...` -Expected: No errors - -**Step 5: Commit** - -```bash -git add pkg/sdk/ -git add go.work go.work.sum -git commit -m "feat(sdk): add SDK package structure with types - -Initial pkg/sdk setup with Config types for OpenAPI SDK generation. -Includes language selection, diff config, and publish config. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 2: Implement OpenAPI Spec Detection - -**Files:** -- Create: `pkg/sdk/detect.go` -- Create: `pkg/sdk/detect_test.go` - -**Step 1: Write the failing test** - -```go -package sdk - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDetectSpec_Good_ConfigPath(t *testing.T) { - // Create temp directory with spec at configured path - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "api", "spec.yaml") - os.MkdirAll(filepath.Dir(specPath), 0755) - os.WriteFile(specPath, []byte("openapi: 3.0.0"), 0644) - - sdk := New(tmpDir, &Config{Spec: "api/spec.yaml"}) - got, err := sdk.DetectSpec() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != specPath { - t.Errorf("got %q, want %q", got, specPath) - } -} - -func TestDetectSpec_Good_CommonPath(t *testing.T) { - // Create temp directory with spec at common path - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "openapi.yaml") - os.WriteFile(specPath, []byte("openapi: 3.0.0"), 0644) - - sdk := New(tmpDir, nil) - got, err := sdk.DetectSpec() - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if got != specPath { - t.Errorf("got %q, want %q", got, specPath) - } -} - -func TestDetectSpec_Bad_NotFound(t *testing.T) { - tmpDir := t.TempDir() - sdk := New(tmpDir, nil) - _, err := sdk.DetectSpec() - if err == nil { - t.Fatal("expected error for missing spec") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/... -run TestDetectSpec -v` -Expected: FAIL (DetectSpec not defined) - -**Step 3: Write minimal implementation** - -```go -package sdk - -import ( - "fmt" - "os" - "path/filepath" -) - -// commonSpecPaths are checked in order when no spec is configured. -var commonSpecPaths = []string{ - "api/openapi.yaml", - "api/openapi.json", - "openapi.yaml", - "openapi.json", - "docs/api.yaml", - "docs/api.json", - "swagger.yaml", - "swagger.json", -} - -// DetectSpec finds the OpenAPI spec file. -// Priority: config path → common paths → Laravel Scramble. -func (s *SDK) DetectSpec() (string, error) { - // 1. Check configured path - if s.config.Spec != "" { - specPath := filepath.Join(s.projectDir, s.config.Spec) - if _, err := os.Stat(specPath); err == nil { - return specPath, nil - } - return "", fmt.Errorf("sdk.DetectSpec: configured spec not found: %s", s.config.Spec) - } - - // 2. Check common paths - for _, p := range commonSpecPaths { - specPath := filepath.Join(s.projectDir, p) - if _, err := os.Stat(specPath); err == nil { - return specPath, nil - } - } - - // 3. Try Laravel Scramble detection - specPath, err := s.detectScramble() - if err == nil { - return specPath, nil - } - - return "", fmt.Errorf("sdk.DetectSpec: no OpenAPI spec found (checked config, common paths, Scramble)") -} - -// detectScramble checks for Laravel Scramble and exports the spec. -func (s *SDK) detectScramble() (string, error) { - composerPath := filepath.Join(s.projectDir, "composer.json") - if _, err := os.Stat(composerPath); err != nil { - return "", fmt.Errorf("no composer.json") - } - - // Check for scramble in composer.json - data, err := os.ReadFile(composerPath) - if err != nil { - return "", err - } - - // Simple check for scramble package - if !containsScramble(data) { - return "", fmt.Errorf("scramble not found in composer.json") - } - - // TODO: Run php artisan scramble:export - return "", fmt.Errorf("scramble export not implemented") -} - -// containsScramble checks if composer.json includes scramble. -func containsScramble(data []byte) bool { - return len(data) > 0 && - (contains(data, "dedoc/scramble") || contains(data, "\"scramble\"")) -} - -// contains is a simple byte slice search. -func contains(data []byte, substr string) bool { - return len(data) >= len(substr) && - string(data) != "" && - indexOf(string(data), substr) >= 0 -} - -func indexOf(s, substr string) int { - for i := 0; i <= len(s)-len(substr); i++ { - if s[i:i+len(substr)] == substr { - return i - } - } - return -1 -} -``` - -**Step 4: Run tests to verify they pass** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/... -run TestDetectSpec -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/sdk/detect.go pkg/sdk/detect_test.go -git commit -m "feat(sdk): add OpenAPI spec detection - -Detects OpenAPI spec via: -1. Configured spec path -2. Common paths (api/openapi.yaml, openapi.yaml, etc.) -3. Laravel Scramble (stub for now) - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 3: Define Generator Interface - -**Files:** -- Create: `pkg/sdk/generators/generator.go` - -**Step 1: Create generator interface** - -```go -// Package generators provides SDK code generators for different languages. -package generators - -import ( - "context" -) - -// Options holds common generation options. -type Options struct { - // SpecPath is the path to the OpenAPI spec file. - SpecPath string - // OutputDir is where to write the generated SDK. - OutputDir string - // PackageName is the package/module name. - PackageName string - // Version is the SDK version. - Version string -} - -// Generator defines the interface for SDK generators. -type Generator interface { - // Language returns the generator's target language identifier. - Language() string - - // Generate creates SDK from OpenAPI spec. - Generate(ctx context.Context, opts Options) error - - // Available checks if generator dependencies are installed. - Available() bool - - // Install returns instructions for installing the generator. - Install() string -} - -// Registry holds available generators. -type Registry struct { - generators map[string]Generator -} - -// NewRegistry creates a registry with all available generators. -func NewRegistry() *Registry { - r := &Registry{ - generators: make(map[string]Generator), - } - // Generators will be registered in subsequent tasks - return r -} - -// Get returns a generator by language. -func (r *Registry) Get(lang string) (Generator, bool) { - g, ok := r.generators[lang] - return g, ok -} - -// Register adds a generator to the registry. -func (r *Registry) Register(g Generator) { - r.generators[g.Language()] = g -} - -// Languages returns all registered language identifiers. -func (r *Registry) Languages() []string { - langs := make([]string, 0, len(r.generators)) - for lang := range r.generators { - langs = append(langs, lang) - } - return langs -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/sdk/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/sdk/generators/generator.go -git commit -m "feat(sdk): add Generator interface and Registry - -Defines the common interface for SDK generators with: -- Generate(), Available(), Install() methods -- Registry for managing multiple generators - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 4: Implement TypeScript Generator - -**Files:** -- Create: `pkg/sdk/generators/typescript.go` -- Create: `pkg/sdk/generators/typescript_test.go` - -**Step 1: Write the failing test** - -```go -package generators - -import ( - "context" - "os" - "os/exec" - "path/filepath" - "testing" -) - -func TestTypeScriptGenerator_Good_Available(t *testing.T) { - g := NewTypeScriptGenerator() - // Just check it doesn't panic - _ = g.Available() - _ = g.Language() - _ = g.Install() -} - -func TestTypeScriptGenerator_Good_Generate(t *testing.T) { - // Skip if no generator available - g := NewTypeScriptGenerator() - if !g.Available() && !dockerAvailable() { - t.Skip("no TypeScript generator available (need openapi-typescript-codegen or Docker)") - } - - // Create temp spec - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "spec.yaml") - spec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - os.WriteFile(specPath, []byte(spec), 0644) - - outputDir := filepath.Join(tmpDir, "sdk", "typescript") - err := g.Generate(context.Background(), Options{ - SpecPath: specPath, - OutputDir: outputDir, - PackageName: "test-api", - Version: "1.0.0", - }) - if err != nil { - t.Fatalf("Generate failed: %v", err) - } - - // Check output exists - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - t.Error("output directory not created") - } -} - -func dockerAvailable() bool { - _, err := exec.LookPath("docker") - return err == nil -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestTypeScriptGenerator -v` -Expected: FAIL (NewTypeScriptGenerator not defined) - -**Step 3: Write implementation** - -```go -package generators - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// TypeScriptGenerator generates TypeScript SDKs using openapi-typescript-codegen. -type TypeScriptGenerator struct{} - -// NewTypeScriptGenerator creates a new TypeScript generator. -func NewTypeScriptGenerator() *TypeScriptGenerator { - return &TypeScriptGenerator{} -} - -// Language returns "typescript". -func (g *TypeScriptGenerator) Language() string { - return "typescript" -} - -// Available checks if openapi-typescript-codegen is installed. -func (g *TypeScriptGenerator) Available() bool { - _, err := exec.LookPath("openapi-typescript-codegen") - if err == nil { - return true - } - // Also check npx availability - _, err = exec.LookPath("npx") - return err == nil -} - -// Install returns installation instructions. -func (g *TypeScriptGenerator) Install() string { - return "npm install -g openapi-typescript-codegen" -} - -// Generate creates TypeScript SDK from OpenAPI spec. -func (g *TypeScriptGenerator) Generate(ctx context.Context, opts Options) error { - // Ensure output directory exists - if err := os.MkdirAll(opts.OutputDir, 0755); err != nil { - return fmt.Errorf("typescript.Generate: failed to create output dir: %w", err) - } - - // Try native generator first - if g.nativeAvailable() { - return g.generateNative(ctx, opts) - } - - // Try npx - if g.npxAvailable() { - return g.generateNpx(ctx, opts) - } - - // Fall back to Docker - return g.generateDocker(ctx, opts) -} - -func (g *TypeScriptGenerator) nativeAvailable() bool { - _, err := exec.LookPath("openapi-typescript-codegen") - return err == nil -} - -func (g *TypeScriptGenerator) npxAvailable() bool { - _, err := exec.LookPath("npx") - return err == nil -} - -func (g *TypeScriptGenerator) generateNative(ctx context.Context, opts Options) error { - cmd := exec.CommandContext(ctx, "openapi-typescript-codegen", - "--input", opts.SpecPath, - "--output", opts.OutputDir, - "--name", opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -func (g *TypeScriptGenerator) generateNpx(ctx context.Context, opts Options) error { - cmd := exec.CommandContext(ctx, "npx", "openapi-typescript-codegen", - "--input", opts.SpecPath, - "--output", opts.OutputDir, - "--name", opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -func (g *TypeScriptGenerator) generateDocker(ctx context.Context, opts Options) error { - // Use openapi-generator via Docker - specDir := filepath.Dir(opts.SpecPath) - specName := filepath.Base(opts.SpecPath) - - cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "-v", specDir+":/spec", - "-v", opts.OutputDir+":/out", - "openapitools/openapi-generator-cli", "generate", - "-i", "/spec/"+specName, - "-g", "typescript-fetch", - "-o", "/out", - "--additional-properties=npmName="+opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("typescript.generateDocker: %w", err) - } - return nil -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestTypeScriptGenerator -v` -Expected: PASS (or skip if no generator available) - -**Step 5: Commit** - -```bash -git add pkg/sdk/generators/typescript.go pkg/sdk/generators/typescript_test.go -git commit -m "feat(sdk): add TypeScript generator - -Uses openapi-typescript-codegen (native or npx) with Docker fallback. -Generates TypeScript-fetch client from OpenAPI spec. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 5: Implement Python Generator - -**Files:** -- Create: `pkg/sdk/generators/python.go` -- Create: `pkg/sdk/generators/python_test.go` - -**Step 1: Write the failing test** - -```go -package generators - -import ( - "context" - "os" - "path/filepath" - "testing" -) - -func TestPythonGenerator_Good_Available(t *testing.T) { - g := NewPythonGenerator() - _ = g.Available() - _ = g.Language() - _ = g.Install() -} - -func TestPythonGenerator_Good_Generate(t *testing.T) { - g := NewPythonGenerator() - if !g.Available() && !dockerAvailable() { - t.Skip("no Python generator available") - } - - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "spec.yaml") - spec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - os.WriteFile(specPath, []byte(spec), 0644) - - outputDir := filepath.Join(tmpDir, "sdk", "python") - err := g.Generate(context.Background(), Options{ - SpecPath: specPath, - OutputDir: outputDir, - PackageName: "test_api", - Version: "1.0.0", - }) - if err != nil { - t.Fatalf("Generate failed: %v", err) - } - - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - t.Error("output directory not created") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestPythonGenerator -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package generators - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// PythonGenerator generates Python SDKs using openapi-python-client. -type PythonGenerator struct{} - -// NewPythonGenerator creates a new Python generator. -func NewPythonGenerator() *PythonGenerator { - return &PythonGenerator{} -} - -// Language returns "python". -func (g *PythonGenerator) Language() string { - return "python" -} - -// Available checks if openapi-python-client is installed. -func (g *PythonGenerator) Available() bool { - _, err := exec.LookPath("openapi-python-client") - return err == nil -} - -// Install returns installation instructions. -func (g *PythonGenerator) Install() string { - return "pip install openapi-python-client" -} - -// Generate creates Python SDK from OpenAPI spec. -func (g *PythonGenerator) Generate(ctx context.Context, opts Options) error { - if err := os.MkdirAll(opts.OutputDir, 0755); err != nil { - return fmt.Errorf("python.Generate: failed to create output dir: %w", err) - } - - if g.Available() { - return g.generateNative(ctx, opts) - } - return g.generateDocker(ctx, opts) -} - -func (g *PythonGenerator) generateNative(ctx context.Context, opts Options) error { - // openapi-python-client creates a directory named after the package - // We need to generate into a temp location then move - parentDir := filepath.Dir(opts.OutputDir) - - cmd := exec.CommandContext(ctx, "openapi-python-client", "generate", - "--path", opts.SpecPath, - "--output-path", opts.OutputDir, - ) - cmd.Dir = parentDir - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} - -func (g *PythonGenerator) generateDocker(ctx context.Context, opts Options) error { - specDir := filepath.Dir(opts.SpecPath) - specName := filepath.Base(opts.SpecPath) - - cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "-v", specDir+":/spec", - "-v", opts.OutputDir+":/out", - "openapitools/openapi-generator-cli", "generate", - "-i", "/spec/"+specName, - "-g", "python", - "-o", "/out", - "--additional-properties=packageName="+opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestPythonGenerator -v` -Expected: PASS (or skip) - -**Step 5: Commit** - -```bash -git add pkg/sdk/generators/python.go pkg/sdk/generators/python_test.go -git commit -m "feat(sdk): add Python generator - -Uses openapi-python-client with Docker fallback. -Generates Python client from OpenAPI spec. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 6: Implement Go Generator - -**Files:** -- Create: `pkg/sdk/generators/go.go` -- Create: `pkg/sdk/generators/go_test.go` - -**Step 1: Write the failing test** - -```go -package generators - -import ( - "context" - "os" - "path/filepath" - "testing" -) - -func TestGoGenerator_Good_Available(t *testing.T) { - g := NewGoGenerator() - _ = g.Available() - _ = g.Language() - _ = g.Install() -} - -func TestGoGenerator_Good_Generate(t *testing.T) { - g := NewGoGenerator() - if !g.Available() && !dockerAvailable() { - t.Skip("no Go generator available") - } - - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "spec.yaml") - spec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - os.WriteFile(specPath, []byte(spec), 0644) - - outputDir := filepath.Join(tmpDir, "sdk", "go") - err := g.Generate(context.Background(), Options{ - SpecPath: specPath, - OutputDir: outputDir, - PackageName: "testapi", - Version: "1.0.0", - }) - if err != nil { - t.Fatalf("Generate failed: %v", err) - } - - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - t.Error("output directory not created") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestGoGenerator -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package generators - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// GoGenerator generates Go SDKs using oapi-codegen. -type GoGenerator struct{} - -// NewGoGenerator creates a new Go generator. -func NewGoGenerator() *GoGenerator { - return &GoGenerator{} -} - -// Language returns "go". -func (g *GoGenerator) Language() string { - return "go" -} - -// Available checks if oapi-codegen is installed. -func (g *GoGenerator) Available() bool { - _, err := exec.LookPath("oapi-codegen") - return err == nil -} - -// Install returns installation instructions. -func (g *GoGenerator) Install() string { - return "go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest" -} - -// Generate creates Go SDK from OpenAPI spec. -func (g *GoGenerator) Generate(ctx context.Context, opts Options) error { - if err := os.MkdirAll(opts.OutputDir, 0755); err != nil { - return fmt.Errorf("go.Generate: failed to create output dir: %w", err) - } - - if g.Available() { - return g.generateNative(ctx, opts) - } - return g.generateDocker(ctx, opts) -} - -func (g *GoGenerator) generateNative(ctx context.Context, opts Options) error { - outputFile := filepath.Join(opts.OutputDir, "client.go") - - cmd := exec.CommandContext(ctx, "oapi-codegen", - "-package", opts.PackageName, - "-generate", "types,client", - "-o", outputFile, - opts.SpecPath, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("go.generateNative: %w", err) - } - - // Create go.mod - goMod := fmt.Sprintf("module %s\n\ngo 1.21\n", opts.PackageName) - return os.WriteFile(filepath.Join(opts.OutputDir, "go.mod"), []byte(goMod), 0644) -} - -func (g *GoGenerator) generateDocker(ctx context.Context, opts Options) error { - specDir := filepath.Dir(opts.SpecPath) - specName := filepath.Base(opts.SpecPath) - - cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "-v", specDir+":/spec", - "-v", opts.OutputDir+":/out", - "openapitools/openapi-generator-cli", "generate", - "-i", "/spec/"+specName, - "-g", "go", - "-o", "/out", - "--additional-properties=packageName="+opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestGoGenerator -v` -Expected: PASS (or skip) - -**Step 5: Commit** - -```bash -git add pkg/sdk/generators/go.go pkg/sdk/generators/go_test.go -git commit -m "feat(sdk): add Go generator - -Uses oapi-codegen with Docker fallback. -Generates Go client and types from OpenAPI spec. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 7: Implement PHP Generator - -**Files:** -- Create: `pkg/sdk/generators/php.go` -- Create: `pkg/sdk/generators/php_test.go` - -**Step 1: Write the failing test** - -```go -package generators - -import ( - "context" - "os" - "path/filepath" - "testing" -) - -func TestPHPGenerator_Good_Available(t *testing.T) { - g := NewPHPGenerator() - _ = g.Available() - _ = g.Language() - _ = g.Install() -} - -func TestPHPGenerator_Good_Generate(t *testing.T) { - g := NewPHPGenerator() - if !g.Available() { - t.Skip("Docker not available for PHP generator") - } - - tmpDir := t.TempDir() - specPath := filepath.Join(tmpDir, "spec.yaml") - spec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - os.WriteFile(specPath, []byte(spec), 0644) - - outputDir := filepath.Join(tmpDir, "sdk", "php") - err := g.Generate(context.Background(), Options{ - SpecPath: specPath, - OutputDir: outputDir, - PackageName: "TestApi", - Version: "1.0.0", - }) - if err != nil { - t.Fatalf("Generate failed: %v", err) - } - - if _, err := os.Stat(outputDir); os.IsNotExist(err) { - t.Error("output directory not created") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestPHPGenerator -v` -Expected: FAIL - -**Step 3: Write implementation** - -```go -package generators - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" -) - -// PHPGenerator generates PHP SDKs using openapi-generator (Docker). -type PHPGenerator struct{} - -// NewPHPGenerator creates a new PHP generator. -func NewPHPGenerator() *PHPGenerator { - return &PHPGenerator{} -} - -// Language returns "php". -func (g *PHPGenerator) Language() string { - return "php" -} - -// Available checks if Docker is available. -func (g *PHPGenerator) Available() bool { - _, err := exec.LookPath("docker") - return err == nil -} - -// Install returns installation instructions. -func (g *PHPGenerator) Install() string { - return "Docker is required for PHP SDK generation" -} - -// Generate creates PHP SDK from OpenAPI spec using Docker. -func (g *PHPGenerator) Generate(ctx context.Context, opts Options) error { - if !g.Available() { - return fmt.Errorf("php.Generate: Docker is required but not available") - } - - if err := os.MkdirAll(opts.OutputDir, 0755); err != nil { - return fmt.Errorf("php.Generate: failed to create output dir: %w", err) - } - - specDir := filepath.Dir(opts.SpecPath) - specName := filepath.Base(opts.SpecPath) - - cmd := exec.CommandContext(ctx, "docker", "run", "--rm", - "-v", specDir+":/spec", - "-v", opts.OutputDir+":/out", - "openapitools/openapi-generator-cli", "generate", - "-i", "/spec/"+specName, - "-g", "php", - "-o", "/out", - "--additional-properties=invokerPackage="+opts.PackageName, - ) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - - if err := cmd.Run(); err != nil { - return fmt.Errorf("php.Generate: %w", err) - } - return nil -} -``` - -**Step 4: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/generators/... -run TestPHPGenerator -v` -Expected: PASS (or skip) - -**Step 5: Commit** - -```bash -git add pkg/sdk/generators/php.go pkg/sdk/generators/php_test.go -git commit -m "feat(sdk): add PHP generator - -Uses openapi-generator via Docker. -Generates PHP client from OpenAPI spec. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 8: Implement Breaking Change Detection - -**Files:** -- Create: `pkg/sdk/diff.go` -- Create: `pkg/sdk/diff_test.go` - -**Step 1: Write the failing test** - -```go -package sdk - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDiff_Good_NoBreaking(t *testing.T) { - tmpDir := t.TempDir() - - baseSpec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - revSpec := `openapi: "3.0.0" -info: - title: Test API - version: "1.1.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK - /status: - get: - operationId: getStatus - responses: - "200": - description: OK -` - basePath := filepath.Join(tmpDir, "base.yaml") - revPath := filepath.Join(tmpDir, "rev.yaml") - os.WriteFile(basePath, []byte(baseSpec), 0644) - os.WriteFile(revPath, []byte(revSpec), 0644) - - result, err := Diff(basePath, revPath) - if err != nil { - t.Fatalf("Diff failed: %v", err) - } - if result.Breaking { - t.Error("expected no breaking changes for adding endpoint") - } -} - -func TestDiff_Good_Breaking(t *testing.T) { - tmpDir := t.TempDir() - - baseSpec := `openapi: "3.0.0" -info: - title: Test API - version: "1.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK - /users: - get: - operationId: getUsers - responses: - "200": - description: OK -` - revSpec := `openapi: "3.0.0" -info: - title: Test API - version: "2.0.0" -paths: - /health: - get: - operationId: getHealth - responses: - "200": - description: OK -` - basePath := filepath.Join(tmpDir, "base.yaml") - revPath := filepath.Join(tmpDir, "rev.yaml") - os.WriteFile(basePath, []byte(baseSpec), 0644) - os.WriteFile(revPath, []byte(revSpec), 0644) - - result, err := Diff(basePath, revPath) - if err != nil { - t.Fatalf("Diff failed: %v", err) - } - if !result.Breaking { - t.Error("expected breaking change for removed endpoint") - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/... -run TestDiff -v` -Expected: FAIL (Diff not defined) - -**Step 3: Add oasdiff dependency** - -Run: `cd /Users/snider/Code/Core/pkg/sdk && go get github.com/tufin/oasdiff@latest github.com/getkin/kin-openapi@latest` - -**Step 4: Write implementation** - -```go -package sdk - -import ( - "fmt" - - "github.com/getkin/kin-openapi/openapi3" - "github.com/tufin/oasdiff/checker" - "github.com/tufin/oasdiff/diff" - "github.com/tufin/oasdiff/load" -) - -// DiffResult holds the result of comparing two OpenAPI specs. -type DiffResult struct { - // Breaking is true if breaking changes were detected. - Breaking bool - // Changes is the list of breaking changes. - Changes []string - // Summary is a human-readable summary. - Summary string -} - -// Diff compares two OpenAPI specs and detects breaking changes. -func Diff(basePath, revisionPath string) (*DiffResult, error) { - loader := openapi3.NewLoader() - loader.IsExternalRefsAllowed = true - - // Load specs - baseSpec, err := load.NewSpecInfo(loader, load.NewSource(basePath)) - if err != nil { - return nil, fmt.Errorf("sdk.Diff: failed to load base spec: %w", err) - } - - revSpec, err := load.NewSpecInfo(loader, load.NewSource(revisionPath)) - if err != nil { - return nil, fmt.Errorf("sdk.Diff: failed to load revision spec: %w", err) - } - - // Compute diff - diffResult, err := diff.Get(diff.NewConfig(), baseSpec.Spec, revSpec.Spec) - if err != nil { - return nil, fmt.Errorf("sdk.Diff: failed to compute diff: %w", err) - } - - // Check for breaking changes - config := checker.GetAllChecks() - breaks := checker.CheckBackwardCompatibilityUntilLevel( - config, - diffResult, - baseSpec.Spec, - revSpec.Spec, - checker.ERR, // Only errors (breaking changes) - ) - - // Build result - result := &DiffResult{ - Breaking: len(breaks) > 0, - Changes: make([]string, 0, len(breaks)), - } - - for _, b := range breaks { - result.Changes = append(result.Changes, b.GetUncolorizedText(checker.NewDefaultLocalizer())) - } - - if result.Breaking { - result.Summary = fmt.Sprintf("%d breaking change(s) detected", len(breaks)) - } else { - result.Summary = "No breaking changes" - } - - return result, nil -} - -// DiffExitCode returns the exit code for CI integration. -// 0 = no breaking changes, 1 = breaking changes, 2 = error -func DiffExitCode(result *DiffResult, err error) int { - if err != nil { - return 2 - } - if result.Breaking { - return 1 - } - return 0 -} -``` - -**Step 5: Run tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/... -run TestDiff -v` -Expected: PASS - -**Step 6: Commit** - -```bash -git add pkg/sdk/diff.go pkg/sdk/diff_test.go pkg/sdk/go.mod pkg/sdk/go.sum -git commit -m "feat(sdk): add breaking change detection with oasdiff - -Compares OpenAPI specs to detect breaking changes: -- Removed endpoints -- Changed required parameters -- Modified response schemas - -Returns CI-friendly exit codes (0=ok, 1=breaking, 2=error). - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 9: Wire Up Generate Command - -**Files:** -- Modify: `pkg/sdk/sdk.go` - -**Step 1: Update SDK.Generate to use generators** - -```go -// Add to sdk.go, replacing the stub Generate method - -import ( - "forge.lthn.ai/core/cli/pkg/sdk/generators" -) - -// Generate generates SDKs for all configured languages. -func (s *SDK) Generate(ctx context.Context) error { - // Detect spec - specPath, err := s.DetectSpec() - if err != nil { - return err - } - - // Create registry with all generators - registry := generators.NewRegistry() - registry.Register(generators.NewTypeScriptGenerator()) - registry.Register(generators.NewPythonGenerator()) - registry.Register(generators.NewGoGenerator()) - registry.Register(generators.NewPHPGenerator()) - - // Generate for each language - for _, lang := range s.config.Languages { - if err := s.GenerateLanguage(ctx, lang); err != nil { - return err - } - } - - return nil -} - -// GenerateLanguage generates SDK for a specific language. -func (s *SDK) GenerateLanguage(ctx context.Context, lang string) error { - specPath, err := s.DetectSpec() - if err != nil { - return err - } - - registry := generators.NewRegistry() - registry.Register(generators.NewTypeScriptGenerator()) - registry.Register(generators.NewPythonGenerator()) - registry.Register(generators.NewGoGenerator()) - registry.Register(generators.NewPHPGenerator()) - - gen, ok := registry.Get(lang) - if !ok { - return fmt.Errorf("sdk.GenerateLanguage: unknown language: %s", lang) - } - - if !gen.Available() { - fmt.Printf("Warning: %s generator not available. Install with: %s\n", lang, gen.Install()) - fmt.Printf("Falling back to Docker...\n") - } - - outputDir := filepath.Join(s.projectDir, s.config.Output, lang) - opts := generators.Options{ - SpecPath: specPath, - OutputDir: outputDir, - PackageName: s.config.Package.Name, - Version: s.config.Package.Version, - } - - fmt.Printf("Generating %s SDK...\n", lang) - if err := gen.Generate(ctx, opts); err != nil { - return fmt.Errorf("sdk.GenerateLanguage: %s generation failed: %w", lang, err) - } - fmt.Printf("Generated %s SDK at %s\n", lang, outputDir) - - return nil -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/sdk/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/sdk/sdk.go -git commit -m "feat(sdk): wire up Generate to use all generators - -SDK.Generate() and SDK.GenerateLanguage() now use the -generator registry to generate SDKs for configured languages. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 10: Add CLI Commands - -**Files:** -- Create: `cmd/core/cmd/sdk.go` - -**Step 1: Create SDK command file** - -```go -package cmd - -import ( - "context" - "fmt" - "os" - - "github.com/charmbracelet/lipgloss" - "forge.lthn.ai/core/cli/pkg/sdk" - "github.com/leaanthony/clir" -) - -var ( - sdkHeaderStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#3b82f6")) - - sdkSuccessStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#22c55e")) - - sdkErrorStyle = lipgloss.NewStyle(). - Bold(true). - Foreground(lipgloss.Color("#ef4444")) - - sdkDimStyle = lipgloss.NewStyle(). - Foreground(lipgloss.Color("#6b7280")) -) - -// AddSDKCommand adds the sdk command and its subcommands. -func AddSDKCommand(app *clir.Cli) { - sdkCmd := app.NewSubCommand("sdk", "Generate and manage API SDKs") - sdkCmd.LongDescription("Generate typed API clients from OpenAPI specs.\n" + - "Supports TypeScript, Python, Go, and PHP.") - - // sdk generate - genCmd := sdkCmd.NewSubCommand("generate", "Generate SDKs from OpenAPI spec") - var specPath, lang string - genCmd.StringFlag("spec", "Path to OpenAPI spec file", &specPath) - genCmd.StringFlag("lang", "Generate only this language", &lang) - genCmd.Action(func() error { - return runSDKGenerate(specPath, lang) - }) - - // sdk diff - diffCmd := sdkCmd.NewSubCommand("diff", "Check for breaking API changes") - var basePath string - diffCmd.StringFlag("base", "Base spec (version tag or file)", &basePath) - diffCmd.StringFlag("spec", "Current spec file", &specPath) - diffCmd.Action(func() error { - return runSDKDiff(basePath, specPath) - }) - - // sdk validate - validateCmd := sdkCmd.NewSubCommand("validate", "Validate OpenAPI spec") - validateCmd.StringFlag("spec", "Path to OpenAPI spec file", &specPath) - validateCmd.Action(func() error { - return runSDKValidate(specPath) - }) -} - -func runSDKGenerate(specPath, lang string) error { - ctx := context.Background() - - projectDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get working directory: %w", err) - } - - // Load config - config := sdk.DefaultConfig() - if specPath != "" { - config.Spec = specPath - } - - s := sdk.New(projectDir, config) - - fmt.Printf("%s Generating SDKs\n", sdkHeaderStyle.Render("SDK:")) - - if lang != "" { - // Generate single language - if err := s.GenerateLanguage(ctx, lang); err != nil { - fmt.Printf("%s %v\n", sdkErrorStyle.Render("Error:"), err) - return err - } - } else { - // Generate all - if err := s.Generate(ctx); err != nil { - fmt.Printf("%s %v\n", sdkErrorStyle.Render("Error:"), err) - return err - } - } - - fmt.Printf("%s SDK generation complete\n", sdkSuccessStyle.Render("Success:")) - return nil -} - -func runSDKDiff(basePath, specPath string) error { - projectDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get working directory: %w", err) - } - - // Detect current spec if not provided - if specPath == "" { - s := sdk.New(projectDir, nil) - specPath, err = s.DetectSpec() - if err != nil { - return err - } - } - - if basePath == "" { - return fmt.Errorf("--base is required (version tag or file path)") - } - - fmt.Printf("%s Checking for breaking changes\n", sdkHeaderStyle.Render("SDK Diff:")) - fmt.Printf(" Base: %s\n", sdkDimStyle.Render(basePath)) - fmt.Printf(" Current: %s\n", sdkDimStyle.Render(specPath)) - fmt.Println() - - result, err := sdk.Diff(basePath, specPath) - if err != nil { - fmt.Printf("%s %v\n", sdkErrorStyle.Render("Error:"), err) - os.Exit(2) - } - - if result.Breaking { - fmt.Printf("%s %s\n", sdkErrorStyle.Render("Breaking:"), result.Summary) - for _, change := range result.Changes { - fmt.Printf(" - %s\n", change) - } - os.Exit(1) - } - - fmt.Printf("%s %s\n", sdkSuccessStyle.Render("OK:"), result.Summary) - return nil -} - -func runSDKValidate(specPath string) error { - projectDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get working directory: %w", err) - } - - s := sdk.New(projectDir, &sdk.Config{Spec: specPath}) - - fmt.Printf("%s Validating OpenAPI spec\n", sdkHeaderStyle.Render("SDK:")) - - detectedPath, err := s.DetectSpec() - if err != nil { - fmt.Printf("%s %v\n", sdkErrorStyle.Render("Error:"), err) - return err - } - - fmt.Printf(" Spec: %s\n", sdkDimStyle.Render(detectedPath)) - fmt.Printf("%s Spec is valid\n", sdkSuccessStyle.Render("OK:")) - return nil -} -``` - -**Step 2: Register command in root.go** - -Add to root.go after other command registrations: -```go -AddSDKCommand(app) -``` - -**Step 3: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./cmd/core/...` -Expected: No errors - -**Step 4: Commit** - -```bash -git add cmd/core/cmd/sdk.go cmd/core/cmd/root.go -git commit -m "feat(cli): add sdk command with generate, diff, validate - -Commands: -- core sdk generate [--spec FILE] [--lang LANG] -- core sdk diff --base VERSION [--spec FILE] -- core sdk validate [--spec FILE] - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 11: Add SDK Config to Release Config - -**Files:** -- Modify: `pkg/release/config.go` - -**Step 1: Add SDK field to Config** - -Add to Config struct in config.go: -```go -// SDK configures SDK generation. -SDK *SDKConfig `yaml:"sdk,omitempty"` -``` - -Add SDKConfig type: -```go -// SDKConfig holds SDK generation configuration. -type SDKConfig struct { - // Spec is the path to the OpenAPI spec file. - Spec string `yaml:"spec,omitempty"` - // Languages to generate. - Languages []string `yaml:"languages,omitempty"` - // Output directory (default: sdk/). - Output string `yaml:"output,omitempty"` - // Package naming. - Package SDKPackageConfig `yaml:"package,omitempty"` - // Diff configuration. - Diff SDKDiffConfig `yaml:"diff,omitempty"` - // Publish configuration. - Publish SDKPublishConfig `yaml:"publish,omitempty"` -} - -// SDKPackageConfig holds package naming configuration. -type SDKPackageConfig struct { - Name string `yaml:"name,omitempty"` - Version string `yaml:"version,omitempty"` -} - -// SDKDiffConfig holds diff configuration. -type SDKDiffConfig struct { - Enabled bool `yaml:"enabled,omitempty"` - FailOnBreaking bool `yaml:"fail_on_breaking,omitempty"` -} - -// SDKPublishConfig holds monorepo publish configuration. -type SDKPublishConfig struct { - Repo string `yaml:"repo,omitempty"` - Path string `yaml:"path,omitempty"` -} -``` - -**Step 2: Verify it compiles** - -Run: `cd /Users/snider/Code/Core && go build ./pkg/release/...` -Expected: No errors - -**Step 3: Commit** - -```bash -git add pkg/release/config.go -git commit -m "feat(release): add SDK configuration to release.yaml - -Adds sdk: section to .core/release.yaml for configuring -OpenAPI SDK generation during releases. - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 12: Add SDK Example to Docs - -**Files:** -- Create: `docs/examples/sdk-full.yaml` - -**Step 1: Create example file** - -```yaml -# Example: Full SDK Configuration -# Generate typed API clients from OpenAPI specs - -sdk: - # OpenAPI spec source (auto-detected if omitted) - spec: api/openapi.yaml - - # Languages to generate - languages: - - typescript - - python - - go - - php - - # Output directory (default: sdk/) - output: sdk/ - - # Package naming - package: - name: myapi - version: "{{.Version}}" - - # Breaking change detection - diff: - enabled: true - fail_on_breaking: true # CI fails on breaking changes - - # Optional: publish to monorepo - publish: - repo: myorg/sdks - path: packages/myapi - -# Required tools (install one per language): -# TypeScript: npm i -g openapi-typescript-codegen (or Docker) -# Python: pip install openapi-python-client (or Docker) -# Go: go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest -# PHP: Docker required -# -# Usage: -# core sdk generate # Generate all configured languages -# core sdk generate --lang go # Generate single language -# core sdk diff --base v1.0.0 # Check for breaking changes -# core sdk validate # Validate spec -``` - -**Step 2: Commit** - -```bash -git add docs/examples/sdk-full.yaml -git commit -m "docs: add SDK configuration example - -Shows full SDK config with all options: -- Language selection -- Breaking change detection -- Monorepo publishing - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -### Task 13: Final Integration Test - -**Step 1: Build and verify CLI** - -Run: `cd /Users/snider/Code/Core && go build -o bin/core ./cmd/core && ./bin/core sdk --help` -Expected: Shows sdk command help - -**Step 2: Run all tests** - -Run: `cd /Users/snider/Code/Core && go test ./pkg/sdk/... -v` -Expected: All tests pass - -**Step 3: Final commit if needed** - -```bash -git add -A -git commit -m "chore(sdk): finalize S3.4 SDK generation - -All SDK generation features complete: -- OpenAPI spec detection -- TypeScript, Python, Go, PHP generators -- Breaking change detection with oasdiff -- CLI commands (generate, diff, validate) -- Integration with release config - -Co-Authored-By: Claude Opus 4.5 " -``` - ---- - -## Summary - -13 tasks covering: -1. Package structure -2. Spec detection -3. Generator interface -4. TypeScript generator -5. Python generator -6. Go generator -7. PHP generator -8. Breaking change detection -9. Wire up Generate -10. CLI commands -11. Release config integration -12. Documentation example -13. Integration test diff --git a/tasks/plans/2026-01-29-sdk-release-design.md b/tasks/plans/2026-01-29-sdk-release-design.md deleted file mode 100644 index 37a26ae4..00000000 --- a/tasks/plans/2026-01-29-sdk-release-design.md +++ /dev/null @@ -1,210 +0,0 @@ -# SDK Release Integration Design (S3.4) - -## Summary - -Add `core release --target sdk` to generate SDKs as a separate release target. Runs breaking change detection before generating, uses release version for SDK versioning, outputs locally for manual publishing. - -## Design Decisions - -- **Separate target**: `--target sdk` runs ONLY SDK generation (no binary builds) -- **Local output**: Generates to `sdk/` directory, user handles publishing -- **Diff first**: Run breaking change detection before generating -- **Match version**: SDK version matches release version from git tags - -## CLI - -```bash -core release --target sdk # Generate SDKs only -core release --target sdk --version v1.2.3 # Explicit version -core release --target sdk --dry-run # Preview what would generate -core release # Normal release (unchanged) -``` - -## Config Schema - -In `.core/release.yaml`: - -```yaml -sdk: - spec: openapi.yaml # or auto-detect - languages: [typescript, python, go, php] - output: sdk # output directory - package: - name: myapi-sdk - diff: - enabled: true - fail_on_breaking: false # warn but continue -``` - -## Flow - -``` -core release --target sdk - ↓ -1. Load release config (.core/release.yaml) - ↓ -2. Check sdk config exists (error if not configured) - ↓ -3. Determine version (git tag or --version flag) - ↓ -4. If diff.enabled: - - Get previous tag - - Run oasdiff against current spec - - If breaking && fail_on_breaking: abort - - If breaking && !fail_on_breaking: warn, continue - ↓ -5. Generate SDKs for each language - - Pass version to generators - - Output to sdk/{language}/ - ↓ -6. Print summary (languages generated, output paths) -``` - -## Package Structure - -``` -pkg/release/ -├── sdk.go # RunSDK() orchestration + diff helper ← NEW -├── release.go # Existing Run() unchanged -└── config.go # Existing SDKConfig unchanged - -pkg/sdk/ -└── sdk.go # Add SetVersion() method ← MODIFY - -cmd/core/cmd/ -└── release.go # Add --target flag ← MODIFY -``` - -## RunSDK Implementation - -```go -// pkg/release/sdk.go - -// RunSDK executes SDK-only release: diff check + generate. -func RunSDK(ctx context.Context, cfg *Config, dryRun bool) (*SDKRelease, error) { - if cfg.SDK == nil { - return nil, fmt.Errorf("sdk not configured in .core/release.yaml") - } - - projectDir := cfg.projectDir - if projectDir == "" { - projectDir = "." - } - - // Determine version - version := cfg.version - if version == "" { - var err error - version, err = DetermineVersion(projectDir) - if err != nil { - return nil, fmt.Errorf("failed to determine version: %w", err) - } - } - - // Run diff check if enabled - if cfg.SDK.Diff.Enabled { - breaking, err := checkBreakingChanges(projectDir, cfg.SDK) - if err != nil { - // Non-fatal: warn and continue - fmt.Printf("Warning: diff check failed: %v\n", err) - } else if breaking { - if cfg.SDK.Diff.FailOnBreaking { - return nil, fmt.Errorf("breaking API changes detected") - } - fmt.Printf("Warning: breaking API changes detected\n") - } - } - - if dryRun { - return &SDKRelease{ - Version: version, - Languages: cfg.SDK.Languages, - Output: cfg.SDK.Output, - }, nil - } - - // Generate SDKs - sdkCfg := toSDKConfig(cfg.SDK) - s := sdk.New(projectDir, sdkCfg) - s.SetVersion(version) - - if err := s.Generate(ctx); err != nil { - return nil, fmt.Errorf("sdk generation failed: %w", err) - } - - return &SDKRelease{ - Version: version, - Languages: cfg.SDK.Languages, - Output: cfg.SDK.Output, - }, nil -} - -// SDKRelease holds the result of an SDK release. -type SDKRelease struct { - Version string - Languages []string - Output string -} -``` - -## CLI Integration - -```go -// cmd/core/cmd/release.go - -var target string -releaseCmd.StringFlag("target", "Release target (sdk)", &target) - -releaseCmd.Action(func() error { - if target == "sdk" { - return runReleaseSDK(dryRun, version) - } - return runRelease(dryRun, version, draft, prerelease) -}) - -func runReleaseSDK(dryRun bool, version string) error { - ctx := context.Background() - projectDir, _ := os.Getwd() - - cfg, err := release.LoadConfig(projectDir) - if err != nil { - return err - } - - if version != "" { - cfg.SetVersion(version) - } - - fmt.Printf("%s Generating SDKs\n", releaseHeaderStyle.Render("SDK Release:")) - if dryRun { - fmt.Printf(" %s\n", releaseDimStyle.Render("(dry-run mode)")) - } - - result, err := release.RunSDK(ctx, cfg, dryRun) - if err != nil { - fmt.Printf("%s %v\n", releaseErrorStyle.Render("Error:"), err) - return err - } - - fmt.Printf("%s SDK generation complete\n", releaseSuccessStyle.Render("Success:")) - fmt.Printf(" Version: %s\n", result.Version) - fmt.Printf(" Languages: %v\n", result.Languages) - fmt.Printf(" Output: %s/\n", result.Output) - - return nil -} -``` - -## Implementation Steps - -1. Add `SetVersion()` method to `pkg/sdk/sdk.go` -2. Create `pkg/release/sdk.go` with `RunSDK()` and helpers -3. Add `--target` flag to `cmd/core/cmd/release.go` -4. Add `runReleaseSDK()` function to CLI -5. Add tests for `pkg/release/sdk_test.go` -6. Final verification and TODO update - -## Dependencies - -- `oasdiff` CLI (for breaking change detection) -- Existing SDK generators (openapi-generator, etc.) diff --git a/tasks/plans/2026-01-29-sdk-release-impl.md b/tasks/plans/2026-01-29-sdk-release-impl.md deleted file mode 100644 index 170541da..00000000 --- a/tasks/plans/2026-01-29-sdk-release-impl.md +++ /dev/null @@ -1,576 +0,0 @@ -# SDK Release Implementation Plan (S3.4) - -> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. - -**Goal:** Add `core release --target sdk` to generate SDKs with version and diff checking - -**Architecture:** Separate release target that runs diff check then SDK generation, outputs locally - -**Tech Stack:** Go, existing pkg/sdk generators, oasdiff for diff - ---- - -## Task 1: Add SetVersion to SDK struct - -**Files:** -- Modify: `pkg/sdk/sdk.go` -- Test: `pkg/sdk/sdk_test.go` (create if needed) - -**Step 1: Write the failing test** - -```go -// pkg/sdk/sdk_test.go -package sdk - -import ( - "testing" -) - -func TestSDK_Good_SetVersion(t *testing.T) { - s := New("/tmp", nil) - s.SetVersion("v1.2.3") - - if s.version != "v1.2.3" { - t.Errorf("expected version v1.2.3, got %s", s.version) - } -} - -func TestSDK_Good_VersionPassedToGenerator(t *testing.T) { - config := &Config{ - Languages: []string{"typescript"}, - Output: "sdk", - Package: PackageConfig{ - Name: "test-sdk", - }, - } - s := New("/tmp", config) - s.SetVersion("v2.0.0") - - // Version should override config - if s.config.Package.Version != "v2.0.0" { - t.Errorf("expected config version v2.0.0, got %s", s.config.Package.Version) - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test ./pkg/sdk/... -run TestSDK_Good_SetVersion -v` -Expected: FAIL with "s.version undefined" or similar - -**Step 3: Write minimal implementation** - -Add to `pkg/sdk/sdk.go`: - -```go -// SDK struct - add version field -type SDK struct { - config *Config - projectDir string - version string // ADD THIS -} - -// SetVersion sets the SDK version, overriding config. -func (s *SDK) SetVersion(version string) { - s.version = version - if s.config != nil { - s.config.Package.Version = version - } -} -``` - -**Step 4: Run test to verify it passes** - -Run: `go test ./pkg/sdk/... -run TestSDK_Good -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/sdk/sdk.go pkg/sdk/sdk_test.go -git commit -m "feat(sdk): add SetVersion method for release integration" -``` - ---- - -## Task 2: Create pkg/release/sdk.go structure - -**Files:** -- Create: `pkg/release/sdk.go` - -**Step 1: Create file with types and helper** - -```go -// pkg/release/sdk.go -package release - -import ( - "context" - "fmt" - - "forge.lthn.ai/core/cli/pkg/sdk" -) - -// SDKRelease holds the result of an SDK release. -type SDKRelease struct { - // Version is the SDK version. - Version string - // Languages that were generated. - Languages []string - // Output directory. - Output string -} - -// toSDKConfig converts release.SDKConfig to sdk.Config. -func toSDKConfig(cfg *SDKConfig) *sdk.Config { - if cfg == nil { - return nil - } - return &sdk.Config{ - Spec: cfg.Spec, - Languages: cfg.Languages, - Output: cfg.Output, - Package: sdk.PackageConfig{ - Name: cfg.Package.Name, - Version: cfg.Package.Version, - }, - Diff: sdk.DiffConfig{ - Enabled: cfg.Diff.Enabled, - FailOnBreaking: cfg.Diff.FailOnBreaking, - }, - } -} -``` - -**Step 2: Verify it compiles** - -Run: `go build ./pkg/release/...` -Expected: Success - -**Step 3: Commit** - -```bash -git add pkg/release/sdk.go -git commit -m "feat(release): add SDK release types and config converter" -``` - ---- - -## Task 3: Implement RunSDK function - -**Files:** -- Modify: `pkg/release/sdk.go` -- Test: `pkg/release/sdk_test.go` - -**Step 1: Write the failing test** - -```go -// pkg/release/sdk_test.go -package release - -import ( - "context" - "testing" -) - -func TestRunSDK_Bad_NoConfig(t *testing.T) { - cfg := &Config{ - SDK: nil, - } - cfg.projectDir = "/tmp" - - _, err := RunSDK(context.Background(), cfg, true) - if err == nil { - t.Error("expected error when SDK config is nil") - } -} - -func TestRunSDK_Good_DryRun(t *testing.T) { - cfg := &Config{ - SDK: &SDKConfig{ - Languages: []string{"typescript", "python"}, - Output: "sdk", - }, - } - cfg.projectDir = "/tmp" - cfg.version = "v1.0.0" - - result, err := RunSDK(context.Background(), cfg, true) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - - if result.Version != "v1.0.0" { - t.Errorf("expected version v1.0.0, got %s", result.Version) - } - if len(result.Languages) != 2 { - t.Errorf("expected 2 languages, got %d", len(result.Languages)) - } -} -``` - -**Step 2: Run test to verify it fails** - -Run: `go test ./pkg/release/... -run TestRunSDK -v` -Expected: FAIL with "RunSDK undefined" - -**Step 3: Write implementation** - -Add to `pkg/release/sdk.go`: - -```go -// RunSDK executes SDK-only release: diff check + generate. -// If dryRun is true, it shows what would be done without generating. -func RunSDK(ctx context.Context, cfg *Config, dryRun bool) (*SDKRelease, error) { - if cfg == nil { - return nil, fmt.Errorf("release.RunSDK: config is nil") - } - if cfg.SDK == nil { - return nil, fmt.Errorf("release.RunSDK: sdk not configured in .core/release.yaml") - } - - projectDir := cfg.projectDir - if projectDir == "" { - projectDir = "." - } - - // Determine version - version := cfg.version - if version == "" { - var err error - version, err = DetermineVersion(projectDir) - if err != nil { - return nil, fmt.Errorf("release.RunSDK: failed to determine version: %w", err) - } - } - - // Run diff check if enabled - if cfg.SDK.Diff.Enabled { - breaking, err := checkBreakingChanges(projectDir, cfg.SDK) - if err != nil { - // Non-fatal: warn and continue - fmt.Printf("Warning: diff check failed: %v\n", err) - } else if breaking { - if cfg.SDK.Diff.FailOnBreaking { - return nil, fmt.Errorf("release.RunSDK: breaking API changes detected") - } - fmt.Printf("Warning: breaking API changes detected\n") - } - } - - // Prepare result - output := cfg.SDK.Output - if output == "" { - output = "sdk" - } - - result := &SDKRelease{ - Version: version, - Languages: cfg.SDK.Languages, - Output: output, - } - - if dryRun { - return result, nil - } - - // Generate SDKs - sdkCfg := toSDKConfig(cfg.SDK) - s := sdk.New(projectDir, sdkCfg) - s.SetVersion(version) - - if err := s.Generate(ctx); err != nil { - return nil, fmt.Errorf("release.RunSDK: generation failed: %w", err) - } - - return result, nil -} - -// checkBreakingChanges runs oasdiff to detect breaking changes. -func checkBreakingChanges(projectDir string, cfg *SDKConfig) (bool, error) { - // Get previous tag for comparison - prevTag, err := getPreviousTag(projectDir) - if err != nil { - return false, fmt.Errorf("no previous tag found: %w", err) - } - - // Detect spec path - specPath := cfg.Spec - if specPath == "" { - s := sdk.New(projectDir, nil) - specPath, err = s.DetectSpec() - if err != nil { - return false, err - } - } - - // Run diff - result, err := sdk.Diff(prevTag, specPath) - if err != nil { - return false, err - } - - return result.Breaking, nil -} - -// getPreviousTag gets the most recent tag before HEAD. -func getPreviousTag(projectDir string) (string, error) { - // Use git describe to get previous tag - // This is a simplified version - may need refinement - cmd := exec.Command("git", "describe", "--tags", "--abbrev=0", "HEAD^") - cmd.Dir = projectDir - out, err := cmd.Output() - if err != nil { - return "", err - } - return strings.TrimSpace(string(out)), nil -} -``` - -Add import for `os/exec` and `strings`. - -**Step 4: Run test to verify it passes** - -Run: `go test ./pkg/release/... -run TestRunSDK -v` -Expected: PASS - -**Step 5: Commit** - -```bash -git add pkg/release/sdk.go pkg/release/sdk_test.go -git commit -m "feat(release): implement RunSDK for SDK-only releases" -``` - ---- - -## Task 4: Add --target flag to CLI - -**Files:** -- Modify: `cmd/core/cmd/release.go` - -**Step 1: Add target flag and routing** - -In `AddReleaseCommand`, add: - -```go -var target string -releaseCmd.StringFlag("target", "Release target (sdk)", &target) - -// Update the action -releaseCmd.Action(func() error { - if target == "sdk" { - return runReleaseSDK(dryRun, version) - } - return runRelease(dryRun, version, draft, prerelease) -}) -``` - -**Step 2: Verify it compiles** - -Run: `go build ./cmd/core/...` -Expected: FAIL with "runReleaseSDK undefined" - -**Step 3: Commit partial progress** - -```bash -git add cmd/core/cmd/release.go -git commit -m "feat(cli): add --target flag to release command" -``` - ---- - -## Task 5: Implement runReleaseSDK CLI function - -**Files:** -- Modify: `cmd/core/cmd/release.go` - -**Step 1: Add the function** - -```go -// runReleaseSDK executes SDK-only release. -func runReleaseSDK(dryRun bool, version string) error { - ctx := context.Background() - - projectDir, err := os.Getwd() - if err != nil { - return fmt.Errorf("failed to get working directory: %w", err) - } - - // Load configuration - cfg, err := release.LoadConfig(projectDir) - if err != nil { - return fmt.Errorf("failed to load config: %w", err) - } - - // Apply CLI overrides - if version != "" { - cfg.SetVersion(version) - } - - // Print header - fmt.Printf("%s Generating SDKs\n", releaseHeaderStyle.Render("SDK Release:")) - if dryRun { - fmt.Printf(" %s\n", releaseDimStyle.Render("(dry-run mode)")) - } - fmt.Println() - - // Run SDK release - result, err := release.RunSDK(ctx, cfg, dryRun) - if err != nil { - fmt.Printf("%s %v\n", releaseErrorStyle.Render("Error:"), err) - return err - } - - // Print summary - fmt.Println() - fmt.Printf("%s SDK generation complete!\n", releaseSuccessStyle.Render("Success:")) - fmt.Printf(" Version: %s\n", releaseValueStyle.Render(result.Version)) - fmt.Printf(" Languages: %v\n", result.Languages) - fmt.Printf(" Output: %s/\n", releaseValueStyle.Render(result.Output)) - - return nil -} -``` - -**Step 2: Verify it compiles and help shows flag** - -Run: `go build -o bin/core ./cmd/core && ./bin/core release --help` -Expected: Shows `--target` flag in help output - -**Step 3: Commit** - -```bash -git add cmd/core/cmd/release.go -git commit -m "feat(cli): implement runReleaseSDK for SDK generation" -``` - ---- - -## Task 6: Add integration tests - -**Files:** -- Modify: `pkg/release/sdk_test.go` - -**Step 1: Add more test cases** - -```go -func TestRunSDK_Good_WithDiffEnabled(t *testing.T) { - cfg := &Config{ - SDK: &SDKConfig{ - Languages: []string{"typescript"}, - Output: "sdk", - Diff: SDKDiffConfig{ - Enabled: true, - FailOnBreaking: false, - }, - }, - } - cfg.projectDir = "/tmp" - cfg.version = "v1.0.0" - - // Dry run should succeed even without git repo - result, err := RunSDK(context.Background(), cfg, true) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if result.Version != "v1.0.0" { - t.Errorf("expected v1.0.0, got %s", result.Version) - } -} - -func TestRunSDK_Good_DefaultOutput(t *testing.T) { - cfg := &Config{ - SDK: &SDKConfig{ - Languages: []string{"go"}, - // Output not set - should default to "sdk" - }, - } - cfg.projectDir = "/tmp" - cfg.version = "v1.0.0" - - result, err := RunSDK(context.Background(), cfg, true) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if result.Output != "sdk" { - t.Errorf("expected default output 'sdk', got %s", result.Output) - } -} - -func TestToSDKConfig_Good_Conversion(t *testing.T) { - relCfg := &SDKConfig{ - Spec: "api.yaml", - Languages: []string{"typescript", "python"}, - Output: "generated", - Package: SDKPackageConfig{ - Name: "my-sdk", - Version: "v2.0.0", - }, - Diff: SDKDiffConfig{ - Enabled: true, - FailOnBreaking: true, - }, - } - - sdkCfg := toSDKConfig(relCfg) - - if sdkCfg.Spec != "api.yaml" { - t.Errorf("expected spec api.yaml, got %s", sdkCfg.Spec) - } - if len(sdkCfg.Languages) != 2 { - t.Errorf("expected 2 languages, got %d", len(sdkCfg.Languages)) - } - if sdkCfg.Package.Name != "my-sdk" { - t.Errorf("expected package name my-sdk, got %s", sdkCfg.Package.Name) - } -} - -func TestToSDKConfig_Good_NilInput(t *testing.T) { - result := toSDKConfig(nil) - if result != nil { - t.Error("expected nil for nil input") - } -} -``` - -**Step 2: Run all tests** - -Run: `go test ./pkg/release/... -v` -Expected: All tests PASS - -**Step 3: Commit** - -```bash -git add pkg/release/sdk_test.go -git commit -m "test(release): add SDK release integration tests" -``` - ---- - -## Task 7: Final verification and TODO update - -**Step 1: Build CLI** - -Run: `go build -o bin/core ./cmd/core` -Expected: Success - -**Step 2: Test help output** - -Run: `./bin/core release --help` -Expected: Shows `--target` flag - -**Step 3: Run all tests** - -Run: `go test ./pkg/release/... ./pkg/sdk/... -v` -Expected: All PASS - -**Step 4: Update TODO.md** - -Mark S3.4 `core release --target sdk` as complete in `tasks/TODO.md`. - -**Step 5: Commit** - -```bash -git add tasks/TODO.md -git commit -m "docs: mark S3.4 SDK release integration as complete" -``` diff --git a/tasks/plans/docs-sync-next-steps.md b/tasks/plans/docs-sync-next-steps.md deleted file mode 100644 index d360a05c..00000000 --- a/tasks/plans/docs-sync-next-steps.md +++ /dev/null @@ -1,43 +0,0 @@ -# Docs Sync Setup - Next Steps - -After moving repo to `~/Code/host-uk/core`: - -## 1. Add to repos.yaml - -Add this to `/Users/snider/Code/host-uk/repos.yaml` under `repos:`: - -```yaml - # CLI (Go) - core: - type: foundation - description: Core CLI - build, release, deploy for Go/Wails/PHP/containers - docs: true - ci: github-actions -``` - -## 2. Test docs sync - -```bash -cd ~/Code/host-uk -core docs list # Should show "core" with docs -core docs sync --dry-run # Preview what syncs -``` - -## 3. Add CLI section to VitePress (core-php) - -Edit `core-php/docs/.vitepress/config.js`: -- Add `/cli/` to nav -- Add sidebar for CLI commands - -## 4. Sync and verify - -```bash -core docs sync --output ../core-php/docs/cli -``` - ---- - -Current state: -- CLI docs written in `docs/cmd/*.md` (12 files) -- `docs/index.md` updated with command table -- All committed to git diff --git a/tools/rag/README.md b/tools/rag/README.md deleted file mode 100644 index e7a4f5d5..00000000 --- a/tools/rag/README.md +++ /dev/null @@ -1,193 +0,0 @@ -# RAG Pipeline for Host UK Documentation - -Store documentation in a vector database so Claude (and local LLMs) can retrieve relevant context without being reminded every conversation. - -## The Problem This Solves - -> "The amount of times I've had to re-tell you how to make a Flux button is crazy" - -Instead of wasting context window on "remember, Flux buttons work like this...", the RAG system: -1. Stores all documentation in Qdrant -2. Claude queries before answering -3. Relevant docs injected automatically -4. No more re-teaching - -## Prerequisites - -**Already running on your lab:** -- Qdrant: `linux.snider.dev:6333` -- Ollama: `linux.snider.dev:11434` (or local) - -**Install Python deps:** -```bash -pip install -r requirements.txt -``` - -**Ensure embedding model is available:** -```bash -ollama pull nomic-embed-text -``` - -## Quick Start - -### 1. Ingest Documentation - -```bash -# Ingest recovered Host UK docs -python ingest.py /Users/snider/Code/host-uk/core/tasks/recovered-hostuk \ - --collection hostuk-docs \ - --recreate - -# Ingest Flux UI docs separately (higher priority) -python ingest.py /path/to/flux-ui-docs \ - --collection flux-ui-docs \ - --recreate -``` - -### 2. Query the Database - -```bash -# Search for Flux button docs -python query.py "how to create a Flux button component" - -# Filter by category -python query.py "path sandboxing" --category architecture - -# Get more results -python query.py "Vi personality" --top 10 - -# Output as JSON -python query.py "brand voice" --format json - -# Output for LLM context injection -python query.py "Flux modal component" --format context -``` - -### 3. List Collections - -```bash -python query.py --list-collections -python query.py --stats --collection flux-ui-docs -``` - -## Collections Strategy - -| Collection | Content | Priority | -|------------|---------|----------| -| `flux-ui-docs` | Flux Pro component docs | High (UI questions) | -| `hostuk-docs` | Recovered implementation docs | Medium | -| `brand-docs` | Vi, brand voice, visual identity | For content generation | -| `lethean-docs` | SASE/dVPN technical docs | Product-specific | - -## Integration with Claude Code - -### Option 1: MCP Server (Best) - -Create an MCP server that Claude can query: - -```go -// In core CLI -func (s *RagServer) Query(query string) ([]Document, error) { - // Query Qdrant - // Return relevant docs -} -``` - -Then Claude can call `rag.query("Flux button")` and get docs automatically. - -### Option 2: CLAUDE.md Instruction - -Add to project CLAUDE.md: - -```markdown -## Before Answering UI Questions - -When asked about Flux UI components, query the RAG database first: -```bash -python /path/to/query.py "your question" --collection flux-ui-docs --format context -``` - -Include the retrieved context in your response. -``` - -### Option 3: Claude Code Hook - -Create a hook that auto-injects context for certain queries. - -## Category Taxonomy - -The ingestion automatically categorizes files: - -| Category | Matches | -|----------|---------| -| `ui-component` | flux, ui/component | -| `brand` | brand, mascot | -| `product-brief` | brief | -| `help-doc` | help, draft | -| `task` | task, plan | -| `architecture` | architecture, migration | -| `documentation` | default | - -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `QDRANT_HOST` | linux.snider.dev | Qdrant server | -| `QDRANT_PORT` | 6333 | Qdrant port | -| `EMBEDDING_MODEL` | nomic-embed-text | Ollama model | -| `CHUNK_SIZE` | 500 | Characters per chunk | -| `CHUNK_OVERLAP` | 50 | Overlap between chunks | - -## Training a Model vs RAG - -**RAG** (what this does): -- Model weights unchanged -- Documents retrieved at query time -- Knowledge updates instantly (re-ingest) -- Good for: facts, API docs, current information - -**Fine-tuning** (separate process): -- Model weights updated -- Knowledge baked into model -- Requires retraining to update -- Good for: style, patterns, conventions - -**For Flux UI**: RAG is perfect. The docs change, API changes, you want current info. - -**For Vi's voice**: Fine-tuning is better. Style doesn't change often, should be "baked in". - -## Vector Math (For Understanding) - -```text -"How do I make a Flux button?" - ↓ Embedding -[0.12, -0.45, 0.78, ...768 floats...] - ↓ Cosine similarity search -Find chunks with similar vectors - ↓ Results -1. doc/ui/flux/components/button.md (score: 0.89) -2. doc/ui/flux/forms.md (score: 0.76) -3. doc/ui/flux/components/input.md (score: 0.71) -``` - -The embedding model converts text to "meaning vectors". Similar meanings = similar vectors = found by search. - -## Troubleshooting - -**"No results found"** -- Lower threshold: `--threshold 0.3` -- Check collection has data: `--stats` -- Verify Ollama is running: `ollama list` - -**"Connection refused"** -- Check Qdrant is running: `curl http://linux.snider.dev:6333/collections` -- Check firewall/network - -**"Embedding model not available"** -```bash -ollama pull nomic-embed-text -``` - ---- - -*Part of the Host UK Core CLI tooling* diff --git a/tools/rag/ingest.py b/tools/rag/ingest.py deleted file mode 100644 index 7755bc26..00000000 --- a/tools/rag/ingest.py +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env python3 -""" -RAG Ingestion Pipeline for Host UK Documentation - -Chunks markdown files, generates embeddings via Ollama, stores in Qdrant. - -Usage: - python ingest.py /path/to/docs --collection hostuk-docs - python ingest.py /path/to/flux-ui --collection flux-ui-docs - -Requirements: - pip install qdrant-client ollama markdown -""" - -import argparse -import hashlib -import json -import os -import re -import sys -from pathlib import Path -from typing import Generator - -try: - from qdrant_client import QdrantClient - from qdrant_client.models import Distance, VectorParams, PointStruct - import ollama -except ImportError: - print("Install dependencies: pip install qdrant-client ollama") - sys.exit(1) - - -# Configuration -QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost") -QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) -EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text") -CHUNK_SIZE = int(os.getenv("CHUNK_SIZE", "500")) # chars -CHUNK_OVERLAP = int(os.getenv("CHUNK_OVERLAP", "50")) # chars -VECTOR_DIM = 768 # nomic-embed-text dimension - - -def chunk_markdown(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> Generator[dict, None, None]: - """ - Chunk markdown by sections (## headers), then by paragraphs if too long. - Preserves context with overlap. - """ - # Split by ## headers first - sections = re.split(r'\n(?=## )', text) - - for section in sections: - if not section.strip(): - continue - - # Extract section title - lines = section.strip().split('\n') - title = lines[0].lstrip('#').strip() if lines[0].startswith('#') else "" - - # If section is small enough, yield as-is - if len(section) <= chunk_size: - yield { - "text": section.strip(), - "section": title, - } - continue - - # Otherwise, chunk by paragraphs - paragraphs = re.split(r'\n\n+', section) - current_chunk = "" - - for para in paragraphs: - if len(current_chunk) + len(para) <= chunk_size: - current_chunk += "\n\n" + para if current_chunk else para - else: - if current_chunk: - yield { - "text": current_chunk.strip(), - "section": title, - } - # Start new chunk with overlap from previous - if overlap and current_chunk: - overlap_text = current_chunk[-overlap:] - current_chunk = overlap_text + "\n\n" + para - else: - current_chunk = para - - # Don't forget the last chunk - if current_chunk.strip(): - yield { - "text": current_chunk.strip(), - "section": title, - } - - -def generate_embedding(text: str) -> list[float]: - """Generate embedding using Ollama.""" - response = ollama.embeddings(model=EMBEDDING_MODEL, prompt=text) - return response["embedding"] - - -def get_file_category(path: str) -> str: - """Determine category from file path.""" - path_lower = path.lower() - - if "flux" in path_lower or "ui/component" in path_lower: - return "ui-component" - elif "brand" in path_lower or "mascot" in path_lower: - return "brand" - elif "brief" in path_lower: - return "product-brief" - elif "help" in path_lower or "draft" in path_lower: - return "help-doc" - elif "task" in path_lower or "plan" in path_lower: - return "task" - elif "architecture" in path_lower or "migration" in path_lower: - return "architecture" - else: - return "documentation" - - -def ingest_directory( - directory: Path, - client: QdrantClient, - collection: str, - verbose: bool = False -) -> dict: - """Ingest all markdown files from directory into Qdrant.""" - - stats = {"files": 0, "chunks": 0, "errors": 0} - points = [] - - # Find all markdown files - md_files = list(directory.rglob("*.md")) - print(f"Found {len(md_files)} markdown files") - - for file_path in md_files: - try: - rel_path = str(file_path.relative_to(directory)) - - with open(file_path, "r", encoding="utf-8", errors="ignore") as f: - content = f.read() - - if not content.strip(): - continue - - # Extract metadata - category = get_file_category(rel_path) - - # Chunk the content - for i, chunk in enumerate(chunk_markdown(content)): - chunk_id = hashlib.md5( - f"{rel_path}:{i}:{chunk['text'][:100]}".encode() - ).hexdigest() - - # Generate embedding - embedding = generate_embedding(chunk["text"]) - - # Create point - point = PointStruct( - id=chunk_id, - vector=embedding, - payload={ - "text": chunk["text"], - "source": rel_path, - "section": chunk["section"], - "category": category, - "chunk_index": i, - } - ) - points.append(point) - stats["chunks"] += 1 - - if verbose: - print(f" [{category}] {rel_path} chunk {i}: {len(chunk['text'])} chars") - - stats["files"] += 1 - if not verbose: - print(f" Processed: {rel_path} ({stats['chunks']} chunks total)") - - except Exception as e: - print(f" Error processing {file_path}: {e}") - stats["errors"] += 1 - - # Batch upsert to Qdrant - if points: - print(f"\nUpserting {len(points)} vectors to Qdrant...") - - # Upsert in batches of 100 - batch_size = 100 - for i in range(0, len(points), batch_size): - batch = points[i:i + batch_size] - client.upsert(collection_name=collection, points=batch) - print(f" Uploaded batch {i // batch_size + 1}/{(len(points) - 1) // batch_size + 1}") - - return stats - - -def main(): - parser = argparse.ArgumentParser(description="Ingest markdown docs into Qdrant") - parser.add_argument("directory", type=Path, help="Directory containing markdown files") - parser.add_argument("--collection", default="hostuk-docs", help="Qdrant collection name") - parser.add_argument("--recreate", action="store_true", help="Delete and recreate collection") - parser.add_argument("--verbose", "-v", action="store_true", help="Verbose output") - parser.add_argument("--qdrant-host", default=QDRANT_HOST, help="Qdrant host") - parser.add_argument("--qdrant-port", type=int, default=QDRANT_PORT, help="Qdrant port") - - args = parser.parse_args() - - if not args.directory.exists(): - print(f"Error: Directory not found: {args.directory}") - sys.exit(1) - - # Connect to Qdrant - print(f"Connecting to Qdrant at {args.qdrant_host}:{args.qdrant_port}...") - client = QdrantClient(host=args.qdrant_host, port=args.qdrant_port) - - # Create or recreate collection - collections = [c.name for c in client.get_collections().collections] - - if args.recreate and args.collection in collections: - print(f"Deleting existing collection: {args.collection}") - client.delete_collection(args.collection) - collections.remove(args.collection) - - if args.collection not in collections: - print(f"Creating collection: {args.collection}") - client.create_collection( - collection_name=args.collection, - vectors_config=VectorParams(size=VECTOR_DIM, distance=Distance.COSINE) - ) - - # Verify Ollama model is available - print(f"Using embedding model: {EMBEDDING_MODEL}") - try: - ollama.embeddings(model=EMBEDDING_MODEL, prompt="test") - except Exception as e: - print(f"Error: Embedding model not available. Run: ollama pull {EMBEDDING_MODEL}") - sys.exit(1) - - # Ingest files - print(f"\nIngesting from: {args.directory}") - stats = ingest_directory(args.directory, client, args.collection, args.verbose) - - # Summary - print(f"\n{'=' * 50}") - print(f"Ingestion complete!") - print(f" Files processed: {stats['files']}") - print(f" Chunks created: {stats['chunks']}") - print(f" Errors: {stats['errors']}") - print(f" Collection: {args.collection}") - print(f"{'=' * 50}") - - -if __name__ == "__main__": - main() diff --git a/tools/rag/query.py b/tools/rag/query.py deleted file mode 100644 index 24846d5c..00000000 --- a/tools/rag/query.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -""" -RAG Query Tool for Host UK Documentation - -Query the vector database and retrieve relevant documentation chunks. - -Usage: - python query.py "how do I create a Flux button" - python query.py "what is Vi's personality" --collection hostuk-docs - python query.py "path sandboxing" --top 10 --category architecture - -Requirements: - pip install qdrant-client ollama -""" - -import argparse -import html -import json -import os -import sys -from typing import Optional - -try: - from qdrant_client import QdrantClient - from qdrant_client.models import Filter, FieldCondition, MatchValue - import ollama -except ImportError: - print("Install dependencies: pip install qdrant-client ollama") - sys.exit(1) - - -# Configuration -QDRANT_HOST = os.getenv("QDRANT_HOST", "localhost") -QDRANT_PORT = int(os.getenv("QDRANT_PORT", "6333")) -EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "nomic-embed-text") - - -def generate_embedding(text: str) -> list[float]: - """Generate embedding using Ollama.""" - response = ollama.embeddings(model=EMBEDDING_MODEL, prompt=text) - return response["embedding"] - - -def query_rag( - query: str, - client: QdrantClient, - collection: str, - top_k: int = 5, - category: Optional[str] = None, - score_threshold: float = 0.5, -) -> list[dict]: - """Query the RAG database and return relevant chunks.""" - - # Generate query embedding - query_embedding = generate_embedding(query) - - # Build filter if category specified - query_filter = None - if category: - query_filter = Filter( - must=[ - FieldCondition(key="category", match=MatchValue(value=category)) - ] - ) - - # Search - results = client.query_points( - collection_name=collection, - query=query_embedding, - query_filter=query_filter, - limit=top_k, - score_threshold=score_threshold, - ).points - - return [ - { - "score": hit.score, - "text": hit.payload["text"], - "source": hit.payload["source"], - "section": hit.payload.get("section", ""), - "category": hit.payload.get("category", ""), - } - for hit in results - ] - - -def format_results(results: list[dict], query: str, format: str = "text") -> str: - """Format results for display.""" - - if format == "json": - return json.dumps(results, indent=2) - - if not results: - return f"No results found for: {query}" - - output = [] - output.append(f"Query: {query}") - output.append(f"Results: {len(results)}") - output.append("=" * 60) - - for i, r in enumerate(results, 1): - output.append(f"\n[{i}] {r['source']} (score: {r['score']:.3f})") - if r['section']: - output.append(f" Section: {r['section']}") - output.append(f" Category: {r['category']}") - output.append("-" * 40) - # Truncate long text for display - text = r['text'] - if len(text) > 500: - text = text[:500] + "..." - output.append(text) - output.append("") - - return "\n".join(output) - - -def format_for_context(results: list[dict], query: str) -> str: - """Format results as context for LLM injection.""" - - if not results: - return "" - - output = [] - output.append(f'') - - for r in results: - output.append(f'\n') - output.append(html.escape(r['text'])) - output.append("") - - output.append("\n") - - return "\n".join(output) - -def main(): - parser = argparse.ArgumentParser(description="Query RAG documentation") - parser.add_argument("query", nargs="?", help="Search query") - parser.add_argument("--collection", default="hostuk-docs", help="Qdrant collection name") - parser.add_argument("--top", "-k", type=int, default=5, help="Number of results") - parser.add_argument("--category", "-c", help="Filter by category") - parser.add_argument("--threshold", "-t", type=float, default=0.5, help="Score threshold") - parser.add_argument("--format", "-f", choices=["text", "json", "context"], default="text") - parser.add_argument("--qdrant-host", default=QDRANT_HOST) - parser.add_argument("--qdrant-port", type=int, default=QDRANT_PORT) - parser.add_argument("--list-collections", action="store_true", help="List available collections") - parser.add_argument("--stats", action="store_true", help="Show collection stats") - - args = parser.parse_args() - - # Connect to Qdrant - client = QdrantClient(host=args.qdrant_host, port=args.qdrant_port) - - # List collections - if args.list_collections: - collections = client.get_collections().collections - print("Available collections:") - for c in collections: - info = client.get_collection(c.name) - print(f" - {c.name}: {info.points_count} vectors") - return - - # Show stats - if args.stats: - try: - info = client.get_collection(args.collection) - print(f"Collection: {args.collection}") - print(f" Vectors: {info.points_count}") - print(f" Status: {info.status}") - except Exception as e: - print(f"Collection not found: {args.collection}") - return - - # Query required - if not args.query: - parser.print_help() - return - - # Execute query - results = query_rag( - query=args.query, - client=client, - collection=args.collection, - top_k=args.top, - category=args.category, - score_threshold=args.threshold, - ) - - # Format output - if args.format == "context": - print(format_for_context(results, args.query)) - else: - print(format_results(results, args.query, args.format)) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tools/rag/requirements.txt b/tools/rag/requirements.txt deleted file mode 100644 index cd4cc3e0..00000000 --- a/tools/rag/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -qdrant-client>=1.12.0,<2.0.0 -ollama>=0.1.0 \ No newline at end of file