diff --git a/Services/AssetTrackerService.php b/Services/AssetTrackerService.php index 5204fd4..51eae32 100644 --- a/Services/AssetTrackerService.php +++ b/Services/AssetTrackerService.php @@ -19,6 +19,47 @@ use Core\Mod\Uptelligence\Models\AssetVersion; */ class AssetTrackerService { + /** + * Valid Composer package name pattern. + * + * Matches vendor/package format with alphanumeric, hyphen, underscore, and dot characters. + */ + protected const COMPOSER_PACKAGE_PATTERN = '/^[a-z0-9]([_.-]?[a-z0-9]+)*\/[a-z0-9](([_.]?|-{0,2})[a-z0-9]+)*$/i'; + + /** + * Valid NPM package name pattern. + * + * Matches scoped (@scope/package) and unscoped package names. + */ + protected const NPM_PACKAGE_PATTERN = '/^(@[a-z0-9-~][a-z0-9-._~]*\/)?[a-z0-9-~][a-z0-9-._~]*$/i'; + + /** + * Validate a package name to prevent shell injection. + * + * Package names should only contain safe characters for CLI usage. + * + * @throws \InvalidArgumentException If the package name contains invalid characters + */ + protected function validatePackageName(string $packageName, string $type): string + { + $pattern = match ($type) { + Asset::TYPE_COMPOSER => self::COMPOSER_PACKAGE_PATTERN, + Asset::TYPE_NPM => self::NPM_PACKAGE_PATTERN, + default => throw new \InvalidArgumentException("Unknown package type: {$type}"), + }; + + if (! preg_match($pattern, $packageName)) { + Log::warning('Uptelligence: Invalid package name rejected', [ + 'package_name' => $packageName, + 'type' => $type, + ]); + + throw new \InvalidArgumentException("Invalid package name format: {$packageName}"); + } + + return $packageName; + } + /** * Check all active assets for updates. */ @@ -142,11 +183,21 @@ class AssetTrackerService /** * Check custom Composer registry (like Flux Pro). + * + * Uses array-based Process invocation to prevent shell injection. */ protected function checkCustomComposerRegistry(Asset $asset): array { + // Validate package name to prevent shell injection + try { + $packageName = $this->validatePackageName($asset->package_name, Asset::TYPE_COMPOSER); + } catch (\InvalidArgumentException $e) { + return ['status' => 'error', 'message' => 'Invalid package name format']; + } + // For licensed packages, we need to check the installed version via composer show - $result = Process::run("composer show {$asset->package_name} --format=json 2>/dev/null"); + // Use array syntax to prevent shell injection + $result = Process::run(['composer', 'show', $packageName, '--format=json']); if ($result->successful()) { $data = json_decode($result->output(), true); @@ -241,7 +292,8 @@ class AssetTrackerService } // Check for scoped/private packages via npm view - $result = Process::run("npm view {$asset->package_name} version 2>/dev/null"); + // Use array syntax to prevent shell injection + $result = Process::run(['npm', 'view', $asset->package_name, 'version']); if ($result->successful()) { $latestVersion = trim($result->output()); if ($latestVersion) { @@ -335,6 +387,8 @@ class AssetTrackerService /** * Update a Composer package. + * + * Uses array-based Process invocation to prevent shell injection. */ protected function updateComposerPackage(Asset $asset): array { @@ -342,13 +396,21 @@ class AssetTrackerService return ['status' => 'error', 'message' => 'No package name']; } + // Validate package name to prevent shell injection + try { + $packageName = $this->validatePackageName($asset->package_name, Asset::TYPE_COMPOSER); + } catch (\InvalidArgumentException $e) { + return ['status' => 'error', 'message' => 'Invalid package name format']; + } + + // Use array syntax to prevent shell injection $result = Process::timeout(300)->run( - "composer update {$asset->package_name} --no-interaction" + ['composer', 'update', $packageName, '--no-interaction'] ); if ($result->successful()) { - // Get new installed version - $showResult = Process::run("composer show {$asset->package_name} --format=json"); + // Get new installed version using array syntax + $showResult = Process::run(['composer', 'show', $packageName, '--format=json']); if ($showResult->successful()) { $data = json_decode($showResult->output(), true); $newVersion = $data['versions'][0] ?? $asset->latest_version; @@ -363,6 +425,8 @@ class AssetTrackerService /** * Update an NPM package. + * + * Uses array-based Process invocation to prevent shell injection. */ protected function updateNpmPackage(Asset $asset): array { @@ -370,7 +434,15 @@ class AssetTrackerService return ['status' => 'error', 'message' => 'No package name']; } - $result = Process::timeout(300)->run("npm update {$asset->package_name}"); + // Validate package name to prevent shell injection + try { + $packageName = $this->validatePackageName($asset->package_name, Asset::TYPE_NPM); + } catch (\InvalidArgumentException $e) { + return ['status' => 'error', 'message' => 'Invalid package name format']; + } + + // Use array syntax to prevent shell injection + $result = Process::timeout(300)->run(['npm', 'update', $packageName]); if ($result->successful()) { $asset->update(['installed_version' => $asset->latest_version]); diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..2bb4994 --- /dev/null +++ b/TODO.md @@ -0,0 +1,367 @@ +# TODO - core-uptelligence + +Upstream vendor tracking and dependency intelligence for Host UK. + +**Previous review:** See `changelog/2026/jan/code-review.md` for historical context. Many issues were fixed in the 2026-01-21 wave. + +## P1 - Critical / Security + +### Migration Mismatch - Uptime Monitoring vs Vendor Tracking +The first migration (`0001_01_01_000001_create_uptelligence_tables.php`) creates uptime monitoring tables (`uptelligence_monitors`, `uptelligence_checks`, `uptelligence_incidents`, `uptelligence_daily_stats`) rather than vendor tracking tables. + +**Note:** The code review from 2026-01-21 mentions migrations were created (`2026_01_21_100000_create_uptelligence_tables.php`), but this file is not present in the repository. The current migration appears to be for a different purpose (uptime monitoring). + +**Files affected:** +- `database/migrations/0001_01_01_000001_create_uptelligence_tables.php` - Contains uptime monitoring tables +- `Models/Vendor.php` - References `vendors` table +- `Models/VersionRelease.php` - References `version_releases` table +- `Models/UpstreamTodo.php` - References `upstream_todos` table +- `Models/DiffCache.php` - References `diff_cache` table +- `Models/AnalysisLog.php` - References `analysis_logs` table +- `Models/Asset.php` - References `assets` table +- `Models/AssetVersion.php` - References `asset_versions` table + +**Acceptance criteria:** +- [ ] Clarify whether uptime monitoring is part of this package or a separate concern +- [ ] If vendor tracking is the focus, replace or supplement migration with vendor tables +- [ ] Ensure all model tables are created with appropriate columns +- [ ] Add indexes as noted in the prior code review + +### Webhook Signature Timing Attack Vulnerability +The `verifyGitLabSignature` method uses direct string comparison which may be vulnerable to timing attacks. + +**File:** `Models/UptelligenceWebhook.php:250-253` +```php +protected function verifyGitLabSignature(string $signature, string $secret): bool +{ + return hash_equals($secret, $signature); // This is correct, but see below +} +``` + +**Note:** The method itself uses `hash_equals`, but verify all callers pass correctly. Additionally, consider constant-time comparison for all providers. + +**Acceptance criteria:** +- [ ] Audit all signature verification paths for timing safety +- [ ] Add unit tests for signature verification edge cases + +### API Key Exposure in Logs +The `AIAnalyzerService` and other services may log sensitive data in error scenarios. + +**Files affected:** +- `Services/AIAnalyzerService.php` - Logs API responses which could contain sensitive context +- `Services/IssueGeneratorService.php` - Logs response bodies on failure + +**Acceptance criteria:** +- [ ] Audit all Log::error calls for sensitive data +- [ ] Truncate or redact sensitive information in logs +- [ ] Never log full API request/response bodies containing credentials + +### Missing Input Validation on Webhook Payloads +The `WebhookController` accepts JSON payloads without size limits or schema validation. + +**File:** `Controllers/Api/WebhookController.php` + +**Acceptance criteria:** +- [ ] Add maximum payload size validation (e.g., 1MB limit) +- [ ] Add basic schema validation for expected payload structure +- [ ] Add protection against deeply nested JSON (DoS vector) + +--- + +## P2 - High Priority + +### Missing Table Name on Vendor Model +The `Vendor` model doesn't explicitly set `$table`, relying on Laravel's convention which would create `vendors` table. + +**File:** `Models/Vendor.php` + +**Acceptance criteria:** +- [ ] Add explicit `protected $table = 'uptelligence_vendors';` for consistency with other models +- [ ] Update migration to match + +### DiffAnalyzerService Constructor Pattern Inconsistency +`DiffAnalyzerService` requires a `Vendor` in constructor unlike other services (which use dependency injection). + +**File:** `Services/DiffAnalyzerService.php:27-33` + +**Acceptance criteria:** +- [ ] Refactor to accept Vendor as method parameter instead of constructor +- [ ] Update all callers +- [ ] Register as singleton like other services + +### ~~Missing Process Shell Injection Protection in AssetTrackerService~~ FIXED +~~The `AssetTrackerService` uses `Process::run()` with string interpolation.~~ + +**FIXED:** Now uses array-based Process invocation with package name validation. + +**Changes made:** +- [x] Use array syntax for Process commands to prevent shell injection +- [x] Validate package names against allowlist pattern (Composer and NPM patterns) +- [x] Added tests for shell injection prevention in `tests/Unit/AssetTrackerServiceTest.php` + +### VendorStorageService Uses Undefined File Facade +The `extractMetadata` and `getDirectorySize` methods use `File::` facade which is not imported. + +**File:** `Services/VendorStorageService.php:391,417,562` + +**Acceptance criteria:** +- [ ] Add `use Illuminate\Support\Facades\File;` import +- [ ] Or refactor to use Storage facade consistently + +### Missing Authorisation Checks +No authorisation checks on admin Livewire components - relies entirely on route middleware. + +**Files affected:** +- `View/Modal/Admin/*.php` - All admin modals + +**Acceptance criteria:** +- [ ] Add explicit authorisation checks in Livewire components +- [ ] Use Gates/Policies for fine-grained access control +- [ ] Consider workspace-level permissions for multi-tenant isolation + +### Missing Vendor Model Workspace Scope +The `Vendor` model doesn't use `BelongsToWorkspace` trait, but `UptelligenceDigest` does. + +**File:** `Models/Vendor.php` + +**Acceptance criteria:** +- [ ] Determine if vendors should be workspace-scoped or global +- [ ] If workspace-scoped, add `BelongsToWorkspace` trait +- [ ] Add `workspace_id` to migration + +--- + +## P3 - Medium Priority + +### TestCase Incorrect Namespace +The test case uses wrong namespace and doesn't extend Orchestra TestCase. + +**File:** `tests/TestCase.php` +```php +namespace Tests; // Should be Core\Mod\Uptelligence\Tests +``` + +**Acceptance criteria:** +- [ ] Fix namespace to `Core\Mod\Uptelligence\Tests` +- [ ] Extend `Orchestra\Testbench\TestCase` +- [ ] Configure package service provider loading + +### Missing Test Coverage +The `tests/Feature/` and `tests/Unit/` directories contain only `.gitkeep` files. + +**Acceptance criteria:** +- [ ] Add unit tests for all Services +- [ ] Add unit tests for Model methods and scopes +- [ ] Add feature tests for webhook endpoints +- [ ] Add feature tests for console commands +- [ ] Target: 80% code coverage + +### Missing Rate Limiter Key Consistency +Different services use different rate limiter key formats. + +**Files affected:** +- `Boot.php` - Defines limiters +- `Services/AIAnalyzerService.php:221` - Uses key without user context +- `Services/IssueGeneratorService.php:91` - Uses same global key + +**Acceptance criteria:** +- [ ] Standardise rate limiter key naming +- [ ] Consider per-vendor or per-user rate limits where appropriate + +### Missing Retry Logic Consistency +Some HTTP calls have retry logic, others don't. + +**File:** `Services/IssueGeneratorService.php:393-406` - `createWeeklyDigest` lacks retry logic unlike `createGitHubIssue` + +**Acceptance criteria:** +- [ ] Add retry logic to all external HTTP calls +- [ ] Extract common HTTP client configuration to shared method + +### DiffCache Table Name Hardcoded +Model sets table name to `diff_cache` but other tables use `uptelligence_` prefix. + +**File:** `Models/DiffCache.php:22` + +**Acceptance criteria:** +- [ ] Rename to `uptelligence_diff_cache` for consistency +- [ ] Update migration + +### Missing Carbon Import in UptelligenceDigest +Uses `\Carbon\Carbon` with full path instead of import. + +**File:** `Models/UptelligenceDigest.php:258` + +**Acceptance criteria:** +- [ ] Add `use Carbon\Carbon;` import +- [ ] Replace `\Carbon\Carbon` with `Carbon` + +### Emoji Usage in Models +Models use emoji characters for icons which may cause encoding issues. + +**Files affected:** +- `Models/Vendor.php:208-215` +- `Models/UpstreamTodo.php:215-228` +- `Models/DiffCache.php:213-250` +- `Models/AnalysisLog.php:156-170` + +**Acceptance criteria:** +- [ ] Replace emojis with icon class names (Font Awesome) +- [ ] Or create separate icon mapping service +- [ ] Ensure UTF-8 encoding throughout + +--- + +## P4 - Low Priority + +### Inconsistent Method Naming +Some methods use British spelling, others American. + +**Examples:** +- `normaliseVersion` (British) in `WebhookReceiverService.php` +- `normaliseVersion` (British) in `VendorUpdateCheckerService.php` +- Consistent, but verify across codebase + +**Acceptance criteria:** +- [ ] Audit all method names for spelling consistency (prefer British) +- [ ] Document spelling conventions in CLAUDE.md + +### Missing PHPDoc Return Types +Some methods lack `@return` documentation. + +**Files affected:** Most service methods + +**Acceptance criteria:** +- [ ] Add comprehensive PHPDoc blocks to all public methods +- [ ] Include `@param`, `@return`, `@throws` annotations + +### Console Commands Missing Progress Bars +Commands don't show progress for long-running operations. + +**Files affected:** +- `Console/CheckCommand.php` +- `Console/AnalyzeCommand.php` +- `Console/IssuesCommand.php` + +**Acceptance criteria:** +- [ ] Add progress bars for iterating over vendors/assets +- [ ] Add `--verbose` option for detailed output + +### Missing Soft Deletes on Some Models +`DiffCache` and `AnalysisLog` lack soft deletes while related models have them. + +**Acceptance criteria:** +- [ ] Add `SoftDeletes` trait to `DiffCache` and `AnalysisLog` +- [ ] Add `deleted_at` column to migrations + +### Missing Model Events +No model observers or events for audit logging. + +**Acceptance criteria:** +- [ ] Add observer for `UpstreamTodo` status changes +- [ ] Add events for version detection, analysis completion +- [ ] Integrate with activity logging system + +--- + +## P5 - Nice to Have + +### Add MCP Tool Integration +The package is designed for AI analysis but has no MCP tool handlers. + +**Acceptance criteria:** +- [ ] Add `McpToolsRegistering` event listener to Boot.php +- [ ] Create MCP tools for: + - Listing pending todos + - Checking vendor status + - Triggering analysis + - Getting quick wins summary + +### Add WebSocket/Broadcast Support +No real-time updates for webhook deliveries or analysis progress. + +**Acceptance criteria:** +- [ ] Add Laravel Echo events for webhook received +- [ ] Broadcast analysis progress updates +- [ ] Add Livewire polling as fallback + +### Add Slack/Discord Notifications +Config has webhook URLs but no implementation. + +**File:** `config.php:223-225` + +**Acceptance criteria:** +- [ ] Implement Slack notification channel +- [ ] Implement Discord notification channel +- [ ] Add notification preferences per user + +### Support GitLab Self-Hosted +`isGiteaUrl` method only checks configured Gitea host. + +**File:** `Services/VendorUpdateCheckerService.php:409-418` + +**Acceptance criteria:** +- [ ] Add support for GitLab self-hosted instances +- [ ] Make git host detection more flexible + +### Add Changelog Parsing +Extract structured changelog data from releases. + +**Acceptance criteria:** +- [ ] Parse Keep a Changelog format +- [ ] Extract breaking changes automatically +- [ ] Link changelog entries to todos + +--- + +## P6+ - Future / Backlog + +### Database Performance Optimisation +- [ ] Add database indexes for common query patterns +- [ ] Implement query caching for dashboard stats +- [ ] Consider read replicas for analytics queries + +### Multi-Language Support +- [ ] Extract all user-facing strings to lang files +- [ ] Add translation support for notifications + +### API Endpoints for External Integration +- [ ] Create REST API for querying todos +- [ ] Add GraphQL support +- [ ] Implement API versioning + +### Archive Management UI +- [ ] Add S3 storage browser in admin +- [ ] Implement bulk archive/restore operations +- [ ] Add storage quota monitoring + +### Advanced AI Analysis +- [ ] Implement multi-file context analysis +- [ ] Add code complexity metrics +- [ ] Generate migration scripts automatically + +### Vendor Dependency Graph +- [ ] Track inter-vendor dependencies +- [ ] Visualise dependency tree +- [ ] Detect cascading update requirements + +--- + +## Completed + +Items completed as part of the 2026-01-21 code review wave (per `changelog/2026/jan/code-review.md`): + +- [x] **Path traversal validation** - Added to `DiffAnalyzerService::validatePath()` to prevent directory traversal attacks +- [x] **Shell injection fix in DiffAnalyzerService** - Now uses array syntax: `Process::run(['diff', '-u', $prevPath, $currPath])` +- [x] **Shell injection fix in AssetTrackerService** - Now uses array syntax with package name validation for all Process::run() calls +- [x] **Rate limiting for AI API calls** - Implemented in `AIAnalyzerService` with configurable limit +- [x] **Retry logic with exponential backoff** - Added to Packagist, NPM, GitHub, Gitea, Anthropic, and OpenAI API calls +- [x] **Enhanced error logging** - Improved logging for API failures beyond just `report($e)` +- [x] **Database transactions** - Added to `DiffAnalyzerService::cacheDiffs()` with rollback on failure +- [x] **Soft deletes** - Added to relevant models (Vendor, VersionRelease, UpstreamTodo, UptelligenceWebhook) +- [x] **Database indexes** - Added indexes on `diff_cache.version_release_id` and `upstream_todos.vendor_id` +- [x] **Target repo validation** - Added validation in `IssueGeneratorService` before using `explode('/', ...)` +- [x] **Storage facade standardisation** - `VendorStorageService` now uses Storage facade consistently +- [x] **Config validation on boot** - Added warnings for missing API keys in `Boot::validateConfig()` +- [x] **Timestamp validation** - Fixed `released_at` parsing in `AssetTrackerService::parseReleaseTimestamp()` +- [x] **Agentic module soft dependency** - `UpstreamPlanGeneratorService` now checks module availability +- [x] **Webhook system** - Implemented inbound webhook endpoints, signature verification, and async processing diff --git a/changelog/2026/jan/code-review.md b/changelog/2026/jan/code-review.md index dcac74a..b71b306 100644 --- a/changelog/2026/jan/code-review.md +++ b/changelog/2026/jan/code-review.md @@ -29,6 +29,7 @@ This module is well-designed architecturally. P1 critical issues fixed in Wave 1 - [x] **Dependency on Agentic module** - FIXED: `UpstreamPlanGeneratorService` now checks `agenticModuleAvailable()` before using Agentic models - [ ] **API keys required** - AI analysis and GitHub/Gitea integration require API keys but no validation or graceful degradation - [x] **DiffAnalyzerService shell injection risk** - FIXED: Now uses `Process::run(['diff', '-u', $prevPath, $currPath])` array syntax +- [x] **AssetTrackerService shell injection risk** - FIXED: Now uses array-based Process invocation with package name validation ## Recommended Improvements diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..1b4146b --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,345 @@ +--- +title: Architecture +description: Technical architecture of core-uptelligence +updated: 2026-01-29 +--- + +# Architecture + +The `core-uptelligence` package provides upstream vendor tracking and dependency intelligence for the Host UK platform. It monitors software vendors, analyses version differences, generates porting tasks, and manages notification digests. + +## Overview + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ External Systems │ +├─────────────┬─────────────┬─────────────┬─────────────┬────────────────────┤ +│ GitHub │ GitLab │ npm │ Packagist │ AI Providers │ +│ Releases │ Releases │ Publish │ Updates │ (Anthropic/OpenAI) │ +└──────┬──────┴──────┬──────┴──────┬──────┴──────┬──────┴─────────┬──────────┘ + │ │ │ │ │ + └─────────────┴─────────────┴─────────────┘ │ + │ │ + ┌──────▼──────┐ │ + │ Webhook │ │ + │ Controller │ │ + └──────┬──────┘ │ + │ │ + ┌──────▼──────┐ ┌──────▼──────┐ + │ Webhook │ │ AI │ + │ Receiver │ │ Analyzer │ + │ Service │ │ Service │ + └──────┬──────┘ └──────┬──────┘ + │ │ + ┌───────────────────┼───────────────────────────────────────┤ + │ │ │ +┌──────▼──────┐ ┌──────▼──────┐ ┌──────────────┐ ┌──────▼──────┐ +│ Vendor │ │ Version │ │ Diff │ │ Upstream │ +│ Model │◄────┤ Release │────►│ Analyzer │───►│ Todo │ +└─────────────┘ └─────────────┘ │ Service │ └─────────────┘ + └──────────────┘ │ + │ + ┌──────────────┐ ┌──────▼──────┐ + │ Issue │◄───────────────────────┤ Issue │ + │ (GitHub/ │ │ Generator │ + │ Gitea) │ │ Service │ + └──────────────┘ └─────────────┘ +``` + +## Module Registration + +The package registers as a Laravel service provider via `Boot.php` and uses the event-driven module loading pattern from `core-php`. + +### Event Listeners + +```php +public static array $listens = [ + AdminPanelBooting::class => 'onAdminPanel', + ApiRoutesRegistering::class => 'onApiRoutes', + ConsoleBooting::class => 'onConsole', +]; +``` + +### Registered Services + +All services are registered as singletons: + +| Service | Purpose | +|---------|---------| +| `VendorStorageService` | Local and S3 file storage for vendor archives | +| `VendorUpdateCheckerService` | Polls registries (GitHub, Packagist, npm) for updates | +| `DiffAnalyzerService` | Generates file diffs between versions | +| `AIAnalyzerService` | Uses AI to analyse diffs and categorise changes | +| `IssueGeneratorService` | Creates GitHub/Gitea issues from todos | +| `UptelligenceDigestService` | Compiles and sends digest notifications | +| `WebhookReceiverService` | Processes incoming vendor release webhooks | +| `AssetTrackerService` | Tracks package dependencies (Composer, npm) | + +## Data Model + +### Core Entities + +``` +┌─────────────┐ ┌─────────────────┐ ┌─────────────┐ +│ Vendor │────►│ VersionRelease │────►│ DiffCache │ +└─────────────┘ └─────────────────┘ └─────────────┘ + │ │ + │ │ + ▼ ▼ +┌─────────────┐ ┌─────────────────┐ +│UpstreamTodo │ │ AnalysisLog │ +└─────────────┘ └─────────────────┘ +``` + +### Vendor + +Represents an upstream software source (licensed, OSS, or plugin). + +**Key attributes:** +- `slug` - Unique identifier +- `source_type` - `licensed`, `oss`, or `plugin` +- `git_repo_url` - Repository URL for OSS vendors +- `path_mapping` - Maps upstream paths to target paths +- `ignored_paths` - Patterns to skip during analysis +- `priority_paths` - High-importance file patterns +- `target_repo` - GitHub/Gitea repo for issue creation + +### VersionRelease + +Tracks a specific version of a vendor's software. + +**Key attributes:** +- `version` / `previous_version` - Version comparison +- `storage_disk` - `local` or `s3` +- `s3_key` - Archive location in cold storage +- `file_hash` - SHA-256 for integrity verification +- `summary` - AI-generated release summary + +### UpstreamTodo + +A porting task generated from version analysis. + +**Key attributes:** +- `type` - `feature`, `bugfix`, `security`, `ui`, `block`, `api`, `refactor`, `dependency` +- `status` - `pending`, `in_progress`, `ported`, `skipped`, `wont_port` +- `priority` - 1-10 scale +- `effort` - `low`, `medium`, `high` +- `ai_analysis` - Raw AI analysis data +- `github_issue_number` - Linked issue + +### DiffCache + +Stores individual file changes for a version release. + +**Key attributes:** +- `change_type` - `added`, `modified`, `removed` +- `category` - Auto-detected: `controller`, `model`, `view`, `security`, etc. +- `diff_content` - Unified diff for modified files + +## Webhook System + +### Architecture + +``` + ┌─────────────────┐ + GitHub/GitLab/npm ─────►│WebhookController│ + └────────┬────────┘ + │ + ┌────────▼────────┐ + │ Validate │ + │ Signature │ + └────────┬────────┘ + │ + ┌────────▼────────┐ + │ Create │ + │ Delivery │ + └────────┬────────┘ + │ + ┌────────▼────────┐ + │ Dispatch │ + │ Job │ + └────────┬────────┘ + │ + ┌────────▼────────┐ + │ Process │ + │ Webhook │ + │ Job │ + └────────┬────────┘ + │ + ┌──────────────┼──────────────┐ + │ │ │ + ┌──────▼───┐ ┌──────▼───┐ ┌──────▼───┐ + │ Parse │ │ Create │ │ Notify │ + │ Payload │ │ Release │ │ Users │ + └──────────┘ └──────────┘ └──────────┘ +``` + +### Supported Providers + +| Provider | Signature Method | Event Types | +|----------|-----------------|-------------| +| GitHub | HMAC-SHA256 (`X-Hub-Signature-256`) | `release.published`, `release.created` | +| GitLab | Token (`X-Gitlab-Token`) | `release.create`, `tag_push` | +| npm | HMAC-SHA256 | `package:publish` | +| Packagist | HMAC-SHA1 | `package.update` | +| Custom | HMAC-SHA256 | Flexible | + +### Secret Rotation + +Webhooks support secret rotation with a configurable grace period (default 24 hours) where both old and new secrets are accepted. + +### Circuit Breaker + +After 10 consecutive failures, a webhook endpoint is automatically disabled to prevent continued processing failures. + +## Storage Architecture + +### Dual Storage Mode + +The package supports both local and S3 storage for vendor archives. + +``` +┌─────────────────────────────────────────────────────────┐ +│ Storage Flow │ +├─────────────────────────────────────────────────────────┤ +│ │ +│ Upload ─► Local Storage ─► Create Archive ─► S3 │ +│ │ │ +│ ▼ │ +│ Delete Local │ +│ (optional, based │ +│ on retention) │ +│ │ +│ Analysis ─► Check Local ─► Not Found ─► Download S3 │ +│ │ │ │ +│ ▼ ▼ │ +│ Use Local Extract │ +│ │ │ +│ ▼ │ +│ Use Local │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +### Retention Policy + +- Keep N most recent versions locally (configurable, default 2) +- Never delete current or previous version locally +- Archive older versions to S3 automatically +- Verify file integrity on S3 download via SHA-256 hash + +### Path Structure + +**Local:** +``` +storage/app/vendors/{vendor-slug}/{version}/ +``` + +**S3:** +``` +{prefix}{vendor-slug}/{version}.tar.gz +``` + +## AI Analysis Pipeline + +### Flow + +``` +VersionRelease + │ + ▼ +┌────────────┐ +│ Group │ ─► Related files grouped together +│ Diffs │ (controller + view + route) +└─────┬──────┘ + │ + ▼ +┌────────────┐ +│ Build │ ─► Construct context with file changes +│ Context │ +└─────┬──────┘ + │ + ▼ +┌────────────┐ +│ Call │ ─► Rate-limited AI API call +│ AI │ (10/minute default) +└─────┬──────┘ + │ + ▼ +┌────────────┐ +│ Parse │ ─► Extract structured JSON +│ Response │ +└─────┬──────┘ + │ + ▼ +┌────────────┐ +│ Create │ ─► Generate UpstreamTodo records +│ Todos │ +└────────────┘ +``` + +### AI Providers + +Supports: +- **Anthropic** (default) - Claude models +- **OpenAI** - GPT models + +Configuration via `upstream.ai.provider` and `upstream.ai.model`. + +### Rate Limiting + +- AI API calls: 10/minute (configurable) +- Registry checks: 30/minute +- Issue creation: 10/minute +- Webhook ingestion: 60/minute per endpoint + +## Console Commands + +| Command | Purpose | +|---------|---------| +| `upstream:check` | Check vendors for updates, display status table | +| `upstream:analyze` | Analyse version diffs and generate todos | +| `upstream:issues` | Create GitHub/Gitea issues from pending todos | +| `upstream:check-updates` | Poll external registries for new versions | +| `upstream:send-digests` | Send scheduled digest emails | + +## Admin UI Components + +All components are Livewire-based and registered under the `uptelligence.admin.*` namespace: + +| Component | Purpose | +|-----------|---------| +| `Dashboard` | Overview of vendors, todos, recent activity | +| `VendorManager` | CRUD for vendor configurations | +| `TodoList` | View and manage porting todos | +| `DiffViewer` | Browse file changes between versions | +| `AssetManager` | Track package dependencies | +| `DigestPreferences` | User notification settings | +| `WebhookManager` | Configure and monitor webhook endpoints | + +## Configuration + +See `/Users/snider/Code/host-uk/core-uptelligence/config.php` for all configuration options. + +Key configuration sections: +- `storage` - Local and S3 storage settings +- `source_types` - Vendor type definitions +- `detection_patterns` - File categorisation patterns +- `ai` - AI provider and model settings +- `github` / `gitea` - Issue tracker integration +- `update_checker` - Auto-checking behaviour +- `notifications` - Slack, Discord, email settings +- `default_vendors` - Pre-configured vendor seeds + +## Dependencies + +### Required +- `host-uk/core` - Foundation framework + +### External Services +- Anthropic API or OpenAI API (for AI analysis) +- GitHub API (for releases and issues) +- Gitea API (for internal git server) +- Packagist API (for Composer package checks) +- npm Registry (for npm package checks) +- S3-compatible storage (for archives) diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 0000000..007ee51 --- /dev/null +++ b/docs/security.md @@ -0,0 +1,307 @@ +--- +title: Security +description: Security considerations and audit notes for core-uptelligence +updated: 2026-01-29 +--- + +# Security Considerations + +This document outlines security considerations, implemented protections, and areas requiring attention for the `core-uptelligence` package. + +## Authentication and Authorisation + +### Webhook Authentication + +Webhooks are authenticated via HMAC signature verification, not API tokens or session auth. + +**Implementation:** `Models/UptelligenceWebhook.php` + +| Provider | Method | Algorithm | +|----------|--------|-----------| +| GitHub | `X-Hub-Signature-256` header | HMAC-SHA256 | +| GitLab | `X-Gitlab-Token` header | Direct comparison | +| npm | `X-Npm-Signature` header | HMAC-SHA256 | +| Packagist | `X-Hub-Signature` header | HMAC-SHA1 | +| Custom | `X-Signature` header | HMAC-SHA256 | + +**Strengths:** +- Uses `hash_equals()` for timing-safe comparison +- Supports secret rotation with grace period +- Previous secret accepted during 24-hour grace period + +**Weaknesses:** +- GitLab uses simple token comparison (not HMAC) +- No IP allowlist for webhook sources +- No request signing for outbound API calls + +### Admin Access + +Admin routes are protected by middleware defined at the application level. The package itself does not enforce authorisation. + +**Risk:** If admin middleware is misconfigured, admin endpoints could be exposed. + +**Recommendation:** Add explicit authorisation checks in Livewire components. + +### API Token Storage + +External API tokens (GitHub, Gitea, AI providers) are stored in environment variables, not in the database. + +**Configuration:** `config.php` +```php +'github' => [ + 'token' => env('GITHUB_TOKEN'), +], +'gitea' => [ + 'token' => env('GITEA_TOKEN'), +], +``` + +**Strength:** Tokens not exposed in database backups. + +## Input Validation + +### Webhook Payloads + +**Current state:** +- JSON parsing with error handling +- Signature verification before processing +- Event type extraction from headers/payload + +**Missing:** +- Maximum payload size validation +- Schema validation for expected structure +- Protection against deeply nested JSON +- Rate limiting per source IP (partially implemented) + +### File Path Validation + +**Implementation:** `Services/DiffAnalyzerService.php:232-274` + +Path traversal protection is implemented: +```php +protected function validatePath(string $path, string $basePath): string +{ + // Check for path traversal attempts + if (str_contains($path, '..') || str_contains($path, "\0")) { + throw new InvalidArgumentException('Invalid path: path traversal not allowed'); + } + // ... realpath validation +} +``` + +**Strengths:** +- Blocks `..` sequences +- Blocks null bytes +- Validates against real filesystem path + +### Shell Command Execution + +**Risk areas:** + +1. `DiffAnalyzerService::generateDiff()` - Uses array syntax (safe) + ```php + Process::run(['diff', '-u', $prevPath, $currPath]); + ``` + +2. `VendorStorageService::createArchive()` - Uses array syntax (safe) + ```php + new Process(['tar', '-czf', $archivePath, '-C', $sourcePath, '.']); + ``` + +3. `AssetTrackerService` - Uses array syntax with validation (FIXED) + ```php + // Package names are validated against allowlist patterns before use + $packageName = $this->validatePackageName($asset->package_name, Asset::TYPE_COMPOSER); + Process::run(['composer', 'show', $packageName, '--format=json']); + ``` + +**Mitigations applied:** +- Array-based Process invocation prevents shell metacharacter interpretation +- Package name validation rejects names containing shell injection characters +- Invalid package names are logged and result in safe error responses + +## Data Protection + +### Sensitive Data Handling + +**Encrypted fields:** +- `UptelligenceWebhook.secret` - Laravel encrypted cast +- `UptelligenceWebhook.previous_secret` - Laravel encrypted cast + +**Hidden from serialisation:** +- Both secret fields are in `$hidden` array + +### Logging Considerations + +**Risk:** API responses and payloads may be logged on error. + +**Current logging:** +```php +Log::error('Uptelligence: GitHub issue creation failed', [ + 'todo_id' => $todo->id, + 'status' => $response->status(), + 'body' => substr($response->body(), 0, 500), // Truncated but may contain sensitive data +]); +``` + +**Recommendation:** +- Redact or exclude response bodies +- Never log full webhook payloads +- Use structured logging with PII filtering + +### File Storage Security + +**S3 Configuration:** +- Uses private bucket by default +- Dual endpoint support for Hetzner Object Store +- Files stored with `application/gzip` content type + +**Local Storage:** +- Files stored under `storage/app/vendors/` +- Not publicly accessible by default +- Temp files cleaned up after 24 hours + +## Rate Limiting + +### Implemented Limiters + +| Limiter | Rate | Scope | +|---------|------|-------| +| `upstream-ai-api` | 10/minute | Global | +| `upstream-registry` | 30/minute | Global | +| `upstream-issues` | 10/minute | Global | +| `uptelligence-webhooks` | 60/minute | Per webhook UUID or IP | + +### Webhook Rate Limiting + +```php +RateLimiter::for('uptelligence-webhooks', function (Request $request) { + $webhook = $request->route('webhook'); + return $webhook + ? Limit::perMinute(60)->by('uptelligence-webhook:'.$webhook) + : Limit::perMinute(30)->by('uptelligence-webhook-ip:'.$request->ip()); +}); +``` + +**Note:** IP-based fallback uses `$request->ip()` which may be spoofed without proper proxy configuration. + +## External Service Integration + +### GitHub API + +**Scopes required:** +- `repo` - For reading releases and creating issues +- `read:org` - If targeting organisation repos + +**Token handling:** +- Sent via `Authorization: Bearer` header +- Token not logged in error scenarios + +### Gitea API + +**Token handling:** +- Sent via `Authorization: token` header +- Similar security profile to GitHub + +### AI Providers (Anthropic/OpenAI) + +**Data sent to AI:** +- Code diffs (potentially containing sensitive patterns) +- File paths +- Vendor names + +**Recommendation:** +- Consider PII scanning before AI submission +- Document data processing in privacy policy +- Offer opt-out for AI analysis + +## Known Vulnerabilities + +### Critical + +1. **Missing database migrations** - Core models reference non-existent tables + - Impact: Application will fail on first use + - Status: Documented in TODO.md P1 + +2. ~~**Shell injection in AssetTrackerService**~~ - FIXED + - Impact: Was arbitrary command execution if package names were malicious + - Status: RESOLVED - Now uses array-based Process invocation with package name validation + +### High + +1. **No authorisation in Livewire components** + - Impact: Relies entirely on route middleware + - Mitigation: Admin routes should be protected at application level + +2. **Missing File facade import in VendorStorageService** + - Impact: Runtime errors in extractMetadata/getDirectorySize + - Status: Documented in TODO.md P2 + +### Medium + +1. **Inconsistent workspace scoping** + - Impact: Vendor data may leak between workspaces + - Status: Design decision needed + +2. **Webhook payloads stored unencrypted** + - Impact: Payload data visible in database + - Mitigation: Payloads are not typically sensitive + +## Security Checklist for Production + +Before deploying to production: + +- [ ] All API tokens set via environment variables +- [ ] S3 bucket configured as private +- [ ] Webhook secrets generated with sufficient entropy (64+ characters) +- [ ] Rate limiters tested and tuned +- [ ] Admin routes protected by authentication middleware +- [ ] Logging configured to exclude sensitive data +- [ ] Database migrations created and run +- [ ] AssetTrackerService shell injection fixed +- [ ] Trusted proxy configuration set (for accurate IP detection) + +## Incident Response + +### Compromised Webhook Secret + +1. Rotate secret immediately via `webhook->rotateSecret()` +2. Grace period allows valid senders to update +3. Monitor for invalid signature attempts +4. After grace period, old secret is invalidated + +### Compromised API Token + +1. Revoke token at provider (GitHub/Gitea) +2. Generate new token +3. Update environment variable +4. Restart application +5. Review logs for unauthorised actions + +### Suspected Data Breach + +1. Check webhook delivery logs for unusual patterns +2. Review AI API call logs for data exfiltration +3. Audit S3 access logs +4. Check for path traversal attempts in logs + +## Compliance Notes + +### GDPR Considerations + +- Webhook payloads may contain contributor names/emails +- AI analysis may process code comments with PII +- Digest notifications sent to user email addresses + +**Recommendations:** +- Document data flows in privacy policy +- Implement data retention policies +- Provide data export/deletion capabilities + +### EUPL-1.2 License + +The package is licensed under EUPL-1.2 (copyleft). + +- Modifications to `Core\` namespace must be shared +- Does not apply to vendor code being analysed +- External API usage subject to provider terms diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..f333a6e --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,283 @@ +--- +title: Storage +description: Vendor file storage and archival system +updated: 2026-01-29 +--- + +# Storage + +The Uptelligence module supports dual storage modes for vendor version archives: local filesystem and S3-compatible object storage. + +## Storage Modes + +### Local Only + +Default mode. All vendor files stored on local filesystem. + +```php +// config.php +'storage' => [ + 'disk' => 'local', +] +``` + +**Paths:** +``` +storage/app/vendors/{vendor-slug}/{version}/ +storage/app/temp/upstream/ +``` + +### S3 Cold Storage + +For archiving older versions to reduce local disk usage. + +```php +// config.php +'storage' => [ + 'disk' => 's3', + 's3' => [ + 'bucket' => 'hostuk', + 'prefix' => 'upstream/vendors/', + 'disk' => 's3-private', + ], +] +``` + +**S3 Structure:** +``` +s3://hostuk/upstream/vendors/{vendor-slug}/{version}.tar.gz +``` + +## Archive Flow + +### Upload and Archive + +When S3 mode is enabled: + +``` +1. Version uploaded to local storage +2. Analysis performed on local files +3. Archive created: tar.gz +4. Upload archive to S3 +5. Record S3 key and file hash +6. Optionally delete local files +``` + +### Retrieval for Analysis + +When analysing an archived version: + +``` +1. Check if local copy exists +2. If not, download from S3 +3. Verify SHA-256 hash +4. Extract to local storage +5. Perform analysis +6. Mark last_downloaded_at +``` + +## Configuration Options + +```php +'storage' => [ + // Primary storage mode + 'disk' => env('UPSTREAM_STORAGE_DISK', 'local'), + + // Local paths + 'base_path' => storage_path('app/vendors'), + 'licensed' => storage_path('app/vendors/licensed'), + 'oss' => storage_path('app/vendors/oss'), + 'plugins' => storage_path('app/vendors/plugins'), + 'temp_path' => storage_path('app/temp/upstream'), + + // S3 settings + 's3' => [ + 'bucket' => env('UPSTREAM_S3_BUCKET', 'hostuk'), + 'prefix' => env('UPSTREAM_S3_PREFIX', 'upstream/vendors/'), + 'region' => env('UPSTREAM_S3_REGION', 'eu-west-2'), + 'private_endpoint' => env('S3_PRIVATE_ENDPOINT'), + 'public_endpoint' => env('S3_PUBLIC_ENDPOINT'), + 'disk' => env('UPSTREAM_S3_DISK', 's3-private'), + ], + + // Archive behaviour + 'archive' => [ + 'auto_archive' => env('UPSTREAM_AUTO_ARCHIVE', true), + 'delete_local_after_archive' => env('UPSTREAM_DELETE_LOCAL', true), + 'keep_local_versions' => env('UPSTREAM_KEEP_LOCAL', 2), + 'cleanup_after_hours' => env('UPSTREAM_CLEANUP_HOURS', 24), + ], + + // Download behaviour + 'download' => [ + 'max_concurrent' => 3, + 'timeout' => 300, + ], +], +``` + +## Retention Policy + +### Local Retention + +By default, the N most recent versions are kept locally: + +```php +'keep_local_versions' => 2, +``` + +Versions are never deleted if they are: +- The vendor's current version +- The vendor's previous version (for diff analysis) + +### Archive Retention + +Archives in S3 are retained indefinitely. No automatic deletion. + +### Temp File Cleanup + +Temporary files older than the configured hours are cleaned up: + +```php +'cleanup_after_hours' => 24, +``` + +Run cleanup: +```bash +php artisan schedule:run # If scheduled +# Or manually via service +$storageService->cleanupTemp(); +``` + +## File Integrity + +### Hash Verification + +Each archived version records a SHA-256 hash: + +```php +$hash = hash_file('sha256', $archivePath); +$release->update(['file_hash' => $hash]); +``` + +On download, the hash is verified: + +```php +$downloadedHash = hash_file('sha256', $tempArchive); +if ($downloadedHash !== $release->file_hash) { + throw new RuntimeException("Hash mismatch"); +} +``` + +### Version Markers + +Each extracted version has a marker file: + +``` +storage/app/vendors/{vendor-slug}/{version}/.version_marker +``` + +Contains the version string. Used to verify extraction state. + +## Service Methods + +### VendorStorageService + +```php +// Check storage status +$service->isS3Enabled(): bool +$service->existsLocally(Vendor $vendor, string $version): bool +$service->existsInS3(Vendor $vendor, string $version): bool + +// Path helpers +$service->getLocalPath(Vendor $vendor, string $version): string +$service->getS3Key(Vendor $vendor, string $version): string +$service->getTempPath(?string $suffix = null): string + +// Archive operations +$service->archiveToS3(VersionRelease $release): bool +$service->downloadFromS3(VersionRelease $release, ?string $targetPath = null): string +$service->ensureLocal(VersionRelease $release): string + +// Cleanup +$service->deleteLocalIfAllowed(VersionRelease $release): bool +$service->cleanupTemp(): int + +// Statistics +$service->getStorageStats(): array +$service->getStorageStatus(VersionRelease $release): array +``` + +## Archive Format + +Archives are created as gzipped tarballs: + +```bash +tar -czf {vendor-slug}-{version}.tar.gz -C {source-path} . +``` + +Extraction: +```bash +tar -xzf {archive-path} -C {target-path} +``` + +**Content type:** `application/gzip` + +## S3 Configuration for Hetzner Object Store + +The module supports Hetzner Object Store with dual endpoints: + +```php +'s3' => [ + // Private endpoint for server-side access + 'private_endpoint' => env('S3_PRIVATE_ENDPOINT'), + // Public endpoint for CDN (not used for vendor archives) + 'public_endpoint' => env('S3_PUBLIC_ENDPOINT'), +], +``` + +Vendor archives should always use the private bucket/endpoint. + +## Dashboard Statistics + +The storage service provides statistics for the admin dashboard: + +```php +$stats = $storageService->getStorageStats(); + +// Returns: +[ + 'total_versions' => 42, + 'local_only' => 5, + 's3_only' => 30, + 'both' => 7, + 'local_size' => 1073741824, // bytes + 's3_size' => 5368709120, // bytes +] +``` + +## Troubleshooting + +### "Version not available locally or in S3" + +1. Check if the version was ever uploaded +2. Verify S3 credentials and bucket access +3. Check `s3_key` in version_releases table + +### Archive Fails with "Failed to create archive" + +1. Check tar is installed and in PATH +2. Verify source directory exists and is readable +3. Check available disk space for temp files + +### Download Fails with "Hash mismatch" + +1. The S3 file may be corrupted +2. Re-upload the version if original exists locally +3. Check for encoding issues in hash comparison + +### Local Files Not Being Deleted + +1. Version may be current or previous (protected) +2. Version may be in recent N versions +3. Check `delete_local_after_archive` config diff --git a/docs/webhooks.md b/docs/webhooks.md new file mode 100644 index 0000000..08845cd --- /dev/null +++ b/docs/webhooks.md @@ -0,0 +1,306 @@ +--- +title: Webhooks +description: Webhook configuration and integration guide +updated: 2026-01-29 +--- + +# Webhooks + +The Uptelligence module can receive webhooks from vendor release systems to automatically track new versions. + +## Supported Providers + +| Provider | Event Types | Signature Method | +|----------|-------------|------------------| +| GitHub | Release published/created | HMAC-SHA256 | +| GitLab | Release created, tag push | Token header | +| npm | Package published | HMAC-SHA256 | +| Packagist | Package updated | HMAC-SHA1 | +| Custom | Flexible | HMAC-SHA256 | + +## Endpoint URL + +Each webhook has a unique endpoint: + +``` +POST /api/uptelligence/webhook/{uuid} +``` + +The UUID is generated when the webhook is created and serves as the identifier. + +## Configuring Webhooks + +### GitHub + +1. Go to repository Settings > Webhooks > Add webhook +2. Set Payload URL to: `https://your-domain.com/api/uptelligence/webhook/{uuid}` +3. Set Content type to: `application/json` +4. Set Secret to the webhook's secret (visible in admin panel) +5. Select "Let me select individual events" +6. Check "Releases" only +7. Save webhook + +**Expected headers:** +- `X-Hub-Signature-256: sha256={signature}` +- `X-GitHub-Event: release` + +### GitLab + +1. Go to project Settings > Webhooks +2. Set URL to: `https://your-domain.com/api/uptelligence/webhook/{uuid}` +3. Set Secret token to the webhook's secret +4. Check "Releases events" and optionally "Tag push events" +5. Save webhook + +**Expected headers:** +- `X-Gitlab-Token: {secret}` +- `X-Gitlab-Event: Release Hook` or `Tag Push Hook` + +### npm + +npm webhooks are configured per package via the npm CLI or website. + +```bash +npm hook add @scope/package https://your-domain.com/api/uptelligence/webhook/{uuid} {secret} +``` + +**Expected headers:** +- `X-Npm-Signature: {signature}` + +### Packagist + +Packagist webhooks are configured in the package settings on packagist.org. + +1. Go to package page > Edit +2. Add webhook URL: `https://your-domain.com/api/uptelligence/webhook/{uuid}` +3. Set secret if required + +**Expected headers:** +- `X-Hub-Signature: sha1={signature}` + +### Custom + +For custom integrations, send a POST request with: + +**Headers:** +- `Content-Type: application/json` +- `X-Signature: sha256={hmac_sha256_of_body}` (optional) + +**Body:** +```json +{ + "version": "1.2.3", + "tag_name": "v1.2.3", + "name": "Release Name", + "body": "Release notes...", + "prerelease": false, + "published_at": "2026-01-29T12:00:00Z" +} +``` + +## Signature Verification + +### HMAC-SHA256 (GitHub, npm, Custom) + +```php +$expectedSignature = hash_hmac('sha256', $payload, $secret); +$valid = hash_equals($expectedSignature, $providedSignature); +``` + +The signature header may have a `sha256=` prefix which is stripped before comparison. + +### Token Comparison (GitLab) + +```php +$valid = hash_equals($secret, $providedToken); +``` + +Direct constant-time comparison of the token. + +### HMAC-SHA1 (Packagist) + +```php +$expectedSignature = hash_hmac('sha1', $payload, $secret); +$valid = hash_equals($expectedSignature, $providedSignature); +``` + +## Secret Management + +### Creating a Webhook + +When a webhook is created, a 64-character random secret is automatically generated. + +### Rotating Secrets + +Secrets can be rotated with a grace period: + +```php +$webhook->rotateSecret(); +``` + +This: +1. Moves current secret to `previous_secret` +2. Generates new 64-character secret +3. Sets `secret_rotated_at` to current time + +During the grace period (default 24 hours), both old and new secrets are accepted. + +### Regenerating Without Grace + +To immediately invalidate the old secret: + +```php +$webhook->regenerateSecret(); +``` + +This: +1. Generates new secret +2. Clears `previous_secret` +3. Clears `secret_rotated_at` + +## Delivery Processing + +### Flow + +1. **Receive** - Webhook received at endpoint +2. **Validate** - Check webhook is active, verify signature +3. **Log** - Create `UptelligenceWebhookDelivery` record +4. **Queue** - Dispatch `ProcessUptelligenceWebhook` job +5. **Parse** - Extract version and metadata from payload +6. **Process** - Create/update version release record +7. **Notify** - Send notifications to subscribed users + +### Delivery Statuses + +| Status | Description | +|--------|-------------| +| `pending` | Queued for processing | +| `processing` | Currently being processed | +| `completed` | Successfully processed | +| `failed` | Processing failed | +| `skipped` | Not a release event or unable to parse | + +### Retry Logic + +Failed deliveries are retried with exponential backoff: + +- Attempt 1: Immediate +- Attempt 2: After 30 seconds +- Attempt 3: After 60 seconds (2^2 * 30) +- Attempt 4: After 120 seconds (2^3 * 30) + +Maximum 3 retries by default. + +## Circuit Breaker + +To prevent continuous processing of failing webhooks: + +- After 10 consecutive failures, the webhook is automatically disabled +- Status changes to "Circuit Open" +- Manual intervention required to re-enable + +Reset the circuit breaker: + +```php +$webhook->update(['is_active' => true, 'failure_count' => 0]); +``` + +## Monitoring + +### Admin Dashboard + +The Webhook Manager component shows: +- Webhook status (Active, Disabled, Circuit Open) +- Last received timestamp +- Failure count +- Recent deliveries with status + +### Deliveries Log + +Each delivery records: +- Event type +- Provider +- Version extracted +- Payload (full JSON) +- Parsed data (normalised) +- Source IP +- Signature status +- Processing time +- Error message (if failed) + +## Rate Limiting + +Webhooks are rate-limited to prevent abuse: + +- **60 requests/minute per webhook UUID** +- **30 requests/minute per IP** (fallback for unknown webhooks) + +Rate limit exceeded returns `429 Too Many Requests`. + +## Testing Webhooks + +### Test Endpoint + +``` +POST /api/uptelligence/webhook/{uuid}/test +``` + +Returns webhook configuration status without processing: + +```json +{ + "status": "ok", + "webhook_id": "uuid-here", + "vendor_id": 1, + "provider": "github", + "is_active": true, + "signature_status": "valid", + "has_secret": true +} +``` + +### Manual Testing + +Use curl to test a webhook: + +```bash +# Generate signature +SECRET="your-webhook-secret" +PAYLOAD='{"tag_name":"v1.0.0","name":"Test Release"}' +SIGNATURE=$(echo -n "$PAYLOAD" | openssl dgst -sha256 -hmac "$SECRET" | cut -d' ' -f2) + +# Send request +curl -X POST \ + -H "Content-Type: application/json" \ + -H "X-Hub-Signature-256: sha256=$SIGNATURE" \ + -d "$PAYLOAD" \ + https://your-domain.com/api/uptelligence/webhook/{uuid} +``` + +## Troubleshooting + +### "Invalid signature" Response + +1. Verify the secret matches between provider and webhook config +2. Check that the payload is sent as raw JSON (not form-encoded) +3. Ensure the signature algorithm matches the provider +4. Check for encoding issues (UTF-8 BOM, etc.) + +### "Webhook disabled" Response + +1. Check if circuit breaker has tripped (failure_count >= 10) +2. Verify `is_active` is true in database +3. Re-enable via admin panel or API + +### Deliveries Not Processing + +1. Check queue worker is running: `php artisan queue:work --queue=uptelligence-webhooks` +2. Check for failed jobs: `php artisan queue:failed` +3. Review delivery status in admin panel + +### Missing Release Record + +1. Verify the event type is supported (release/publish, not push) +2. Check parsed_data in delivery record +3. Ensure version extraction succeeded +4. Check if version already exists (duplicate webhooks are de-duplicated) diff --git a/tests/Unit/AssetTrackerServiceTest.php b/tests/Unit/AssetTrackerServiceTest.php new file mode 100644 index 0000000..4325c18 --- /dev/null +++ b/tests/Unit/AssetTrackerServiceTest.php @@ -0,0 +1,270 @@ +service = new AssetTrackerService; + } + + protected function getPackageProviders($app): array + { + return []; + } + + /** + * Test that valid Composer package names are accepted. + */ + #[Test] + #[DataProvider('validComposerPackageNames')] + public function it_accepts_valid_composer_package_names(string $packageName): void + { + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('validatePackageName'); + $method->setAccessible(true); + + $result = $method->invoke($this->service, $packageName, Asset::TYPE_COMPOSER); + + $this->assertEquals($packageName, $result); + } + + /** + * Test that valid NPM package names are accepted. + */ + #[Test] + #[DataProvider('validNpmPackageNames')] + public function it_accepts_valid_npm_package_names(string $packageName): void + { + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('validatePackageName'); + $method->setAccessible(true); + + $result = $method->invoke($this->service, $packageName, Asset::TYPE_NPM); + + $this->assertEquals($packageName, $result); + } + + /** + * Test that shell injection attempts in Composer package names are rejected. + */ + #[Test] + #[DataProvider('shellInjectionAttempts')] + public function it_rejects_shell_injection_in_composer_package_names(string $maliciousInput): void + { + Log::shouldReceive('warning')->once(); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('validatePackageName'); + $method->setAccessible(true); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('Invalid package name format'); + + $method->invoke($this->service, $maliciousInput, Asset::TYPE_COMPOSER); + } + + /** + * Test that shell injection attempts in NPM package names are rejected. + */ + #[Test] + #[DataProvider('shellInjectionAttempts')] + public function it_rejects_shell_injection_in_npm_package_names(string $maliciousInput): void + { + Log::shouldReceive('warning')->once(); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('validatePackageName'); + $method->setAccessible(true); + + $this->expectException(\InvalidArgumentException::class); + $this->expectExceptionMessage('Invalid package name format'); + + $method->invoke($this->service, $maliciousInput, Asset::TYPE_NPM); + } + + /** + * Test that updateComposerPackage returns error for invalid package name. + */ + #[Test] + public function it_returns_error_for_invalid_composer_package_in_update(): void + { + Log::shouldReceive('warning')->once(); + + $asset = new Asset([ + 'package_name' => 'vendor/package; rm -rf /', + 'type' => Asset::TYPE_COMPOSER, + ]); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('updateComposerPackage'); + $method->setAccessible(true); + + $result = $method->invoke($this->service, $asset); + + $this->assertEquals('error', $result['status']); + $this->assertEquals('Invalid package name format', $result['message']); + } + + /** + * Test that updateNpmPackage returns error for invalid package name. + */ + #[Test] + public function it_returns_error_for_invalid_npm_package_in_update(): void + { + Log::shouldReceive('warning')->once(); + + $asset = new Asset([ + 'package_name' => 'package`whoami`', + 'type' => Asset::TYPE_NPM, + ]); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('updateNpmPackage'); + $method->setAccessible(true); + + $result = $method->invoke($this->service, $asset); + + $this->assertEquals('error', $result['status']); + $this->assertEquals('Invalid package name format', $result['message']); + } + + /** + * Test that checkCustomComposerRegistry returns error for invalid package name. + */ + #[Test] + public function it_returns_error_for_invalid_package_in_custom_registry_check(): void + { + Log::shouldReceive('warning')->once(); + + $asset = new Asset([ + 'package_name' => '$(cat /etc/passwd)', + 'type' => Asset::TYPE_COMPOSER, + 'registry_url' => 'https://custom.registry.com', + ]); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('checkCustomComposerRegistry'); + $method->setAccessible(true); + + $result = $method->invoke($this->service, $asset); + + $this->assertEquals('error', $result['status']); + $this->assertEquals('Invalid package name format', $result['message']); + } + + /** + * Test that Process::run is called with array syntax for valid packages. + */ + #[Test] + public function it_uses_array_syntax_for_process_run(): void + { + Process::fake([ + '*' => Process::result(output: '{"versions":["1.0.0"]}'), + ]); + + $asset = new Asset([ + 'package_name' => 'vendor/package', + 'type' => Asset::TYPE_COMPOSER, + 'registry_url' => 'https://custom.registry.com', + ]); + + $reflection = new \ReflectionClass($this->service); + $method = $reflection->getMethod('checkCustomComposerRegistry'); + $method->setAccessible(true); + + $method->invoke($this->service, $asset); + + // Verify array syntax was used (not string interpolation) + Process::assertRan(function ($process) { + // The command should be an array, not a string with interpolation + return is_array($process->command) && + $process->command === ['composer', 'show', 'vendor/package', '--format=json']; + }); + } + + /** + * Data provider for valid Composer package names. + */ + public static function validComposerPackageNames(): array + { + return [ + 'simple package' => ['vendor/package'], + 'with hyphen' => ['my-vendor/my-package'], + 'with underscore' => ['my_vendor/my_package'], + 'with dots' => ['vendor.name/package.name'], + 'with numbers' => ['vendor123/package456'], + 'laravel package' => ['laravel/framework'], + 'symfony component' => ['symfony/console'], + 'complex name' => ['my-vendor123/complex_package.name'], + 'livewire flux' => ['livewire/flux-pro'], + ]; + } + + /** + * Data provider for valid NPM package names. + */ + public static function validNpmPackageNames(): array + { + return [ + 'simple package' => ['lodash'], + 'with hyphen' => ['my-package'], + 'with underscore' => ['my_package'], + 'with dot' => ['package.js'], + 'scoped package' => ['@scope/package'], + 'scoped with hyphen' => ['@my-scope/my-package'], + 'scoped complex' => ['@angular/core'], + 'alpinejs' => ['alpinejs'], + 'tailwindcss' => ['tailwindcss'], + 'vue' => ['vue'], + ]; + } + + /** + * Data provider for shell injection attempts. + */ + public static function shellInjectionAttempts(): array + { + return [ + 'command substitution with backticks' => ['package`whoami`'], + 'command substitution with $()' => ['$(cat /etc/passwd)'], + 'semicolon injection' => ['vendor/package; rm -rf /'], + 'pipe injection' => ['vendor/package | cat /etc/passwd'], + 'ampersand injection' => ['vendor/package && rm -rf /'], + 'or injection' => ['vendor/package || rm -rf /'], + 'newline injection' => ["vendor/package\nrm -rf /"], + 'redirect injection' => ['vendor/package > /tmp/pwned'], + 'redirect input injection' => ['vendor/package < /etc/passwd'], + 'single quote escape' => ["vendor/package'; rm -rf /'"], + 'double quote escape' => ['vendor/package"; rm -rf /"'], + 'space injection' => ['vendor/package rm -rf /'], + 'glob injection' => ['vendor/*'], + 'question mark glob' => ['vendor/pack?ge'], + 'bracket glob' => ['vendor/pack[a]ge'], + 'curly brace expansion' => ['vendor/{a,b}'], + 'tilde expansion' => ['~/../../etc/passwd'], + 'null byte injection' => ["vendor/package\x00rm"], + 'env variable injection' => ['$HOME/package'], + 'backtick in scoped npm' => ['@scope`whoami`/package'], + ]; + } +}