forked from core/php-mcp
Cover all three circuit states (closed, open, half-open), failure threshold logic, reset timing, manual reset, success tracking, recoverable error detection, and per-service isolation. Fixes #6 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
605 lines
21 KiB
PHP
605 lines
21 KiB
PHP
<?php
|
|
|
|
/*
|
|
* Core MCP Package
|
|
*
|
|
* Licensed under the European Union Public Licence (EUPL) v1.2.
|
|
* See LICENSE file for details.
|
|
*/
|
|
|
|
declare(strict_types=1);
|
|
|
|
use Core\Mcp\Exceptions\CircuitOpenException;
|
|
use Core\Mcp\Services\CircuitBreaker;
|
|
use Illuminate\Support\Facades\Cache;
|
|
use Illuminate\Support\Facades\Config;
|
|
use Illuminate\Support\Facades\Log;
|
|
|
|
beforeEach(function () {
|
|
$this->breaker = new CircuitBreaker();
|
|
$this->service = 'test-service';
|
|
|
|
// Ensure clean state before each test
|
|
Cache::flush();
|
|
|
|
// Set default config values
|
|
Config::set('mcp.circuit_breaker.default_threshold', 5);
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 60);
|
|
Config::set('mcp.circuit_breaker.default_failure_window', 120);
|
|
});
|
|
|
|
describe('CircuitBreaker', function () {
|
|
|
|
describe('closed state (pass-through)', function () {
|
|
it('starts in closed state by default', function () {
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_CLOSED);
|
|
});
|
|
|
|
it('passes operations through when circuit is closed', function () {
|
|
$result = $this->breaker->call($this->service, fn () => 'success');
|
|
|
|
expect($result)->toBe('success');
|
|
});
|
|
|
|
it('returns the operation result', function () {
|
|
$result = $this->breaker->call($this->service, fn () => ['data' => 42]);
|
|
|
|
expect($result)->toBe(['data' => 42]);
|
|
});
|
|
|
|
it('reports service as available when closed', function () {
|
|
expect($this->breaker->isAvailable($this->service))->toBeTrue();
|
|
});
|
|
|
|
it('re-throws exceptions from the operation', function () {
|
|
expect(fn () => $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('operation failed'),
|
|
))->toThrow(RuntimeException::class, 'operation failed');
|
|
});
|
|
|
|
it('uses fallback on recoverable error when circuit is closed', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('Connection refused'),
|
|
fn () => 'fallback-value',
|
|
);
|
|
|
|
expect($result)->toBe('fallback-value');
|
|
});
|
|
|
|
it('does not use fallback on non-recoverable error', function () {
|
|
expect(fn () => $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('some random error'),
|
|
fn () => 'fallback-value',
|
|
))->toThrow(RuntimeException::class, 'some random error');
|
|
});
|
|
});
|
|
|
|
describe('open state (fast fail with CircuitOpenException)', function () {
|
|
it('throws CircuitOpenException when circuit is open', function () {
|
|
// Trip the circuit by exceeding the failure threshold
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
|
|
expect(fn () => $this->breaker->call(
|
|
$this->service,
|
|
fn () => 'should not execute',
|
|
))->toThrow(CircuitOpenException::class);
|
|
});
|
|
|
|
it('does not execute the operation when circuit is open', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
$executed = false;
|
|
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
function () use (&$executed) {
|
|
$executed = true;
|
|
|
|
return 'should not run';
|
|
},
|
|
);
|
|
} catch (CircuitOpenException) {
|
|
// Expected
|
|
}
|
|
|
|
expect($executed)->toBeFalse();
|
|
});
|
|
|
|
it('uses fallback when circuit is open and fallback is provided', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => 'should not execute',
|
|
fn () => 'fallback-result',
|
|
);
|
|
|
|
expect($result)->toBe('fallback-result');
|
|
});
|
|
|
|
it('reports service as unavailable when open', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->isAvailable($this->service))->toBeFalse();
|
|
});
|
|
|
|
it('includes service name in CircuitOpenException', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
try {
|
|
$this->breaker->call($this->service, fn () => 'nope');
|
|
test()->fail('Expected CircuitOpenException');
|
|
} catch (CircuitOpenException $e) {
|
|
expect($e->service)->toBe($this->service);
|
|
expect($e->getMessage())->toContain($this->service);
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('half-open state (probe)', function () {
|
|
it('transitions from open to half-open after reset timeout', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 1);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
|
|
// Simulate time passing by manipulating the opened_at cache value
|
|
Cache::put('circuit_breaker:'.$this->service.':opened_at', time() - 2, 86400);
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_HALF_OPEN);
|
|
});
|
|
|
|
it('closes circuit on successful probe in half-open state', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 1);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
// Simulate reset timeout elapsed
|
|
Cache::put('circuit_breaker:'.$this->service.':opened_at', time() - 2, 86400);
|
|
|
|
// Successful probe should close the circuit
|
|
$result = $this->breaker->call($this->service, fn () => 'recovered');
|
|
|
|
expect($result)->toBe('recovered');
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_CLOSED);
|
|
});
|
|
|
|
it('re-opens circuit on failed probe in half-open state', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 1);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('service down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
// Simulate reset timeout elapsed
|
|
Cache::put('circuit_breaker:'.$this->service.':opened_at', time() - 2, 86400);
|
|
|
|
// Failed probe
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('still down'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
|
|
// Should be open again (failure count exceeds threshold)
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
});
|
|
});
|
|
|
|
describe('failure threshold', function () {
|
|
it('does not trip circuit below threshold', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 5);
|
|
|
|
// Cause 4 failures (below threshold of 5)
|
|
for ($i = 0; $i < 4; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_CLOSED);
|
|
});
|
|
|
|
it('trips circuit at exactly the threshold', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
});
|
|
|
|
it('respects per-service threshold configuration', function () {
|
|
Config::set('mcp.circuit_breaker.custom-svc.threshold', 2);
|
|
|
|
// 2 failures should trip with per-service threshold of 2
|
|
for ($i = 0; $i < 2; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
'custom-svc',
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState('custom-svc'))->toBe(CircuitBreaker::STATE_OPEN);
|
|
});
|
|
|
|
it('tracks failure count in stats', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 5);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
$stats = $this->breaker->getStats($this->service);
|
|
|
|
expect($stats['failures'])->toBe(3);
|
|
expect($stats['service'])->toBe($this->service);
|
|
expect($stats['threshold'])->toBe(5);
|
|
});
|
|
|
|
it('records last failure details', function () {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('specific error message'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
|
|
$stats = $this->breaker->getStats($this->service);
|
|
|
|
expect($stats['last_failure'])->not->toBeNull();
|
|
expect($stats['last_failure']['message'])->toBe('specific error message');
|
|
expect($stats['last_failure']['class'])->toBe(RuntimeException::class);
|
|
});
|
|
});
|
|
|
|
describe('reset timing', function () {
|
|
it('respects the default reset timeout', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 60);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
// Set opened_at to 30 seconds ago (less than 60s timeout)
|
|
Cache::put('circuit_breaker:'.$this->service.':opened_at', time() - 30, 86400);
|
|
|
|
// Should still be open
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
|
|
// Set opened_at to 61 seconds ago (past 60s timeout)
|
|
Cache::put('circuit_breaker:'.$this->service.':opened_at', time() - 61, 86400);
|
|
|
|
// Should now be half-open
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_HALF_OPEN);
|
|
});
|
|
|
|
it('respects per-service reset timeout', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
Config::set('mcp.circuit_breaker.custom-svc.threshold', 3);
|
|
Config::set('mcp.circuit_breaker.custom-svc.reset_timeout', 10);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
'custom-svc',
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
// Set opened_at to 11 seconds ago (past 10s per-service timeout)
|
|
Cache::put('circuit_breaker:custom-svc:opened_at', time() - 11, 86400);
|
|
|
|
expect($this->breaker->getState('custom-svc'))->toBe(CircuitBreaker::STATE_HALF_OPEN);
|
|
});
|
|
|
|
it('reports reset timeout in stats', function () {
|
|
Config::set('mcp.circuit_breaker.default_reset_timeout', 90);
|
|
|
|
$stats = $this->breaker->getStats($this->service);
|
|
|
|
expect($stats['reset_timeout'])->toBe(90);
|
|
});
|
|
});
|
|
|
|
describe('manual reset', function () {
|
|
it('resets an open circuit to closed state', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
// Trip the circuit
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_OPEN);
|
|
|
|
$this->breaker->reset($this->service);
|
|
|
|
expect($this->breaker->getState($this->service))->toBe(CircuitBreaker::STATE_CLOSED);
|
|
expect($this->breaker->isAvailable($this->service))->toBeTrue();
|
|
});
|
|
|
|
it('clears failure counters on reset', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
$this->breaker->reset($this->service);
|
|
|
|
$stats = $this->breaker->getStats($this->service);
|
|
|
|
expect($stats['failures'])->toBe(0);
|
|
expect($stats['last_failure'])->toBeNull();
|
|
expect($stats['opened_at'])->toBeNull();
|
|
});
|
|
});
|
|
|
|
describe('success tracking', function () {
|
|
it('increments success counter on successful call', function () {
|
|
$this->breaker->call($this->service, fn () => 'ok');
|
|
$this->breaker->call($this->service, fn () => 'ok');
|
|
|
|
$stats = $this->breaker->getStats($this->service);
|
|
|
|
expect($stats['successes'])->toBe(2);
|
|
});
|
|
|
|
it('decays failure count on successful call', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 5);
|
|
|
|
// Cause 3 failures
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
$statsBefore = $this->breaker->getStats($this->service);
|
|
expect($statsBefore['failures'])->toBe(3);
|
|
|
|
// One success should decay failure count
|
|
$this->breaker->call($this->service, fn () => 'ok');
|
|
|
|
$statsAfter = $this->breaker->getStats($this->service);
|
|
expect($statsAfter['failures'])->toBeLessThan(3);
|
|
});
|
|
});
|
|
|
|
describe('recoverable error detection', function () {
|
|
it('treats SQLSTATE errors as recoverable', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('SQLSTATE[HY000]: General error'),
|
|
fn () => 'fallback',
|
|
);
|
|
|
|
expect($result)->toBe('fallback');
|
|
});
|
|
|
|
it('treats connection refused as recoverable', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('Connection refused'),
|
|
fn () => 'fallback',
|
|
);
|
|
|
|
expect($result)->toBe('fallback');
|
|
});
|
|
|
|
it('treats connection timed out as recoverable', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('Connection timed out'),
|
|
fn () => 'fallback',
|
|
);
|
|
|
|
expect($result)->toBe('fallback');
|
|
});
|
|
|
|
it('treats too many connections as recoverable', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('Too many connections'),
|
|
fn () => 'fallback',
|
|
);
|
|
|
|
expect($result)->toBe('fallback');
|
|
});
|
|
|
|
it('treats table not found as recoverable', function () {
|
|
$result = $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException("Base table or view not found: Table 'users' doesn't exist"),
|
|
fn () => 'fallback',
|
|
);
|
|
|
|
expect($result)->toBe('fallback');
|
|
});
|
|
|
|
it('does not treat generic errors as recoverable', function () {
|
|
expect(fn () => $this->breaker->call(
|
|
$this->service,
|
|
fn () => throw new RuntimeException('something unexpected'),
|
|
fn () => 'fallback',
|
|
))->toThrow(RuntimeException::class, 'something unexpected');
|
|
});
|
|
});
|
|
|
|
describe('circuit isolation', function () {
|
|
it('maintains independent state per service', function () {
|
|
Config::set('mcp.circuit_breaker.default_threshold', 3);
|
|
|
|
// Trip circuit for service-a
|
|
for ($i = 0; $i < 3; $i++) {
|
|
try {
|
|
$this->breaker->call(
|
|
'service-a',
|
|
fn () => throw new RuntimeException('failure'),
|
|
);
|
|
} catch (RuntimeException) {
|
|
// Expected
|
|
}
|
|
}
|
|
|
|
expect($this->breaker->getState('service-a'))->toBe(CircuitBreaker::STATE_OPEN);
|
|
expect($this->breaker->getState('service-b'))->toBe(CircuitBreaker::STATE_CLOSED);
|
|
expect($this->breaker->isAvailable('service-b'))->toBeTrue();
|
|
});
|
|
});
|
|
});
|