2ff65938ca
Implement a circuit breaker for the website collector to prevent hammering domains that are consistently failing. The circuit breaker has three states: CLOSED, OPEN, and HALF-OPEN. It tracks failures per-domain and will open the circuit after a configurable number of consecutive failures. After a cooldown period, the circuit will transition to HALF-OPEN and allow a limited number of test requests to check for recovery. The following command-line flags have been added to the `collect website` command: - `--no-circuit-breaker`: Disable the circuit breaker - `--circuit-failures`: Number of failures to trip the circuit breaker - `--circuit-cooldown`: Cooldown time for the circuit breaker - `--circuit-success-threshold`: Number of successes to close the circuit breaker - `--circuit-half-open-requests`: Number of test requests in the half-open state The implementation also includes: - A new `circuitbreaker` package with the core logic - Integration into the `website` package with per-domain tracking - Improved logging to include the domain name and state changes - Integration tests to verify the circuit breaker's behavior Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
101 lines
2.3 KiB
Go
101 lines
2.3 KiB
Go
|
|
package circuitbreaker
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestCircuitBreaker(t *testing.T) {
|
|
settings := Settings{
|
|
FailureThreshold: 2,
|
|
SuccessThreshold: 2,
|
|
Cooldown: 100 * time.Millisecond,
|
|
HalfOpenRequests: 2,
|
|
}
|
|
cb := New("test.com", settings)
|
|
|
|
// Initially closed
|
|
_, err := cb.Execute(func() (interface{}, error) {
|
|
return "success", nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("Expected success, got %v", err)
|
|
}
|
|
|
|
// Trip the breaker
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return nil, errors.New("failure 1")
|
|
})
|
|
if err == nil {
|
|
t.Error("Expected failure, got nil")
|
|
}
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return nil, errors.New("failure 2")
|
|
})
|
|
if err == nil {
|
|
t.Error("Expected failure, got nil")
|
|
}
|
|
|
|
// Now open
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return "should not be called", nil
|
|
})
|
|
if err == nil || err.Error() != "circuit is open for: failure 2" {
|
|
t.Errorf("Expected open circuit error, got %v", err)
|
|
}
|
|
|
|
// Wait for cooldown
|
|
time.Sleep(150 * time.Millisecond)
|
|
|
|
// Half-open, should succeed
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return "success", nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("Expected success in half-open, got %v", err)
|
|
}
|
|
|
|
// Still half-open, need another success
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return "success", nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("Expected success in half-open, got %v", err)
|
|
}
|
|
|
|
// Now closed again
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return "success", nil
|
|
})
|
|
if err != nil {
|
|
t.Errorf("Expected success in closed state, got %v", err)
|
|
}
|
|
|
|
// Trip again to test half-open failure
|
|
cb.Execute(func() (interface{}, error) {
|
|
return nil, errors.New("failure 1")
|
|
})
|
|
cb.Execute(func() (interface{}, error) {
|
|
return nil, errors.New("failure 2")
|
|
})
|
|
|
|
time.Sleep(150 * time.Millisecond)
|
|
|
|
// Half-open, but fail
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return nil, errors.New("half-open failure")
|
|
})
|
|
if err == nil {
|
|
t.Error("Expected failure in half-open, got nil")
|
|
}
|
|
|
|
// Should be open again
|
|
_, err = cb.Execute(func() (interface{}, error) {
|
|
return "should not be called", nil
|
|
})
|
|
if err == nil || err.Error() != "circuit is open for: half-open failure" {
|
|
t.Errorf("Expected open circuit error, got %v", err)
|
|
}
|
|
}
|