Borg/pkg/httpclient/client_test.go
google-labs-jules[bot] e057ddfa7f feat: Add Proxy and Tor Support for Website Collector
This commit introduces support for routing website collection requests through HTTP/SOCKS5 proxies or Tor.

New flags for the `borg collect website` command:
- `--proxy <url>`: Route requests through a single HTTP or SOCKS5 proxy.
- `--proxy-list <file>`: Select a random proxy from a file for all requests.
- `--tor`: Route requests through a Tor SOCKS5 proxy (defaults to 127.0.0.1:9050).

Key changes:
- Created a new `pkg/httpclient` to centralize the creation of proxy-configured `http.Client` instances.
- Refactored `pkg/website` to use a `Downloader` interface, allowing for dependency injection of the HTTP client.
- Added validation to ensure the new proxy flags are mutually exclusive.
- Implemented support for SOCKS5 authentication via credentials in the proxy URL.
- Added comprehensive unit tests for the new functionality.

Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
2026-02-02 00:52:32 +00:00

124 lines
3.2 KiB
Go

package httpclient
import (
"net/http"
"net/url"
"os"
"testing"
)
func TestNewClient(t *testing.T) {
// Test case for no proxy
t.Run("NoProxy", func(t *testing.T) {
client, err := NewClient("", "", false)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if client.Transport != nil {
t.Errorf("Expected nil Transport, got %T", client.Transport)
}
})
// Test case for Tor
t.Run("Tor", func(t *testing.T) {
client, err := NewClient("", "", true)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
transport, ok := client.Transport.(*http.Transport)
if !ok {
t.Fatalf("Expected http.Transport, got %T", client.Transport)
}
if transport.Dial == nil {
t.Error("Expected a custom dialer for Tor, got nil")
}
})
// Test case for a single proxy
t.Run("SingleProxy", func(t *testing.T) {
proxyURL := "http://localhost:8080"
client, err := NewClient(proxyURL, "", false)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
transport, ok := client.Transport.(*http.Transport)
if !ok {
t.Fatalf("Expected http.Transport, got %T", client.Transport)
}
proxyFunc := transport.Proxy
if proxyFunc == nil {
t.Fatal("Expected a proxy function, got nil")
}
req, _ := http.NewRequest("GET", "http://example.com", nil)
proxy, err := proxyFunc(req)
if err != nil {
t.Fatalf("Expected no error from proxy func, got %v", err)
}
expectedProxy, _ := url.Parse(proxyURL)
if proxy.String() != expectedProxy.String() {
t.Errorf("Expected proxy %s, got %s", expectedProxy, proxy)
}
})
// Test case for a proxy list
t.Run("ProxyList", func(t *testing.T) {
proxies := []string{
"http://localhost:8081",
"http://localhost:8082",
"http://localhost:8083",
}
proxyFile, err := os.CreateTemp("", "proxies.txt")
if err != nil {
t.Fatalf("Failed to create temp proxy file: %v", err)
}
defer os.Remove(proxyFile.Name())
for _, p := range proxies {
proxyFile.WriteString(p + "\n")
}
proxyFile.Close()
client, err := NewClient("", proxyFile.Name(), false)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
transport, ok := client.Transport.(*http.Transport)
if !ok {
t.Fatalf("Expected http.Transport, got %T", client.Transport)
}
proxyFunc := transport.Proxy
if proxyFunc == nil {
t.Fatal("Expected a proxy function, got nil")
}
req, _ := http.NewRequest("GET", "http://example.com", nil)
proxy, err := proxyFunc(req)
if err != nil {
t.Fatalf("Expected no error from proxy func, got %v", err)
}
found := false
for _, p := range proxies {
if proxy.String() == p {
found = true
break
}
}
if !found {
t.Errorf("Expected proxy to be one of %v, got %s", proxies, proxy)
}
})
t.Run("SOCKS5WithAuth", func(t *testing.T) {
proxyURL := "socks5://user:password@localhost:1080"
client, err := NewClient(proxyURL, "", false)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
transport, ok := client.Transport.(*http.Transport)
if !ok {
t.Fatalf("Expected http.Transport, got %T", client.Transport)
}
if transport.Dial == nil {
t.Fatal("Expected a custom dialer for SOCKS5, got nil")
}
})
}