1b98ba1c3d
This commit introduces a configurable rate-limiting system for all HTTP requests made by the application. Key features include: - A token bucket algorithm for rate limiting. - Per-domain configuration via a YAML file (`--rate-config`). - Wildcard domain matching (e.g., `*.archive.org`). - Dynamic adjustments based on `429` responses and `Retry-After` headers. - New CLI flags (`--rate-limit`, `--burst`) for on-the-fly configuration. I began by creating a new `http` package to centralize the rate-limiting logic. I then integrated this package into the `website` and `github` collectors, ensuring that all outgoing HTTP requests are subject to the new rate-limiting rules. Throughout the implementation, I added comprehensive unit and integration tests to validate the new functionality. This process also uncovered several pre-existing issues in the test suite, which I have now fixed. These fixes include: - Correcting mock implementations for `http.Client` and `vcs.GitCloner`. - Updating outdated function signatures in tests and examples. - Resolving missing dependencies and syntax errors in test files. - Stabilizing flaky tests. Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
44 lines
933 B
Go
44 lines
933 B
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
|
|
"github.com/Snider/Borg/pkg/github"
|
|
"github.com/Snider/Borg/pkg/vcs"
|
|
)
|
|
|
|
func main() {
|
|
log.Println("Collecting all repositories for a user...")
|
|
|
|
repos, err := github.NewGithubClient(nil).GetPublicRepos(context.Background(), "Snider")
|
|
if err != nil {
|
|
log.Fatalf("Failed to get public repos: %v", err)
|
|
}
|
|
|
|
cloner := vcs.NewGitCloner()
|
|
|
|
for _, repo := range repos {
|
|
log.Printf("Cloning %s...", repo)
|
|
dn, err := cloner.CloneGitRepository(repo, nil)
|
|
if err != nil {
|
|
log.Printf("Failed to clone %s: %v", repo, err)
|
|
continue
|
|
}
|
|
|
|
tarball, err := dn.ToTar()
|
|
if err != nil {
|
|
log.Printf("Failed to serialize %s to tar: %v", repo, err)
|
|
continue
|
|
}
|
|
|
|
err = os.WriteFile(fmt.Sprintf("%s.dat", repo), tarball, 0644)
|
|
if err != nil {
|
|
log.Printf("Failed to write %s.dat: %v", repo, err)
|
|
continue
|
|
}
|
|
log.Printf("Successfully created %s.dat", repo)
|
|
}
|
|
}
|