e3efb59d98
This commit introduces a deduplication cache to avoid re-downloading files across multiple collection jobs. Key changes include: - A new `pkg/cache` package that provides content-addressable storage using SHA256 hashes of the file content. - Integration of the cache into the `collect website` command. Downloads are now skipped if the content already exists in the cache. - The addition of `--no-cache` and `--cache-dir` flags to give users control over the caching behavior. - New `borg cache stats` and `borg cache clear` commands to allow users to manage the cache. - A performance improvement to the cache implementation, which now only writes the URL-to-hash index file once at the end of the collection process, rather than on every file download. - Centralized logic for determining the default cache directory, removing code duplication. - Improved error handling and refactored duplicated cache-checking logic in the website collector. - Added comprehensive unit tests for the new cache package and an integration test to verify that the website collector correctly uses the cache. The implementation of cache size limiting and LRU eviction is still pending and will be addressed in a future commit. Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
87 lines
1.9 KiB
Go
87 lines
1.9 KiB
Go
package cache
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func TestCache_Good(t *testing.T) {
|
|
cacheDir := t.TempDir()
|
|
c, err := New(cacheDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create cache: %v", err)
|
|
}
|
|
|
|
// Test Put and Get
|
|
url := "https://example.com"
|
|
data := []byte("hello world")
|
|
err = c.Put(url, data)
|
|
if err != nil {
|
|
t.Fatalf("Failed to put data in cache: %v", err)
|
|
}
|
|
|
|
cachedData, ok, err := c.Get(url)
|
|
if err != nil {
|
|
t.Fatalf("Failed to get data from cache: %v", err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("Expected data to be in cache, but it wasn't")
|
|
}
|
|
if string(cachedData) != string(data) {
|
|
t.Errorf("Expected data to be '%s', but got '%s'", string(data), string(cachedData))
|
|
}
|
|
|
|
// Test persistence
|
|
err = c.Close()
|
|
if err != nil {
|
|
t.Fatalf("Failed to close cache: %v", err)
|
|
}
|
|
|
|
c2, err := New(cacheDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create new cache: %v", err)
|
|
}
|
|
cachedData, ok, err = c2.Get(url)
|
|
if err != nil {
|
|
t.Fatalf("Failed to get data from new cache: %v", err)
|
|
}
|
|
if !ok {
|
|
t.Fatal("Expected data to be in new cache, but it wasn't")
|
|
}
|
|
if string(cachedData) != string(data) {
|
|
t.Errorf("Expected data to be '%s', but got '%s'", string(data), string(cachedData))
|
|
}
|
|
}
|
|
|
|
func TestCache_Clear(t *testing.T) {
|
|
cacheDir := t.TempDir()
|
|
c, err := New(cacheDir)
|
|
if err != nil {
|
|
t.Fatalf("Failed to create cache: %v", err)
|
|
}
|
|
|
|
// Put some data in the cache
|
|
url := "https://example.com"
|
|
data := []byte("hello world")
|
|
err = c.Put(url, data)
|
|
if err != nil {
|
|
t.Fatalf("Failed to put data in cache: %v", err)
|
|
}
|
|
err = c.Close()
|
|
if err != nil {
|
|
t.Fatalf("Failed to close cache: %v", err)
|
|
}
|
|
|
|
// Clear the cache
|
|
err = c.Clear()
|
|
if err != nil {
|
|
t.Fatalf("Failed to clear cache: %v", err)
|
|
}
|
|
|
|
// Check that the cache directory is gone
|
|
_, err = os.Stat(cacheDir)
|
|
if !os.IsNotExist(err) {
|
|
t.Fatal("Expected cache directory to be gone, but it wasn't")
|
|
}
|
|
}
|