e3efb59d98
This commit introduces a deduplication cache to avoid re-downloading files across multiple collection jobs. Key changes include: - A new `pkg/cache` package that provides content-addressable storage using SHA256 hashes of the file content. - Integration of the cache into the `collect website` command. Downloads are now skipped if the content already exists in the cache. - The addition of `--no-cache` and `--cache-dir` flags to give users control over the caching behavior. - New `borg cache stats` and `borg cache clear` commands to allow users to manage the cache. - A performance improvement to the cache implementation, which now only writes the URL-to-hash index file once at the end of the collection process, rather than on every file download. - Centralized logic for determining the default cache directory, removing code duplication. - Improved error handling and refactored duplicated cache-checking logic in the website collector. - Added comprehensive unit tests for the new cache package and an integration test to verify that the website collector correctly uses the cache. The implementation of cache size limiting and LRU eviction is still pending and will be addressed in a future commit. Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
46 lines
1.2 KiB
Go
46 lines
1.2 KiB
Go
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
func NewRootCmd() *cobra.Command {
|
|
rootCmd := &cobra.Command{
|
|
Use: "borg",
|
|
Short: "A tool for collecting and managing data.",
|
|
Long: `Borg Data Collector is a command-line tool for cloning Git repositories,
|
|
packaging their contents into a single file, and managing the data within.`,
|
|
}
|
|
|
|
rootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging")
|
|
return rootCmd
|
|
}
|
|
|
|
// RootCmd represents the base command when called without any subcommands
|
|
var RootCmd = NewRootCmd()
|
|
|
|
// Execute adds all child commands to the root command and sets flags appropriately.
|
|
// This is called by main.main(). It only needs to happen once to the rootCmd.
|
|
func Execute(log *slog.Logger) error {
|
|
RootCmd.SetContext(context.WithValue(context.Background(), "logger", log))
|
|
return RootCmd.Execute()
|
|
}
|
|
|
|
func GetCacheDir(cmd *cobra.Command) (string, error) {
|
|
cacheDir, _ := cmd.Flags().GetString("cache-dir")
|
|
if cacheDir != "" {
|
|
return cacheDir, nil
|
|
}
|
|
|
|
home, err := os.UserHomeDir()
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to get user home directory: %w", err)
|
|
}
|
|
return filepath.Join(home, ".borg", "cache"), nil
|
|
}
|