diff --git a/cmd/all.go b/cmd/all.go index fe152e6..02767ef 100644 --- a/cmd/all.go +++ b/cmd/all.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/vcs" "github.com/spf13/cobra" @@ -31,8 +32,10 @@ var allCmd = &cobra.Command{ for _, repoURL := range repos { log.Info("cloning repository", "url", repoURL) + bar := ui.NewProgressBar(-1, "Cloning repository") + defer bar.Finish() - dn, err := vcs.CloneGitRepository(repoURL) + dn, err := vcs.CloneGitRepository(repoURL, bar) if err != nil { log.Error("failed to clone repository", "url", repoURL, "err", err) continue diff --git a/cmd/collect_github_repo.go b/cmd/collect_github_repo.go index b4f873d..e66f7aa 100644 --- a/cmd/collect_github_repo.go +++ b/cmd/collect_github_repo.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/vcs" "github.com/spf13/cobra" @@ -19,7 +20,10 @@ var collectGithubRepoCmd = &cobra.Command{ repoURL := args[0] outputFile, _ := cmd.Flags().GetString("output") - dn, err := vcs.CloneGitRepository(repoURL) + bar := ui.NewProgressBar(-1, "Cloning repository") + defer bar.Finish() + + dn, err := vcs.CloneGitRepository(repoURL, bar) if err != nil { fmt.Printf("Error cloning repository: %v\n", err) return diff --git a/cmd/collect_pwa.go b/cmd/collect_pwa.go index 99c595d..000ac53 100644 --- a/cmd/collect_pwa.go +++ b/cmd/collect_pwa.go @@ -5,6 +5,7 @@ import ( "os" "github.com/Snider/Borg/pkg/pwa" + "github.com/Snider/Borg/pkg/ui" "github.com/spf13/cobra" ) @@ -26,16 +27,16 @@ Example: return } - fmt.Println("Finding PWA manifest...") + bar := ui.NewProgressBar(-1, "Finding PWA manifest") + defer bar.Finish() + manifestURL, err := pwa.FindManifest(pwaURL) if err != nil { fmt.Printf("Error finding manifest: %v\n", err) return } - fmt.Printf("Found manifest: %s\n", manifestURL) - - fmt.Println("Downloading and packaging PWA...") - dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL) + bar.Describe("Downloading and packaging PWA") + dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL, bar) if err != nil { fmt.Printf("Error downloading and packaging PWA: %v\n", err) return diff --git a/cmd/collect_website.go b/cmd/collect_website.go index 9222da6..b11803f 100644 --- a/cmd/collect_website.go +++ b/cmd/collect_website.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/website" "github.com/spf13/cobra" @@ -20,7 +21,10 @@ var collectWebsiteCmd = &cobra.Command{ outputFile, _ := cmd.Flags().GetString("output") depth, _ := cmd.Flags().GetInt("depth") - dn, err := website.DownloadAndPackageWebsite(websiteURL, depth) + bar := ui.NewProgressBar(-1, "Crawling website") + defer bar.Finish() + + dn, err := website.DownloadAndPackageWebsite(websiteURL, depth, bar) if err != nil { fmt.Printf("Error downloading and packaging website: %v\n", err) return diff --git a/pkg/pwa/pwa.go b/pkg/pwa/pwa.go index 9c55ff2..640f096 100644 --- a/pkg/pwa/pwa.go +++ b/pkg/pwa/pwa.go @@ -9,6 +9,7 @@ import ( "path" "github.com/Snider/Borg/pkg/datanode" + "github.com/schollz/progressbar/v3" "golang.org/x/net/html" ) @@ -80,7 +81,7 @@ func FindManifest(pageURL string) (string, error) { } // DownloadAndPackagePWA downloads all assets of a PWA and packages them into a DataNode. -func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNode, error) { +func DownloadAndPackagePWA(baseURL string, manifestURL string, bar *progressbar.ProgressBar) (*datanode.DataNode, error) { manifestAbsURL, err := resolveURL(baseURL, manifestURL) if err != nil { return nil, fmt.Errorf("could not resolve manifest URL: %w", err) @@ -110,7 +111,7 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo if err != nil { return nil, fmt.Errorf("could not resolve start_url: %w", err) } - err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL) + err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL, bar) if err != nil { return nil, fmt.Errorf("failed to download start_url asset: %w", err) } @@ -122,14 +123,14 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo fmt.Printf("Warning: could not resolve icon URL %s: %v\n", icon.Src, err) continue } - err = downloadAndAddFile(dn, iconURLAbs, icon.Src) + err = downloadAndAddFile(dn, iconURLAbs, icon.Src, bar) if err != nil { fmt.Printf("Warning: failed to download icon %s: %v\n", icon.Src, err) } } baseURLAbs, _ := url.Parse(baseURL) - err = downloadAndAddFile(dn, baseURLAbs, "index.html") + err = downloadAndAddFile(dn, baseURLAbs, "index.html", bar) if err != nil { return nil, fmt.Errorf("failed to download base HTML: %w", err) } @@ -149,7 +150,7 @@ func resolveURL(base, ref string) (*url.URL, error) { return baseURL.ResolveReference(refURL), nil } -func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string) error { +func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string, bar *progressbar.ProgressBar) error { resp, err := http.Get(fileURL.String()) if err != nil { return err @@ -165,5 +166,6 @@ func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath st return err } dn.AddData(path.Clean(internalPath), data) + bar.Add(1) return nil } diff --git a/pkg/pwa/pwa_test.go b/pkg/pwa/pwa_test.go index f90fdb3..81728c8 100644 --- a/pkg/pwa/pwa_test.go +++ b/pkg/pwa/pwa_test.go @@ -4,6 +4,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/schollz/progressbar/v3" ) func TestFindManifest(t *testing.T) { @@ -78,7 +80,8 @@ func TestDownloadAndPackagePWA(t *testing.T) { })) defer server.Close() - dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json") + bar := progressbar.New(1) + dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json", bar) if err != nil { t.Fatalf("DownloadAndPackagePWA failed: %v", err) } diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index b43549f..b820f22 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -1,6 +1,7 @@ package vcs import ( + "io" "os" "path/filepath" @@ -10,7 +11,7 @@ import ( ) // CloneGitRepository clones a Git repository from a URL and packages it into a DataNode. -func CloneGitRepository(repoURL string) (*datanode.DataNode, error) { +func CloneGitRepository(repoURL string, progress io.Writer) (*datanode.DataNode, error) { tempPath, err := os.MkdirTemp("", "borg-clone-*") if err != nil { return nil, err @@ -19,7 +20,7 @@ func CloneGitRepository(repoURL string) (*datanode.DataNode, error) { _, err = git.PlainClone(tempPath, false, &git.CloneOptions{ URL: repoURL, - Progress: os.Stdout, + Progress: progress, }) if err != nil { return nil, err diff --git a/pkg/vcs/git_test.go b/pkg/vcs/git_test.go index bc55d2e..d36e310 100644 --- a/pkg/vcs/git_test.go +++ b/pkg/vcs/git_test.go @@ -69,7 +69,7 @@ func TestCloneGitRepository(t *testing.T) { } // Clone the repository using the function we're testing - dn, err := CloneGitRepository("file://" + bareRepoPath) + dn, err := CloneGitRepository("file://"+bareRepoPath, os.Stdout) if err != nil { t.Fatalf("CloneGitRepository failed: %v", err) } diff --git a/pkg/website/website.go b/pkg/website/website.go index 94a783d..635fccf 100644 --- a/pkg/website/website.go +++ b/pkg/website/website.go @@ -32,7 +32,7 @@ func NewDownloader(maxDepth int) *Downloader { } // DownloadAndPackageWebsite downloads a website and packages it into a DataNode. -func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNode, error) { +func DownloadAndPackageWebsite(startURL string, maxDepth int, bar *progressbar.ProgressBar) (*datanode.DataNode, error) { baseURL, err := url.Parse(startURL) if err != nil { return nil, err @@ -40,9 +40,7 @@ func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNod d := NewDownloader(maxDepth) d.baseURL = baseURL - - fmt.Println("Downloading website...") - d.progressBar = progressbar.NewOptions(1, progressbar.OptionSetDescription("Downloading")) + d.progressBar = bar d.crawl(startURL, 0) return d.dn, nil diff --git a/pkg/website/website_test.go b/pkg/website/website_test.go index 9a8ec85..aac7d94 100644 --- a/pkg/website/website_test.go +++ b/pkg/website/website_test.go @@ -4,6 +4,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/schollz/progressbar/v3" ) func TestDownloadAndPackageWebsite(t *testing.T) { @@ -64,7 +66,8 @@ func TestDownloadAndPackageWebsite(t *testing.T) { })) defer server.Close() - dn, err := DownloadAndPackageWebsite(server.URL, 2) + bar := progressbar.New(1) + dn, err := DownloadAndPackageWebsite(server.URL, 2, bar) if err != nil { t.Fatalf("DownloadAndPackageWebsite failed: %v", err) }