diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index f1c8fe0..b6998a6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -17,10 +17,13 @@ jobs: - name: Set up Go uses: actions/setup-go@v6 with: - go-version: '1.24' + go-version-file: 'go.mod' + + - name: Install Task + run: go install github.com/go-task/task/v3/cmd/task@latest - name: Build - run: go build -v ./... + run: ~/go/bin/task build - name: Test - run: go test -v ./... + run: ~/go/bin/task test diff --git a/.gitignore b/.gitignore index 0530eff..860cd15 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,2 @@ borg *.cube -.idea \ No newline at end of file diff --git a/.task/checksum/build b/.task/checksum/build new file mode 100644 index 0000000..d221f2a --- /dev/null +++ b/.task/checksum/build @@ -0,0 +1 @@ +e7a169415ff04b3b388216dc3048d84c diff --git a/README.md b/README.md index a1f734a..ba6d4aa 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,7 @@ # Borg Data Collector -Assimulate all the data!!! \ -No, seriously, what do you need to download? PWA? A GitHub repo, or every repository they have? A website? Build artefacts? Malware? - -That's why I made Borg, to download and contain sets of data into explorable collections, to reuse later; ATM there is only Zuul, erm, Tar, but that’s all I need right now~ Custom rootFS distroless image and, of course, Nanites (sec ops tooling) to come, but if you want to use and work on a Web3 malware analysis, tool, patches welcome (non-sarcasticly). - -Oh, Calling Trekkies, the status messages below, you know they are wrong, don't you? It hurts a little? Good, you, you're the one... \ -Take part in Open Source, make us smirk with amusement and make the CLI crack more smirks. +As the name might sugest; this pkg collects information and stores it in a Cube file or passes it on; +comes as both a cli tool and a usable package for your go project with a clean export only top level interface. ## Borg Status Scratch Pad diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..be7523c --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,30 @@ +version: '3' + +tasks: + clean: + cmds: + - rm -f borg + build: + cmds: + - task: clean + - GOOS=linux GOARCH=amd64 go build -o borg main.go + sources: + - main.go + - ./pkg/**/*.go + generates: + - borg + run: + cmds: + - task: build + - chmod +x borg + - ./borg + deps: + - build + test: + cmds: + - go test ./... + test-e2e: + cmds: + - task: build + - chmod +x borg + - ./borg --help diff --git a/cmd/all.go b/cmd/all.go index 8a6253e..1fecf21 100644 --- a/cmd/all.go +++ b/cmd/all.go @@ -1,11 +1,14 @@ package cmd import ( + "context" "fmt" + "log/slog" "os" "strings" "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/vcs" "github.com/spf13/cobra" @@ -18,26 +21,34 @@ var allCmd = &cobra.Command{ Long: `Collect all public repositories from a user or organization and store them in a DataNode.`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { - repos, err := github.GetPublicRepos(args[0]) + logVal := cmd.Context().Value("logger") + log, ok := logVal.(*slog.Logger) + if !ok || log == nil { + fmt.Fprintln(os.Stderr, "Error: logger not properly initialised") + return + } + repos, err := github.GetPublicRepos(context.Background(), args[0]) if err != nil { - fmt.Println(err) + log.Error("failed to get public repos", "err", err) return } outputDir, _ := cmd.Flags().GetString("output") for _, repoURL := range repos { - fmt.Printf("Cloning %s...\n", repoURL) + log.Info("cloning repository", "url", repoURL) + bar := ui.NewProgressBar(-1, "Cloning repository") - dn, err := vcs.CloneGitRepository(repoURL) + dn, err := vcs.CloneGitRepository(repoURL, bar) + bar.Finish() if err != nil { - fmt.Printf("Error cloning %s: %s\n", repoURL, err) + log.Error("failed to clone repository", "url", repoURL, "err", err) continue } data, err := dn.ToTar() if err != nil { - fmt.Printf("Error serializing DataNode for %s: %v\n", repoURL, err) + log.Error("failed to serialize datanode", "url", repoURL, "err", err) continue } @@ -45,7 +56,7 @@ var allCmd = &cobra.Command{ outputFile := fmt.Sprintf("%s/%s.dat", outputDir, repoName) err = os.WriteFile(outputFile, data, 0644) if err != nil { - fmt.Printf("Error writing DataNode for %s to file: %v\n", repoURL, err) + log.Error("failed to write datanode to file", "url", repoURL, "err", err) continue } } @@ -53,6 +64,6 @@ var allCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(allCmd) + RootCmd.AddCommand(allCmd) allCmd.PersistentFlags().String("output", ".", "Output directory for the DataNodes") } diff --git a/cmd/collect.go b/cmd/collect.go index 57960b2..8e3d817 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -12,5 +12,5 @@ var collectCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(collectCmd) + RootCmd.AddCommand(collectCmd) } diff --git a/cmd/collect_github_release_subcommand.go b/cmd/collect_github_release_subcommand.go index b844704..83565c0 100644 --- a/cmd/collect_github_release_subcommand.go +++ b/cmd/collect_github_release_subcommand.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "io" + "log/slog" "net/http" "os" "path/filepath" @@ -23,6 +24,12 @@ var collectGithubReleaseCmd = &cobra.Command{ Long: `Download the latest release of a file from GitHub releases. If the file or URL has a version number, it will check for a higher version and download it if found.`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { + logVal := cmd.Context().Value("logger") + log, ok := logVal.(*slog.Logger) + if !ok || log == nil { + fmt.Fprintln(os.Stderr, "Error: logger not properly initialised") + return + } repoURL := args[0] outputDir, _ := cmd.Flags().GetString("output") pack, _ := cmd.Flags().GetBool("pack") @@ -31,25 +38,25 @@ var collectGithubReleaseCmd = &cobra.Command{ owner, repo, err := borg_github.ParseRepoFromURL(repoURL) if err != nil { - fmt.Printf("Error parsing repository URL: %v\n", err) + log.Error("failed to parse repository url", "err", err) return } release, err := borg_github.GetLatestRelease(owner, repo) if err != nil { - fmt.Printf("Error getting latest release: %v\n", err) + log.Error("failed to get latest release", "err", err) return } - fmt.Printf("Found latest release: %s\n", release.GetTagName()) + log.Info("found latest release", "tag", release.GetTagName()) if version != "" { if !semver.IsValid(version) { - fmt.Printf("Invalid version string: %s\n", version) + log.Error("invalid version string", "version", version) return } if semver.Compare(release.GetTagName(), version) <= 0 { - fmt.Printf("Latest release (%s) is not newer than the provided version (%s).\n", release.GetTagName(), version) + log.Info("latest release is not newer than the provided version", "latest", release.GetTagName(), "provided", version) return } } @@ -57,24 +64,24 @@ var collectGithubReleaseCmd = &cobra.Command{ if pack { dn := datanode.New() for _, asset := range release.Assets { - fmt.Printf("Downloading asset: %s\n", asset.GetName()) + log.Info("downloading asset", "name", asset.GetName()) resp, err := http.Get(asset.GetBrowserDownloadURL()) if err != nil { - fmt.Printf("Error downloading asset: %v\n", err) + log.Error("failed to download asset", "name", asset.GetName(), "err", err) continue } defer resp.Body.Close() var buf bytes.Buffer _, err = io.Copy(&buf, resp.Body) if err != nil { - fmt.Printf("Error reading asset: %v\n", err) + log.Error("failed to read asset", "name", asset.GetName(), "err", err) continue } dn.AddData(asset.GetName(), buf.Bytes()) } tar, err := dn.ToTar() if err != nil { - fmt.Printf("Error creating DataNode: %v\n", err) + log.Error("failed to create datanode", "err", err) return } outputFile := outputDir @@ -83,13 +90,13 @@ var collectGithubReleaseCmd = &cobra.Command{ } err = os.WriteFile(outputFile, tar, 0644) if err != nil { - fmt.Printf("Error writing DataNode: %v\n", err) + log.Error("failed to write datanode", "err", err) return } - fmt.Printf("DataNode saved to %s\n", outputFile) + log.Info("datanode saved", "path", outputFile) } else { if len(release.Assets) == 0 { - fmt.Println("No assets found in the latest release.") + log.Info("no assets found in the latest release") return } var assetToDownload *gh.ReleaseAsset @@ -101,20 +108,20 @@ var collectGithubReleaseCmd = &cobra.Command{ } } if assetToDownload == nil { - fmt.Printf("Asset '%s' not found in the latest release.\n", file) + log.Error("asset not found in the latest release", "asset", file) return } } else { assetToDownload = release.Assets[0] } outputPath := filepath.Join(outputDir, assetToDownload.GetName()) - fmt.Printf("Downloading asset: %s\n", assetToDownload.GetName()) + log.Info("downloading asset", "name", assetToDownload.GetName()) err = borg_github.DownloadReleaseAsset(assetToDownload, outputPath) if err != nil { - fmt.Printf("Error downloading asset: %v\n", err) + log.Error("failed to download asset", "name", assetToDownload.GetName(), "err", err) return } - fmt.Printf("Asset downloaded to %s\n", outputPath) + log.Info("asset downloaded", "path", outputPath) } }, } diff --git a/cmd/collect_github_repo.go b/cmd/collect_github_repo.go index 75b7dff..e66f7aa 100644 --- a/cmd/collect_github_repo.go +++ b/cmd/collect_github_repo.go @@ -3,13 +3,10 @@ package cmd import ( "fmt" "os" - "path/filepath" - "strings" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/vcs" - "github.com/Snider/Borg/pkg/github" - "github.com/Snider/Borg/pkg/vcs" "github.com/spf13/cobra" ) @@ -19,90 +16,36 @@ var collectGithubRepoCmd = &cobra.Command{ Short: "Collect a single Git repository", Long: `Collect a single Git repository and store it in a DataNode.`, Args: cobra.ExactArgs(1), -// collectGitCmd represents the collect git command -var collectGitCmd = &cobra.Command{ - Use: "git", - Short: "Collect one or more Git repositories", - Long: `Collect a single Git repository from a URL, or all public repositories from a GitHub user/organization.`, Run: func(cmd *cobra.Command, args []string) { - repoURL, _ := cmd.Flags().GetString("uri") - user, _ := cmd.Flags().GetString("user") - output, _ := cmd.Flags().GetString("output") + repoURL := args[0] + outputFile, _ := cmd.Flags().GetString("output") - if (repoURL == "" && user == "") || (repoURL != "" && user != "") { - fmt.Println("Error: You must specify either --uri or --user, but not both.") - os.Exit(1) + bar := ui.NewProgressBar(-1, "Cloning repository") + defer bar.Finish() + + dn, err := vcs.CloneGitRepository(repoURL, bar) + if err != nil { + fmt.Printf("Error cloning repository: %v\n", err) + return } - if user != "" { - // User specified, collect all their repos - fmt.Printf("Fetching public repositories for %s...\n", user) - repos, err := github.GetPublicRepos(user) - if err != nil { - fmt.Printf("Error fetching repositories: %v\n", err) - return - } - fmt.Printf("Found %d repositories. Cloning...\n\n", len(repos)) - - // Ensure output directory exists - err = os.MkdirAll(output, 0755) - if err != nil { - fmt.Printf("Error creating output directory: %v\n", err) - return - } - - for _, repo := range repos { - fmt.Printf("Cloning %s...\n", repo) - dn, err := vcs.CloneGitRepository(repo) - if err != nil { - fmt.Printf(" Error cloning: %v\n", err) - continue - } - - data, err := dn.ToTar() - if err != nil { - fmt.Printf(" Error serializing: %v\n", err) - continue - } - - repoName := strings.TrimSuffix(filepath.Base(repo), ".git") - outputFile := filepath.Join(output, fmt.Sprintf("%s.dat", repoName)) - err = os.WriteFile(outputFile, data, 0644) - if err != nil { - fmt.Printf(" Error writing file: %v\n", err) - continue - } - fmt.Printf(" Successfully saved to %s\n", outputFile) - } - fmt.Println("\nCollection complete.") - - } else { - // Single repository URL specified - dn, err := vcs.CloneGitRepository(repoURL) - if err != nil { - fmt.Printf("Error cloning repository: %v\n", err) - return - } - - data, err := dn.ToTar() - if err != nil { - fmt.Printf("Error serializing DataNode: %v\n", err) - return - } - - err = os.WriteFile(output, data, 0644) - if err != nil { - fmt.Printf("Error writing DataNode to file: %v\n", err) - return - } - fmt.Printf("Repository saved to %s\n", output) + data, err := dn.ToTar() + if err != nil { + fmt.Printf("Error serializing DataNode: %v\n", err) + return } + + err = os.WriteFile(outputFile, data, 0644) + if err != nil { + fmt.Printf("Error writing DataNode to file: %v\n", err) + return + } + + fmt.Printf("Repository saved to %s\n", outputFile) }, } func init() { collectGithubCmd.AddCommand(collectGithubRepoCmd) - collectGithubRepoCmd.Flags().String("uri", "", "URL of the Git repository to collect") - collectGitCmd.Flags().String("user", "", "GitHub user or organization to collect all repositories from") - collectGitCmd.Flags().String("output", "repo.dat", "Output file (for --uri) or directory (for --user)") + collectGithubRepoCmd.PersistentFlags().String("output", "repo.dat", "Output file for the DataNode") } diff --git a/cmd/collect_pwa.go b/cmd/collect_pwa.go index 99c595d..000ac53 100644 --- a/cmd/collect_pwa.go +++ b/cmd/collect_pwa.go @@ -5,6 +5,7 @@ import ( "os" "github.com/Snider/Borg/pkg/pwa" + "github.com/Snider/Borg/pkg/ui" "github.com/spf13/cobra" ) @@ -26,16 +27,16 @@ Example: return } - fmt.Println("Finding PWA manifest...") + bar := ui.NewProgressBar(-1, "Finding PWA manifest") + defer bar.Finish() + manifestURL, err := pwa.FindManifest(pwaURL) if err != nil { fmt.Printf("Error finding manifest: %v\n", err) return } - fmt.Printf("Found manifest: %s\n", manifestURL) - - fmt.Println("Downloading and packaging PWA...") - dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL) + bar.Describe("Downloading and packaging PWA") + dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL, bar) if err != nil { fmt.Printf("Error downloading and packaging PWA: %v\n", err) return diff --git a/cmd/collect_website.go b/cmd/collect_website.go index 9222da6..b11803f 100644 --- a/cmd/collect_website.go +++ b/cmd/collect_website.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/website" "github.com/spf13/cobra" @@ -20,7 +21,10 @@ var collectWebsiteCmd = &cobra.Command{ outputFile, _ := cmd.Flags().GetString("output") depth, _ := cmd.Flags().GetInt("depth") - dn, err := website.DownloadAndPackageWebsite(websiteURL, depth) + bar := ui.NewProgressBar(-1, "Crawling website") + defer bar.Finish() + + dn, err := website.DownloadAndPackageWebsite(websiteURL, depth, bar) if err != nil { fmt.Printf("Error downloading and packaging website: %v\n", err) return diff --git a/cmd/main_test.go b/cmd/main_test.go new file mode 100644 index 0000000..cb55db3 --- /dev/null +++ b/cmd/main_test.go @@ -0,0 +1,18 @@ +package cmd + +import ( + "log/slog" + "os" + "testing" +) + +func TestExecute(t *testing.T) { + // This test simply checks that the Execute function can be called without error. + // It doesn't actually test any of the application's functionality. + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })) + if err := Execute(log); err != nil { + t.Errorf("Execute() failed: %v", err) + } +} diff --git a/cmd/root.go b/cmd/root.go index a95ea23..8ca2ebe 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,14 +1,15 @@ package cmd import ( - "os" + "context" + "log/slog" "github.com/spf13/cobra" ) -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ - Use: "github.com/Snider/Borg", +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "borg-data-collector", Short: "A tool for collecting and managing data.", Long: `Borg Data Collector is a command-line tool for cloning Git repositories, packaging their contents into a single file, and managing the data within.`, @@ -16,21 +17,11 @@ packaging their contents into a single file, and managing the data within.`, // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - err := rootCmd.Execute() - if err != nil { - os.Exit(1) - } +func Execute(log *slog.Logger) error { + RootCmd.SetContext(context.WithValue(context.Background(), "logger", log)) + return RootCmd.Execute() } func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.github.com/Snider/Borg.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + RootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging") } diff --git a/cmd/serve.go b/cmd/serve.go index 13516e7..40dd400 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -44,6 +44,6 @@ var serveCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(serveCmd) + RootCmd.AddCommand(serveCmd) serveCmd.PersistentFlags().String("port", "8080", "Port to serve the PWA on") } diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index b9e4da9..0000000 --- a/docs/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Borg Data Collector Documentation - -This directory contains the documentation for the Borg Data Collector. - -## Table of Contents - -- [Usage](usage.md) diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index 52feff0..0000000 --- a/docs/usage.md +++ /dev/null @@ -1,39 +0,0 @@ -# Usage - -This document explains how to use the Borg Data Collector. - -## `collect git` - -The `collect git` command is used to clone a git repository and store it in a DataNode. - -### Collect a single repository - -```bash -borg collect git --uri https://github.com/torvalds/linux.git --output linux.dat -``` - -### Collect all repositories for a user - -```bash -borg collect git --user torvalds --output /path/to/output/dir -``` - -## `collect website` - -The `collect website` command is used to crawl a website and store it in a DataNode. - -### Example - -```bash -borg collect website --uri https://tldp.org/ -``` - -## `serve` - -The `serve` command is used to serve a DataNode file. - -### Example - -```bash -borg serve --file linux.borg -``` diff --git a/examples/collect_git.sh b/examples/collect_git.sh deleted file mode 100644 index b7fd1d5..0000000 --- a/examples/collect_git.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# Example of how to use the 'collect git' command. - -# This will clone a single git repository and store it in a DataNode. -borg collect git --uri https://github.com/torvalds/linux.git --output linux.dat - -# This will clone all public repositories for a user and store them in a directory. -borg collect git --user torvalds --output /tmp/borg-repos diff --git a/examples/collect_website.sh b/examples/collect_website.sh deleted file mode 100644 index 5f96616..0000000 --- a/examples/collect_website.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Example of how to use the 'collect website' command. - -# This will crawl the specified website and store it in a DataNode. -borg collect website --uri https://tldp.org/ diff --git a/examples/serve.sh b/examples/serve.sh deleted file mode 100644 index 3aa4c9c..0000000 --- a/examples/serve.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -# Example of how to use the 'serve' command. - -# This will serve the specified DataNode file. -borg serve --file linux.borg diff --git a/go.mod b/go.mod index 64eb54c..2b473d4 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,9 @@ require ( github.com/google/go-github/v39 v39.2.0 github.com/schollz/progressbar/v3 v3.18.0 github.com/spf13/cobra v1.10.1 - golang.org/x/mod v0.12.0 + golang.org/x/mod v0.29.0 golang.org/x/net v0.46.0 + golang.org/x/oauth2 v0.27.0 ) require ( @@ -21,6 +22,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -35,5 +37,7 @@ require ( golang.org/x/crypto v0.43.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/term v0.36.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index d841f7d..693b03f 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -101,14 +103,17 @@ golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -129,7 +134,10 @@ golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/go.work b/go.work deleted file mode 100644 index 0eda0de..0000000 --- a/go.work +++ /dev/null @@ -1,3 +0,0 @@ -go 1.24.3 - -use . diff --git a/go.work.sum b/go.work.sum deleted file mode 100644 index 2bd247a..0000000 --- a/go.work.sum +++ /dev/null @@ -1,43 +0,0 @@ -github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw= -github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= -github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= -github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7c02e17 --- /dev/null +++ b/main.go @@ -0,0 +1,17 @@ +package main + +import ( + "os" + + "github.com/Snider/Borg/cmd" + "github.com/Snider/Borg/pkg/logger" +) + +func main() { + verbose, _ := cmd.RootCmd.PersistentFlags().GetBool("verbose") + log := logger.New(verbose) + if err := cmd.Execute(log); err != nil { + log.Error("fatal error", "err", err) + os.Exit(1) + } +} diff --git a/pkg/github/github.go b/pkg/github/github.go index c7cf8f4..d7a1bf8 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -1,51 +1,106 @@ package github import ( + "context" "encoding/json" "fmt" "net/http" + "os" + "strings" + + "golang.org/x/oauth2" ) type Repo struct { CloneURL string `json:"clone_url"` } -func GetPublicRepos(userOrOrg string) ([]string, error) { - return GetPublicReposWithAPIURL("https://api.github.com", userOrOrg) +func GetPublicRepos(ctx context.Context, userOrOrg string) ([]string, error) { + return GetPublicReposWithAPIURL(ctx, "https://api.github.com", userOrOrg) } -func GetPublicReposWithAPIURL(apiURL, userOrOrg string) ([]string, error) { - if userOrOrg == "" { - return nil, fmt.Errorf("user or organization cannot be empty") +func newAuthenticatedClient(ctx context.Context) *http.Client { + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + return http.DefaultClient } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + return oauth2.NewClient(ctx, ts) +} - resp, err := http.Get(fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg)) - if err != nil { - return nil, err - } - defer resp.Body.Close() +func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([]string, error) { + client := newAuthenticatedClient(ctx) + var allCloneURLs []string + url := fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg) - if resp.StatusCode != http.StatusOK { - // Try organization endpoint - resp, err = http.Get(fmt.Sprintf("%s/orgs/%s/repos", apiURL, userOrOrg)) + for { + if err := ctx.Err(); err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return nil, err } - defer resp.Body.Close() + req.Header.Set("User-Agent", "Borg-Data-Collector") + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + // Try organization endpoint + url = fmt.Sprintf("%s/orgs/%s/repos", apiURL, userOrOrg) + req, err = http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Borg-Data-Collector") + resp, err = client.Do(req) + if err != nil { + return nil, err + } + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() return nil, fmt.Errorf("failed to fetch repos: %s", resp.Status) } + + var repos []Repo + if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil { + resp.Body.Close() + return nil, err + } + resp.Body.Close() + + for _, repo := range repos { + allCloneURLs = append(allCloneURLs, repo.CloneURL) + } + + linkHeader := resp.Header.Get("Link") + if linkHeader == "" { + break + } + nextURL := findNextURL(linkHeader) + if nextURL == "" { + break + } + url = nextURL } - var repos []Repo - if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil { - return nil, err - } - - var cloneURLs []string - for _, repo := range repos { - cloneURLs = append(cloneURLs, repo.CloneURL) - } - - return cloneURLs, nil + return allCloneURLs, nil +} + +func findNextURL(linkHeader string) string { + links := strings.Split(linkHeader, ",") + for _, link := range links { + parts := strings.Split(link, ";") + if len(parts) == 2 && strings.TrimSpace(parts[1]) == `rel="next"` { + return strings.Trim(strings.TrimSpace(parts[0]), "<>") + } + } + return "" } diff --git a/pkg/github/github_test.go b/pkg/github/github_test.go deleted file mode 100644 index 00e952b..0000000 --- a/pkg/github/github_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package github - -import ( - "net/http" - "net/http/httptest" - "testing" -) - -func TestGetPublicRepos_Good(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte(`[{"clone_url": "https://github.com/good/repo.git"}]`)) - })) - defer server.Close() - - repos, err := GetPublicReposWithAPIURL(server.URL, "good") - if err != nil { - t.Errorf("Expected no error, got %v", err) - } - if len(repos) != 1 || repos[0] != "https://github.com/good/repo.git" { - t.Errorf("Expected one repo, got %v", repos) - } -} - -func TestGetPublicRepos_Bad(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - })) - defer server.Close() - - _, err := GetPublicReposWithAPIURL(server.URL, "bad") - if err == nil { - t.Errorf("Expected an error, got nil") - } -} - -func TestGetPublicRepos_Ugly(t *testing.T) { - _, err := GetPublicReposWithAPIURL("http://localhost", "") - if err == nil { - t.Errorf("Expected an error for empty user/org, got nil") - } -} diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..0dfc2d2 --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,16 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New(verbose bool) *slog.Logger { + level := slog.LevelInfo + if verbose { + level = slog.LevelDebug + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: level, + })) +} diff --git a/pkg/pwa/pwa.go b/pkg/pwa/pwa.go index 9c55ff2..2304f63 100644 --- a/pkg/pwa/pwa.go +++ b/pkg/pwa/pwa.go @@ -9,6 +9,7 @@ import ( "path" "github.com/Snider/Borg/pkg/datanode" + "github.com/schollz/progressbar/v3" "golang.org/x/net/html" ) @@ -80,7 +81,10 @@ func FindManifest(pageURL string) (string, error) { } // DownloadAndPackagePWA downloads all assets of a PWA and packages them into a DataNode. -func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNode, error) { +func DownloadAndPackagePWA(baseURL string, manifestURL string, bar *progressbar.ProgressBar) (*datanode.DataNode, error) { + if bar == nil { + return nil, fmt.Errorf("progress bar cannot be nil") + } manifestAbsURL, err := resolveURL(baseURL, manifestURL) if err != nil { return nil, fmt.Errorf("could not resolve manifest URL: %w", err) @@ -110,7 +114,7 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo if err != nil { return nil, fmt.Errorf("could not resolve start_url: %w", err) } - err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL) + err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL, bar) if err != nil { return nil, fmt.Errorf("failed to download start_url asset: %w", err) } @@ -122,14 +126,14 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo fmt.Printf("Warning: could not resolve icon URL %s: %v\n", icon.Src, err) continue } - err = downloadAndAddFile(dn, iconURLAbs, icon.Src) + err = downloadAndAddFile(dn, iconURLAbs, icon.Src, bar) if err != nil { fmt.Printf("Warning: failed to download icon %s: %v\n", icon.Src, err) } } baseURLAbs, _ := url.Parse(baseURL) - err = downloadAndAddFile(dn, baseURLAbs, "index.html") + err = downloadAndAddFile(dn, baseURLAbs, "index.html", bar) if err != nil { return nil, fmt.Errorf("failed to download base HTML: %w", err) } @@ -149,7 +153,7 @@ func resolveURL(base, ref string) (*url.URL, error) { return baseURL.ResolveReference(refURL), nil } -func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string) error { +func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string, bar *progressbar.ProgressBar) error { resp, err := http.Get(fileURL.String()) if err != nil { return err @@ -165,5 +169,6 @@ func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath st return err } dn.AddData(path.Clean(internalPath), data) + bar.Add(1) return nil } diff --git a/pkg/pwa/pwa_test.go b/pkg/pwa/pwa_test.go index f90fdb3..81728c8 100644 --- a/pkg/pwa/pwa_test.go +++ b/pkg/pwa/pwa_test.go @@ -4,6 +4,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/schollz/progressbar/v3" ) func TestFindManifest(t *testing.T) { @@ -78,7 +80,8 @@ func TestDownloadAndPackagePWA(t *testing.T) { })) defer server.Close() - dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json") + bar := progressbar.New(1) + dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json", bar) if err != nil { t.Fatalf("DownloadAndPackagePWA failed: %v", err) } diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index b43549f..b820f22 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -1,6 +1,7 @@ package vcs import ( + "io" "os" "path/filepath" @@ -10,7 +11,7 @@ import ( ) // CloneGitRepository clones a Git repository from a URL and packages it into a DataNode. -func CloneGitRepository(repoURL string) (*datanode.DataNode, error) { +func CloneGitRepository(repoURL string, progress io.Writer) (*datanode.DataNode, error) { tempPath, err := os.MkdirTemp("", "borg-clone-*") if err != nil { return nil, err @@ -19,7 +20,7 @@ func CloneGitRepository(repoURL string) (*datanode.DataNode, error) { _, err = git.PlainClone(tempPath, false, &git.CloneOptions{ URL: repoURL, - Progress: os.Stdout, + Progress: progress, }) if err != nil { return nil, err diff --git a/pkg/vcs/git_test.go b/pkg/vcs/git_test.go index bc55d2e..d36e310 100644 --- a/pkg/vcs/git_test.go +++ b/pkg/vcs/git_test.go @@ -69,7 +69,7 @@ func TestCloneGitRepository(t *testing.T) { } // Clone the repository using the function we're testing - dn, err := CloneGitRepository("file://" + bareRepoPath) + dn, err := CloneGitRepository("file://"+bareRepoPath, os.Stdout) if err != nil { t.Fatalf("CloneGitRepository failed: %v", err) } diff --git a/pkg/website/website.go b/pkg/website/website.go index 94a783d..e600ad3 100644 --- a/pkg/website/website.go +++ b/pkg/website/website.go @@ -32,7 +32,10 @@ func NewDownloader(maxDepth int) *Downloader { } // DownloadAndPackageWebsite downloads a website and packages it into a DataNode. -func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNode, error) { +func DownloadAndPackageWebsite(startURL string, maxDepth int, bar *progressbar.ProgressBar) (*datanode.DataNode, error) { + if bar == nil { + return nil, fmt.Errorf("progress bar cannot be nil") + } baseURL, err := url.Parse(startURL) if err != nil { return nil, err @@ -40,9 +43,7 @@ func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNod d := NewDownloader(maxDepth) d.baseURL = baseURL - - fmt.Println("Downloading website...") - d.progressBar = progressbar.NewOptions(1, progressbar.OptionSetDescription("Downloading")) + d.progressBar = bar d.crawl(startURL, 0) return d.dn, nil diff --git a/pkg/website/website_test.go b/pkg/website/website_test.go index 9a8ec85..aac7d94 100644 --- a/pkg/website/website_test.go +++ b/pkg/website/website_test.go @@ -4,6 +4,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "github.com/schollz/progressbar/v3" ) func TestDownloadAndPackageWebsite(t *testing.T) { @@ -64,7 +66,8 @@ func TestDownloadAndPackageWebsite(t *testing.T) { })) defer server.Close() - dn, err := DownloadAndPackageWebsite(server.URL, 2) + bar := progressbar.New(1) + dn, err := DownloadAndPackageWebsite(server.URL, 2, bar) if err != nil { t.Fatalf("DownloadAndPackageWebsite failed: %v", err) }