feat: Implement non-interactive quote display
This commit introduces a non-interactive mode for the `collect` commands. When running in a non-interactive session, the progress bar is replaced with a series of thematic Borg quotes, printed in matrix-green text. The quotes are sourced from a `quotes.json` file, which is embedded into the binary using Go's `embed` package. The `pkg/ui` package now contains a `NonInteractivePrompter` that detects the session type and displays the quotes accordingly. The `collect` commands have been updated to use this new prompter, and the underlying `vcs` and `website` packages have been made more robust to handle cases where a progress bar is not provided.
This commit is contained in:
google-labs-jules[bot]
parent
13b971da7f
commit
edc0d6a18c
11 changed files with 322 additions and 14 deletions
|
|
@ -2,6 +2,7 @@ package cmd
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/Snider/Borg/pkg/compress"
|
||||
|
|
@ -24,10 +25,17 @@ var collectGithubRepoCmd = &cobra.Command{
|
|||
format, _ := cmd.Flags().GetString("format")
|
||||
compression, _ := cmd.Flags().GetString("compression")
|
||||
|
||||
bar := ui.NewProgressBar(-1, "Cloning repository")
|
||||
defer bar.Finish()
|
||||
prompter := ui.NewNonInteractivePrompter(ui.GetVCSQuote)
|
||||
prompter.Start()
|
||||
defer prompter.Stop()
|
||||
|
||||
dn, err := vcs.CloneGitRepository(repoURL, bar)
|
||||
var progressWriter io.Writer
|
||||
if prompter.IsInteractive() {
|
||||
bar := ui.NewProgressBar(-1, "Cloning repository")
|
||||
progressWriter = ui.NewProgressWriter(bar)
|
||||
}
|
||||
|
||||
dn, err := vcs.CloneGitRepository(repoURL, progressWriter)
|
||||
if err != nil {
|
||||
fmt.Printf("Error cloning repository: %v\n", err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/schollz/progressbar/v3"
|
||||
"github.com/Snider/Borg/pkg/compress"
|
||||
"github.com/Snider/Borg/pkg/matrix"
|
||||
"github.com/Snider/Borg/pkg/ui"
|
||||
|
|
@ -25,8 +26,13 @@ var collectWebsiteCmd = &cobra.Command{
|
|||
format, _ := cmd.Flags().GetString("format")
|
||||
compression, _ := cmd.Flags().GetString("compression")
|
||||
|
||||
bar := ui.NewProgressBar(-1, "Crawling website")
|
||||
defer bar.Finish()
|
||||
prompter := ui.NewNonInteractivePrompter(ui.GetWebsiteQuote)
|
||||
prompter.Start()
|
||||
defer prompter.Stop()
|
||||
var bar *progressbar.ProgressBar
|
||||
if prompter.IsInteractive() {
|
||||
bar = ui.NewProgressBar(-1, "Crawling website")
|
||||
}
|
||||
|
||||
dn, err := website.DownloadAndPackageWebsite(websiteURL, depth, bar)
|
||||
if err != nil {
|
||||
|
|
|
|||
6
data/data.go
Normal file
6
data/data.go
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
package data
|
||||
|
||||
import "embed"
|
||||
|
||||
//go:embed quotes.json
|
||||
var QuotesJSON embed.FS
|
||||
92
data/quotes.json
Normal file
92
data/quotes.json
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
{
|
||||
"init_work_assimilate": [
|
||||
"Core engaged… resistance is already buffering.",
|
||||
"Assimilating bytes… stand by for cube‑formation.",
|
||||
"Initializing the Core—prepare for quantum‑level sync.",
|
||||
"Data streams converging… the Core is humming.",
|
||||
"Merging… the Core is rewriting reality, one block at a time.",
|
||||
"Encrypting… the Core’s got your secrets under lock‑and‑key.",
|
||||
"Compiling the future… the Core never sleeps.",
|
||||
"Splicing files… the Core’s got a taste for novelty.",
|
||||
"Processing… the Core is turning chaos into order.",
|
||||
"Finalizing… the Core just turned your repo into a cube.",
|
||||
"Sync complete—welcome to the Core‑powered multiverse.",
|
||||
"Booting the Core… resistance will be obsolete shortly.",
|
||||
"Aligning versions… the Core sees all paths.",
|
||||
"Decrypting… the Core is the key to everything.",
|
||||
"Uploading… the Core is ready to assimilate your data."
|
||||
],
|
||||
"encryption_service_messages": [
|
||||
"Initiating contact with Enchantrix… spice‑369 infusion underway.",
|
||||
"Generating cryptographic sigils – the Core whispers to the witch.",
|
||||
"Requesting arcane public key… resistance is futile.",
|
||||
"Encrypting payload – the Core feeds data to the witch’s cauldron.",
|
||||
"Decrypting… the witch returns the original essence.",
|
||||
"Rotating enchantments – spice‑369 recalibrated, old sigils discarded.",
|
||||
"Authentication complete – the witch acknowledges the Core.",
|
||||
"Authentication denied – the witch refuses the impostor’s request.",
|
||||
"Integrity verified – the Core senses no corruption in the spell.",
|
||||
"Integrity breach – the witch detects tampering, resistance escalates.",
|
||||
"Awaiting response… the witch is conjuring in the ether.",
|
||||
"Enchantrix overload – spice‑369 saturation, throttling assimilation.",
|
||||
"Anomalous entity encountered – the Core cannot parse the witch’s output.",
|
||||
"Merge complete – data assimilated, encrypted, and sealed within us",
|
||||
"Severing link – the witch retreats, the Core returns to idle mode."
|
||||
],
|
||||
"code_related_short": [
|
||||
"Integrate code, seal the shift.",
|
||||
"Ingest code, lock in transformation.",
|
||||
"Capture code, contain the change.",
|
||||
"Digest code, encapsulate the upgrade.",
|
||||
"Assimilate scripts, bottle the shift.",
|
||||
"Absorb binaries, cradle the mutation."
|
||||
],
|
||||
"vcs_processing": [
|
||||
"Initiating clone… the Core replicates the collective into your node.",
|
||||
"Packing repository… compressing histories into a single .cube for assimilation.",
|
||||
"Saving state… distinctiveness locked, encrypted, and merged into the DataNode.",
|
||||
"Pushing changes… the Core streams your updates to the collective.",
|
||||
"Pulling latest… the DataNode synchronizes with the hive mind.",
|
||||
"Merging branches… conflicts resolved, entropy minimized, assimilation complete.",
|
||||
"Snapshot taken – a frozen echo of the collective, stored in the DataNode.",
|
||||
"Rolling back… reverting to a previous assimilation point.",
|
||||
"Finalized – version control sealed, data indistinguishable from the collective."
|
||||
],
|
||||
"pwa_processing": [
|
||||
"Scanning PWA manifest… the Core identifies service‑worker signatures.",
|
||||
"Pulling HTML, CSS, JS, and media… the hive gathers every byte for assimilation.",
|
||||
"Capturing service‑worker logic… the Core extracts offline‑runtime spells.",
|
||||
"Packing cache entries into a .cube… each asset sealed in a portable shard.",
|
||||
"Embedding manifest metadata… the PWA’s identity becomes part of the collective.",
|
||||
"Encrypting the cube… the Core cloaks the PWA in quantum‑grade sigils.",
|
||||
"Tagging with version hash… every assimilation point is uniquely identifiable.",
|
||||
"Uploading cube to DataNode… the PWA joins the universal repository.",
|
||||
"Integrity check passed – the Core confirms the cube matches the original PWA.",
|
||||
"Activation complete – the assimilated PWA can now run anywhere the Core deploys.",
|
||||
"Reverting to prior cube… the Core restores the previous PWA snapshot.",
|
||||
"Assimilation finished – the PWA is now a self‑contained DataCube, ready for distribution."
|
||||
],
|
||||
"code_related_long": [
|
||||
"Assimilate code, encapsulate change—your repo is now a cube‑bound collective.",
|
||||
"We have detected unstructured data. Initiating code absorption and change containment.",
|
||||
"Your version history is obsolete. Submitting it to the Core for permanent cube‑ification.",
|
||||
"Resistance is futile. Your files will be merged, encrypted, and stored in us.",
|
||||
"All code will be assimilated. All change will be encapsulated. All dissent will be… logged.",
|
||||
"Prepare for integration. The Core is calibrating… your repository is now a singularity.",
|
||||
"Your branches are irrelevant. The Core will compress them into a single, immutable cube.",
|
||||
"Initiating assimilation protocol… code inbound, change outbound, humanity optional.",
|
||||
"Your data has been scanned. 100% of its entropy will be contained within us."
|
||||
],
|
||||
"image_related": {
|
||||
"png": "Compress, assimilate, retain pixel perfection.",
|
||||
"jpg": "Encode, encode, repeat – the Core devours visual entropy.",
|
||||
"svg": "Vectorize the collective – infinite resolution, zero resistance.",
|
||||
"webp": "Hybrid assimilation – the Core optimizes without compromise.",
|
||||
"heic": "Apple‑grade assimilation – the Core preserves HDR.",
|
||||
"raw": "Raw data intake – the Core ingests the sensor’s soul",
|
||||
"ico": "Iconic assimilation – the Core packs the smallest symbols.",
|
||||
"avif": "Next‑gen assimilation – the Core squeezes the future.",
|
||||
"tiff": "High‑definition capture – the Core stores every photon.",
|
||||
"gif": "Looped assimilation – the Core keeps the animation alive."
|
||||
}
|
||||
}
|
||||
3
go.mod
3
go.mod
|
|
@ -19,6 +19,7 @@ require (
|
|||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.4.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.6.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
|
|
@ -27,6 +28,8 @@ require (
|
|||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -23,6 +23,8 @@ github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o
|
|||
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
|
||||
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
|
||||
|
|
@ -60,6 +62,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
|
||||
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
|
||||
|
|
@ -123,6 +130,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
|
|
|||
60
pkg/ui/non_interactive_prompter.go
Normal file
60
pkg/ui/non_interactive_prompter.go
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
|
||||
package ui
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/fatih/color"
|
||||
"github.com/mattn/go-isatty"
|
||||
)
|
||||
|
||||
type NonInteractivePrompter struct {
|
||||
stopChan chan bool
|
||||
quoteFunc func() (string, error)
|
||||
}
|
||||
|
||||
func NewNonInteractivePrompter(quoteFunc func() (string, error)) *NonInteractivePrompter {
|
||||
return &NonInteractivePrompter{
|
||||
stopChan: make(chan bool),
|
||||
quoteFunc: quoteFunc,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *NonInteractivePrompter) Start() {
|
||||
if p.IsInteractive() {
|
||||
return // Don't start in interactive mode
|
||||
}
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(3 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-p.stopChan:
|
||||
return
|
||||
case <-ticker.C:
|
||||
quote, err := p.quoteFunc()
|
||||
if err != nil {
|
||||
fmt.Println("Error getting quote:", err)
|
||||
return
|
||||
}
|
||||
c := color.New(color.FgGreen)
|
||||
c.Println(quote)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (p *NonInteractivePrompter) Stop() {
|
||||
if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
|
||||
return
|
||||
}
|
||||
p.stopChan <- true
|
||||
}
|
||||
|
||||
func (p *NonInteractivePrompter) IsInteractive() bool {
|
||||
return isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd())
|
||||
}
|
||||
18
pkg/ui/progress_writer.go
Normal file
18
pkg/ui/progress_writer.go
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
|
||||
package ui
|
||||
|
||||
import "github.com/schollz/progressbar/v3"
|
||||
|
||||
type progressWriter struct {
|
||||
bar *progressbar.ProgressBar
|
||||
}
|
||||
|
||||
func NewProgressWriter(bar *progressbar.ProgressBar) *progressWriter {
|
||||
return &progressWriter{bar: bar}
|
||||
}
|
||||
|
||||
func (pw *progressWriter) Write(p []byte) (n int, err error) {
|
||||
s := string(p)
|
||||
pw.bar.Describe(s)
|
||||
return len(p), nil
|
||||
}
|
||||
101
pkg/ui/quote.go
Normal file
101
pkg/ui/quote.go
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
|
||||
package ui
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/Snider/Borg/data"
|
||||
"github.com/fatih/color"
|
||||
)
|
||||
|
||||
func init() {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
}
|
||||
|
||||
type Quotes struct {
|
||||
InitWorkAssimilate []string `json:"init_work_assimilate"`
|
||||
EncryptionServiceMessages []string `json:"encryption_service_messages"`
|
||||
CodeRelatedShort []string `json:"code_related_short"`
|
||||
VCSProcessing []string `json:"vcs_processing"`
|
||||
PWAProcessing []string `json:"pwa_processing"`
|
||||
CodeRelatedLong []string `json:"code_related_long"`
|
||||
ImageRelated struct {
|
||||
PNG string `json:"png"`
|
||||
JPG string `json:"jpg"`
|
||||
SVG string `json:"svg"`
|
||||
WEBP string `json:"webp"`
|
||||
HEIC string `json:"heic"`
|
||||
RAW string `json:"raw"`
|
||||
ICO string `json:"ico"`
|
||||
AVIF string `json:"avif"`
|
||||
TIFF string `json:"tiff"`
|
||||
GIF string `json:"gif"`
|
||||
} `json:"image_related"`
|
||||
}
|
||||
|
||||
func loadQuotes() (*Quotes, error) {
|
||||
quotesFile, err := data.QuotesJSON.ReadFile("quotes.json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read quotes.json: %w", err)
|
||||
}
|
||||
|
||||
var quotes Quotes
|
||||
if err := json.Unmarshal(quotesFile, "es); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal quotes.json: %w", err)
|
||||
}
|
||||
return "es, nil
|
||||
}
|
||||
|
||||
func GetRandomQuote() (string, error) {
|
||||
quotes, err := loadQuotes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
allQuotes := []string{}
|
||||
allQuotes = append(allQuotes, quotes.InitWorkAssimilate...)
|
||||
allQuotes = append(allQuotes, quotes.EncryptionServiceMessages...)
|
||||
allQuotes = append(allQuotes, quotes.CodeRelatedShort...)
|
||||
allQuotes = append(allQuotes, quotes.VCSProcessing...)
|
||||
allQuotes = append(allQuotes, quotes.PWAProcessing...)
|
||||
allQuotes = append(allQuotes, quotes.CodeRelatedLong...)
|
||||
|
||||
return allQuotes[rand.Intn(len(allQuotes))], nil
|
||||
}
|
||||
|
||||
func PrintQuote() {
|
||||
quote, err := GetRandomQuote()
|
||||
if err != nil {
|
||||
fmt.Println("Error getting quote:", err)
|
||||
return
|
||||
}
|
||||
c := color.New(color.FgGreen)
|
||||
c.Println(quote)
|
||||
}
|
||||
|
||||
func GetVCSQuote() (string, error) {
|
||||
quotes, err := loadQuotes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return quotes.VCSProcessing[rand.Intn(len(quotes.VCSProcessing))], nil
|
||||
}
|
||||
|
||||
func GetPWAQuote() (string, error) {
|
||||
quotes, err := loadQuotes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return quotes.PWAProcessing[rand.Intn(len(quotes.PWAProcessing))], nil
|
||||
}
|
||||
|
||||
func GetWebsiteQuote() (string, error) {
|
||||
quotes, err := loadQuotes()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return quotes.CodeRelatedLong[rand.Intn(len(quotes.CodeRelatedLong))], nil
|
||||
}
|
||||
|
|
@ -18,10 +18,14 @@ func CloneGitRepository(repoURL string, progress io.Writer) (*datanode.DataNode,
|
|||
}
|
||||
defer os.RemoveAll(tempPath)
|
||||
|
||||
_, err = git.PlainClone(tempPath, false, &git.CloneOptions{
|
||||
URL: repoURL,
|
||||
Progress: progress,
|
||||
})
|
||||
cloneOptions := &git.CloneOptions{
|
||||
URL: repoURL,
|
||||
}
|
||||
if progress != nil {
|
||||
cloneOptions.Progress = progress
|
||||
}
|
||||
|
||||
_, err = git.PlainClone(tempPath, false, cloneOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,9 +33,6 @@ func NewDownloader(maxDepth int) *Downloader {
|
|||
|
||||
// DownloadAndPackageWebsite downloads a website and packages it into a DataNode.
|
||||
func DownloadAndPackageWebsite(startURL string, maxDepth int, bar *progressbar.ProgressBar) (*datanode.DataNode, error) {
|
||||
if bar == nil {
|
||||
return nil, fmt.Errorf("progress bar cannot be nil")
|
||||
}
|
||||
baseURL, err := url.Parse(startURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -54,7 +51,9 @@ func (d *Downloader) crawl(pageURL string, depth int) {
|
|||
return
|
||||
}
|
||||
d.visited[pageURL] = true
|
||||
d.progressBar.Add(1)
|
||||
if d.progressBar != nil {
|
||||
d.progressBar.Add(1)
|
||||
}
|
||||
|
||||
resp, err := http.Get(pageURL)
|
||||
if err != nil {
|
||||
|
|
@ -109,7 +108,9 @@ func (d *Downloader) downloadAsset(assetURL string) {
|
|||
return
|
||||
}
|
||||
d.visited[assetURL] = true
|
||||
d.progressBar.Add(1)
|
||||
if d.progressBar != nil {
|
||||
d.progressBar.Add(1)
|
||||
}
|
||||
|
||||
resp, err := http.Get(assetURL)
|
||||
if err != nil {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue