Snider/Borg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #5 from Snider/feature-github-release-downloader

Browse source 
fix: Resolve broken build and improve testing
This commit is contained in:
Snider 2025-11-02 03:20:17 +00:00 • committed by GitHub
commit a1b29043fc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 299 additions and 338 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.github/workflows/go.yml vendored
View file

@ -17,10 +17,13 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v6
with:
go-version: '1.24'
go-version-file: 'go.mod'
- name: Install Task
run: go install github.com/go-task/task/v3/cmd/task@latest
- name: Build
run: go build -v ./...
run: ~/go/bin/task build
- name: Test
run: go test -v ./...
run: ~/go/bin/task test

1
.gitignore vendored
View file

@ -1,3 +1,2 @@
borg
*.cube
.idea

1
.task/checksum/build Normal file
View file

@ -0,0 +1 @@
e7a169415ff04b3b388216dc3048d84c

9
README.md
View file

@ -1,12 +1,7 @@
# Borg Data Collector
Assimulate all the data!!! \
No, seriously, what do you need to download? PWA? A GitHub repo, or every repository they have? A website? Build artefacts? Malware?
That's why I made Borg, to download and contain sets of data into explorable collections, to reuse later; ATM there is only Zuul, erm, Tar, but that’s all I need right now~ Custom rootFS distroless image and, of course, Nanites (sec ops tooling) to come, but if you want to use and work on a Web3 malware analysis, tool, patches welcome (non-sarcasticly).
Oh, Calling Trekkies, the status messages below, you know they are wrong, don't you? It hurts a little? Good, you, you're the one... \
Take part in Open Source, make us smirk with amusement and make the CLI crack more smirks.
As the name might sugest; this pkg collects information and stores it in a Cube file or passes it on;
comes as both a cli tool and a usable package for your go project with a clean export only top level interface.
## Borg Status Scratch Pad

30
Taskfile.yml Normal file
View file

@ -0,0 +1,30 @@
version: '3'
tasks:
clean:
cmds:
- rm -f borg
build:
cmds:
- task: clean
- GOOS=linux GOARCH=amd64 go build -o borg main.go
sources:
- main.go
- ./pkg/**/*.go
generates:
- borg
run:
cmds:
- task: build
- chmod +x borg
- ./borg
deps:
- build
test:
cmds:
- go test ./...
test-e2e:
cmds:
- task: build
- chmod +x borg
- ./borg --help

27
cmd/all.go
View file

@ -1,11 +1,14 @@
package cmd
import (
"context"
"fmt"
"log/slog"
"os"
"strings"
"github.com/Snider/Borg/pkg/github"
"github.com/Snider/Borg/pkg/ui"
"github.com/Snider/Borg/pkg/vcs"
"github.com/spf13/cobra"
@ -18,26 +21,34 @@ var allCmd = &cobra.Command{
Long: `Collect all public repositories from a user or organization and store them in a DataNode.`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
repos, err := github.GetPublicRepos(args[0])
logVal := cmd.Context().Value("logger")
log, ok := logVal.(*slog.Logger)
if !ok || log == nil {
fmt.Fprintln(os.Stderr, "Error: logger not properly initialised")
return
}
repos, err := github.GetPublicRepos(context.Background(), args[0])
if err != nil {
fmt.Println(err)
log.Error("failed to get public repos", "err", err)
return
}
outputDir, _ := cmd.Flags().GetString("output")
for _, repoURL := range repos {
fmt.Printf("Cloning %s...\n", repoURL)
log.Info("cloning repository", "url", repoURL)
bar := ui.NewProgressBar(-1, "Cloning repository")
dn, err := vcs.CloneGitRepository(repoURL)
dn, err := vcs.CloneGitRepository(repoURL, bar)
bar.Finish()
if err != nil {
fmt.Printf("Error cloning %s: %s\n", repoURL, err)
log.Error("failed to clone repository", "url", repoURL, "err", err)
continue
}
data, err := dn.ToTar()
if err != nil {
fmt.Printf("Error serializing DataNode for %s: %v\n", repoURL, err)
log.Error("failed to serialize datanode", "url", repoURL, "err", err)
continue
}
@ -45,7 +56,7 @@ var allCmd = &cobra.Command{
outputFile := fmt.Sprintf("%s/%s.dat", outputDir, repoName)
err = os.WriteFile(outputFile, data, 0644)
if err != nil {
fmt.Printf("Error writing DataNode for %s to file: %v\n", repoURL, err)
log.Error("failed to write datanode to file", "url", repoURL, "err", err)
continue
}
}
@ -53,6 +64,6 @@ var allCmd = &cobra.Command{
}
func init() {
rootCmd.AddCommand(allCmd)
RootCmd.AddCommand(allCmd)
allCmd.PersistentFlags().String("output", ".", "Output directory for the DataNodes")
}

2
cmd/collect.go
View file

@ -12,5 +12,5 @@ var collectCmd = &cobra.Command{
}
func init() {
rootCmd.AddCommand(collectCmd)
RootCmd.AddCommand(collectCmd)
}

39
cmd/collect_github_release_subcommand.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
@ -23,6 +24,12 @@ var collectGithubReleaseCmd = &cobra.Command{
Long: `Download the latest release of a file from GitHub releases. If the file or URL has a version number, it will check for a higher version and download it if found.`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
logVal := cmd.Context().Value("logger")
log, ok := logVal.(*slog.Logger)
if !ok || log == nil {
fmt.Fprintln(os.Stderr, "Error: logger not properly initialised")
return
}
repoURL := args[0]
outputDir, _ := cmd.Flags().GetString("output")
pack, _ := cmd.Flags().GetBool("pack")
@ -31,25 +38,25 @@ var collectGithubReleaseCmd = &cobra.Command{
owner, repo, err := borg_github.ParseRepoFromURL(repoURL)
if err != nil {
fmt.Printf("Error parsing repository URL: %v\n", err)
log.Error("failed to parse repository url", "err", err)
return
}
release, err := borg_github.GetLatestRelease(owner, repo)
if err != nil {
fmt.Printf("Error getting latest release: %v\n", err)
log.Error("failed to get latest release", "err", err)
return
}
fmt.Printf("Found latest release: %s\n", release.GetTagName())
log.Info("found latest release", "tag", release.GetTagName())
if version != "" {
if !semver.IsValid(version) {
fmt.Printf("Invalid version string: %s\n", version)
log.Error("invalid version string", "version", version)
return
}
if semver.Compare(release.GetTagName(), version) <= 0 {
fmt.Printf("Latest release (%s) is not newer than the provided version (%s).\n", release.GetTagName(), version)
log.Info("latest release is not newer than the provided version", "latest", release.GetTagName(), "provided", version)
return
}
}
@ -57,24 +64,24 @@ var collectGithubReleaseCmd = &cobra.Command{
if pack {
dn := datanode.New()
for _, asset := range release.Assets {
fmt.Printf("Downloading asset: %s\n", asset.GetName())
log.Info("downloading asset", "name", asset.GetName())
resp, err := http.Get(asset.GetBrowserDownloadURL())
if err != nil {
fmt.Printf("Error downloading asset: %v\n", err)
log.Error("failed to download asset", "name", asset.GetName(), "err", err)
continue
}
defer resp.Body.Close()
var buf bytes.Buffer
_, err = io.Copy(&buf, resp.Body)
if err != nil {
fmt.Printf("Error reading asset: %v\n", err)
log.Error("failed to read asset", "name", asset.GetName(), "err", err)
continue
}
dn.AddData(asset.GetName(), buf.Bytes())
}
tar, err := dn.ToTar()
if err != nil {
fmt.Printf("Error creating DataNode: %v\n", err)
log.Error("failed to create datanode", "err", err)
return
}
outputFile := outputDir
@ -83,13 +90,13 @@ var collectGithubReleaseCmd = &cobra.Command{
}
err = os.WriteFile(outputFile, tar, 0644)
if err != nil {
fmt.Printf("Error writing DataNode: %v\n", err)
log.Error("failed to write datanode", "err", err)
return
}
fmt.Printf("DataNode saved to %s\n", outputFile)
log.Info("datanode saved", "path", outputFile)
} else {
if len(release.Assets) == 0 {
fmt.Println("No assets found in the latest release.")
log.Info("no assets found in the latest release")
return
}
var assetToDownload *gh.ReleaseAsset
@ -101,20 +108,20 @@ var collectGithubReleaseCmd = &cobra.Command{
}
}
if assetToDownload == nil {
fmt.Printf("Asset '%s' not found in the latest release.\n", file)
log.Error("asset not found in the latest release", "asset", file)
return
}
} else {
assetToDownload = release.Assets[0]
}
outputPath := filepath.Join(outputDir, assetToDownload.GetName())
fmt.Printf("Downloading asset: %s\n", assetToDownload.GetName())
log.Info("downloading asset", "name", assetToDownload.GetName())
err = borg_github.DownloadReleaseAsset(assetToDownload, outputPath)
if err != nil {
fmt.Printf("Error downloading asset: %v\n", err)
log.Error("failed to download asset", "name", assetToDownload.GetName(), "err", err)
return
}
fmt.Printf("Asset downloaded to %s\n", outputPath)
log.Info("asset downloaded", "path", outputPath)
}
},
}

103
cmd/collect_github_repo.go
View file

@ -3,13 +3,10 @@ package cmd
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/Snider/Borg/pkg/ui"
"github.com/Snider/Borg/pkg/vcs"
"github.com/Snider/Borg/pkg/github"
"github.com/Snider/Borg/pkg/vcs"
"github.com/spf13/cobra"
)
@ -19,90 +16,36 @@ var collectGithubRepoCmd = &cobra.Command{
Short: "Collect a single Git repository",
Long: `Collect a single Git repository and store it in a DataNode.`,
Args: cobra.ExactArgs(1),
// collectGitCmd represents the collect git command
var collectGitCmd = &cobra.Command{
Use: "git",
Short: "Collect one or more Git repositories",
Long: `Collect a single Git repository from a URL, or all public repositories from a GitHub user/organization.`,
Run: func(cmd *cobra.Command, args []string) {
repoURL, _ := cmd.Flags().GetString("uri")
user, _ := cmd.Flags().GetString("user")
output, _ := cmd.Flags().GetString("output")
repoURL := args[0]
outputFile, _ := cmd.Flags().GetString("output")
if (repoURL == "" && user == "") || (repoURL != "" && user != "") {
fmt.Println("Error: You must specify either --uri or --user, but not both.")
os.Exit(1)
bar := ui.NewProgressBar(-1, "Cloning repository")
defer bar.Finish()
dn, err := vcs.CloneGitRepository(repoURL, bar)
if err != nil {
fmt.Printf("Error cloning repository: %v\n", err)
return
}
if user != "" {
// User specified, collect all their repos
fmt.Printf("Fetching public repositories for %s...\n", user)
repos, err := github.GetPublicRepos(user)
if err != nil {
fmt.Printf("Error fetching repositories: %v\n", err)
return
}
fmt.Printf("Found %d repositories. Cloning...\n\n", len(repos))
// Ensure output directory exists
err = os.MkdirAll(output, 0755)
if err != nil {
fmt.Printf("Error creating output directory: %v\n", err)
return
}
for _, repo := range repos {
fmt.Printf("Cloning %s...\n", repo)
dn, err := vcs.CloneGitRepository(repo)
if err != nil {
fmt.Printf(" Error cloning: %v\n", err)
continue
}
data, err := dn.ToTar()
if err != nil {
fmt.Printf(" Error serializing: %v\n", err)
continue
}
repoName := strings.TrimSuffix(filepath.Base(repo), ".git")
outputFile := filepath.Join(output, fmt.Sprintf("%s.dat", repoName))
err = os.WriteFile(outputFile, data, 0644)
if err != nil {
fmt.Printf(" Error writing file: %v\n", err)
continue
}
fmt.Printf(" Successfully saved to %s\n", outputFile)
}
fmt.Println("\nCollection complete.")
} else {
// Single repository URL specified
dn, err := vcs.CloneGitRepository(repoURL)
if err != nil {
fmt.Printf("Error cloning repository: %v\n", err)
return
}
data, err := dn.ToTar()
if err != nil {
fmt.Printf("Error serializing DataNode: %v\n", err)
return
}
err = os.WriteFile(output, data, 0644)
if err != nil {
fmt.Printf("Error writing DataNode to file: %v\n", err)
return
}
fmt.Printf("Repository saved to %s\n", output)
data, err := dn.ToTar()
if err != nil {
fmt.Printf("Error serializing DataNode: %v\n", err)
return
}
err = os.WriteFile(outputFile, data, 0644)
if err != nil {
fmt.Printf("Error writing DataNode to file: %v\n", err)
return
}
fmt.Printf("Repository saved to %s\n", outputFile)
},
}
func init() {
collectGithubCmd.AddCommand(collectGithubRepoCmd)
collectGithubRepoCmd.Flags().String("uri", "", "URL of the Git repository to collect")
collectGitCmd.Flags().String("user", "", "GitHub user or organization to collect all repositories from")
collectGitCmd.Flags().String("output", "repo.dat", "Output file (for --uri) or directory (for --user)")
collectGithubRepoCmd.PersistentFlags().String("output", "repo.dat", "Output file for the DataNode")
}

11
cmd/collect_pwa.go
View file

@ -5,6 +5,7 @@ import (
"os"
"github.com/Snider/Borg/pkg/pwa"
"github.com/Snider/Borg/pkg/ui"
"github.com/spf13/cobra"
)
@ -26,16 +27,16 @@ Example:
return
}
fmt.Println("Finding PWA manifest...")
bar := ui.NewProgressBar(-1, "Finding PWA manifest")
defer bar.Finish()
manifestURL, err := pwa.FindManifest(pwaURL)
if err != nil {
fmt.Printf("Error finding manifest: %v\n", err)
return
}
fmt.Printf("Found manifest: %s\n", manifestURL)
fmt.Println("Downloading and packaging PWA...")
dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL)
bar.Describe("Downloading and packaging PWA")
dn, err := pwa.DownloadAndPackagePWA(pwaURL, manifestURL, bar)
if err != nil {
fmt.Printf("Error downloading and packaging PWA: %v\n", err)
return

6
cmd/collect_website.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"github.com/Snider/Borg/pkg/ui"
"github.com/Snider/Borg/pkg/website"
"github.com/spf13/cobra"
@ -20,7 +21,10 @@ var collectWebsiteCmd = &cobra.Command{
outputFile, _ := cmd.Flags().GetString("output")
depth, _ := cmd.Flags().GetInt("depth")
dn, err := website.DownloadAndPackageWebsite(websiteURL, depth)
bar := ui.NewProgressBar(-1, "Crawling website")
defer bar.Finish()
dn, err := website.DownloadAndPackageWebsite(websiteURL, depth, bar)
if err != nil {
fmt.Printf("Error downloading and packaging website: %v\n", err)
return

18
cmd/main_test.go Normal file
View file

@ -0,0 +1,18 @@
package cmd
import (
"log/slog"
"os"
"testing"
)
func TestExecute(t *testing.T) {
// This test simply checks that the Execute function can be called without error.
// It doesn't actually test any of the application's functionality.
log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
Level: slog.LevelDebug,
}))
if err := Execute(log); err != nil {
t.Errorf("Execute() failed: %v", err)
}
}

27
cmd/root.go
View file

@ -1,14 +1,15 @@
package cmd
import (
"os"
"context"
"log/slog"
"github.com/spf13/cobra"
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "github.com/Snider/Borg",
// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
Use: "borg-data-collector",
Short: "A tool for collecting and managing data.",
Long: `Borg Data Collector is a command-line tool for cloning Git repositories,
packaging their contents into a single file, and managing the data within.`,
@ -16,21 +17,11 @@ packaging their contents into a single file, and managing the data within.`,
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
err := rootCmd.Execute()
if err != nil {
os.Exit(1)
}
func Execute(log *slog.Logger) error {
RootCmd.SetContext(context.WithValue(context.Background(), "logger", log))
return RootCmd.Execute()
}
func init() {
// Here you will define your flags and configuration settings.
// Cobra supports persistent flags, which, if defined here,
// will be global for your application.
// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.github.com/Snider/Borg.yaml)")
// Cobra also supports local flags, which will only run
// when this action is called directly.
rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
RootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging")
}

2
cmd/serve.go
View file

@ -44,6 +44,6 @@ var serveCmd = &cobra.Command{
}
func init() {
rootCmd.AddCommand(serveCmd)
RootCmd.AddCommand(serveCmd)
serveCmd.PersistentFlags().String("port", "8080", "Port to serve the PWA on")
}

7
docs/README.md
View file

@ -1,7 +0,0 @@
# Borg Data Collector Documentation
This directory contains the documentation for the Borg Data Collector.
## Table of Contents
- [Usage](usage.md)

39
docs/usage.md
View file

@ -1,39 +0,0 @@
# Usage
This document explains how to use the Borg Data Collector.
## `collect git`
The `collect git` command is used to clone a git repository and store it in a DataNode.
### Collect a single repository
```bash
borg collect git --uri https://github.com/torvalds/linux.git --output linux.dat
```
### Collect all repositories for a user
```bash
borg collect git --user torvalds --output /path/to/output/dir
```
## `collect website`
The `collect website` command is used to crawl a website and store it in a DataNode.
### Example
```bash
borg collect website --uri https://tldp.org/
```
## `serve`
The `serve` command is used to serve a DataNode file.
### Example
```bash
borg serve --file linux.borg
```

9
examples/collect_git.sh
View file

@ -1,9 +0,0 @@
#!/bin/bash
# Example of how to use the 'collect git' command.
# This will clone a single git repository and store it in a DataNode.
borg collect git --uri https://github.com/torvalds/linux.git --output linux.dat
# This will clone all public repositories for a user and store them in a directory.
borg collect git --user torvalds --output /tmp/borg-repos

6
examples/collect_website.sh
View file

@ -1,6 +0,0 @@
#!/bin/bash
# Example of how to use the 'collect website' command.
# This will crawl the specified website and store it in a DataNode.
borg collect website --uri https://tldp.org/

6
examples/serve.sh
View file

@ -1,6 +0,0 @@
#!/bin/bash
# Example of how to use the 'serve' command.
# This will serve the specified DataNode file.
borg serve --file linux.borg

6
go.mod
View file

@ -7,8 +7,9 @@ require (
github.com/google/go-github/v39 v39.2.0
github.com/schollz/progressbar/v3 v3.18.0
github.com/spf13/cobra v1.10.1
golang.org/x/mod v0.12.0
golang.org/x/mod v0.29.0
golang.org/x/net v0.46.0
golang.org/x/oauth2 v0.27.0
)
require (
@ -21,6 +22,7 @@ require (
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@ -35,5 +37,7 @@ require (
golang.org/x/crypto v0.43.0 // indirect
golang.org/x/sys v0.37.0 // indirect
golang.org/x/term v0.36.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
)

12
go.sum
View file

@ -37,6 +37,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@ -101,14 +103,17 @@ golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M=
golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -129,7 +134,10 @@ golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

3
go.work
View file

@ -1,3 +0,0 @@
go 1.24.3
use .

43
go.work.sum
View file

@ -1,43 +0,0 @@
github.com/bwesterb/go-ristretto v1.2.3 h1:1w53tCkGhCQ5djbat3+MH0BAQ5Kfgbt56UZQ/JMzngw=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ=
github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8=
golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

17
main.go Normal file
View file

@ -0,0 +1,17 @@
package main
import (
"os"
"github.com/Snider/Borg/cmd"
"github.com/Snider/Borg/pkg/logger"
)
func main() {
verbose, _ := cmd.RootCmd.PersistentFlags().GetBool("verbose")
log := logger.New(verbose)
if err := cmd.Execute(log); err != nil {
log.Error("fatal error", "err", err)
os.Exit(1)
}
}

105
pkg/github/github.go
View file

@ -1,51 +1,106 @@
package github
import (
"context"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
"golang.org/x/oauth2"
)
type Repo struct {
CloneURL string `json:"clone_url"`
}
func GetPublicRepos(userOrOrg string) ([]string, error) {
return GetPublicReposWithAPIURL("https://api.github.com", userOrOrg)
func GetPublicRepos(ctx context.Context, userOrOrg string) ([]string, error) {
return GetPublicReposWithAPIURL(ctx, "https://api.github.com", userOrOrg)
}
func GetPublicReposWithAPIURL(apiURL, userOrOrg string) ([]string, error) {
if userOrOrg == "" {
return nil, fmt.Errorf("user or organization cannot be empty")
func newAuthenticatedClient(ctx context.Context) *http.Client {
token := os.Getenv("GITHUB_TOKEN")
if token == "" {
return http.DefaultClient
}
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
return oauth2.NewClient(ctx, ts)
}
resp, err := http.Get(fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg))
if err != nil {
return nil, err
}
defer resp.Body.Close()
func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([]string, error) {
client := newAuthenticatedClient(ctx)
var allCloneURLs []string
url := fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg)
if resp.StatusCode != http.StatusOK {
// Try organization endpoint
resp, err = http.Get(fmt.Sprintf("%s/orgs/%s/repos", apiURL, userOrOrg))
for {
if err := ctx.Err(); err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
}
defer resp.Body.Close()
req.Header.Set("User-Agent", "Borg-Data-Collector")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
// Try organization endpoint
url = fmt.Sprintf("%s/orgs/%s/repos", apiURL, userOrOrg)
req, err = http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Borg-Data-Collector")
resp, err = client.Do(req)
if err != nil {
return nil, err
}
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, fmt.Errorf("failed to fetch repos: %s", resp.Status)
}
var repos []Repo
if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
resp.Body.Close()
return nil, err
}
resp.Body.Close()
for _, repo := range repos {
allCloneURLs = append(allCloneURLs, repo.CloneURL)
}
linkHeader := resp.Header.Get("Link")
if linkHeader == "" {
break
}
nextURL := findNextURL(linkHeader)
if nextURL == "" {
break
}
url = nextURL
}
var repos []Repo
if err := json.NewDecoder(resp.Body).Decode(&repos); err != nil {
return nil, err
}
var cloneURLs []string
for _, repo := range repos {
cloneURLs = append(cloneURLs, repo.CloneURL)
}
return cloneURLs, nil
return allCloneURLs, nil
}
func findNextURL(linkHeader string) string {
links := strings.Split(linkHeader, ",")
for _, link := range links {
parts := strings.Split(link, ";")
if len(parts) == 2 && strings.TrimSpace(parts[1]) == `rel="next"` {
return strings.Trim(strings.TrimSpace(parts[0]), "<>")
}
}
return ""
}

42
pkg/github/github_test.go
View file

@ -1,42 +0,0 @@
package github
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestGetPublicRepos_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`[{"clone_url": "https://github.com/good/repo.git"}]`))
}))
defer server.Close()
repos, err := GetPublicReposWithAPIURL(server.URL, "good")
if err != nil {
t.Errorf("Expected no error, got %v", err)
}
if len(repos) != 1 || repos[0] != "https://github.com/good/repo.git" {
t.Errorf("Expected one repo, got %v", repos)
}
}
func TestGetPublicRepos_Bad(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
_, err := GetPublicReposWithAPIURL(server.URL, "bad")
if err == nil {
t.Errorf("Expected an error, got nil")
}
}
func TestGetPublicRepos_Ugly(t *testing.T) {
_, err := GetPublicReposWithAPIURL("http://localhost", "")
if err == nil {
t.Errorf("Expected an error for empty user/org, got nil")
}
}

16
pkg/logger/logger.go Normal file
View file

@ -0,0 +1,16 @@
package logger
import (
"log/slog"
"os"
)
func New(verbose bool) *slog.Logger {
level := slog.LevelInfo
if verbose {
level = slog.LevelDebug
}
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
Level: level,
}))
}

15
pkg/pwa/pwa.go
View file

@ -9,6 +9,7 @@ import (
"path"
"github.com/Snider/Borg/pkg/datanode"
"github.com/schollz/progressbar/v3"
"golang.org/x/net/html"
)
@ -80,7 +81,10 @@ func FindManifest(pageURL string) (string, error) {
}
// DownloadAndPackagePWA downloads all assets of a PWA and packages them into a DataNode.
func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNode, error) {
func DownloadAndPackagePWA(baseURL string, manifestURL string, bar *progressbar.ProgressBar) (*datanode.DataNode, error) {
if bar == nil {
return nil, fmt.Errorf("progress bar cannot be nil")
}
manifestAbsURL, err := resolveURL(baseURL, manifestURL)
if err != nil {
return nil, fmt.Errorf("could not resolve manifest URL: %w", err)
@ -110,7 +114,7 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo
if err != nil {
return nil, fmt.Errorf("could not resolve start_url: %w", err)
}
err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL)
err = downloadAndAddFile(dn, startURLAbs, manifest.StartURL, bar)
if err != nil {
return nil, fmt.Errorf("failed to download start_url asset: %w", err)
}
@ -122,14 +126,14 @@ func DownloadAndPackagePWA(baseURL string, manifestURL string) (*datanode.DataNo
fmt.Printf("Warning: could not resolve icon URL %s: %v\n", icon.Src, err)
continue
}
err = downloadAndAddFile(dn, iconURLAbs, icon.Src)
err = downloadAndAddFile(dn, iconURLAbs, icon.Src, bar)
if err != nil {
fmt.Printf("Warning: failed to download icon %s: %v\n", icon.Src, err)
}
}
baseURLAbs, _ := url.Parse(baseURL)
err = downloadAndAddFile(dn, baseURLAbs, "index.html")
err = downloadAndAddFile(dn, baseURLAbs, "index.html", bar)
if err != nil {
return nil, fmt.Errorf("failed to download base HTML: %w", err)
}
@ -149,7 +153,7 @@ func resolveURL(base, ref string) (*url.URL, error) {
return baseURL.ResolveReference(refURL), nil
}
func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string) error {
func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath string, bar *progressbar.ProgressBar) error {
resp, err := http.Get(fileURL.String())
if err != nil {
return err
@ -165,5 +169,6 @@ func downloadAndAddFile(dn *datanode.DataNode, fileURL *url.URL, internalPath st
return err
}
dn.AddData(path.Clean(internalPath), data)
bar.Add(1)
return nil
}

5
pkg/pwa/pwa_test.go
View file

@ -4,6 +4,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"github.com/schollz/progressbar/v3"
)
func TestFindManifest(t *testing.T) {
@ -78,7 +80,8 @@ func TestDownloadAndPackagePWA(t *testing.T) {
}))
defer server.Close()
dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json")
bar := progressbar.New(1)
dn, err := DownloadAndPackagePWA(server.URL, server.URL+"/manifest.json", bar)
if err != nil {
t.Fatalf("DownloadAndPackagePWA failed: %v", err)
}

5
pkg/vcs/git.go
View file

@ -1,6 +1,7 @@
package vcs
import (
"io"
"os"
"path/filepath"
@ -10,7 +11,7 @@ import (
)
// CloneGitRepository clones a Git repository from a URL and packages it into a DataNode.
func CloneGitRepository(repoURL string) (*datanode.DataNode, error) {
func CloneGitRepository(repoURL string, progress io.Writer) (*datanode.DataNode, error) {
tempPath, err := os.MkdirTemp("", "borg-clone-*")
if err != nil {
return nil, err
@ -19,7 +20,7 @@ func CloneGitRepository(repoURL string) (*datanode.DataNode, error) {
_, err = git.PlainClone(tempPath, false, &git.CloneOptions{
URL: repoURL,
Progress: os.Stdout,
Progress: progress,
})
if err != nil {
return nil, err

2
pkg/vcs/git_test.go
View file

@ -69,7 +69,7 @@ func TestCloneGitRepository(t *testing.T) {
}
// Clone the repository using the function we're testing
dn, err := CloneGitRepository("file://" + bareRepoPath)
dn, err := CloneGitRepository("file://"+bareRepoPath, os.Stdout)
if err != nil {
t.Fatalf("CloneGitRepository failed: %v", err)
}

9
pkg/website/website.go
View file

@ -32,7 +32,10 @@ func NewDownloader(maxDepth int) *Downloader {
}
// DownloadAndPackageWebsite downloads a website and packages it into a DataNode.
func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNode, error) {
func DownloadAndPackageWebsite(startURL string, maxDepth int, bar *progressbar.ProgressBar) (*datanode.DataNode, error) {
if bar == nil {
return nil, fmt.Errorf("progress bar cannot be nil")
}
baseURL, err := url.Parse(startURL)
if err != nil {
return nil, err
@ -40,9 +43,7 @@ func DownloadAndPackageWebsite(startURL string, maxDepth int) (*datanode.DataNod
d := NewDownloader(maxDepth)
d.baseURL = baseURL
fmt.Println("Downloading website...")
d.progressBar = progressbar.NewOptions(1, progressbar.OptionSetDescription("Downloading"))
d.progressBar = bar
d.crawl(startURL, 0)
return d.dn, nil

5
pkg/website/website_test.go
View file

@ -4,6 +4,8 @@ import (
"net/http"
"net/http/httptest"
"testing"
"github.com/schollz/progressbar/v3"
)
func TestDownloadAndPackageWebsite(t *testing.T) {
@ -64,7 +66,8 @@ func TestDownloadAndPackageWebsite(t *testing.T) {
}))
defer server.Close()
dn, err := DownloadAndPackageWebsite(server.URL, 2)
bar := progressbar.New(1)
dn, err := DownloadAndPackageWebsite(server.URL, 2, bar)
if err != nil {
t.Fatalf("DownloadAndPackageWebsite failed: %v", err)
}