feat: Add authenticated GitHub access and structured logging

This commit introduces two key improvements to the application:

1.  **Authenticated GitHub API Access:** The GitHub client now uses a personal access token (PAT) from the `GITHUB_TOKEN` environment variable if it is available. This increases the rate limit for GitHub API requests, making the tool more robust for users who need to collect a large number of repositories.

2.  **Structured Logging:** The application now uses the standard library's `slog` package for structured logging. A `--verbose` flag has been added to the root command to control the log level, allowing for more detailed output when needed. This makes the application's output more consistent and easier to parse.

cmd/all.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"os"
 	"strings"
 
@@ -19,26 +20,27 @@ var allCmd = &cobra.Command{
 	Long:  `Collect all public repositories from a user or organization and store them in a DataNode.`,
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
+		log := cmd.Context().Value("logger").(*slog.Logger)
 		repos, err := github.GetPublicRepos(context.Background(), args[0])
 		if err != nil {
-			fmt.Println(err)
+			log.Error("failed to get public repos", "err", err)
 			return
 		}
 
 		outputDir, _ := cmd.Flags().GetString("output")
 
 		for _, repoURL := range repos {
-			fmt.Printf("Cloning %s...\n", repoURL)
+			log.Info("cloning repository", "url", repoURL)
 
 			dn, err := vcs.CloneGitRepository(repoURL)
 			if err != nil {
-				fmt.Printf("Error cloning %s: %s\n", repoURL, err)
+				log.Error("failed to clone repository", "url", repoURL, "err", err)
 				continue
 			}
 
 			data, err := dn.ToTar()
 			if err != nil {
-				fmt.Printf("Error serializing DataNode for %s: %v\n", repoURL, err)
+				log.Error("failed to serialize datanode", "url", repoURL, "err", err)
 				continue
 			}
 
@@ -46,7 +48,7 @@ var allCmd = &cobra.Command{
 			outputFile := fmt.Sprintf("%s/%s.dat", outputDir, repoName)
 			err = os.WriteFile(outputFile, data, 0644)
 			if err != nil {
-				fmt.Printf("Error writing DataNode for %s to file: %v\n", repoURL, err)
+				log.Error("failed to write datanode to file", "url", repoURL, "err", err)
 				continue
 			}
 		}
@@ -54,6 +56,6 @@ var allCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(allCmd)
+	RootCmd.AddCommand(allCmd)
 	allCmd.PersistentFlags().String("output", ".", "Output directory for the DataNodes")
 }

cmd/collect.go

@@ -12,5 +12,5 @@ var collectCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(collectCmd)
+	RootCmd.AddCommand(collectCmd)
 }

cmd/collect_github_release_subcommand.go

@@ -2,8 +2,8 @@ package cmd
 
 import (
 	"bytes"
-	"fmt"
 	"io"
+	"log/slog"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -23,6 +23,7 @@ var collectGithubReleaseCmd = &cobra.Command{
 	Long:  `Download the latest release of a file from GitHub releases. If the file or URL has a version number, it will check for a higher version and download it if found.`,
 	Args:  cobra.ExactArgs(1),
 	Run: func(cmd *cobra.Command, args []string) {
+		log := cmd.Context().Value("logger").(*slog.Logger)
 		repoURL := args[0]
 		outputDir, _ := cmd.Flags().GetString("output")
 		pack, _ := cmd.Flags().GetBool("pack")
@@ -31,25 +32,25 @@ var collectGithubReleaseCmd = &cobra.Command{
 
 		owner, repo, err := borg_github.ParseRepoFromURL(repoURL)
 		if err != nil {
-			fmt.Printf("Error parsing repository URL: %v\n", err)
+			log.Error("failed to parse repository url", "err", err)
 			return
 		}
 
 		release, err := borg_github.GetLatestRelease(owner, repo)
 		if err != nil {
-			fmt.Printf("Error getting latest release: %v\n", err)
+			log.Error("failed to get latest release", "err", err)
 			return
 		}
 
-		fmt.Printf("Found latest release: %s\n", release.GetTagName())
+		log.Info("found latest release", "tag", release.GetTagName())
 
 		if version != "" {
 			if !semver.IsValid(version) {
-				fmt.Printf("Invalid version string: %s\n", version)
+				log.Error("invalid version string", "version", version)
 				return
 			}
 			if semver.Compare(release.GetTagName(), version) <= 0 {
-				fmt.Printf("Latest release (%s) is not newer than the provided version (%s).\n", release.GetTagName(), version)
+				log.Info("latest release is not newer than the provided version", "latest", release.GetTagName(), "provided", version)
 				return
 			}
 		}
@@ -57,24 +58,24 @@ var collectGithubReleaseCmd = &cobra.Command{
 		if pack {
 			dn := datanode.New()
 			for _, asset := range release.Assets {
-				fmt.Printf("Downloading asset: %s\n", asset.GetName())
+				log.Info("downloading asset", "name", asset.GetName())
 				resp, err := http.Get(asset.GetBrowserDownloadURL())
 				if err != nil {
-					fmt.Printf("Error downloading asset: %v\n", err)
+					log.Error("failed to download asset", "name", asset.GetName(), "err", err)
 					continue
 				}
 				defer resp.Body.Close()
 				var buf bytes.Buffer
 				_, err = io.Copy(&buf, resp.Body)
 				if err != nil {
-					fmt.Printf("Error reading asset: %v\n", err)
+					log.Error("failed to read asset", "name", asset.GetName(), "err", err)
 					continue
 				}
 				dn.AddData(asset.GetName(), buf.Bytes())
 			}
 			tar, err := dn.ToTar()
 			if err != nil {
-				fmt.Printf("Error creating DataNode: %v\n", err)
+				log.Error("failed to create datanode", "err", err)
 				return
 			}
 			outputFile := outputDir
@@ -83,13 +84,13 @@ var collectGithubReleaseCmd = &cobra.Command{
 			}
 			err = os.WriteFile(outputFile, tar, 0644)
 			if err != nil {
-				fmt.Printf("Error writing DataNode: %v\n", err)
+				log.Error("failed to write datanode", "err", err)
 				return
 			}
-			fmt.Printf("DataNode saved to %s\n", outputFile)
+			log.Info("datanode saved", "path", outputFile)
 		} else {
 			if len(release.Assets) == 0 {
-				fmt.Println("No assets found in the latest release.")
+				log.Info("no assets found in the latest release")
 				return
 			}
 			var assetToDownload *gh.ReleaseAsset
@@ -101,20 +102,20 @@ var collectGithubReleaseCmd = &cobra.Command{
 					}
 				}
 				if assetToDownload == nil {
-					fmt.Printf("Asset '%s' not found in the latest release.\n", file)
+					log.Error("asset not found in the latest release", "asset", file)
 					return
 				}
 			} else {
 				assetToDownload = release.Assets[0]
 			}
 			outputPath := filepath.Join(outputDir, assetToDownload.GetName())
-			fmt.Printf("Downloading asset: %s\n", assetToDownload.GetName())
+			log.Info("downloading asset", "name", assetToDownload.GetName())
 			err = borg_github.DownloadReleaseAsset(assetToDownload, outputPath)
 			if err != nil {
-				fmt.Printf("Error downloading asset: %v\n", err)
+				log.Error("failed to download asset", "name", assetToDownload.GetName(), "err", err)
 				return
 			}
-			fmt.Printf("Asset downloaded to %s\n", outputPath)
+			log.Info("asset downloaded", "path", outputPath)
 		}
 	},
 }

cmd/main_test.go

@@ -1,18 +1,18 @@
 package cmd
 
 import (
+	"log/slog"
+	"os"
 	"testing"
 )
 
 func TestExecute(t *testing.T) {
 	// This test simply checks that the Execute function can be called without error.
 	// It doesn't actually test any of the application's functionality.
-	rootCmd.SetArgs([]string{})
-	t.Cleanup(func() {
-		rootCmd.SetArgs(nil)
-	})
-	if err := Execute(); err != nil {
+	log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+	}))
+	if err := Execute(log); err != nil {
 		t.Errorf("Execute() failed: %v", err)
 	}
 }
-}

cmd/root.go

@@ -1,11 +1,14 @@
 package cmd
 
 import (
+	"context"
+	"log/slog"
+
 	"github.com/spf13/cobra"
 )
 
-// rootCmd represents the base command when called without any subcommands
-var rootCmd = &cobra.Command{
+// RootCmd represents the base command when called without any subcommands
+var RootCmd = &cobra.Command{
 	Use:   "borg-data-collector",
 	Short: "A tool for collecting and managing data.",
 	Long: `Borg Data Collector is a command-line tool for cloning Git repositories,
@@ -14,18 +17,11 @@ packaging their contents into a single file, and managing the data within.`,
 
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
-func Execute() error {
-	return rootCmd.Execute()
+func Execute(log *slog.Logger) error {
+	RootCmd.SetContext(context.WithValue(context.Background(), "logger", log))
+	return RootCmd.Execute()
 }
 
 func init() {
-	// Here you will define your flags and configuration settings.
-	// Cobra supports persistent flags, which, if defined here,
-	// will be global for your application.
-
-	// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.borg-data-collector.yaml)")
-
-	// Cobra also supports local flags, which will only run
-	// when this action is called directly.
-	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+	RootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging")
 }

cmd/serve.go

@@ -44,6 +44,6 @@ var serveCmd = &cobra.Command{
 }
 
 func init() {
-	rootCmd.AddCommand(serveCmd)
+	RootCmd.AddCommand(serveCmd)
 	serveCmd.PersistentFlags().String("port", "8080", "Port to serve the PWA on")
 }

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/spf13/cobra v1.10.1
 	golang.org/x/mod v0.29.0
 	golang.org/x/net v0.46.0
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
 )
 
 require (
@@ -21,6 +22,7 @@ require (
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-git/go-billy/v5 v5.6.2 // indirect
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -35,5 +37,7 @@ require (
 	golang.org/x/crypto v0.43.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect
 	golang.org/x/term v0.36.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

go.sum

@@ -37,6 +37,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J
 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
@@ -108,6 +110,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
 golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -129,7 +132,10 @@ golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
 golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

main.go

@@ -1,15 +1,17 @@
 package main
 
 import (
-	"fmt"
 	"os"
 
 	"github.com/Snider/Borg/cmd"
+	"github.com/Snider/Borg/pkg/logger"
 )
 
 func main() {
-	if err := cmd.Execute(); err != nil {
-		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+	verbose, _ := cmd.RootCmd.PersistentFlags().GetBool("verbose")
+	log := logger.New(verbose)
+	if err := cmd.Execute(log); err != nil {
+		log.Error("fatal error", "err", err)
 		os.Exit(1)
 	}
 }

pkg/github/github.go

@@ -5,7 +5,10 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
+
+	"golang.org/x/oauth2"
 )
 
 type Repo struct {
@@ -16,7 +19,19 @@ func GetPublicRepos(ctx context.Context, userOrOrg string) ([]string, error) {
 	return GetPublicReposWithAPIURL(ctx, "https://api.github.com", userOrOrg)
 }
 
+func newAuthenticatedClient(ctx context.Context) *http.Client {
+	token := os.Getenv("GITHUB_TOKEN")
+	if token == "" {
+		return http.DefaultClient
+	}
+	ts := oauth2.StaticTokenSource(
+		&oauth2.Token{AccessToken: token},
+	)
+	return oauth2.NewClient(ctx, ts)
+}
+
 func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([]string, error) {
+	client := newAuthenticatedClient(ctx)
 	var allCloneURLs []string
 	url := fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg)
 
@@ -26,7 +41,7 @@ func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([]
 			return nil, err
 		}
 		req.Header.Set("User-Agent", "Borg-Data-Collector")
-		resp, err := http.DefaultClient.Do(req)
+		resp, err := client.Do(req)
 		if err != nil {
 			return nil, err
 		}

pkg/logger/logger.go

@@ -0,0 +1,16 @@
+package logger
+
+import (
+	"log/slog"
+	"os"
+)
+
+func New(verbose bool) *slog.Logger {
+	level := slog.LevelInfo
+	if verbose {
+		level = slog.LevelDebug
+	}
+	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: level,
+	}))
+}