From beaeb04f037ebf1dd7c511a0d4c9074d582c2b24 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sun, 2 Nov 2025 00:59:46 +0000 Subject: [PATCH] feat: Add authenticated GitHub access and structured logging This commit introduces two key improvements to the application: 1. **Authenticated GitHub API Access:** The GitHub client now uses a personal access token (PAT) from the `GITHUB_TOKEN` environment variable if it is available. This increases the rate limit for GitHub API requests, making the tool more robust for users who need to collect a large number of repositories. 2. **Structured Logging:** The application now uses the standard library's `slog` package for structured logging. A `--verbose` flag has been added to the root command to control the log level, allowing for more detailed output when needed. This makes the application's output more consistent and easier to parse. --- cmd/all.go | 14 ++++++---- cmd/collect.go | 2 +- cmd/collect_github_release_subcommand.go | 35 ++++++++++++------------ cmd/main_test.go | 12 ++++---- cmd/root.go | 22 ++++++--------- cmd/serve.go | 2 +- go.mod | 4 +++ go.sum | 6 ++++ main.go | 8 ++++-- pkg/github/github.go | 17 +++++++++++- pkg/logger/logger.go | 16 +++++++++++ 11 files changed, 90 insertions(+), 48 deletions(-) create mode 100644 pkg/logger/logger.go diff --git a/cmd/all.go b/cmd/all.go index d5364a1..fe152e6 100644 --- a/cmd/all.go +++ b/cmd/all.go @@ -3,6 +3,7 @@ package cmd import ( "context" "fmt" + "log/slog" "os" "strings" @@ -19,26 +20,27 @@ var allCmd = &cobra.Command{ Long: `Collect all public repositories from a user or organization and store them in a DataNode.`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { + log := cmd.Context().Value("logger").(*slog.Logger) repos, err := github.GetPublicRepos(context.Background(), args[0]) if err != nil { - fmt.Println(err) + log.Error("failed to get public repos", "err", err) return } outputDir, _ := cmd.Flags().GetString("output") for _, repoURL := range repos { - fmt.Printf("Cloning %s...\n", repoURL) + log.Info("cloning repository", "url", repoURL) dn, err := vcs.CloneGitRepository(repoURL) if err != nil { - fmt.Printf("Error cloning %s: %s\n", repoURL, err) + log.Error("failed to clone repository", "url", repoURL, "err", err) continue } data, err := dn.ToTar() if err != nil { - fmt.Printf("Error serializing DataNode for %s: %v\n", repoURL, err) + log.Error("failed to serialize datanode", "url", repoURL, "err", err) continue } @@ -46,7 +48,7 @@ var allCmd = &cobra.Command{ outputFile := fmt.Sprintf("%s/%s.dat", outputDir, repoName) err = os.WriteFile(outputFile, data, 0644) if err != nil { - fmt.Printf("Error writing DataNode for %s to file: %v\n", repoURL, err) + log.Error("failed to write datanode to file", "url", repoURL, "err", err) continue } } @@ -54,6 +56,6 @@ var allCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(allCmd) + RootCmd.AddCommand(allCmd) allCmd.PersistentFlags().String("output", ".", "Output directory for the DataNodes") } diff --git a/cmd/collect.go b/cmd/collect.go index 57960b2..8e3d817 100644 --- a/cmd/collect.go +++ b/cmd/collect.go @@ -12,5 +12,5 @@ var collectCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(collectCmd) + RootCmd.AddCommand(collectCmd) } diff --git a/cmd/collect_github_release_subcommand.go b/cmd/collect_github_release_subcommand.go index b844704..e1ef814 100644 --- a/cmd/collect_github_release_subcommand.go +++ b/cmd/collect_github_release_subcommand.go @@ -2,8 +2,8 @@ package cmd import ( "bytes" - "fmt" "io" + "log/slog" "net/http" "os" "path/filepath" @@ -23,6 +23,7 @@ var collectGithubReleaseCmd = &cobra.Command{ Long: `Download the latest release of a file from GitHub releases. If the file or URL has a version number, it will check for a higher version and download it if found.`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { + log := cmd.Context().Value("logger").(*slog.Logger) repoURL := args[0] outputDir, _ := cmd.Flags().GetString("output") pack, _ := cmd.Flags().GetBool("pack") @@ -31,25 +32,25 @@ var collectGithubReleaseCmd = &cobra.Command{ owner, repo, err := borg_github.ParseRepoFromURL(repoURL) if err != nil { - fmt.Printf("Error parsing repository URL: %v\n", err) + log.Error("failed to parse repository url", "err", err) return } release, err := borg_github.GetLatestRelease(owner, repo) if err != nil { - fmt.Printf("Error getting latest release: %v\n", err) + log.Error("failed to get latest release", "err", err) return } - fmt.Printf("Found latest release: %s\n", release.GetTagName()) + log.Info("found latest release", "tag", release.GetTagName()) if version != "" { if !semver.IsValid(version) { - fmt.Printf("Invalid version string: %s\n", version) + log.Error("invalid version string", "version", version) return } if semver.Compare(release.GetTagName(), version) <= 0 { - fmt.Printf("Latest release (%s) is not newer than the provided version (%s).\n", release.GetTagName(), version) + log.Info("latest release is not newer than the provided version", "latest", release.GetTagName(), "provided", version) return } } @@ -57,24 +58,24 @@ var collectGithubReleaseCmd = &cobra.Command{ if pack { dn := datanode.New() for _, asset := range release.Assets { - fmt.Printf("Downloading asset: %s\n", asset.GetName()) + log.Info("downloading asset", "name", asset.GetName()) resp, err := http.Get(asset.GetBrowserDownloadURL()) if err != nil { - fmt.Printf("Error downloading asset: %v\n", err) + log.Error("failed to download asset", "name", asset.GetName(), "err", err) continue } defer resp.Body.Close() var buf bytes.Buffer _, err = io.Copy(&buf, resp.Body) if err != nil { - fmt.Printf("Error reading asset: %v\n", err) + log.Error("failed to read asset", "name", asset.GetName(), "err", err) continue } dn.AddData(asset.GetName(), buf.Bytes()) } tar, err := dn.ToTar() if err != nil { - fmt.Printf("Error creating DataNode: %v\n", err) + log.Error("failed to create datanode", "err", err) return } outputFile := outputDir @@ -83,13 +84,13 @@ var collectGithubReleaseCmd = &cobra.Command{ } err = os.WriteFile(outputFile, tar, 0644) if err != nil { - fmt.Printf("Error writing DataNode: %v\n", err) + log.Error("failed to write datanode", "err", err) return } - fmt.Printf("DataNode saved to %s\n", outputFile) + log.Info("datanode saved", "path", outputFile) } else { if len(release.Assets) == 0 { - fmt.Println("No assets found in the latest release.") + log.Info("no assets found in the latest release") return } var assetToDownload *gh.ReleaseAsset @@ -101,20 +102,20 @@ var collectGithubReleaseCmd = &cobra.Command{ } } if assetToDownload == nil { - fmt.Printf("Asset '%s' not found in the latest release.\n", file) + log.Error("asset not found in the latest release", "asset", file) return } } else { assetToDownload = release.Assets[0] } outputPath := filepath.Join(outputDir, assetToDownload.GetName()) - fmt.Printf("Downloading asset: %s\n", assetToDownload.GetName()) + log.Info("downloading asset", "name", assetToDownload.GetName()) err = borg_github.DownloadReleaseAsset(assetToDownload, outputPath) if err != nil { - fmt.Printf("Error downloading asset: %v\n", err) + log.Error("failed to download asset", "name", assetToDownload.GetName(), "err", err) return } - fmt.Printf("Asset downloaded to %s\n", outputPath) + log.Info("asset downloaded", "path", outputPath) } }, } diff --git a/cmd/main_test.go b/cmd/main_test.go index 9f77255..cb55db3 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -1,18 +1,18 @@ package cmd import ( + "log/slog" + "os" "testing" ) func TestExecute(t *testing.T) { // This test simply checks that the Execute function can be called without error. // It doesn't actually test any of the application's functionality. - rootCmd.SetArgs([]string{}) - t.Cleanup(func() { - rootCmd.SetArgs(nil) - }) - if err := Execute(); err != nil { + log := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: slog.LevelDebug, + })) + if err := Execute(log); err != nil { t.Errorf("Execute() failed: %v", err) } } -} diff --git a/cmd/root.go b/cmd/root.go index 7475996..8ca2ebe 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,11 +1,14 @@ package cmd import ( + "context" + "log/slog" + "github.com/spf13/cobra" ) -// rootCmd represents the base command when called without any subcommands -var rootCmd = &cobra.Command{ +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ Use: "borg-data-collector", Short: "A tool for collecting and managing data.", Long: `Borg Data Collector is a command-line tool for cloning Git repositories, @@ -14,18 +17,11 @@ packaging their contents into a single file, and managing the data within.`, // Execute adds all child commands to the root command and sets flags appropriately. // This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() error { - return rootCmd.Execute() +func Execute(log *slog.Logger) error { + RootCmd.SetContext(context.WithValue(context.Background(), "logger", log)) + return RootCmd.Execute() } func init() { - // Here you will define your flags and configuration settings. - // Cobra supports persistent flags, which, if defined here, - // will be global for your application. - - // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.borg-data-collector.yaml)") - - // Cobra also supports local flags, which will only run - // when this action is called directly. - rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + RootCmd.PersistentFlags().BoolP("verbose", "v", false, "Enable verbose logging") } diff --git a/cmd/serve.go b/cmd/serve.go index 13516e7..40dd400 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -44,6 +44,6 @@ var serveCmd = &cobra.Command{ } func init() { - rootCmd.AddCommand(serveCmd) + RootCmd.AddCommand(serveCmd) serveCmd.PersistentFlags().String("port", "8080", "Port to serve the PWA on") } diff --git a/go.mod b/go.mod index cbb03b5..f5be154 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/spf13/cobra v1.10.1 golang.org/x/mod v0.29.0 golang.org/x/net v0.46.0 + golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be ) require ( @@ -21,6 +22,7 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -35,5 +37,7 @@ require ( golang.org/x/crypto v0.43.0 // indirect golang.org/x/sys v0.37.0 // indirect golang.org/x/term v0.36.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index e2688d7..dc266b6 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8J github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -108,6 +110,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -129,7 +132,10 @@ golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/main.go b/main.go index c9cc74f..7c02e17 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,17 @@ package main import ( - "fmt" "os" "github.com/Snider/Borg/cmd" + "github.com/Snider/Borg/pkg/logger" ) func main() { - if err := cmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) + verbose, _ := cmd.RootCmd.PersistentFlags().GetBool("verbose") + log := logger.New(verbose) + if err := cmd.Execute(log); err != nil { + log.Error("fatal error", "err", err) os.Exit(1) } } diff --git a/pkg/github/github.go b/pkg/github/github.go index 0b77968..66ea5c1 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -5,7 +5,10 @@ import ( "encoding/json" "fmt" "net/http" + "os" "strings" + + "golang.org/x/oauth2" ) type Repo struct { @@ -16,7 +19,19 @@ func GetPublicRepos(ctx context.Context, userOrOrg string) ([]string, error) { return GetPublicReposWithAPIURL(ctx, "https://api.github.com", userOrOrg) } +func newAuthenticatedClient(ctx context.Context) *http.Client { + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + return http.DefaultClient + } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + return oauth2.NewClient(ctx, ts) +} + func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([]string, error) { + client := newAuthenticatedClient(ctx) var allCloneURLs []string url := fmt.Sprintf("%s/users/%s/repos", apiURL, userOrOrg) @@ -26,7 +41,7 @@ func GetPublicReposWithAPIURL(ctx context.Context, apiURL, userOrOrg string) ([] return nil, err } req.Header.Set("User-Agent", "Borg-Data-Collector") - resp, err := http.DefaultClient.Do(req) + resp, err := client.Do(req) if err != nil { return nil, err } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go new file mode 100644 index 0000000..0dfc2d2 --- /dev/null +++ b/pkg/logger/logger.go @@ -0,0 +1,16 @@ +package logger + +import ( + "log/slog" + "os" +) + +func New(verbose bool) *slog.Logger { + level := slog.LevelInfo + if verbose { + level = slog.LevelDebug + } + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: level, + })) +}