3d7c7c4634
This commit introduces configurable timeouts for HTTP requests made by the `collect` commands. Key changes: - Created a new `pkg/httpclient` package with a `NewClient` function that returns an `http.Client` with configurable timeouts for total, connect, TLS, and header stages. - Added `--timeout`, `--connect-timeout`, `--tls-timeout`, and `--header-timeout` persistent flags to the `collect` command, making them available to all its subcommands. - Refactored the `pkg/website`, `pkg/pwa`, and `pkg/github` packages to accept and use a custom `http.Client`, allowing the timeout configurations to be injected. - Updated the `collect website`, `collect pwa`, and `collect github repos` commands to create a configured HTTP client based on the new flags and pass it to the respective packages. - Added unit tests for the `pkg/httpclient` package to verify correct timeout configuration. - Fixed all test and build failures that resulted from the refactoring. - Addressed an unrelated build failure by creating a placeholder file (`pkg/player/frontend/demo-track.smsg`). This work addresses the initial requirement for configurable timeouts via command-line flags. Further work is needed to implement per-domain overrides from a configuration file and idle timeouts for large file downloads. Co-authored-by: Snider <631881+Snider@users.noreply.github.com>
32 lines
911 B
Go
32 lines
911 B
Go
package cmd
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
)
|
|
|
|
// collectCmd represents the collect command
|
|
var collectCmd = NewCollectCmd()
|
|
|
|
func init() {
|
|
RootCmd.AddCommand(GetCollectCmd())
|
|
}
|
|
func NewCollectCmd() *cobra.Command {
|
|
cmd := &cobra.Command{
|
|
Use: "collect",
|
|
Short: "Collect a resource from a URI.",
|
|
Long: `Collect a resource from a URI and store it in a DataNode.`,
|
|
}
|
|
|
|
cmd.PersistentFlags().Duration("timeout", 0, "Total request timeout (e.g., 60s). 0 means no timeout.")
|
|
cmd.PersistentFlags().Duration("connect-timeout", 10*time.Second, "TCP connection establishment timeout (e.g., 10s)")
|
|
cmd.PersistentFlags().Duration("tls-timeout", 10*time.Second, "TLS handshake timeout (e.g., 10s)")
|
|
cmd.PersistentFlags().Duration("header-timeout", 30*time.Second, "Time to receive response headers timeout (e.g., 30s)")
|
|
|
|
return cmd
|
|
}
|
|
|
|
func GetCollectCmd() *cobra.Command {
|
|
return collectCmd
|
|
}
|