diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..ca38516 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,97 @@ +# Borg Data Collector + +Borg Data Collector is a command-line tool for collecting and managing data from various sources. + +## Commands + +### `collect` + +This command is used to collect resources from different sources and store them in a DataNode. + +#### `collect github repo` + +Collects a single Git repository and stores it in a DataNode. + +**Usage:** +``` +borg collect github repo [repository-url] [flags] +``` + +**Flags:** +- `--output string`: Output file for the DataNode (default "repo.dat") + +**Example:** +``` +./borg collect github repo https://github.com/Snider/Borg --output borg.dat +``` + +#### `collect website` + +Collects a single website and stores it in a DataNode. + +**Usage:** +``` +borg collect website [url] [flags] +``` + +**Flags:** +- `--output string`: Output file for the DataNode (default "website.dat") +- `--depth int`: Recursion depth for downloading (default 2) + +**Example:** +``` +./borg collect website https://google.com --output website.dat --depth 1 +``` + +#### `collect pwa` + +Collects a single PWA and stores it in a DataNode. + +**Usage:** +``` +borg collect pwa [flags] +``` + +**Flags:** +- `--uri string`: The URI of the PWA to collect +- `--output string`: Output file for the DataNode (default "pwa.dat") + +**Example:** +``` +./borg collect pwa --uri https://squoosh.app --output squoosh.dat +``` + +### `serve` + +Serves the contents of a packaged DataNode file using a static file server. + +**Usage:** +``` +borg serve [file] [flags] +``` + +**Flags:** +- `--port string`: Port to serve the DataNode on (default "8080") + +**Example:** +``` +./borg serve squoosh.dat --port 8888 +``` + +## Inspecting a DataNode + +The `examples` directory contains a Go program that can be used to inspect the contents of a `.dat` file. + +**Usage:** +``` +go run examples/inspect_datanode.go +``` + +**Example:** +``` +# First, create a .dat file +./borg collect github repo https://github.com/Snider/Borg --output borg.dat + +# Then, inspect it +go run examples/inspect_datanode.go borg.dat +``` diff --git a/examples/collect_github_repo.sh b/examples/collect_github_repo.sh new file mode 100755 index 0000000..a07b047 --- /dev/null +++ b/examples/collect_github_repo.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Example of using the 'borg collect github repo' command. + +# This script clones the specified Git repository and saves it as a .dat file. +# The main executable 'borg' is built from the project's root. +# Make sure you have built the project by running 'go build -o borg main.go' in the root directory. + +./borg collect github repo https://github.com/Snider/Borg --output borg.dat diff --git a/examples/collect_pwa.sh b/examples/collect_pwa.sh new file mode 100755 index 0000000..bbcc53b --- /dev/null +++ b/examples/collect_pwa.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Example of using the 'borg collect pwa' command. + +# This script downloads the specified PWA and saves it as a .dat file. +# The main executable 'borg' is built from the project's root. +# Make sure you have built the project by running 'go build -o borg main.go' in the root directory. + +./borg collect pwa --uri https://squoosh.app --output squoosh.dat diff --git a/examples/collect_website.sh b/examples/collect_website.sh new file mode 100755 index 0000000..fa80dd1 --- /dev/null +++ b/examples/collect_website.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Example of using the 'borg collect website' command. + +# This script crawls the specified website and saves it as a .dat file. +# The main executable 'borg' is built from the project's root. +# Make sure you have built the project by running 'go build -o borg main.go' in the root directory. + +./borg collect website https://google.com --output website.dat --depth 1 diff --git a/examples/inspect_datanode.go b/examples/inspect_datanode.go new file mode 100644 index 0000000..9c97ede --- /dev/null +++ b/examples/inspect_datanode.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "io/fs" + "os" + + "github.com/Snider/Borg/pkg/datanode" +) + +func main() { + if len(os.Args) != 2 { + fmt.Println("Usage: go run inspect_datanode.go ") + os.Exit(1) + } + + datFile := os.Args[1] + + data, err := os.ReadFile(datFile) + if err != nil { + fmt.Printf("Error reading .dat file: %v\n", err) + os.Exit(1) + } + + dn, err := datanode.FromTar(data) + if err != nil { + fmt.Printf("Error creating DataNode from tarball: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Contents of %s:\n", datFile) + err = dn.Walk(".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + fmt.Println(path) + return nil + }) + if err != nil { + fmt.Printf("Error walking DataNode: %v\n", err) + os.Exit(1) + } +}