Snider/Borg
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

Merge pull request #15 from Snider/docs-update-examples-and-matrix

Browse source 
Feat: Implement Go examples and refactor matrix execution
This commit is contained in:
Snider 2025-11-14 11:21:38 +00:00 • committed by GitHub
commit bbf9bddbcc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 487 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
cmd/run.go
View file

@ -1,11 +1,7 @@
package cmd
import (
"archive/tar"
"io"
"os"
"path/filepath"
"github.com/Snider/Borg/pkg/matrix"
"github.com/spf13/cobra"
)
@ -17,57 +13,7 @@ func NewRunCmd() *cobra.Command {
Short: "Run a Terminal Isolation Matrix.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
matrixFile := args[0]
// Create a temporary directory to unpack the matrix file.
tempDir, err := os.MkdirTemp("", "borg-run-*")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
// Unpack the matrix file.
file, err := os.Open(matrixFile)
if err != nil {
return err
}
defer file.Close()
tr := tar.NewReader(file)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
path := filepath.Join(tempDir, header.Name)
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
continue
}
outFile, err := os.Create(path)
if err != nil {
return err
}
defer outFile.Close()
if _, err := io.Copy(outFile, tr); err != nil {
return err
}
}
// Run the matrix.
runc := execCommand("runc", "run", "borg-container")
runc.Dir = tempDir
runc.Stdout = os.Stdout
runc.Stderr = os.Stderr
runc.Stdin = os.Stdin
return runc.Run()
return matrix.Run(args[0])
},
}
}

33
cmd/run_test.go
View file

@ -10,30 +10,21 @@ import (
"github.com/Snider/Borg/pkg/matrix"
)
func helperProcess(command string, args ...string) *exec.Cmd {
cs := []string{"-test.run=TestHelperProcess", "--", command}
cs = append(cs, args...)
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
return cmd
}
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
os.Exit(0)
}
func TestRunCmd_Good(t *testing.T) {
// Create a dummy matrix file.
matrixPath := createDummyMatrix(t)
// Mock the exec.Command function.
origExecCommand := execCommand
execCommand = helperProcess
// Mock the exec.Command function in the matrix package.
origExecCommand := matrix.ExecCommand
matrix.ExecCommand = func(command string, args ...string) *exec.Cmd {
cs := []string{"-test.run=TestHelperProcess", "--", command}
cs = append(cs, args...)
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
return cmd
}
t.Cleanup(func() {
execCommand = origExecCommand
matrix.ExecCommand = origExecCommand
})
// Run the run command.
@ -90,8 +81,8 @@ func createDummyMatrix(t *testing.T) string {
tw := tar.NewWriter(matrixFile)
// Add a dummy config.json.
configContent := []byte(matrix.DefaultConfigJSON)
// Add a dummy config.json. This is not a valid config, but it's enough to test the run command.
configContent := []byte(`{}`)
hdr := &tar.Header{
Name: "config.json",
Mode: 0600,

169
docs/index.md
View file

@ -47,6 +47,44 @@ borg collect website [url] [flags]
./borg collect website https://google.com --output website.dat --depth 1
```
#### `collect github repos`
Collects all public repositories for a user or organization.
**Usage:**
```
borg collect github repos [user-or-org] [flags]
```
**Example:**
```
./borg collect github repos Snider
```
#### `collect github release`
Downloads the latest release of a file from GitHub releases.
**Usage:**
```
borg collect github release [repository-url] [flags]
```
**Flags:**
- `--output string`: Output directory for the downloaded file (default ".")
- `--pack`: Pack all assets into a DataNode
- `--file string`: The file to download from the release
- `--version string`: The version to check against
**Example:**
```
# Download the latest release of the 'borg' executable
./borg collect github release https://github.com/Snider/Borg --file borg
# Pack all assets from the latest release into a DataNode
./borg collect github release https://github.com/Snider/Borg --pack --output borg-release.dat
```
#### `collect pwa`
Collects a single PWA and stores it in a DataNode.
@ -67,6 +105,24 @@ borg collect pwa [flags]
./borg collect pwa --uri https://squoosh.app --output squoosh.dat
```
### `compile`
Compiles a `Borgfile` into a Terminal Isolation Matrix.
**Usage:**
```
borg compile [flags]
```
**Flags:**
- `--file string`: Path to the Borgfile (default "Borgfile")
- `--output string`: Path to the output matrix file (default "a.matrix")
**Example:**
```
./borg compile -f my-borgfile -o my-app.matrix
```
### `serve`
Serves the contents of a packaged DataNode or Terminal Isolation Matrix file using a static file server.
@ -116,33 +172,94 @@ To create a Matrix, use the `--format matrix` flag with any of the `collect` sub
./borg collect github repo https://github.com/Snider/Borg --output borg.matrix --format matrix
```
You can then execute the Matrix with `runc`:
```
# Create a directory for the bundle
mkdir borg-bundle
# Unpack the matrix into the bundle directory
tar -xf borg.matrix -C borg-bundle
# Run the bundle
cd borg-bundle
runc run borg
```
## Inspecting a DataNode
The `examples` directory contains a Go program that can be used to inspect the contents of a `.dat` file.
**Usage:**
```
go run examples/inspect_datanode.go <path to .dat file>
```
The `borg run` command is used to execute a Terminal Isolation Matrix. This command handles the unpacking and execution of the matrix in a secure, isolated environment using `runc`. This ensures that the payload can be safely analyzed without affecting the host system.
**Example:**
```
# First, create a .dat file
./borg collect github repo https://github.com/Snider/Borg --output borg.dat
# Then, inspect it
go run examples/inspect_datanode.go borg.dat
./borg run borg.matrix
```
## Programmatic Usage
The `examples` directory contains a number of Go programs that demonstrate how to use the `borg` package programmatically.
### Inspecting a DataNode
The `inspect_datanode` example demonstrates how to read, decompress, and walk a `.dat` file.
**Usage:**
```
go run examples/inspect_datanode/main.go <path to .dat file>
```
### Creating a Matrix Programmatically
The `create_matrix_programmatically` example demonstrates how to create a Terminal Isolation Matrix from scratch.
**Usage:**
```
go run examples/create_matrix_programmatically/main.go
```
### Running a Matrix Programmatically
The `run_matrix_programmatically` example demonstrates how to run a Terminal Isolation Matrix using the `borg` package.
**Usage:**
```
go run examples/run_matrix_programmatically/main.go
```
### Collecting a Website
The `collect_website` example demonstrates how to collect a website and package it into a `.dat` file.
**Usage:**
```
go run examples/collect_website/main.go
```
### Collecting a GitHub Release
The `collect_github_release` example demonstrates how to collect the latest release of a GitHub repository.
**Usage:**
```
go run examples/collect_github_release/main.go
```
### Collecting All Repositories for a User
The `all` example demonstrates how to collect all public repositories for a GitHub user.
**Usage:**
```
go run examples/all/main.go
```
### Collecting a PWA
The `collect_pwa` example demonstrates how to collect a Progressive Web App and package it into a `.dat` file.
**Usage:**
```
go run examples/collect_pwa/main.go
```
### Collecting a GitHub Repository
The `collect_github_repo` example demonstrates how to clone a GitHub repository and package it into a `.dat` file.
**Usage:**
```
go run examples/collect_github_repo/main.go
```
### Serving a DataNode
The `serve` example demonstrates how to serve the contents of a `.dat` file over HTTP.
**Usage:**
```
go run examples/serve/main.go
```

41
examples/all/main.go
View file

@ -1,7 +1,44 @@
package main
import "fmt"
import (
"context"
"fmt"
"log"
"os"
"github.com/Snider/Borg/pkg/github"
"github.com/Snider/Borg/pkg/vcs"
)
func main() {
fmt.Println("This is a placeholder for the 'all' example.")
log.Println("Collecting all repositories for a user...")
repos, err := github.NewGithubClient().GetPublicRepos(context.Background(), "Snider")
if err != nil {
log.Fatalf("Failed to get public repos: %v", err)
}
cloner := vcs.NewGitCloner()
for _, repo := range repos {
log.Printf("Cloning %s...", repo)
dn, err := cloner.CloneGitRepository(fmt.Sprintf("https://github.com/%s", repo), nil)
if err != nil {
log.Printf("Failed to clone %s: %v", repo, err)
continue
}
tarball, err := dn.ToTar()
if err != nil {
log.Printf("Failed to serialize %s to tar: %v", repo, err)
continue
}
err = os.WriteFile(fmt.Sprintf("%s.dat", repo), tarball, 0644)
if err != nil {
log.Printf("Failed to write %s.dat: %v", repo, err)
continue
}
log.Printf("Successfully created %s.dat", repo)
}
}

39
examples/collect_github_release/main.go
View file

@ -1,7 +1,42 @@
package main
import "fmt"
import (
"log"
"os"
"github.com/Snider/Borg/pkg/github"
)
func main() {
fmt.Println("This is a placeholder for the 'collect github release' example.")
log.Println("Collecting GitHub release...")
owner, repo, err := github.ParseRepoFromURL("https://github.com/Snider/Borg")
if err != nil {
log.Fatalf("Failed to parse repo from URL: %v", err)
}
release, err := github.GetLatestRelease(owner, repo)
if err != nil {
log.Fatalf("Failed to get latest release: %v", err)
}
if len(release.Assets) == 0 {
log.Println("No assets found in the latest release.")
return
}
asset := release.Assets[0]
log.Printf("Downloading asset: %s", asset.GetName())
data, err := github.DownloadReleaseAsset(asset)
if err != nil {
log.Fatalf("Failed to download asset: %v", err)
}
err = os.WriteFile(asset.GetName(), data, 0644)
if err != nil {
log.Fatalf("Failed to write asset to file: %v", err)
}
log.Printf("Successfully downloaded asset to %s", asset.GetName())
}

27
examples/collect_github_repo/main.go
View file

@ -1,7 +1,30 @@
package main
import "fmt"
import (
"log"
"os"
"github.com/Snider/Borg/pkg/vcs"
)
func main() {
fmt.Println("This is a placeholder for the 'collect github repo' example.")
log.Println("Collecting GitHub repo...")
cloner := vcs.NewGitCloner()
dn, err := cloner.CloneGitRepository("https://github.com/Snider/Borg", nil)
if err != nil {
log.Fatalf("Failed to clone repository: %v", err)
}
tarball, err := dn.ToTar()
if err != nil {
log.Fatalf("Failed to serialize datanode to tar: %v", err)
}
err = os.WriteFile("repo.dat", tarball, 0644)
if err != nil {
log.Fatalf("Failed to write datanode file: %v", err)
}
log.Println("Successfully created repo.dat")
}

34
examples/collect_pwa/main.go
View file

@ -1,7 +1,37 @@
package main
import "fmt"
import (
"log"
"os"
"github.com/Snider/Borg/pkg/pwa"
)
func main() {
fmt.Println("This is a placeholder for the 'collect pwa' example.")
log.Println("Collecting PWA...")
client := pwa.NewPWAClient()
pwaURL := "https://squoosh.app"
manifestURL, err := client.FindManifest(pwaURL)
if err != nil {
log.Fatalf("Failed to find manifest: %v", err)
}
dn, err := client.DownloadAndPackagePWA(pwaURL, manifestURL, nil)
if err != nil {
log.Fatalf("Failed to download and package PWA: %v", err)
}
tarball, err := dn.ToTar()
if err != nil {
log.Fatalf("Failed to serialize datanode to tar: %v", err)
}
err = os.WriteFile("pwa.dat", tarball, 0644)
if err != nil {
log.Fatalf("Failed to write datanode file: %v", err)
}
log.Println("Successfully created pwa.dat")
}

29
examples/collect_website/main.go
View file

@ -1,7 +1,32 @@
package main
import "fmt"
import (
"log"
"os"
"github.com/Snider/Borg/pkg/website"
)
func main() {
fmt.Println("This is a placeholder for the 'collect website' example.")
log.Println("Collecting website...")
// Download and package the website.
dn, err := website.DownloadAndPackageWebsite("https://example.com", 2, nil)
if err != nil {
log.Fatalf("Failed to collect website: %v", err)
}
// Serialize the DataNode to a tarball.
tarball, err := dn.ToTar()
if err != nil {
log.Fatalf("Failed to serialize datanode to tar: %v", err)
}
// Write the tarball to a file.
err = os.WriteFile("website.dat", tarball, 0644)
if err != nil {
log.Fatalf("Failed to write datanode file: %v", err)
}
log.Println("Successfully created website.dat")
}

67
examples/run_matrix_programmatically/main.go
View file

@ -1,73 +1,16 @@
package main
import (
"archive/tar"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"github.com/Snider/Borg/pkg/matrix"
)
func main() {
// Open the matrix file.
matrixFile, err := os.Open("programmatic.matrix")
if err != nil {
log.Fatalf("Failed to open matrix file (run create_matrix_programmatically first): %v", err)
}
defer matrixFile.Close()
log.Println("Executing matrix with Borg...")
// Create a temporary directory to unpack the matrix.
tempDir, err := os.MkdirTemp("", "borg-run-example-*")
if err != nil {
log.Fatalf("Failed to create temporary directory: %v", err)
}
defer os.RemoveAll(tempDir)
log.Printf("Unpacking matrix to %s", tempDir)
// Unpack the tar archive.
tr := tar.NewReader(matrixFile)
for {
header, err := tr.Next()
if err == io.EOF {
break // End of archive
}
if err != nil {
log.Fatalf("Failed to read tar header: %v", err)
}
target := filepath.Join(tempDir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
if err := os.MkdirAll(target, 0755); err != nil {
log.Fatalf("Failed to create directory: %v", err)
}
case tar.TypeReg:
outFile, err := os.Create(target)
if err != nil {
log.Fatalf("Failed to create file: %v", err)
}
if _, err := io.Copy(outFile, tr); err != nil {
log.Fatalf("Failed to write file content: %v", err)
}
outFile.Close()
default:
log.Printf("Skipping unsupported type: %c in %s", header.Typeflag, header.Name)
}
}
log.Println("Executing matrix with runc...")
// Execute the matrix using runc.
cmd := exec.Command("runc", "run", "borg-container-example")
cmd.Dir = tempDir
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Stdin = os.Stdin
if err := cmd.Run(); err != nil {
// Execute the matrix using the Borg package.
if err := matrix.Run("programmatic.matrix"); err != nil {
log.Fatalf("Failed to run matrix: %v", err)
}

39
examples/serve/main.go
View file

@ -1,7 +1,42 @@
package main
import "fmt"
import (
"log"
"net/http"
"os"
"github.com/Snider/Borg/pkg/compress"
"github.com/Snider/Borg/pkg/tarfs"
)
func main() {
fmt.Println("This is a placeholder for the 'serve' example.")
log.Println("Serving datanode...")
// Create a dummy datanode for serving.
err := os.WriteFile("serve.dat", []byte{}, 0644)
if err != nil {
log.Fatalf("Failed to create dummy datanode: %v", err)
}
data, err := os.ReadFile("serve.dat")
if err != nil {
log.Fatalf("Failed to read datanode: %v", err)
}
decompressed, err := compress.Decompress(data)
if err != nil {
log.Fatalf("Failed to decompress datanode: %v", err)
}
fs, err := tarfs.New(decompressed)
if err != nil {
log.Fatalf("Failed to create tarfs: %v", err)
}
http.Handle("/", http.FileServer(fs))
log.Println("Listening on :8080...")
err = http.ListenAndServe(":8080", nil)
if err != nil {
log.Fatalf("Failed to start server: %v", err)
}
}

65
pkg/matrix/run.go Normal file
View file

@ -0,0 +1,65 @@
package matrix
import (
"archive/tar"
"io"
"os"
"os/exec"
"path/filepath"
)
// ExecCommand is a wrapper around exec.Command that can be overridden for testing.
var ExecCommand = exec.Command
// Run executes a Terminal Isolation Matrix from a given path.
func Run(matrixPath string) error {
// Create a temporary directory to unpack the matrix file.
tempDir, err := os.MkdirTemp("", "borg-run-*")
if err != nil {
return err
}
defer os.RemoveAll(tempDir)
// Unpack the matrix file.
file, err := os.Open(matrixPath)
if err != nil {
return err
}
defer file.Close()
tr := tar.NewReader(file)
for {
header, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
path := filepath.Join(tempDir, header.Name)
if header.Typeflag == tar.TypeDir {
if err := os.MkdirAll(path, 0755); err != nil {
return err
}
continue
}
outFile, err := os.Create(path)
if err != nil {
return err
}
defer outFile.Close()
if _, err := io.Copy(outFile, tr); err != nil {
return err
}
}
// Run the matrix.
runc := ExecCommand("runc", "run", "borg-container")
runc.Dir = tempDir
runc.Stdout = os.Stdout
runc.Stderr = os.Stderr
runc.Stdin = os.Stdin
return runc.Run()
}

63
pkg/matrix/run_test.go Normal file
View file

@ -0,0 +1,63 @@
package matrix
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
)
func fakeExecCommand(command string, args ...string) *exec.Cmd {
cs := []string{"-test.run=TestHelperProcess", "--", command}
cs = append(cs, args...)
cmd := exec.Command(os.Args[0], cs...)
cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
return cmd
}
func TestRun_Good(t *testing.T) {
// Create a dummy matrix file.
file, err := os.CreateTemp("", "matrix-*.matrix")
if err != nil {
t.Fatal(err)
}
defer os.Remove(file.Name())
ExecCommand = fakeExecCommand
defer func() { ExecCommand = exec.Command }()
err = Run(file.Name())
if err != nil {
t.Errorf("Run() failed: %v", err)
}
}
func TestHelperProcess(t *testing.T) {
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
return
}
defer os.Exit(0)
args := os.Args
for len(args) > 0 {
if args[0] == "--" {
args = args[1:]
break
}
args = args[1:]
}
if len(args) == 0 {
fmt.Fprintf(os.Stderr, "No command\n")
os.Exit(2)
}
cmd, args := args[0], args[1:]
if cmd == "runc" && args[0] == "run" {
fmt.Println("Success")
os.Exit(0)
} else {
fmt.Fprintf(os.Stderr, "Unknown command %s %s\n", cmd, strings.Join(args, " "))
os.Exit(1)
}
}