From 4e5257ce4a01ce90e987ae377d405eb1d556fb63 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Sun, 2 Nov 2025 12:39:46 +0000 Subject: [PATCH] feat: Add Terminal Isolation Matrix feature This change introduces the "Terminal Isolation Matrix", a new output format that creates a runc-compatible container bundle. This allows for the collected files to be run in an isolated environment. A --format flag has been added to all collect commands to support this new format. --- cmd/collect_github_repo.go | 25 ++++- cmd/collect_pwa.go | 27 +++++- cmd/collect_website.go | 27 +++++- docs/README.md | 27 ++++++ examples/create_matrix.sh | 8 ++ pkg/matrix/config.go | 190 +++++++++++++++++++++++++++++++++++++ pkg/matrix/matrix.go | 116 ++++++++++++++++++++++ 7 files changed, 406 insertions(+), 14 deletions(-) create mode 100755 examples/create_matrix.sh create mode 100644 pkg/matrix/config.go create mode 100644 pkg/matrix/matrix.go diff --git a/cmd/collect_github_repo.go b/cmd/collect_github_repo.go index e66f7aa..e48e505 100644 --- a/cmd/collect_github_repo.go +++ b/cmd/collect_github_repo.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/matrix" "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/vcs" @@ -19,6 +20,7 @@ var collectGithubRepoCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { repoURL := args[0] outputFile, _ := cmd.Flags().GetString("output") + format, _ := cmd.Flags().GetString("format") bar := ui.NewProgressBar(-1, "Cloning repository") defer bar.Finish() @@ -29,10 +31,24 @@ var collectGithubRepoCmd = &cobra.Command{ return } - data, err := dn.ToTar() - if err != nil { - fmt.Printf("Error serializing DataNode: %v\n", err) - return + var data []byte + if format == "matrix" { + matrix, err := matrix.FromDataNode(dn) + if err != nil { + fmt.Printf("Error creating matrix: %v\n", err) + return + } + data, err = matrix.ToTar() + if err != nil { + fmt.Printf("Error serializing matrix: %v\n", err) + return + } + } else { + data, err = dn.ToTar() + if err != nil { + fmt.Printf("Error serializing DataNode: %v\n", err) + return + } } err = os.WriteFile(outputFile, data, 0644) @@ -48,4 +64,5 @@ var collectGithubRepoCmd = &cobra.Command{ func init() { collectGithubCmd.AddCommand(collectGithubRepoCmd) collectGithubRepoCmd.PersistentFlags().String("output", "repo.dat", "Output file for the DataNode") + collectGithubRepoCmd.PersistentFlags().String("format", "datanode", "Output format (datanode or matrix)") } diff --git a/cmd/collect_pwa.go b/cmd/collect_pwa.go index 000ac53..7c371ff 100644 --- a/cmd/collect_pwa.go +++ b/cmd/collect_pwa.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/matrix" "github.com/Snider/Borg/pkg/pwa" "github.com/Snider/Borg/pkg/ui" @@ -21,6 +22,7 @@ Example: Run: func(cmd *cobra.Command, args []string) { pwaURL, _ := cmd.Flags().GetString("uri") outputFile, _ := cmd.Flags().GetString("output") + format, _ := cmd.Flags().GetString("format") if pwaURL == "" { fmt.Println("Error: uri is required") @@ -42,13 +44,27 @@ Example: return } - pwaData, err := dn.ToTar() - if err != nil { - fmt.Printf("Error converting PWA to bytes: %v\n", err) - return + var data []byte + if format == "matrix" { + matrix, err := matrix.FromDataNode(dn) + if err != nil { + fmt.Printf("Error creating matrix: %v\n", err) + return + } + data, err = matrix.ToTar() + if err != nil { + fmt.Printf("Error serializing matrix: %v\n", err) + return + } + } else { + data, err = dn.ToTar() + if err != nil { + fmt.Printf("Error serializing DataNode: %v\n", err) + return + } } - err = os.WriteFile(outputFile, pwaData, 0644) + err = os.WriteFile(outputFile, data, 0644) if err != nil { fmt.Printf("Error writing PWA to file: %v\n", err) return @@ -62,4 +78,5 @@ func init() { collectCmd.AddCommand(collectPWACmd) collectPWACmd.Flags().String("uri", "", "The URI of the PWA to collect") collectPWACmd.Flags().String("output", "pwa.dat", "Output file for the DataNode") + collectPWACmd.Flags().String("format", "datanode", "Output format (datanode or matrix)") } diff --git a/cmd/collect_website.go b/cmd/collect_website.go index b11803f..1a964af 100644 --- a/cmd/collect_website.go +++ b/cmd/collect_website.go @@ -4,6 +4,7 @@ import ( "fmt" "os" + "github.com/Snider/Borg/pkg/matrix" "github.com/Snider/Borg/pkg/ui" "github.com/Snider/Borg/pkg/website" @@ -20,6 +21,7 @@ var collectWebsiteCmd = &cobra.Command{ websiteURL := args[0] outputFile, _ := cmd.Flags().GetString("output") depth, _ := cmd.Flags().GetInt("depth") + format, _ := cmd.Flags().GetString("format") bar := ui.NewProgressBar(-1, "Crawling website") defer bar.Finish() @@ -30,13 +32,27 @@ var collectWebsiteCmd = &cobra.Command{ return } - websiteData, err := dn.ToTar() - if err != nil { - fmt.Printf("Error converting website to bytes: %v\n", err) - return + var data []byte + if format == "matrix" { + matrix, err := matrix.FromDataNode(dn) + if err != nil { + fmt.Printf("Error creating matrix: %v\n", err) + return + } + data, err = matrix.ToTar() + if err != nil { + fmt.Printf("Error serializing matrix: %v\n", err) + return + } + } else { + data, err = dn.ToTar() + if err != nil { + fmt.Printf("Error serializing DataNode: %v\n", err) + return + } } - err = os.WriteFile(outputFile, websiteData, 0644) + err = os.WriteFile(outputFile, data, 0644) if err != nil { fmt.Printf("Error writing website to file: %v\n", err) return @@ -50,4 +66,5 @@ func init() { collectCmd.AddCommand(collectWebsiteCmd) collectWebsiteCmd.PersistentFlags().String("output", "website.dat", "Output file for the DataNode") collectWebsiteCmd.PersistentFlags().Int("depth", 2, "Recursion depth for downloading") + collectWebsiteCmd.PersistentFlags().String("format", "datanode", "Output format (datanode or matrix)") } diff --git a/docs/README.md b/docs/README.md index ca38516..aea826a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,6 +19,7 @@ borg collect github repo [repository-url] [flags] **Flags:** - `--output string`: Output file for the DataNode (default "repo.dat") +- `--format string`: Output format (datanode or matrix) (default "datanode") **Example:** ``` @@ -37,6 +38,7 @@ borg collect website [url] [flags] **Flags:** - `--output string`: Output file for the DataNode (default "website.dat") - `--depth int`: Recursion depth for downloading (default 2) +- `--format string`: Output format (datanode or matrix) (default "datanode") **Example:** ``` @@ -55,6 +57,7 @@ borg collect pwa [flags] **Flags:** - `--uri string`: The URI of the PWA to collect - `--output string`: Output file for the DataNode (default "pwa.dat") +- `--format string`: Output format (datanode or matrix) (default "datanode") **Example:** ``` @@ -78,6 +81,30 @@ borg serve [file] [flags] ./borg serve squoosh.dat --port 8888 ``` +## Terminal Isolation Matrix + +The `matrix` format creates a `runc` compatible bundle. This bundle can be executed by `runc` to create a container with the collected files. This is useful for creating isolated environments for testing or analysis. + +To create a Matrix, use the `--format matrix` flag with any of the `collect` subcommands. + +**Example:** +``` +./borg collect github repo https://github.com/Snider/Borg --output borg.matrix --format matrix +``` + +You can then execute the Matrix with `runc`: +``` +# Create a directory for the bundle +mkdir borg-bundle + +# Unpack the matrix into the bundle directory +tar -xf borg.matrix -C borg-bundle + +# Run the bundle +cd borg-bundle +runc run borg +``` + ## Inspecting a DataNode The `examples` directory contains a Go program that can be used to inspect the contents of a `.dat` file. diff --git a/examples/create_matrix.sh b/examples/create_matrix.sh new file mode 100755 index 0000000..db19b48 --- /dev/null +++ b/examples/create_matrix.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# Example of using the 'borg collect' command with the '--format matrix' flag. + +# This script clones the specified Git repository and saves it as a .matrix file. +# The main executable 'borg' is built from the project's root. +# Make sure you have built the project by running 'go build -o borg main.go' in the root directory. + +./borg collect github repo https://github.com/Snider/Borg --output borg.matrix --format matrix diff --git a/pkg/matrix/config.go b/pkg/matrix/config.go new file mode 100644 index 0000000..3ba19f3 --- /dev/null +++ b/pkg/matrix/config.go @@ -0,0 +1,190 @@ +package matrix + +import ( + "encoding/json" +) + +// This is the default runc spec, generated by `runc spec`. +const defaultConfigJSON = `{ + "ociVersion": "1.2.1", + "process": { + "terminal": true, + "user": { + "uid": 0, + "gid": 0 + }, + "args": [ + "sh" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "TERM=xterm" + ], + "cwd": "/", + "capabilities": { + "bounding": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "effective": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "permitted": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ] + }, + "rlimits": [ + { + "type": "RLIMIT_NOFILE", + "hard": 1024, + "soft": 1024 + } + ], + "noNewPrivileges": true + }, + "root": { + "path": "rootfs", + "readonly": true + }, + "hostname": "runc", + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc" + }, + { + "destination": "/dev", + "type": "tmpfs", + "source": "tmpfs", + "options": [ + "nosuid", + "strictatime", + "mode=755", + "size=65536k" + ] + }, + { + "destination": "/dev/pts", + "type": "devpts", + "source": "devpts", + "options": [ + "nosuid", + "noexec", + "newinstance", + "ptmxmode=0666", + "mode=0620", + "gid":5 + ] + }, + { + "destination": "/dev/shm", + "type": "tmpfs", + "source": "shm", + "options": [ + "nosuid", + "noexec", + "nodev", + "mode=1777", + "size=65536k" + ] + }, + { + "destination": "/dev/mqueue", + "type": "mqueue", + "source": "mqueue", + "options": [ + "nosuid", + "noexec", + "nodev" + ] + }, + { + "destination": "/sys", + "type": "sysfs", + "source": "sysfs", + "options": [ + "nosuid", + "noexec", + "nodev", + "ro" + ] + }, + { + "destination": "/sys/fs/cgroup", + "type": "cgroup", + "source": "cgroup", + "options": [ + "nosuid", + "noexec", + "nodev", + "relatime", + "ro" + ] + } + ], + "linux": { + "resources": { + "devices": [ + { + "allow": false, + "access": "rwm" + } + ] + }, + "namespaces": [ + { + "type": "pid" + }, + { + "type": "network" + }, + { + "type": "ipc" + }, + { + "type": "uts" + }, + { + "type": "mount" + }, + { + "type": "cgroup" + } + ], + "maskedPaths": [ + "/proc/acpi", + "/proc/asound", + "/proc/kcore", + "/proc/keys", + "/proc/latency_stats", + "/proc/timer_list", + "/proc/timer_stats", + "/proc/sched_debug", + "/sys/firmware", + "/proc/scsi" + ], + "readonlyPaths": [ + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger" + ] + } +}` + +// defaultConfig returns the default runc spec. +func defaultConfig() (map[string]interface{}, error) { + var spec map[string]interface{} + err := json.Unmarshal([]byte(defaultConfigJSON), &spec) + if err != nil { + return nil, err + } + return spec, nil +} diff --git a/pkg/matrix/matrix.go b/pkg/matrix/matrix.go new file mode 100644 index 0000000..3738577 --- /dev/null +++ b/pkg/matrix/matrix.go @@ -0,0 +1,116 @@ +package matrix + +import ( + "archive/tar" + "bytes" + "encoding/json" + "io/fs" + + "github.com/Snider/Borg/pkg/datanode" +) + +// TerminalIsolationMatrix represents a runc bundle. +type TerminalIsolationMatrix struct { + Config []byte + RootFS *datanode.DataNode +} + +// New creates a new, empty TerminalIsolationMatrix. +func New() (*TerminalIsolationMatrix, error) { + // Use the default runc spec as a starting point. + // This can be customized later. + spec, err := defaultConfig() + if err != nil { + return nil, err + } + + specBytes, err := json.Marshal(spec) + if err != nil { + return nil, err + } + + return &TerminalIsolationMatrix{ + Config: specBytes, + RootFS: datanode.New(), + }, nil +} + +// FromDataNode creates a new TerminalIsolationMatrix from a DataNode. +func FromDataNode(dn *datanode.DataNode) (*TerminalIsolationMatrix, error) { + m, err := New() + if err != nil { + return nil, err + } + m.RootFS = dn + return m, nil +} + +// ToTar serializes the TerminalIsolationMatrix to a tarball. +func (m *TerminalIsolationMatrix) ToTar() ([]byte, error) { + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + + // Add the config.json file. + hdr := &tar.Header{ + Name: "config.json", + Mode: 0600, + Size: int64(len(m.Config)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, err + } + if _, err := tw.Write(m.Config); err != nil { + return nil, err + } + + // Add the rootfs files. + err := m.RootFS.Walk(".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + file, err := m.RootFS.Open(path) + if err != nil { + return err + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return err + } + + hdr := &tar.Header{ + Name: "rootfs/" + path, + Mode: 0600, + Size: info.Size(), + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + buf := new(bytes.Buffer) + if _, err := buf.ReadFrom(file); err != nil { + return err + } + + if _, err := tw.Write(buf.Bytes()); err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + if err := tw.Close(); err != nil { + return nil, err + } + + return buf.Bytes(), nil +}