Merge ed38992830 into a77024aad4

cmd/all.go

@@ -11,6 +11,7 @@ import (
 	"github.com/Snider/Borg/pkg/compress"
 	"github.com/Snider/Borg/pkg/datanode"
 	"github.com/Snider/Borg/pkg/github"
+	"github.com/Snider/Borg/pkg/manifest"
 	"github.com/Snider/Borg/pkg/tim"
 	"github.com/Snider/Borg/pkg/trix"
 	"github.com/Snider/Borg/pkg/ui"
@@ -18,7 +19,10 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var allCmd = NewAllCmd()
+var (
+	allCmd    = NewAllCmd()
+	allCloner = vcs.NewGitCloner()
+)
 
 func NewAllCmd() *cobra.Command {
 	allCmd := &cobra.Command{
@@ -32,6 +36,7 @@ func NewAllCmd() *cobra.Command {
 			format, _ := cmd.Flags().GetString("format")
 			compression, _ := cmd.Flags().GetString("compression")
 			password, _ := cmd.Flags().GetString("password")
+			generateManifest, _ := cmd.Flags().GetBool("manifest")
 
 			if format != "datanode" && format != "tim" && format != "trix" {
 				return fmt.Errorf("invalid format: %s (must be 'datanode', 'tim', or 'trix')", format)
@@ -57,11 +62,10 @@ func NewAllCmd() *cobra.Command {
 				progressWriter = ui.NewProgressWriter(bar)
 			}
 
-			cloner := vcs.NewGitCloner()
 			allDataNodes := datanode.New()
 
 			for _, repoURL := range repos {
-				dn, err := cloner.CloneGitRepository(repoURL, progressWriter)
+				dn, err := allCloner.CloneGitRepository(repoURL, progressWriter)
 				if err != nil {
 					// Log the error and continue
 					fmt.Fprintln(cmd.ErrOrStderr(), "Error cloning repository:", err)
@@ -103,6 +107,18 @@ func NewAllCmd() *cobra.Command {
 				}
 			}
 
+			if generateManifest {
+				m, err := manifest.Generate(allDataNodes, url, format, password != "")
+				if err != nil {
+					return fmt.Errorf("error generating manifest: %w", err)
+				}
+				manifestData, err := m.ToJSON()
+				if err != nil {
+					return fmt.Errorf("error marshalling manifest: %w", err)
+				}
+				allDataNodes.AddData("MANIFEST.json", manifestData)
+			}
+
 			var data []byte
 			if format == "tim" {
 				tim, err := tim.FromDataNode(allDataNodes)
@@ -144,6 +160,7 @@ func NewAllCmd() *cobra.Command {
 	allCmd.PersistentFlags().String("format", "datanode", "Output format (datanode, tim, or trix)")
 	allCmd.PersistentFlags().String("compression", "none", "Compression format (none, gz, or xz)")
 	allCmd.PersistentFlags().String("password", "", "Password for encryption")
+	allCmd.PersistentFlags().Bool("manifest", false, "Generate a manifest.json file")
 	return allCmd
 }
 

cmd/all_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"io"
 	"net/http"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -114,3 +115,62 @@ func TestAllCmd_Ugly(t *testing.T) {
 		}
 	})
 }
+
+func TestAllCmd_WithManifest_Good(t *testing.T) {
+	// Setup mock HTTP client for GitHub API
+	mockGithubClient := mocks.NewMockClient(map[string]*http.Response{
+		"https://api.github.com/users/testuser/repos": {
+			StatusCode: http.StatusOK,
+			Header:     http.Header{"Content-Type": []string{"application/json"}},
+			Body:       io.NopCloser(bytes.NewBufferString(`[{"clone_url": "https://github.com/testuser/repo1.git"}]`)),
+		},
+	})
+	oldNewAuthenticatedClient := github.NewAuthenticatedClient
+	github.NewAuthenticatedClient = func(ctx context.Context) *http.Client {
+		return mockGithubClient
+	}
+	t.Cleanup(func() {
+		github.NewAuthenticatedClient = oldNewAuthenticatedClient
+	})
+
+	// Setup mock Git cloner
+	mockCloner := &mocks.MockGitCloner{
+		DN:  datanode.New(),
+		Err: nil,
+	}
+	mockCloner.DN.AddData("README.md", []byte("# repo1"))
+	oldAllCloner := allCloner
+	allCloner = mockCloner
+	t.Cleanup(func() {
+		allCloner = oldAllCloner
+	})
+
+	rootCmd := NewRootCmd()
+	rootCmd.AddCommand(GetAllCmd())
+
+	// Execute command
+	out := filepath.Join(t.TempDir(), "out.dat")
+	_, err := executeCommand(rootCmd, "all", "https://github.com/testuser", "--output", out, "--manifest")
+	if err != nil {
+		t.Fatalf("all command failed: %v", err)
+	}
+
+	// Verify MANIFEST.json exists
+	data, err := os.ReadFile(out)
+	if err != nil {
+		t.Fatalf("failed to read output file: %v", err)
+	}
+
+	dn, err := datanode.FromTar(data)
+	if err != nil {
+		t.Fatalf("failed to create datanode from tar: %v", err)
+	}
+
+	exists, err := dn.Exists("MANIFEST.json")
+	if err != nil {
+		t.Fatalf("failed to check for manifest: %v", err)
+	}
+	if !exists {
+		t.Fatal("MANIFEST.json not found in the output datanode")
+	}
+}

cmd/collect_github_repo.go

@@ -77,7 +77,7 @@ func NewCollectGithubRepoCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("error creating tim: %w", err)
 				}
-				data, err = t.ToSigil(password)
+				data, err = t.ToSigil(password, nil)
 				if err != nil {
 					return fmt.Errorf("error encrypting stim: %w", err)
 				}

cmd/collect_pwa.go

@@ -105,7 +105,7 @@ func CollectPWA(client pwa.PWAClient, pwaURL string, outputFile string, format s
 		if err != nil {
 			return "", fmt.Errorf("error creating tim: %w", err)
 		}
-		data, err = t.ToSigil(password)
+		data, err = t.ToSigil(password, nil)
 		if err != nil {
 			return "", fmt.Errorf("error encrypting stim: %w", err)
 		}

cmd/compile.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/Snider/Borg/pkg/manifest"
 	"github.com/Snider/Borg/pkg/tim"
 	"github.com/spf13/cobra"
 )
@@ -12,6 +13,7 @@ import (
 var borgfile string
 var output string
 var encryptPassword string
+var publicManifest bool
 
 var compileCmd = NewCompileCmd()
 
@@ -55,7 +57,19 @@ func NewCompileCmd() *cobra.Command {
 
 			// If encryption is requested, output as .stim
 			if encryptPassword != "" {
-				stimData, err := m.ToSigil(encryptPassword)
+				var manifestData []byte
+				if publicManifest {
+					m, err := manifest.Generate(m.RootFS, borgfile, "stim", true)
+					if err != nil {
+						return fmt.Errorf("error generating manifest: %w", err)
+					}
+					manifestData, err = m.ToJSON()
+					if err != nil {
+						return fmt.Errorf("error marshalling manifest: %w", err)
+					}
+				}
+
+				stimData, err := m.ToSigil(encryptPassword, manifestData)
 				if err != nil {
 					return err
 				}
@@ -80,6 +94,7 @@ func NewCompileCmd() *cobra.Command {
 	compileCmd.Flags().StringVarP(&borgfile, "file", "f", "Borgfile", "Path to the Borgfile.")
 	compileCmd.Flags().StringVarP(&output, "output", "o", "a.tim", "Path to the output tim file.")
 	compileCmd.Flags().StringVarP(&encryptPassword, "encrypt", "e", "", "Encrypt with ChaCha20-Poly1305 using this password (outputs .stim)")
+	compileCmd.Flags().BoolVar(&publicManifest, "public-manifest", false, "Embed a public manifest in the .stim header")
 	return compileCmd
 }
 

cmd/compile_test.go

@@ -4,6 +4,9 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/Snider/Enchantrix/pkg/trix"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestCompileCmd(t *testing.T) {
@@ -120,3 +123,37 @@ func TestCompileCmd(t *testing.T) {
 		}
 	})
 }
+
+func TestCompileCmd_WithPublicManifest_Good(t *testing.T) {
+	tempDir := t.TempDir()
+	outputStimPath := filepath.Join(tempDir, "test.stim")
+	borgfilePath := filepath.Join(tempDir, "Borgfile")
+	dummyFilePath := filepath.Join(tempDir, "dummy.txt")
+
+	// Create a dummy file to add to the tim.
+	err := os.WriteFile(dummyFilePath, []byte("dummy content"), 0644)
+	assert.NoError(t, err)
+
+	// Create a Borgfile.
+	borgfileContent := "ADD " + dummyFilePath + " /dummy.txt"
+	err = os.WriteFile(borgfilePath, []byte(borgfileContent), 0644)
+	assert.NoError(t, err)
+
+	// Execute the compile command.
+	cmd := NewCompileCmd()
+	cmd.SetArgs([]string{"--file", borgfilePath, "--output", outputStimPath, "--encrypt", "password", "--public-manifest"})
+	err = cmd.Execute()
+	assert.NoError(t, err)
+
+	// Verify the output stim file.
+	data, err := os.ReadFile(outputStimPath)
+	assert.NoError(t, err)
+
+	decodedTrix, err := trix.Decode(data, "STIM", nil)
+	assert.NoError(t, err)
+	assert.NotNil(t, decodedTrix)
+
+	manifest, ok := decodedTrix.Header["public_manifest"].(string)
+	assert.True(t, ok)
+	assert.Contains(t, manifest, `"total_files": 1`)
+}

cmd/console.go

@@ -79,7 +79,7 @@ Required files: index.html, support-reply.html, stmf.wasm, wasm_exec.js`,
 			}
 
 			// Encrypt to STIM
-			stim, err := m.ToSigil(password)
+			stim, err := m.ToSigil(password, nil)
 			if err != nil {
 				return fmt.Errorf("encrypting STIM: %w", err)
 			}

cmd/manifest.go

@@ -0,0 +1,73 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"strings"
+
+	"github.com/Snider/Borg/pkg/compress"
+	"github.com/Snider/Borg/pkg/datanode"
+	"github.com/Snider/Borg/pkg/manifest"
+	"github.com/Snider/Enchantrix/pkg/trix"
+	"github.com/spf13/cobra"
+)
+
+var manifestCmd = NewManifestCmd()
+
+func NewManifestCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:   "manifest [archive]",
+		Short: "Generate a manifest from an archive.",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			archivePath := args[0]
+			data, err := os.ReadFile(archivePath)
+			if err != nil {
+				return fmt.Errorf("error reading archive: %w", err)
+			}
+
+			if strings.HasSuffix(archivePath, ".stim") {
+				t, err := trix.Decode(data, "STIM", nil)
+				if err == nil {
+					if manifest, ok := t.Header["public_manifest"].(string); ok {
+						fmt.Fprintln(cmd.OutOrStdout(), manifest)
+						return nil
+					}
+				}
+			}
+
+			decompressedData, err := compress.Decompress(data)
+			if err != nil {
+				return fmt.Errorf("error decompressing archive: %w", err)
+			}
+
+			dn, err := datanode.FromTar(decompressedData)
+			if err != nil {
+				return fmt.Errorf("error reading datanode from archive: %w", err)
+			}
+
+			m, err := manifest.Generate(dn, archivePath, "unknown", false)
+			if err != nil {
+				return fmt.Errorf("error generating manifest: %w", err)
+			}
+
+			manifestData, err := m.ToJSON()
+			if err != nil {
+				return fmt.Errorf("error marshalling manifest: %w", err)
+			}
+
+			fmt.Fprintln(cmd.OutOrStdout(), string(manifestData))
+
+			return nil
+		},
+	}
+}
+
+func GetManifestCmd() *cobra.Command {
+	return manifestCmd
+}
+
+func init() {
+	RootCmd.AddCommand(GetManifestCmd())
+}

cmd/manifest_test.go

@@ -0,0 +1,34 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/Snider/Borg/pkg/datanode"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestManifestCmd_Good(t *testing.T) {
+	// Create a test archive
+	dn := datanode.New()
+	dn.AddData("file1.txt", []byte("hello"))
+	dn.AddData("file2.txt", []byte("world"))
+	tarball, err := dn.ToTar()
+	assert.NoError(t, err)
+
+	tempDir := t.TempDir()
+	archivePath := filepath.Join(tempDir, "test.dat")
+	err = os.WriteFile(archivePath, tarball, 0644)
+	assert.NoError(t, err)
+
+	rootCmd := NewRootCmd()
+	rootCmd.AddCommand(GetManifestCmd())
+
+	output, err := executeCommand(rootCmd, "manifest", archivePath)
+	assert.NoError(t, err)
+
+	// Verify output
+	assert.Contains(t, output, `"total_files": 2`)
+	assert.Contains(t, output, `"total_size": "10 B"`)
+}

pkg/manifest/manifest.go

@@ -0,0 +1,121 @@
+package manifest
+
+import (
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/fs"
+	"path/filepath"
+	"time"
+
+	"github.com/Snider/Borg/pkg/datanode"
+)
+
+type Manifest struct {
+	CollectedAt string    `json:"collected_at"`
+	Source      string    `json:"source"`
+	Format      string    `json:"format"`
+	Encrypted   bool      `json:"encrypted"`
+	Files       []File    `json:"files"`
+	Stats       Stats     `json:"stats"`
+}
+
+type File struct {
+	Path   string `json:"path"`
+	Size   int64  `json:"size"`
+	SHA256 string `json:"sha256"`
+	Type   string `json:"type"`
+}
+
+type Stats struct {
+	TotalFiles int            `json:"total_files"`
+	TotalSize  string         `json:"total_size"`
+	ByType     map[string]int `json:"by_type"`
+}
+
+func Generate(dn *datanode.DataNode, source, format string, encrypted bool) (*Manifest, error) {
+	manifest := &Manifest{
+		CollectedAt: time.Now().UTC().Format(time.RFC3339),
+		Source:      source,
+		Format:      format,
+		Encrypted:   encrypted,
+		Files:       []File{},
+		Stats: Stats{
+			ByType: make(map[string]int),
+		},
+	}
+
+	var totalSize int64
+	err := dn.Walk(".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() {
+			return nil
+		}
+
+		info, err := d.Info()
+		if err != nil {
+			return err
+		}
+
+		file, err := dn.Open(path)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		content, err := io.ReadAll(file)
+		if err != nil {
+			return err
+		}
+
+		hasher := sha256.New()
+		if _, err := hasher.Write(content); err != nil {
+			return err
+		}
+
+		fileType := filepath.Ext(path)
+		if fileType != "" {
+			fileType = fileType[1:]
+		}
+
+		manifest.Files = append(manifest.Files, File{
+			Path:   path,
+			Size:   info.Size(),
+			SHA256: fmt.Sprintf("%x", hasher.Sum(nil)),
+			Type:   fileType,
+		})
+
+		totalSize += info.Size()
+		manifest.Stats.ByType[fileType]++
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	manifest.Stats.TotalFiles = len(manifest.Files)
+	manifest.Stats.TotalSize = formatBytes(totalSize)
+
+	return manifest, nil
+}
+
+func (m *Manifest) ToJSON() ([]byte, error) {
+	return json.MarshalIndent(m, "", "  ")
+}
+
+func formatBytes(b int64) string {
+	const unit = 1024
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
+}

pkg/manifest/manifest_test.go

@@ -0,0 +1,26 @@
+package manifest
+
+import (
+	"testing"
+
+	"github.com/Snider/Borg/pkg/datanode"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerate(t *testing.T) {
+	dn := datanode.New()
+	dn.AddData("file1.txt", []byte("hello"))
+	dn.AddData("file2.txt", []byte("world"))
+	dn.AddData("dir/file3.go", []byte("package main"))
+
+	manifest, err := Generate(dn, "test", "datanode", false)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "test", manifest.Source)
+	assert.Equal(t, "datanode", manifest.Format)
+	assert.False(t, manifest.Encrypted)
+	assert.Len(t, manifest.Files, 3)
+	assert.Equal(t, 3, manifest.Stats.TotalFiles)
+	assert.Equal(t, "22 B", manifest.Stats.TotalSize)
+	assert.Equal(t, map[string]int{"txt": 2, "go": 1}, manifest.Stats.ByType)
+}

pkg/tim/cache.go

@@ -28,7 +28,7 @@ func NewCache(dir, password string) (*Cache, error) {
 
 // Store encrypts and saves a TIM to the cache.
 func (c *Cache) Store(name string, m *TerminalIsolationMatrix) error {
-	data, err := m.ToSigil(c.Password)
+	data, err := m.ToSigil(c.Password, nil)
 	if err != nil {
 		return err
 	}

pkg/tim/sigil_test.go

@@ -20,7 +20,7 @@ func TestToFromSigil(t *testing.T) {
 	password := "testpassword123"
 
 	// Encrypt
-	stim, err := m.ToSigil(password)
+	stim, err := m.ToSigil(password, nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}
@@ -55,7 +55,7 @@ func TestFromSigilWrongPassword(t *testing.T) {
 		t.Fatalf("New() error = %v", err)
 	}
 
-	stim, err := m.ToSigil("correct")
+	stim, err := m.ToSigil("correct", nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}
@@ -72,7 +72,7 @@ func TestToSigilEmptyPassword(t *testing.T) {
 		t.Fatalf("New() error = %v", err)
 	}
 
-	_, err = m.ToSigil("")
+	_, err = m.ToSigil("", nil)
 	if err != ErrPasswordRequired {
 		t.Errorf("Expected ErrPasswordRequired, got %v", err)
 	}
@@ -84,7 +84,7 @@ func TestFromSigilEmptyPassword(t *testing.T) {
 		t.Fatalf("New() error = %v", err)
 	}
 
-	stim, err := m.ToSigil("password")
+	stim, err := m.ToSigil("password", nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}
@@ -216,7 +216,7 @@ func TestToSigilWithLargeData(t *testing.T) {
 	password := "largetest"
 
 	// Encrypt
-	stim, err := m.ToSigil(password)
+	stim, err := m.ToSigil(password, nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}
@@ -251,7 +251,7 @@ func TestRunEncryptedWithTempFile(t *testing.T) {
 
 	// Encrypt
 	password := "runtest"
-	stim, err := m.ToSigil(password)
+	stim, err := m.ToSigil(password, nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}
@@ -399,7 +399,7 @@ func TestToSigilNilConfig(t *testing.T) {
 		RootFS: nil,
 	}
 
-	_, err := m.ToSigil("password")
+	_, err := m.ToSigil("password", nil)
 	if err != ErrConfigIsNil {
 		t.Errorf("Expected ErrConfigIsNil, got %v", err)
 	}
@@ -419,7 +419,7 @@ func TestFromSigilTruncatedPayload(t *testing.T) {
 		t.Fatalf("New() error = %v", err)
 	}
 
-	stim, err := m.ToSigil("password")
+	stim, err := m.ToSigil("password", nil)
 	if err != nil {
 		t.Fatalf("ToSigil() error = %v", err)
 	}

pkg/tim/tim.go

@@ -199,7 +199,7 @@ func (m *TerminalIsolationMatrix) ToTar() ([]byte, error) {
 // The output format is a Trix container with "STIM" magic containing:
 // - Header: {"encryption_algorithm": "chacha20poly1305", "tim": true}
 // - Payload: [config_size(4 bytes)][encrypted_config][encrypted_rootfs]
-func (m *TerminalIsolationMatrix) ToSigil(password string) ([]byte, error) {
+func (m *TerminalIsolationMatrix) ToSigil(password string, publicManifest []byte) ([]byte, error) {
 	if password == "" {
 		return nil, ErrPasswordRequired
 	}
@@ -249,6 +249,10 @@ func (m *TerminalIsolationMatrix) ToSigil(password string) ([]byte, error) {
 		Payload: payload,
 	}
 
+	if publicManifest != nil {
+		t.Header["public_manifest"] = string(publicManifest)
+	}
+
 	return trix.Encode(t, "STIM", nil)
 }