diff --git a/cmd/all.go b/cmd/all.go index 84a06db..fddfe4f 100644 --- a/cmd/all.go +++ b/cmd/all.go @@ -11,6 +11,7 @@ import ( "github.com/Snider/Borg/pkg/compress" "github.com/Snider/Borg/pkg/datanode" "github.com/Snider/Borg/pkg/github" + "github.com/Snider/Borg/pkg/manifest" "github.com/Snider/Borg/pkg/tim" "github.com/Snider/Borg/pkg/trix" "github.com/Snider/Borg/pkg/ui" @@ -18,7 +19,10 @@ import ( "github.com/spf13/cobra" ) -var allCmd = NewAllCmd() +var ( + allCmd = NewAllCmd() + allCloner = vcs.NewGitCloner() +) func NewAllCmd() *cobra.Command { allCmd := &cobra.Command{ @@ -32,6 +36,7 @@ func NewAllCmd() *cobra.Command { format, _ := cmd.Flags().GetString("format") compression, _ := cmd.Flags().GetString("compression") password, _ := cmd.Flags().GetString("password") + generateManifest, _ := cmd.Flags().GetBool("manifest") if format != "datanode" && format != "tim" && format != "trix" { return fmt.Errorf("invalid format: %s (must be 'datanode', 'tim', or 'trix')", format) @@ -57,11 +62,10 @@ func NewAllCmd() *cobra.Command { progressWriter = ui.NewProgressWriter(bar) } - cloner := vcs.NewGitCloner() allDataNodes := datanode.New() for _, repoURL := range repos { - dn, err := cloner.CloneGitRepository(repoURL, progressWriter) + dn, err := allCloner.CloneGitRepository(repoURL, progressWriter) if err != nil { // Log the error and continue fmt.Fprintln(cmd.ErrOrStderr(), "Error cloning repository:", err) @@ -103,6 +107,18 @@ func NewAllCmd() *cobra.Command { } } + if generateManifest { + m, err := manifest.Generate(allDataNodes, url, format, password != "") + if err != nil { + return fmt.Errorf("error generating manifest: %w", err) + } + manifestData, err := m.ToJSON() + if err != nil { + return fmt.Errorf("error marshalling manifest: %w", err) + } + allDataNodes.AddData("MANIFEST.json", manifestData) + } + var data []byte if format == "tim" { tim, err := tim.FromDataNode(allDataNodes) @@ -144,6 +160,7 @@ func NewAllCmd() *cobra.Command { allCmd.PersistentFlags().String("format", "datanode", "Output format (datanode, tim, or trix)") allCmd.PersistentFlags().String("compression", "none", "Compression format (none, gz, or xz)") allCmd.PersistentFlags().String("password", "", "Password for encryption") + allCmd.PersistentFlags().Bool("manifest", false, "Generate a manifest.json file") return allCmd } diff --git a/cmd/all_test.go b/cmd/all_test.go index 66b4af1..dd911a9 100644 --- a/cmd/all_test.go +++ b/cmd/all_test.go @@ -5,6 +5,7 @@ import ( "context" "io" "net/http" + "os" "path/filepath" "testing" @@ -114,3 +115,62 @@ func TestAllCmd_Ugly(t *testing.T) { } }) } + +func TestAllCmd_WithManifest_Good(t *testing.T) { + // Setup mock HTTP client for GitHub API + mockGithubClient := mocks.NewMockClient(map[string]*http.Response{ + "https://api.github.com/users/testuser/repos": { + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(bytes.NewBufferString(`[{"clone_url": "https://github.com/testuser/repo1.git"}]`)), + }, + }) + oldNewAuthenticatedClient := github.NewAuthenticatedClient + github.NewAuthenticatedClient = func(ctx context.Context) *http.Client { + return mockGithubClient + } + t.Cleanup(func() { + github.NewAuthenticatedClient = oldNewAuthenticatedClient + }) + + // Setup mock Git cloner + mockCloner := &mocks.MockGitCloner{ + DN: datanode.New(), + Err: nil, + } + mockCloner.DN.AddData("README.md", []byte("# repo1")) + oldAllCloner := allCloner + allCloner = mockCloner + t.Cleanup(func() { + allCloner = oldAllCloner + }) + + rootCmd := NewRootCmd() + rootCmd.AddCommand(GetAllCmd()) + + // Execute command + out := filepath.Join(t.TempDir(), "out.dat") + _, err := executeCommand(rootCmd, "all", "https://github.com/testuser", "--output", out, "--manifest") + if err != nil { + t.Fatalf("all command failed: %v", err) + } + + // Verify MANIFEST.json exists + data, err := os.ReadFile(out) + if err != nil { + t.Fatalf("failed to read output file: %v", err) + } + + dn, err := datanode.FromTar(data) + if err != nil { + t.Fatalf("failed to create datanode from tar: %v", err) + } + + exists, err := dn.Exists("MANIFEST.json") + if err != nil { + t.Fatalf("failed to check for manifest: %v", err) + } + if !exists { + t.Fatal("MANIFEST.json not found in the output datanode") + } +} diff --git a/cmd/collect_github_repo.go b/cmd/collect_github_repo.go index c25df3b..2c18e0e 100644 --- a/cmd/collect_github_repo.go +++ b/cmd/collect_github_repo.go @@ -77,7 +77,7 @@ func NewCollectGithubRepoCmd() *cobra.Command { if err != nil { return fmt.Errorf("error creating tim: %w", err) } - data, err = t.ToSigil(password) + data, err = t.ToSigil(password, nil) if err != nil { return fmt.Errorf("error encrypting stim: %w", err) } diff --git a/cmd/collect_pwa.go b/cmd/collect_pwa.go index 8b5ef8c..ec5a924 100644 --- a/cmd/collect_pwa.go +++ b/cmd/collect_pwa.go @@ -105,7 +105,7 @@ func CollectPWA(client pwa.PWAClient, pwaURL string, outputFile string, format s if err != nil { return "", fmt.Errorf("error creating tim: %w", err) } - data, err = t.ToSigil(password) + data, err = t.ToSigil(password, nil) if err != nil { return "", fmt.Errorf("error encrypting stim: %w", err) } diff --git a/cmd/compile.go b/cmd/compile.go index 37e6016..526dbad 100644 --- a/cmd/compile.go +++ b/cmd/compile.go @@ -5,6 +5,7 @@ import ( "os" "strings" + "github.com/Snider/Borg/pkg/manifest" "github.com/Snider/Borg/pkg/tim" "github.com/spf13/cobra" ) @@ -12,6 +13,7 @@ import ( var borgfile string var output string var encryptPassword string +var publicManifest bool var compileCmd = NewCompileCmd() @@ -55,7 +57,19 @@ func NewCompileCmd() *cobra.Command { // If encryption is requested, output as .stim if encryptPassword != "" { - stimData, err := m.ToSigil(encryptPassword) + var manifestData []byte + if publicManifest { + m, err := manifest.Generate(m.RootFS, borgfile, "stim", true) + if err != nil { + return fmt.Errorf("error generating manifest: %w", err) + } + manifestData, err = m.ToJSON() + if err != nil { + return fmt.Errorf("error marshalling manifest: %w", err) + } + } + + stimData, err := m.ToSigil(encryptPassword, manifestData) if err != nil { return err } @@ -80,6 +94,7 @@ func NewCompileCmd() *cobra.Command { compileCmd.Flags().StringVarP(&borgfile, "file", "f", "Borgfile", "Path to the Borgfile.") compileCmd.Flags().StringVarP(&output, "output", "o", "a.tim", "Path to the output tim file.") compileCmd.Flags().StringVarP(&encryptPassword, "encrypt", "e", "", "Encrypt with ChaCha20-Poly1305 using this password (outputs .stim)") + compileCmd.Flags().BoolVar(&publicManifest, "public-manifest", false, "Embed a public manifest in the .stim header") return compileCmd } diff --git a/cmd/compile_test.go b/cmd/compile_test.go index d66eb86..fb5bb9c 100644 --- a/cmd/compile_test.go +++ b/cmd/compile_test.go @@ -4,6 +4,9 @@ import ( "os" "path/filepath" "testing" + + "github.com/Snider/Enchantrix/pkg/trix" + "github.com/stretchr/testify/assert" ) func TestCompileCmd(t *testing.T) { @@ -120,3 +123,37 @@ func TestCompileCmd(t *testing.T) { } }) } + +func TestCompileCmd_WithPublicManifest_Good(t *testing.T) { + tempDir := t.TempDir() + outputStimPath := filepath.Join(tempDir, "test.stim") + borgfilePath := filepath.Join(tempDir, "Borgfile") + dummyFilePath := filepath.Join(tempDir, "dummy.txt") + + // Create a dummy file to add to the tim. + err := os.WriteFile(dummyFilePath, []byte("dummy content"), 0644) + assert.NoError(t, err) + + // Create a Borgfile. + borgfileContent := "ADD " + dummyFilePath + " /dummy.txt" + err = os.WriteFile(borgfilePath, []byte(borgfileContent), 0644) + assert.NoError(t, err) + + // Execute the compile command. + cmd := NewCompileCmd() + cmd.SetArgs([]string{"--file", borgfilePath, "--output", outputStimPath, "--encrypt", "password", "--public-manifest"}) + err = cmd.Execute() + assert.NoError(t, err) + + // Verify the output stim file. + data, err := os.ReadFile(outputStimPath) + assert.NoError(t, err) + + decodedTrix, err := trix.Decode(data, "STIM", nil) + assert.NoError(t, err) + assert.NotNil(t, decodedTrix) + + manifest, ok := decodedTrix.Header["public_manifest"].(string) + assert.True(t, ok) + assert.Contains(t, manifest, `"total_files": 1`) +} diff --git a/cmd/console.go b/cmd/console.go index 5309c34..d78613b 100644 --- a/cmd/console.go +++ b/cmd/console.go @@ -79,7 +79,7 @@ Required files: index.html, support-reply.html, stmf.wasm, wasm_exec.js`, } // Encrypt to STIM - stim, err := m.ToSigil(password) + stim, err := m.ToSigil(password, nil) if err != nil { return fmt.Errorf("encrypting STIM: %w", err) } diff --git a/cmd/manifest.go b/cmd/manifest.go new file mode 100644 index 0000000..60d8ffb --- /dev/null +++ b/cmd/manifest.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "fmt" + "os" + + "strings" + + "github.com/Snider/Borg/pkg/compress" + "github.com/Snider/Borg/pkg/datanode" + "github.com/Snider/Borg/pkg/manifest" + "github.com/Snider/Enchantrix/pkg/trix" + "github.com/spf13/cobra" +) + +var manifestCmd = NewManifestCmd() + +func NewManifestCmd() *cobra.Command { + return &cobra.Command{ + Use: "manifest [archive]", + Short: "Generate a manifest from an archive.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + archivePath := args[0] + data, err := os.ReadFile(archivePath) + if err != nil { + return fmt.Errorf("error reading archive: %w", err) + } + + if strings.HasSuffix(archivePath, ".stim") { + t, err := trix.Decode(data, "STIM", nil) + if err == nil { + if manifest, ok := t.Header["public_manifest"].(string); ok { + fmt.Fprintln(cmd.OutOrStdout(), manifest) + return nil + } + } + } + + decompressedData, err := compress.Decompress(data) + if err != nil { + return fmt.Errorf("error decompressing archive: %w", err) + } + + dn, err := datanode.FromTar(decompressedData) + if err != nil { + return fmt.Errorf("error reading datanode from archive: %w", err) + } + + m, err := manifest.Generate(dn, archivePath, "unknown", false) + if err != nil { + return fmt.Errorf("error generating manifest: %w", err) + } + + manifestData, err := m.ToJSON() + if err != nil { + return fmt.Errorf("error marshalling manifest: %w", err) + } + + fmt.Fprintln(cmd.OutOrStdout(), string(manifestData)) + + return nil + }, + } +} + +func GetManifestCmd() *cobra.Command { + return manifestCmd +} + +func init() { + RootCmd.AddCommand(GetManifestCmd()) +} diff --git a/cmd/manifest_test.go b/cmd/manifest_test.go new file mode 100644 index 0000000..4a650ca --- /dev/null +++ b/cmd/manifest_test.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/Snider/Borg/pkg/datanode" + "github.com/stretchr/testify/assert" +) + +func TestManifestCmd_Good(t *testing.T) { + // Create a test archive + dn := datanode.New() + dn.AddData("file1.txt", []byte("hello")) + dn.AddData("file2.txt", []byte("world")) + tarball, err := dn.ToTar() + assert.NoError(t, err) + + tempDir := t.TempDir() + archivePath := filepath.Join(tempDir, "test.dat") + err = os.WriteFile(archivePath, tarball, 0644) + assert.NoError(t, err) + + rootCmd := NewRootCmd() + rootCmd.AddCommand(GetManifestCmd()) + + output, err := executeCommand(rootCmd, "manifest", archivePath) + assert.NoError(t, err) + + // Verify output + assert.Contains(t, output, `"total_files": 2`) + assert.Contains(t, output, `"total_size": "10 B"`) +} diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go new file mode 100644 index 0000000..85debc6 --- /dev/null +++ b/pkg/manifest/manifest.go @@ -0,0 +1,121 @@ +package manifest + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "io/fs" + "path/filepath" + "time" + + "github.com/Snider/Borg/pkg/datanode" +) + +type Manifest struct { + CollectedAt string `json:"collected_at"` + Source string `json:"source"` + Format string `json:"format"` + Encrypted bool `json:"encrypted"` + Files []File `json:"files"` + Stats Stats `json:"stats"` +} + +type File struct { + Path string `json:"path"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Type string `json:"type"` +} + +type Stats struct { + TotalFiles int `json:"total_files"` + TotalSize string `json:"total_size"` + ByType map[string]int `json:"by_type"` +} + +func Generate(dn *datanode.DataNode, source, format string, encrypted bool) (*Manifest, error) { + manifest := &Manifest{ + CollectedAt: time.Now().UTC().Format(time.RFC3339), + Source: source, + Format: format, + Encrypted: encrypted, + Files: []File{}, + Stats: Stats{ + ByType: make(map[string]int), + }, + } + + var totalSize int64 + err := dn.Walk(".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + + info, err := d.Info() + if err != nil { + return err + } + + file, err := dn.Open(path) + if err != nil { + return err + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + return err + } + + hasher := sha256.New() + if _, err := hasher.Write(content); err != nil { + return err + } + + fileType := filepath.Ext(path) + if fileType != "" { + fileType = fileType[1:] + } + + manifest.Files = append(manifest.Files, File{ + Path: path, + Size: info.Size(), + SHA256: fmt.Sprintf("%x", hasher.Sum(nil)), + Type: fileType, + }) + + totalSize += info.Size() + manifest.Stats.ByType[fileType]++ + return nil + }) + + if err != nil { + return nil, err + } + + manifest.Stats.TotalFiles = len(manifest.Files) + manifest.Stats.TotalSize = formatBytes(totalSize) + + return manifest, nil +} + +func (m *Manifest) ToJSON() ([]byte, error) { + return json.MarshalIndent(m, "", " ") +} + +func formatBytes(b int64) string { + const unit = 1024 + if b < unit { + return fmt.Sprintf("%d B", b) + } + div, exp := int64(unit), 0 + for n := b / unit; n >= unit; n /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp]) +} diff --git a/pkg/manifest/manifest_test.go b/pkg/manifest/manifest_test.go new file mode 100644 index 0000000..498b347 --- /dev/null +++ b/pkg/manifest/manifest_test.go @@ -0,0 +1,26 @@ +package manifest + +import ( + "testing" + + "github.com/Snider/Borg/pkg/datanode" + "github.com/stretchr/testify/assert" +) + +func TestGenerate(t *testing.T) { + dn := datanode.New() + dn.AddData("file1.txt", []byte("hello")) + dn.AddData("file2.txt", []byte("world")) + dn.AddData("dir/file3.go", []byte("package main")) + + manifest, err := Generate(dn, "test", "datanode", false) + assert.NoError(t, err) + + assert.Equal(t, "test", manifest.Source) + assert.Equal(t, "datanode", manifest.Format) + assert.False(t, manifest.Encrypted) + assert.Len(t, manifest.Files, 3) + assert.Equal(t, 3, manifest.Stats.TotalFiles) + assert.Equal(t, "22 B", manifest.Stats.TotalSize) + assert.Equal(t, map[string]int{"txt": 2, "go": 1}, manifest.Stats.ByType) +} diff --git a/pkg/tim/cache.go b/pkg/tim/cache.go index fcd929e..93df0d7 100644 --- a/pkg/tim/cache.go +++ b/pkg/tim/cache.go @@ -28,7 +28,7 @@ func NewCache(dir, password string) (*Cache, error) { // Store encrypts and saves a TIM to the cache. func (c *Cache) Store(name string, m *TerminalIsolationMatrix) error { - data, err := m.ToSigil(c.Password) + data, err := m.ToSigil(c.Password, nil) if err != nil { return err } diff --git a/pkg/tim/sigil_test.go b/pkg/tim/sigil_test.go index b2d6420..b4e5064 100644 --- a/pkg/tim/sigil_test.go +++ b/pkg/tim/sigil_test.go @@ -20,7 +20,7 @@ func TestToFromSigil(t *testing.T) { password := "testpassword123" // Encrypt - stim, err := m.ToSigil(password) + stim, err := m.ToSigil(password, nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } @@ -55,7 +55,7 @@ func TestFromSigilWrongPassword(t *testing.T) { t.Fatalf("New() error = %v", err) } - stim, err := m.ToSigil("correct") + stim, err := m.ToSigil("correct", nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } @@ -72,7 +72,7 @@ func TestToSigilEmptyPassword(t *testing.T) { t.Fatalf("New() error = %v", err) } - _, err = m.ToSigil("") + _, err = m.ToSigil("", nil) if err != ErrPasswordRequired { t.Errorf("Expected ErrPasswordRequired, got %v", err) } @@ -84,7 +84,7 @@ func TestFromSigilEmptyPassword(t *testing.T) { t.Fatalf("New() error = %v", err) } - stim, err := m.ToSigil("password") + stim, err := m.ToSigil("password", nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } @@ -216,7 +216,7 @@ func TestToSigilWithLargeData(t *testing.T) { password := "largetest" // Encrypt - stim, err := m.ToSigil(password) + stim, err := m.ToSigil(password, nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } @@ -251,7 +251,7 @@ func TestRunEncryptedWithTempFile(t *testing.T) { // Encrypt password := "runtest" - stim, err := m.ToSigil(password) + stim, err := m.ToSigil(password, nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } @@ -399,7 +399,7 @@ func TestToSigilNilConfig(t *testing.T) { RootFS: nil, } - _, err := m.ToSigil("password") + _, err := m.ToSigil("password", nil) if err != ErrConfigIsNil { t.Errorf("Expected ErrConfigIsNil, got %v", err) } @@ -419,7 +419,7 @@ func TestFromSigilTruncatedPayload(t *testing.T) { t.Fatalf("New() error = %v", err) } - stim, err := m.ToSigil("password") + stim, err := m.ToSigil("password", nil) if err != nil { t.Fatalf("ToSigil() error = %v", err) } diff --git a/pkg/tim/tim.go b/pkg/tim/tim.go index d1a9648..b350171 100644 --- a/pkg/tim/tim.go +++ b/pkg/tim/tim.go @@ -199,7 +199,7 @@ func (m *TerminalIsolationMatrix) ToTar() ([]byte, error) { // The output format is a Trix container with "STIM" magic containing: // - Header: {"encryption_algorithm": "chacha20poly1305", "tim": true} // - Payload: [config_size(4 bytes)][encrypted_config][encrypted_rootfs] -func (m *TerminalIsolationMatrix) ToSigil(password string) ([]byte, error) { +func (m *TerminalIsolationMatrix) ToSigil(password string, publicManifest []byte) ([]byte, error) { if password == "" { return nil, ErrPasswordRequired } @@ -249,6 +249,10 @@ func (m *TerminalIsolationMatrix) ToSigil(password string) ([]byte, error) { Payload: payload, } + if publicManifest != nil { + t.Header["public_manifest"] = string(publicManifest) + } + return trix.Encode(t, "STIM", nil) }