feat: Complete feature gap analysis audit

This commit adds a new file, AUDIT-FEATURES.md, which contains a thorough audit comparing dapp-fm's features against similar data collection tools.

The audit focuses on:
- Missing core features
- Competitive advantages
- Integration opportunities
- User workflow gaps

The comparison includes wget/curl, HTTrack, ArchiveBox, SingleFile, and rclone. This audit will help guide future development and strategic decisions.

AUDIT-FEATURES.md

@@ -0,0 +1,60 @@
+
+# Feature Audit: dapp-fm vs. Competitors
+
+This audit compares the features of `dapp-fm` against popular data collection and archiving tools.
+
+## Feature Comparison Matrix
+
+| Feature                      | dapp-fm | wget/curl | HTTrack | ArchiveBox | SingleFile | rclone |
+| ---------------------------- | ------- | --------- | ------- | ---------- | ---------- | ------ |
+| **General**                  |         |           |         |            |            |        |
+| Target                       | Websites, Git Repos, PWAs | Files, Websites | Websites | Websites | Webpages | Cloud Storage |
+| Output Format                | datanode, tim, trix, stim | Files | HTML | HTML, WARC, etc. | HTML | Files |
+| **Website Archiving**        |         |           |         |            |            |        |
+| Recursive Download           | Yes     | Yes       | Yes     | Yes        | No         | N/A    |
+| Asset Capture (JS, CSS, etc.)| Yes     | Yes       | Yes     | Yes        | Yes        | N/A    |
+| MHTML/WARC Output            | No      | No        | No      | Yes        | No         | N/A    |
+| Single Page Archive          | Yes     | Yes       | Yes     | Yes        | Yes        | N/A    |
+| **Data Sources**             |         |           |         |            |            |        |
+| Git Repositories             | Yes     | No        | No      | Yes        | No         | No     |
+| GitHub Releases              | Yes     | No        | No      | No         | No         | No     |
+| Progressive Web Apps (PWAs)  | Yes     | No        | No      | No         | No         | No     |
+| **Storage & Backend**        |         |           |         |            |            |        |
+| Cloud Storage Sync           | No      | No        | No      | No         | No         | Yes    |
+| **Advanced Features**        |         |           |         |            |            |        |
+| Headless Browser             | No      | No        | No      | Yes        | Yes        | N/A    |
+| Authentication               | No      | Yes       | Yes     | Yes        | No         | Yes    |
+| Rate Limiting                | No      | Yes       | Yes     | Yes        | No         | Yes    |
+| Filtering (Include/Exclude)  | No      | Yes       | Yes     | Yes        | No         | Yes    |
+| Scheduling                   | No      | No        | No      | Yes        | No         | No     |
+| **Usability**                |         |           |         |            |            |        |
+| CLI Interface                | Yes     | Yes       | Yes     | Yes        | No         | Yes    |
+| GUI Interface                | No      | No        | Yes     | Yes        | Yes (Browser Ext) | No |
+
+## Analysis
+
+### Missing Core Features
+
+*   **Headless Browser Rendering:** `dapp-fm` doesn't render pages in a headless browser, which means it may not capture content from single-page applications (SPAs) or websites that rely heavily on JavaScript.
+*   **Standard Archive Formats:** The tool doesn't export to standard formats like WARC or MHTML, which are widely used in web archiving.
+*   **Authentication and Rate Limiting:** `dapp-fm` lacks built-in support for handling websites that require logins or have rate limits.
+*   **Cloud Storage Integration:** Unlike `rclone`, `dapp-fm` cannot sync archives to various cloud storage providers.
+*   **Scheduling:** There's no built-in mechanism for scheduling recurring captures.
+
+### Competitive Advantages
+
+*   **Diverse Data Sources:** `dapp-fm`'s ability to collect not just websites but also Git repositories and Progressive Web Apps gives it a unique advantage.
+*   **Proprietary Archiving Formats:** The `.trix` and `.stim` formats, with their encryption and compression capabilities, offer a secure and efficient way to store and share archives.
+*   **Simplicity and Focus:** `dapp-fm` has a clear focus on collecting specific types of online resources and packaging them into a portable format.
+
+### Integration Opportunities
+
+*   **Browser Extension:** A browser extension, similar to `SingleFile`, could streamline the process of capturing single pages.
+*   **Cloud Storage Providers:** Integrating with services like Amazon S3, Google Cloud Storage, or Dropbox would make it easier for users to store and manage their archives.
+*   **CI/CD Integration:** `dapp-fm` could be integrated into CI/CD pipelines to automatically archive websites or applications after deployment.
+
+### User Workflow Gaps
+
+*   **No GUI:** The lack of a graphical interface makes `dapp-fm` less accessible to non-technical users.
+*   **Limited Configuration:** The tool offers limited configuration options for things like filtering content, setting user agents, or handling cookies.
+*   **Post-Archival Management:** `dapp-fm` doesn't provide any tools for managing, searching, or viewing archives after they've been created.

cmd/collect_local.go

@@ -1,333 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/Snider/Borg/pkg/compress"
-	"github.com/Snider/Borg/pkg/datanode"
-	"github.com/Snider/Borg/pkg/tim"
-	"github.com/Snider/Borg/pkg/trix"
-	"github.com/Snider/Borg/pkg/ui"
-
-	"github.com/spf13/cobra"
-)
-
-type CollectLocalCmd struct {
-	cobra.Command
-}
-
-// NewCollectLocalCmd creates a new collect local command
-func NewCollectLocalCmd() *CollectLocalCmd {
-	c := &CollectLocalCmd{}
-	c.Command = cobra.Command{
-		Use:   "local [directory]",
-		Short: "Collect files from a local directory",
-		Long: `Collect files from a local directory and store them in a DataNode.
-
-If no directory is specified, the current working directory is used.
-
-Examples:
-  borg collect local
-  borg collect local ./src
-  borg collect local /path/to/project --output project.tar
-  borg collect local . --format stim --password secret
-  borg collect local . --exclude "*.log" --exclude "node_modules"`,
-		Args: cobra.MaximumNArgs(1),
-		RunE: func(cmd *cobra.Command, args []string) error {
-			directory := "."
-			if len(args) > 0 {
-				directory = args[0]
-			}
-
-			outputFile, _ := cmd.Flags().GetString("output")
-			format, _ := cmd.Flags().GetString("format")
-			compression, _ := cmd.Flags().GetString("compression")
-			password, _ := cmd.Flags().GetString("password")
-			excludes, _ := cmd.Flags().GetStringSlice("exclude")
-			includeHidden, _ := cmd.Flags().GetBool("hidden")
-			respectGitignore, _ := cmd.Flags().GetBool("gitignore")
-
-			finalPath, err := CollectLocal(directory, outputFile, format, compression, password, excludes, includeHidden, respectGitignore)
-			if err != nil {
-				return err
-			}
-			fmt.Fprintln(cmd.OutOrStdout(), "Files saved to", finalPath)
-			return nil
-		},
-	}
-	c.Flags().String("output", "", "Output file for the DataNode")
-	c.Flags().String("format", "datanode", "Output format (datanode, tim, trix, or stim)")
-	c.Flags().String("compression", "none", "Compression format (none, gz, or xz)")
-	c.Flags().String("password", "", "Password for encryption (required for stim/trix format)")
-	c.Flags().StringSlice("exclude", nil, "Patterns to exclude (can be specified multiple times)")
-	c.Flags().Bool("hidden", false, "Include hidden files and directories")
-	c.Flags().Bool("gitignore", true, "Respect .gitignore files (default: true)")
-	return c
-}
-
-func init() {
-	collectCmd.AddCommand(&NewCollectLocalCmd().Command)
-}
-
-// CollectLocal collects files from a local directory into a DataNode
-func CollectLocal(directory string, outputFile string, format string, compression string, password string, excludes []string, includeHidden bool, respectGitignore bool) (string, error) {
-	// Validate format
-	if format != "datanode" && format != "tim" && format != "trix" && format != "stim" {
-		return "", fmt.Errorf("invalid format: %s (must be 'datanode', 'tim', 'trix', or 'stim')", format)
-	}
-	if (format == "stim" || format == "trix") && password == "" {
-		return "", fmt.Errorf("password is required for %s format", format)
-	}
-	if compression != "none" && compression != "gz" && compression != "xz" {
-		return "", fmt.Errorf("invalid compression: %s (must be 'none', 'gz', or 'xz')", compression)
-	}
-
-	// Resolve directory path
-	absDir, err := filepath.Abs(directory)
-	if err != nil {
-		return "", fmt.Errorf("error resolving directory path: %w", err)
-	}
-
-	info, err := os.Stat(absDir)
-	if err != nil {
-		return "", fmt.Errorf("error accessing directory: %w", err)
-	}
-	if !info.IsDir() {
-		return "", fmt.Errorf("not a directory: %s", absDir)
-	}
-
-	// Load gitignore patterns if enabled
-	var gitignorePatterns []string
-	if respectGitignore {
-		gitignorePatterns = loadGitignore(absDir)
-	}
-
-	// Create DataNode and collect files
-	dn := datanode.New()
-	var fileCount int
-
-	bar := ui.NewProgressBar(-1, "Scanning files")
-	defer bar.Finish()
-
-	err = filepath.WalkDir(absDir, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Get relative path
-		relPath, err := filepath.Rel(absDir, path)
-		if err != nil {
-			return err
-		}
-
-		// Skip root
-		if relPath == "." {
-			return nil
-		}
-
-		// Skip hidden files/dirs unless explicitly included
-		if !includeHidden && isHidden(relPath) {
-			if d.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
-		// Check gitignore patterns
-		if respectGitignore && matchesGitignore(relPath, d.IsDir(), gitignorePatterns) {
-			if d.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
-		// Check exclude patterns
-		if matchesExclude(relPath, excludes) {
-			if d.IsDir() {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-
-		// Skip directories (they're implicit in DataNode)
-		if d.IsDir() {
-			return nil
-		}
-
-		// Read file content
-		content, err := os.ReadFile(path)
-		if err != nil {
-			return fmt.Errorf("error reading %s: %w", relPath, err)
-		}
-
-		// Add to DataNode with forward slashes (tar convention)
-		dn.AddData(filepath.ToSlash(relPath), content)
-		fileCount++
-		bar.Describe(fmt.Sprintf("Collected %d files", fileCount))
-
-		return nil
-	})
-
-	if err != nil {
-		return "", fmt.Errorf("error walking directory: %w", err)
-	}
-
-	if fileCount == 0 {
-		return "", fmt.Errorf("no files found in %s", directory)
-	}
-
-	bar.Describe(fmt.Sprintf("Packaging %d files", fileCount))
-
-	// Convert to output format
-	var data []byte
-	if format == "tim" {
-		t, err := tim.FromDataNode(dn)
-		if err != nil {
-			return "", fmt.Errorf("error creating tim: %w", err)
-		}
-		data, err = t.ToTar()
-		if err != nil {
-			return "", fmt.Errorf("error serializing tim: %w", err)
-		}
-	} else if format == "stim" {
-		t, err := tim.FromDataNode(dn)
-		if err != nil {
-			return "", fmt.Errorf("error creating tim: %w", err)
-		}
-		data, err = t.ToSigil(password)
-		if err != nil {
-			return "", fmt.Errorf("error encrypting stim: %w", err)
-		}
-	} else if format == "trix" {
-		data, err = trix.ToTrix(dn, password)
-		if err != nil {
-			return "", fmt.Errorf("error serializing trix: %w", err)
-		}
-	} else {
-		data, err = dn.ToTar()
-		if err != nil {
-			return "", fmt.Errorf("error serializing DataNode: %w", err)
-		}
-	}
-
-	// Apply compression
-	compressedData, err := compress.Compress(data, compression)
-	if err != nil {
-		return "", fmt.Errorf("error compressing data: %w", err)
-	}
-
-	// Determine output filename
-	if outputFile == "" {
-		baseName := filepath.Base(absDir)
-		if baseName == "." || baseName == "/" {
-			baseName = "local"
-		}
-		outputFile = baseName + "." + format
-		if compression != "none" {
-			outputFile += "." + compression
-		}
-	}
-
-	err = os.WriteFile(outputFile, compressedData, 0644)
-	if err != nil {
-		return "", fmt.Errorf("error writing output file: %w", err)
-	}
-
-	return outputFile, nil
-}
-
-// isHidden checks if a path component starts with a dot
-func isHidden(path string) bool {
-	parts := strings.Split(filepath.ToSlash(path), "/")
-	for _, part := range parts {
-		if strings.HasPrefix(part, ".") {
-			return true
-		}
-	}
-	return false
-}
-
-// loadGitignore loads patterns from .gitignore if it exists
-func loadGitignore(dir string) []string {
-	var patterns []string
-
-	gitignorePath := filepath.Join(dir, ".gitignore")
-	content, err := os.ReadFile(gitignorePath)
-	if err != nil {
-		return patterns
-	}
-
-	lines := strings.Split(string(content), "\n")
-	for _, line := range lines {
-		line = strings.TrimSpace(line)
-		// Skip empty lines and comments
-		if line == "" || strings.HasPrefix(line, "#") {
-			continue
-		}
-		patterns = append(patterns, line)
-	}
-
-	return patterns
-}
-
-// matchesGitignore checks if a path matches any gitignore pattern
-func matchesGitignore(path string, isDir bool, patterns []string) bool {
-	for _, pattern := range patterns {
-		// Handle directory-only patterns
-		if strings.HasSuffix(pattern, "/") {
-			if !isDir {
-				continue
-			}
-			pattern = strings.TrimSuffix(pattern, "/")
-		}
-
-		// Handle negation (simplified - just skip negated patterns)
-		if strings.HasPrefix(pattern, "!") {
-			continue
-		}
-
-		// Match against path components
-		matched, _ := filepath.Match(pattern, filepath.Base(path))
-		if matched {
-			return true
-		}
-
-		// Also try matching the full path
-		matched, _ = filepath.Match(pattern, path)
-		if matched {
-			return true
-		}
-
-		// Handle ** patterns (simplified)
-		if strings.Contains(pattern, "**") {
-			simplePattern := strings.ReplaceAll(pattern, "**", "*")
-			matched, _ = filepath.Match(simplePattern, path)
-			if matched {
-				return true
-			}
-		}
-	}
-	return false
-}
-
-// matchesExclude checks if a path matches any exclude pattern
-func matchesExclude(path string, excludes []string) bool {
-	for _, pattern := range excludes {
-		// Match against basename
-		matched, _ := filepath.Match(pattern, filepath.Base(path))
-		if matched {
-			return true
-		}
-
-		// Match against full path
-		matched, _ = filepath.Match(pattern, path)
-		if matched {
-			return true
-		}
-	}
-	return false
-}

go.mod

@@ -60,7 +60,7 @@ require (
 	github.com/wailsapp/go-webview2 v1.0.22 // indirect
 	github.com/wailsapp/mimetype v1.4.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
+	golang.org/x/crypto v0.44.0 // indirect
 	golang.org/x/sys v0.38.0 // indirect
 	golang.org/x/term v0.37.0 // indirect
 	golang.org/x/text v0.31.0 // indirect

go.sum

@@ -155,8 +155,8 @@ github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU=
+golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
 golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk=