From 8a7bf71f59973b3e3319bbcf01405ace1f2b58d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Feb 2026 12:44:12 +0000 Subject: [PATCH] feat(datanode): add AddPath for filesystem directory collection Co-Authored-By: Claude Opus 4.6 --- pkg/datanode/addpath_test.go | 197 +++++++++++++++++++++++++++++++++++ pkg/datanode/datanode.go | 98 +++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100644 pkg/datanode/addpath_test.go diff --git a/pkg/datanode/addpath_test.go b/pkg/datanode/addpath_test.go new file mode 100644 index 0000000..0e1fe64 --- /dev/null +++ b/pkg/datanode/addpath_test.go @@ -0,0 +1,197 @@ +package datanode + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestAddPath_Good(t *testing.T) { + // Create a temp directory with files and a nested subdirectory. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "hello.txt"), []byte("hello"), 0644); err != nil { + t.Fatal(err) + } + subdir := filepath.Join(dir, "sub") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(subdir, "world.txt"), []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + dn := New() + if err := dn.AddPath(dir, AddPathOptions{}); err != nil { + t.Fatalf("AddPath failed: %v", err) + } + + // Verify files are stored with paths relative to dir, using forward slashes. + file, ok := dn.files["hello.txt"] + if !ok { + t.Fatal("hello.txt not found in datanode") + } + if string(file.content) != "hello" { + t.Errorf("expected content 'hello', got %q", file.content) + } + + file, ok = dn.files["sub/world.txt"] + if !ok { + t.Fatal("sub/world.txt not found in datanode") + } + if string(file.content) != "world" { + t.Errorf("expected content 'world', got %q", file.content) + } + + // Directories should not be stored explicitly. + if _, ok := dn.files["sub"]; ok { + t.Error("directories should not be stored as explicit entries") + } + if _, ok := dn.files["sub/"]; ok { + t.Error("directories should not be stored as explicit entries") + } +} + +func TestAddPath_SkipBrokenSymlinks_Good(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks not reliably supported on Windows") + } + + dir := t.TempDir() + + // Create a real file. + if err := os.WriteFile(filepath.Join(dir, "real.txt"), []byte("real"), 0644); err != nil { + t.Fatal(err) + } + + // Create a broken symlink (target does not exist). + if err := os.Symlink("/nonexistent/target", filepath.Join(dir, "broken.txt")); err != nil { + t.Fatal(err) + } + + dn := New() + err := dn.AddPath(dir, AddPathOptions{SkipBrokenSymlinks: true}) + if err != nil { + t.Fatalf("AddPath should not error with SkipBrokenSymlinks: %v", err) + } + + // The real file should be present. + if _, ok := dn.files["real.txt"]; !ok { + t.Error("real.txt should be present") + } + + // The broken symlink should be skipped. + if _, ok := dn.files["broken.txt"]; ok { + t.Error("broken.txt should have been skipped") + } +} + +func TestAddPath_ExcludePatterns_Good(t *testing.T) { + dir := t.TempDir() + + if err := os.WriteFile(filepath.Join(dir, "app.go"), []byte("package main"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "debug.log"), []byte("log data"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "error.log"), []byte("error data"), 0644); err != nil { + t.Fatal(err) + } + + dn := New() + err := dn.AddPath(dir, AddPathOptions{ + ExcludePatterns: []string{"*.log"}, + }) + if err != nil { + t.Fatalf("AddPath failed: %v", err) + } + + // app.go should be present. + if _, ok := dn.files["app.go"]; !ok { + t.Error("app.go should be present") + } + + // .log files should be excluded. + if _, ok := dn.files["debug.log"]; ok { + t.Error("debug.log should have been excluded") + } + if _, ok := dn.files["error.log"]; ok { + t.Error("error.log should have been excluded") + } +} + +func TestAddPath_Bad(t *testing.T) { + dn := New() + err := dn.AddPath("/nonexistent/path/that/does/not/exist", AddPathOptions{}) + if err == nil { + t.Fatal("expected error for nonexistent directory, got nil") + } +} + +func TestAddPath_ValidSymlink_Good(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks not reliably supported on Windows") + } + + dir := t.TempDir() + + // Create a real file. + if err := os.WriteFile(filepath.Join(dir, "target.txt"), []byte("target content"), 0644); err != nil { + t.Fatal(err) + } + + // Create a valid symlink pointing to the real file. + if err := os.Symlink("target.txt", filepath.Join(dir, "link.txt")); err != nil { + t.Fatal(err) + } + + // Default behavior (FollowSymlinks=false): store as symlink. + dn := New() + err := dn.AddPath(dir, AddPathOptions{}) + if err != nil { + t.Fatalf("AddPath failed: %v", err) + } + + // The target file should be a regular file. + targetFile, ok := dn.files["target.txt"] + if !ok { + t.Fatal("target.txt not found") + } + if targetFile.isSymlink() { + t.Error("target.txt should not be a symlink") + } + if string(targetFile.content) != "target content" { + t.Errorf("expected content 'target content', got %q", targetFile.content) + } + + // The symlink should be stored as a symlink entry. + linkFile, ok := dn.files["link.txt"] + if !ok { + t.Fatal("link.txt not found") + } + if !linkFile.isSymlink() { + t.Error("link.txt should be a symlink") + } + if linkFile.symlink != "target.txt" { + t.Errorf("expected symlink target 'target.txt', got %q", linkFile.symlink) + } + + // Test with FollowSymlinks=true: store as regular file with target content. + dn2 := New() + err = dn2.AddPath(dir, AddPathOptions{FollowSymlinks: true}) + if err != nil { + t.Fatalf("AddPath with FollowSymlinks failed: %v", err) + } + + linkFile2, ok := dn2.files["link.txt"] + if !ok { + t.Fatal("link.txt not found with FollowSymlinks") + } + if linkFile2.isSymlink() { + t.Error("link.txt should NOT be a symlink when FollowSymlinks is true") + } + if string(linkFile2.content) != "target content" { + t.Errorf("expected content 'target content', got %q", linkFile2.content) + } +} diff --git a/pkg/datanode/datanode.go b/pkg/datanode/datanode.go index 7e75be5..e51846d 100644 --- a/pkg/datanode/datanode.go +++ b/pkg/datanode/datanode.go @@ -8,6 +8,7 @@ import ( "io/fs" "os" "path" + "path/filepath" "sort" "strings" "time" @@ -131,6 +132,103 @@ func (d *DataNode) AddSymlink(name, target string) { } } +// AddPathOptions configures the behaviour of AddPath. +type AddPathOptions struct { + SkipBrokenSymlinks bool // skip broken symlinks instead of erroring + FollowSymlinks bool // follow symlinks and store target content (default false = store as symlinks) + ExcludePatterns []string // glob patterns to exclude (matched against basename) +} + +// AddPath walks a real directory and adds its files to the DataNode. +// Paths are stored relative to dir, normalized with forward slashes. +// Directories are implicit and not stored. +func (d *DataNode) AddPath(dir string, opts AddPathOptions) error { + absDir, err := filepath.Abs(dir) + if err != nil { + return err + } + + return filepath.WalkDir(absDir, func(p string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + + // Skip the root directory itself. + if p == absDir { + return nil + } + + // Compute relative path and normalize to forward slashes. + rel, err := filepath.Rel(absDir, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + + // Skip directories — they are implicit in DataNode. + isSymlink := entry.Type()&fs.ModeSymlink != 0 + if entry.IsDir() { + return nil + } + + // Apply exclude patterns against basename. + base := filepath.Base(p) + for _, pattern := range opts.ExcludePatterns { + matched, matchErr := filepath.Match(pattern, base) + if matchErr != nil { + return matchErr + } + if matched { + return nil + } + } + + // Handle symlinks. + if isSymlink { + linkTarget, err := os.Readlink(p) + if err != nil { + return err + } + + // Resolve the symlink target to check if it exists. + absTarget := linkTarget + if !filepath.IsAbs(absTarget) { + absTarget = filepath.Join(filepath.Dir(p), linkTarget) + } + + _, statErr := os.Stat(absTarget) + if statErr != nil { + // Broken symlink. + if opts.SkipBrokenSymlinks { + return nil + } + return statErr + } + + if opts.FollowSymlinks { + // Read the target content and store as regular file. + content, err := os.ReadFile(absTarget) + if err != nil { + return err + } + d.AddData(rel, content) + } else { + // Store as symlink. + d.AddSymlink(rel, linkTarget) + } + return nil + } + + // Regular file: read content and add. + content, err := os.ReadFile(p) + if err != nil { + return err + } + d.AddData(rel, content) + return nil + }) +} + // Open opens a file from the DataNode. func (d *DataNode) Open(name string) (fs.File, error) { name = strings.TrimPrefix(name, "/")