From 4e2327b0c95d846c60a79fe8fc127c9d01394d66 Mon Sep 17 00:00:00 2001 From: Vi Date: Thu, 5 Feb 2026 20:45:45 +0000 Subject: [PATCH] feat(io): add S3 and SQLite Medium backends (#347) (#355) Implement two new storage backends for the io.Medium interface: - pkg/io/s3: S3-backed Medium using AWS SDK v2 with interface-based mocking for tests. Supports prefix-based namespacing via WithPrefix option. All 18 Medium methods implemented with proper S3 semantics (e.g. EnsureDir is no-op, IsDir checks prefix existence). - pkg/io/sqlite: SQLite-backed Medium using modernc.org/sqlite (pure Go, no CGo). Uses a single table schema with path, content, mode, is_dir, and mtime columns. Supports custom table names via WithTable option. All tests use :memory: databases. Both packages include comprehensive test suites following the _Good/_Bad/_Ugly naming convention with 87 tests total (36 S3, 51 SQLite). Co-authored-by: Claude Co-authored-by: Claude Opus 4.6 --- go.mod | 18 + go.sum | 36 ++ pkg/io/s3/s3.go | 625 ++++++++++++++++++++++++++++++++ pkg/io/s3/s3_test.go | 646 +++++++++++++++++++++++++++++++++ pkg/io/sqlite/sqlite.go | 669 +++++++++++++++++++++++++++++++++++ pkg/io/sqlite/sqlite_test.go | 653 ++++++++++++++++++++++++++++++++++ 6 files changed, 2647 insertions(+) create mode 100644 pkg/io/s3/s3.go create mode 100644 pkg/io/s3/s3_test.go create mode 100644 pkg/io/sqlite/sqlite.go create mode 100644 pkg/io/sqlite/sqlite_test.go diff --git a/go.mod b/go.mod index ea9b957..df985d4 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,17 @@ require ( github.com/ProtonMail/go-crypto v1.3.0 // indirect github.com/TwiN/go-color v1.4.1 // indirect github.com/adrg/xdg v0.5.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bep/debounce v1.2.1 // indirect github.com/brianvoe/gofakeit/v6 v6.28.0 // indirect @@ -46,6 +57,7 @@ require ( github.com/cyphar/filepath-securejoin v0.6.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/ebitengine/purego v0.9.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.18.0 // indirect @@ -78,6 +90,7 @@ require ( github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect @@ -85,6 +98,7 @@ require ( github.com/pjbgf/sha1cd v0.5.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/samber/lo v1.52.0 // indirect @@ -118,4 +132,8 @@ require ( google.golang.org/grpc v1.76.0 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + modernc.org/libc v1.67.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.44.3 // indirect ) diff --git a/go.sum b/go.sum index 58a940c..47c905e 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,28 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFI github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -45,6 +67,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= @@ -160,6 +184,8 @@ github.com/modelcontextprotocol/go-sdk v1.2.0 h1:Y23co09300CEk8iZ/tMxIX1dVmKZkzo github.com/modelcontextprotocol/go-sdk v1.2.0/go.mod h1:6fM3LCm3yV7pAs8isnKLn07oKtB0MP9LHd3DfAcKw10= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oasdiff/oasdiff v1.11.9 h1:M/pIY4K1MWnML0DkAdUQU/CnJdNDr2z2hpD0lpKSccM= github.com/oasdiff/oasdiff v1.11.9/go.mod h1:4qorAPsG2EE/lXEs+FGzAJcYHXS3G7XghfqkCFPKzNQ= github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY= @@ -185,6 +211,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/qdrant/go-client v1.16.2 h1:UUMJJfvXTByhwhH1DwWdbkhZ2cTdvSqVkXSIfBrVWSg= github.com/qdrant/go-client v1.16.2/go.mod h1:I+EL3h4HRoRTeHtbfOd/4kDXwCukZfkd41j/9wryGkw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -336,3 +364,11 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.44.3 h1:+39JvV/HWMcYslAwRxHb8067w+2zowvFOUrOWIy9PjY= +modernc.org/sqlite v1.44.3/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= diff --git a/pkg/io/s3/s3.go b/pkg/io/s3/s3.go new file mode 100644 index 0000000..1c7bb94 --- /dev/null +++ b/pkg/io/s3/s3.go @@ -0,0 +1,625 @@ +// Package s3 provides an S3-backed implementation of the io.Medium interface. +package s3 + +import ( + "bytes" + "context" + "fmt" + goio "io" + "io/fs" + "os" + "path" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + + coreerr "github.com/host-uk/core/pkg/framework/core" +) + +// s3API is the subset of the S3 client API used by this package. +// This allows for interface-based mocking in tests. +type s3API interface { + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + DeleteObjects(ctx context.Context, params *s3.DeleteObjectsInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) + CopyObject(ctx context.Context, params *s3.CopyObjectInput, optFns ...func(*s3.Options)) (*s3.CopyObjectOutput, error) +} + +// Medium is an S3-backed storage backend implementing the io.Medium interface. +type Medium struct { + client s3API + bucket string + prefix string +} + +// Option configures a Medium. +type Option func(*Medium) + +// WithPrefix sets an optional key prefix for all operations. +func WithPrefix(prefix string) Option { + return func(m *Medium) { + // Ensure prefix ends with "/" if non-empty + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + m.prefix = prefix + } +} + +// WithClient sets the S3 client for dependency injection. +func WithClient(client *s3.Client) Option { + return func(m *Medium) { + m.client = client + } +} + +// withAPI sets the s3API interface directly (for testing with mocks). +func withAPI(api s3API) Option { + return func(m *Medium) { + m.client = api + } +} + +// New creates a new S3 Medium for the given bucket. +func New(bucket string, opts ...Option) (*Medium, error) { + if bucket == "" { + return nil, coreerr.E("s3.New", "bucket name is required", nil) + } + m := &Medium{bucket: bucket} + for _, opt := range opts { + opt(m) + } + if m.client == nil { + return nil, coreerr.E("s3.New", "S3 client is required (use WithClient option)", nil) + } + return m, nil +} + +// key returns the full S3 object key for a given path. +func (m *Medium) key(p string) string { + // Clean the path using a leading "/" to sandbox traversal attempts, + // then strip the "/" prefix. This ensures ".." can't escape. + clean := path.Clean("/" + p) + if clean == "/" { + clean = "" + } + clean = strings.TrimPrefix(clean, "/") + + if m.prefix == "" { + return clean + } + if clean == "" { + return m.prefix + } + return m.prefix + clean +} + +// Read retrieves the content of a file as a string. +func (m *Medium) Read(p string) (string, error) { + key := m.key(p) + if key == "" { + return "", coreerr.E("s3.Read", "path is required", os.ErrInvalid) + } + + out, err := m.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err != nil { + return "", coreerr.E("s3.Read", "failed to get object: "+key, err) + } + defer out.Body.Close() + + data, err := goio.ReadAll(out.Body) + if err != nil { + return "", coreerr.E("s3.Read", "failed to read body: "+key, err) + } + return string(data), nil +} + +// Write saves the given content to a file, overwriting it if it exists. +func (m *Medium) Write(p, content string) error { + key := m.key(p) + if key == "" { + return coreerr.E("s3.Write", "path is required", os.ErrInvalid) + } + + _, err := m.client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + Body: strings.NewReader(content), + }) + if err != nil { + return coreerr.E("s3.Write", "failed to put object: "+key, err) + } + return nil +} + +// EnsureDir is a no-op for S3 (S3 has no real directories). +func (m *Medium) EnsureDir(_ string) error { + return nil +} + +// IsFile checks if a path exists and is a regular file (not a "directory" prefix). +func (m *Medium) IsFile(p string) bool { + key := m.key(p) + if key == "" { + return false + } + // A "file" in S3 is an object whose key does not end with "/" + if strings.HasSuffix(key, "/") { + return false + } + _, err := m.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + return err == nil +} + +// FileGet is a convenience function that reads a file from the medium. +func (m *Medium) FileGet(p string) (string, error) { + return m.Read(p) +} + +// FileSet is a convenience function that writes a file to the medium. +func (m *Medium) FileSet(p, content string) error { + return m.Write(p, content) +} + +// Delete removes a single object. +func (m *Medium) Delete(p string) error { + key := m.key(p) + if key == "" { + return coreerr.E("s3.Delete", "path is required", os.ErrInvalid) + } + + _, err := m.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err != nil { + return coreerr.E("s3.Delete", "failed to delete object: "+key, err) + } + return nil +} + +// DeleteAll removes all objects under the given prefix. +func (m *Medium) DeleteAll(p string) error { + key := m.key(p) + if key == "" { + return coreerr.E("s3.DeleteAll", "path is required", os.ErrInvalid) + } + + // First, try deleting the exact key + _, _ = m.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + + // Then delete all objects under the prefix + prefix := key + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + paginator := true + var continuationToken *string + + for paginator { + listOut, err := m.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(m.bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return coreerr.E("s3.DeleteAll", "failed to list objects: "+prefix, err) + } + + if len(listOut.Contents) == 0 { + break + } + + objects := make([]types.ObjectIdentifier, len(listOut.Contents)) + for i, obj := range listOut.Contents { + objects[i] = types.ObjectIdentifier{Key: obj.Key} + } + + _, err = m.client.DeleteObjects(context.Background(), &s3.DeleteObjectsInput{ + Bucket: aws.String(m.bucket), + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, + }) + if err != nil { + return coreerr.E("s3.DeleteAll", "failed to delete objects", err) + } + + if listOut.IsTruncated != nil && *listOut.IsTruncated { + continuationToken = listOut.NextContinuationToken + } else { + paginator = false + } + } + + return nil +} + +// Rename moves an object by copying then deleting the original. +func (m *Medium) Rename(oldPath, newPath string) error { + oldKey := m.key(oldPath) + newKey := m.key(newPath) + if oldKey == "" || newKey == "" { + return coreerr.E("s3.Rename", "both old and new paths are required", os.ErrInvalid) + } + + copySource := m.bucket + "/" + oldKey + + _, err := m.client.CopyObject(context.Background(), &s3.CopyObjectInput{ + Bucket: aws.String(m.bucket), + CopySource: aws.String(copySource), + Key: aws.String(newKey), + }) + if err != nil { + return coreerr.E("s3.Rename", "failed to copy object: "+oldKey+" -> "+newKey, err) + } + + _, err = m.client.DeleteObject(context.Background(), &s3.DeleteObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(oldKey), + }) + if err != nil { + return coreerr.E("s3.Rename", "failed to delete source object: "+oldKey, err) + } + + return nil +} + +// List returns directory entries for the given path using ListObjectsV2 with delimiter. +func (m *Medium) List(p string) ([]fs.DirEntry, error) { + prefix := m.key(p) + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + var entries []fs.DirEntry + + listOut, err := m.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(m.bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + }) + if err != nil { + return nil, coreerr.E("s3.List", "failed to list objects: "+prefix, err) + } + + // Common prefixes are "directories" + for _, cp := range listOut.CommonPrefixes { + if cp.Prefix == nil { + continue + } + name := strings.TrimPrefix(*cp.Prefix, prefix) + name = strings.TrimSuffix(name, "/") + if name == "" { + continue + } + entries = append(entries, &dirEntry{ + name: name, + isDir: true, + mode: fs.ModeDir | 0755, + info: &fileInfo{ + name: name, + isDir: true, + mode: fs.ModeDir | 0755, + }, + }) + } + + // Contents are "files" (excluding the prefix itself) + for _, obj := range listOut.Contents { + if obj.Key == nil { + continue + } + name := strings.TrimPrefix(*obj.Key, prefix) + if name == "" || strings.Contains(name, "/") { + continue + } + var size int64 + if obj.Size != nil { + size = *obj.Size + } + var modTime time.Time + if obj.LastModified != nil { + modTime = *obj.LastModified + } + entries = append(entries, &dirEntry{ + name: name, + isDir: false, + mode: 0644, + info: &fileInfo{ + name: name, + size: size, + mode: 0644, + modTime: modTime, + }, + }) + } + + return entries, nil +} + +// Stat returns file information for the given path using HeadObject. +func (m *Medium) Stat(p string) (fs.FileInfo, error) { + key := m.key(p) + if key == "" { + return nil, coreerr.E("s3.Stat", "path is required", os.ErrInvalid) + } + + out, err := m.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, coreerr.E("s3.Stat", "failed to head object: "+key, err) + } + + var size int64 + if out.ContentLength != nil { + size = *out.ContentLength + } + var modTime time.Time + if out.LastModified != nil { + modTime = *out.LastModified + } + + name := path.Base(key) + return &fileInfo{ + name: name, + size: size, + mode: 0644, + modTime: modTime, + }, nil +} + +// Open opens the named file for reading. +func (m *Medium) Open(p string) (fs.File, error) { + key := m.key(p) + if key == "" { + return nil, coreerr.E("s3.Open", "path is required", os.ErrInvalid) + } + + out, err := m.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, coreerr.E("s3.Open", "failed to get object: "+key, err) + } + + data, err := goio.ReadAll(out.Body) + out.Body.Close() + if err != nil { + return nil, coreerr.E("s3.Open", "failed to read body: "+key, err) + } + + var size int64 + if out.ContentLength != nil { + size = *out.ContentLength + } + var modTime time.Time + if out.LastModified != nil { + modTime = *out.LastModified + } + + return &s3File{ + name: path.Base(key), + content: data, + size: size, + modTime: modTime, + }, nil +} + +// Create creates or truncates the named file. Returns a writer that +// uploads the content on Close. +func (m *Medium) Create(p string) (goio.WriteCloser, error) { + key := m.key(p) + if key == "" { + return nil, coreerr.E("s3.Create", "path is required", os.ErrInvalid) + } + return &s3WriteCloser{ + medium: m, + key: key, + }, nil +} + +// Append opens the named file for appending. It downloads the existing +// content (if any) and re-uploads the combined content on Close. +func (m *Medium) Append(p string) (goio.WriteCloser, error) { + key := m.key(p) + if key == "" { + return nil, coreerr.E("s3.Append", "path is required", os.ErrInvalid) + } + + var existing []byte + out, err := m.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err == nil { + existing, _ = goio.ReadAll(out.Body) + out.Body.Close() + } + + return &s3WriteCloser{ + medium: m, + key: key, + data: existing, + }, nil +} + +// ReadStream returns a reader for the file content. +func (m *Medium) ReadStream(p string) (goio.ReadCloser, error) { + key := m.key(p) + if key == "" { + return nil, coreerr.E("s3.ReadStream", "path is required", os.ErrInvalid) + } + + out, err := m.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, coreerr.E("s3.ReadStream", "failed to get object: "+key, err) + } + return out.Body, nil +} + +// WriteStream returns a writer for the file content. Content is uploaded on Close. +func (m *Medium) WriteStream(p string) (goio.WriteCloser, error) { + return m.Create(p) +} + +// Exists checks if a path exists (file or directory prefix). +func (m *Medium) Exists(p string) bool { + key := m.key(p) + if key == "" { + return false + } + + // Check as an exact object + _, err := m.client.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(m.bucket), + Key: aws.String(key), + }) + if err == nil { + return true + } + + // Check as a "directory" prefix + prefix := key + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + listOut, err := m.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(m.bucket), + Prefix: aws.String(prefix), + MaxKeys: aws.Int32(1), + }) + if err != nil { + return false + } + return len(listOut.Contents) > 0 || len(listOut.CommonPrefixes) > 0 +} + +// IsDir checks if a path exists and is a directory (has objects under it as a prefix). +func (m *Medium) IsDir(p string) bool { + key := m.key(p) + if key == "" { + return false + } + + prefix := key + if !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + listOut, err := m.client.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ + Bucket: aws.String(m.bucket), + Prefix: aws.String(prefix), + MaxKeys: aws.Int32(1), + }) + if err != nil { + return false + } + return len(listOut.Contents) > 0 || len(listOut.CommonPrefixes) > 0 +} + +// --- Internal types --- + +// fileInfo implements fs.FileInfo for S3 objects. +type fileInfo struct { + name string + size int64 + mode fs.FileMode + modTime time.Time + isDir bool +} + +func (fi *fileInfo) Name() string { return fi.name } +func (fi *fileInfo) Size() int64 { return fi.size } +func (fi *fileInfo) Mode() fs.FileMode { return fi.mode } +func (fi *fileInfo) ModTime() time.Time { return fi.modTime } +func (fi *fileInfo) IsDir() bool { return fi.isDir } +func (fi *fileInfo) Sys() any { return nil } + +// dirEntry implements fs.DirEntry for S3 listings. +type dirEntry struct { + name string + isDir bool + mode fs.FileMode + info fs.FileInfo +} + +func (de *dirEntry) Name() string { return de.name } +func (de *dirEntry) IsDir() bool { return de.isDir } +func (de *dirEntry) Type() fs.FileMode { return de.mode.Type() } +func (de *dirEntry) Info() (fs.FileInfo, error) { return de.info, nil } + +// s3File implements fs.File for S3 objects. +type s3File struct { + name string + content []byte + offset int64 + size int64 + modTime time.Time +} + +func (f *s3File) Stat() (fs.FileInfo, error) { + return &fileInfo{ + name: f.name, + size: int64(len(f.content)), + mode: 0644, + modTime: f.modTime, + }, nil +} + +func (f *s3File) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.content)) { + return 0, goio.EOF + } + n := copy(b, f.content[f.offset:]) + f.offset += int64(n) + return n, nil +} + +func (f *s3File) Close() error { + return nil +} + +// s3WriteCloser buffers writes and uploads to S3 on Close. +type s3WriteCloser struct { + medium *Medium + key string + data []byte +} + +func (w *s3WriteCloser) Write(p []byte) (int, error) { + w.data = append(w.data, p...) + return len(p), nil +} + +func (w *s3WriteCloser) Close() error { + _, err := w.medium.client.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String(w.medium.bucket), + Key: aws.String(w.key), + Body: bytes.NewReader(w.data), + }) + if err != nil { + return fmt.Errorf("s3: failed to upload on close: %w", err) + } + return nil +} diff --git a/pkg/io/s3/s3_test.go b/pkg/io/s3/s3_test.go new file mode 100644 index 0000000..1f226e7 --- /dev/null +++ b/pkg/io/s3/s3_test.go @@ -0,0 +1,646 @@ +package s3 + +import ( + "bytes" + "context" + "fmt" + goio "io" + "io/fs" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockS3 is an in-memory mock implementing the s3API interface. +type mockS3 struct { + mu sync.RWMutex + objects map[string][]byte + mtimes map[string]time.Time +} + +func newMockS3() *mockS3 { + return &mockS3{ + objects: make(map[string][]byte), + mtimes: make(map[string]time.Time), + } +} + +func (m *mockS3) GetObject(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + key := aws.ToString(params.Key) + data, ok := m.objects[key] + if !ok { + return nil, fmt.Errorf("NoSuchKey: key %q not found", key) + } + mtime := m.mtimes[key] + return &s3.GetObjectOutput{ + Body: goio.NopCloser(bytes.NewReader(data)), + ContentLength: aws.Int64(int64(len(data))), + LastModified: &mtime, + }, nil +} + +func (m *mockS3) PutObject(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + + key := aws.ToString(params.Key) + data, err := goio.ReadAll(params.Body) + if err != nil { + return nil, err + } + m.objects[key] = data + m.mtimes[key] = time.Now() + return &s3.PutObjectOutput{}, nil +} + +func (m *mockS3) DeleteObject(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + + key := aws.ToString(params.Key) + delete(m.objects, key) + delete(m.mtimes, key) + return &s3.DeleteObjectOutput{}, nil +} + +func (m *mockS3) DeleteObjects(_ context.Context, params *s3.DeleteObjectsInput, _ ...func(*s3.Options)) (*s3.DeleteObjectsOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + + for _, obj := range params.Delete.Objects { + key := aws.ToString(obj.Key) + delete(m.objects, key) + delete(m.mtimes, key) + } + return &s3.DeleteObjectsOutput{}, nil +} + +func (m *mockS3) HeadObject(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + key := aws.ToString(params.Key) + data, ok := m.objects[key] + if !ok { + return nil, fmt.Errorf("NotFound: key %q not found", key) + } + mtime := m.mtimes[key] + return &s3.HeadObjectOutput{ + ContentLength: aws.Int64(int64(len(data))), + LastModified: &mtime, + }, nil +} + +func (m *mockS3) ListObjectsV2(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + prefix := aws.ToString(params.Prefix) + delimiter := aws.ToString(params.Delimiter) + maxKeys := int32(1000) + if params.MaxKeys != nil { + maxKeys = *params.MaxKeys + } + + // Collect all matching keys sorted + var allKeys []string + for k := range m.objects { + if strings.HasPrefix(k, prefix) { + allKeys = append(allKeys, k) + } + } + sort.Strings(allKeys) + + var contents []types.Object + commonPrefixes := make(map[string]bool) + + for _, k := range allKeys { + rest := strings.TrimPrefix(k, prefix) + + if delimiter != "" { + if idx := strings.Index(rest, delimiter); idx >= 0 { + // This key has a delimiter after the prefix -> common prefix + cp := prefix + rest[:idx+len(delimiter)] + commonPrefixes[cp] = true + continue + } + } + + if int32(len(contents)) >= maxKeys { + break + } + + data := m.objects[k] + mtime := m.mtimes[k] + contents = append(contents, types.Object{ + Key: aws.String(k), + Size: aws.Int64(int64(len(data))), + LastModified: &mtime, + }) + } + + var cpSlice []types.CommonPrefix + // Sort common prefixes for deterministic output + var cpKeys []string + for cp := range commonPrefixes { + cpKeys = append(cpKeys, cp) + } + sort.Strings(cpKeys) + for _, cp := range cpKeys { + cpSlice = append(cpSlice, types.CommonPrefix{Prefix: aws.String(cp)}) + } + + return &s3.ListObjectsV2Output{ + Contents: contents, + CommonPrefixes: cpSlice, + IsTruncated: aws.Bool(false), + }, nil +} + +func (m *mockS3) CopyObject(_ context.Context, params *s3.CopyObjectInput, _ ...func(*s3.Options)) (*s3.CopyObjectOutput, error) { + m.mu.Lock() + defer m.mu.Unlock() + + // CopySource is "bucket/key" + source := aws.ToString(params.CopySource) + parts := strings.SplitN(source, "/", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid CopySource: %s", source) + } + srcKey := parts[1] + + data, ok := m.objects[srcKey] + if !ok { + return nil, fmt.Errorf("NoSuchKey: source key %q not found", srcKey) + } + + destKey := aws.ToString(params.Key) + m.objects[destKey] = append([]byte{}, data...) + m.mtimes[destKey] = time.Now() + + return &s3.CopyObjectOutput{}, nil +} + +// --- Helper --- + +func newTestMedium(t *testing.T) (*Medium, *mockS3) { + t.Helper() + mock := newMockS3() + m, err := New("test-bucket", withAPI(mock)) + require.NoError(t, err) + return m, mock +} + +// --- Tests --- + +func TestNew_Good(t *testing.T) { + mock := newMockS3() + m, err := New("my-bucket", withAPI(mock)) + require.NoError(t, err) + assert.Equal(t, "my-bucket", m.bucket) + assert.Equal(t, "", m.prefix) +} + +func TestNew_Bad_NoBucket(t *testing.T) { + _, err := New("") + assert.Error(t, err) + assert.Contains(t, err.Error(), "bucket name is required") +} + +func TestNew_Bad_NoClient(t *testing.T) { + _, err := New("bucket") + assert.Error(t, err) + assert.Contains(t, err.Error(), "S3 client is required") +} + +func TestWithPrefix_Good(t *testing.T) { + mock := newMockS3() + m, err := New("bucket", withAPI(mock), WithPrefix("data/")) + require.NoError(t, err) + assert.Equal(t, "data/", m.prefix) + + // Prefix without trailing slash gets one added + m2, err := New("bucket", withAPI(mock), WithPrefix("data")) + require.NoError(t, err) + assert.Equal(t, "data/", m2.prefix) +} + +func TestReadWrite_Good(t *testing.T) { + m, _ := newTestMedium(t) + + err := m.Write("hello.txt", "world") + require.NoError(t, err) + + content, err := m.Read("hello.txt") + require.NoError(t, err) + assert.Equal(t, "world", content) +} + +func TestReadWrite_Bad_NotFound(t *testing.T) { + m, _ := newTestMedium(t) + + _, err := m.Read("nonexistent.txt") + assert.Error(t, err) +} + +func TestReadWrite_Bad_EmptyPath(t *testing.T) { + m, _ := newTestMedium(t) + + _, err := m.Read("") + assert.Error(t, err) + + err = m.Write("", "content") + assert.Error(t, err) +} + +func TestReadWrite_Good_WithPrefix(t *testing.T) { + mock := newMockS3() + m, err := New("bucket", withAPI(mock), WithPrefix("pfx")) + require.NoError(t, err) + + err = m.Write("file.txt", "data") + require.NoError(t, err) + + // Verify the key has the prefix + _, ok := mock.objects["pfx/file.txt"] + assert.True(t, ok, "object should be stored with prefix") + + content, err := m.Read("file.txt") + require.NoError(t, err) + assert.Equal(t, "data", content) +} + +func TestEnsureDir_Good(t *testing.T) { + m, _ := newTestMedium(t) + // EnsureDir is a no-op for S3 + err := m.EnsureDir("any/path") + assert.NoError(t, err) +} + +func TestIsFile_Good(t *testing.T) { + m, _ := newTestMedium(t) + + err := m.Write("file.txt", "content") + require.NoError(t, err) + + assert.True(t, m.IsFile("file.txt")) + assert.False(t, m.IsFile("nonexistent.txt")) + assert.False(t, m.IsFile("")) +} + +func TestFileGetFileSet_Good(t *testing.T) { + m, _ := newTestMedium(t) + + err := m.FileSet("key.txt", "value") + require.NoError(t, err) + + val, err := m.FileGet("key.txt") + require.NoError(t, err) + assert.Equal(t, "value", val) +} + +func TestDelete_Good(t *testing.T) { + m, _ := newTestMedium(t) + + err := m.Write("to-delete.txt", "content") + require.NoError(t, err) + assert.True(t, m.Exists("to-delete.txt")) + + err = m.Delete("to-delete.txt") + require.NoError(t, err) + assert.False(t, m.IsFile("to-delete.txt")) +} + +func TestDelete_Bad_EmptyPath(t *testing.T) { + m, _ := newTestMedium(t) + err := m.Delete("") + assert.Error(t, err) +} + +func TestDeleteAll_Good(t *testing.T) { + m, _ := newTestMedium(t) + + // Create nested structure + require.NoError(t, m.Write("dir/file1.txt", "a")) + require.NoError(t, m.Write("dir/sub/file2.txt", "b")) + require.NoError(t, m.Write("other.txt", "c")) + + err := m.DeleteAll("dir") + require.NoError(t, err) + + assert.False(t, m.IsFile("dir/file1.txt")) + assert.False(t, m.IsFile("dir/sub/file2.txt")) + assert.True(t, m.IsFile("other.txt")) +} + +func TestDeleteAll_Bad_EmptyPath(t *testing.T) { + m, _ := newTestMedium(t) + err := m.DeleteAll("") + assert.Error(t, err) +} + +func TestRename_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("old.txt", "content")) + assert.True(t, m.IsFile("old.txt")) + + err := m.Rename("old.txt", "new.txt") + require.NoError(t, err) + + assert.False(t, m.IsFile("old.txt")) + assert.True(t, m.IsFile("new.txt")) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "content", content) +} + +func TestRename_Bad_EmptyPath(t *testing.T) { + m, _ := newTestMedium(t) + err := m.Rename("", "new.txt") + assert.Error(t, err) + + err = m.Rename("old.txt", "") + assert.Error(t, err) +} + +func TestRename_Bad_SourceNotFound(t *testing.T) { + m, _ := newTestMedium(t) + err := m.Rename("nonexistent.txt", "new.txt") + assert.Error(t, err) +} + +func TestList_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("dir/file1.txt", "a")) + require.NoError(t, m.Write("dir/file2.txt", "b")) + require.NoError(t, m.Write("dir/sub/file3.txt", "c")) + + entries, err := m.List("dir") + require.NoError(t, err) + + names := make(map[string]bool) + for _, e := range entries { + names[e.Name()] = true + } + + assert.True(t, names["file1.txt"], "should list file1.txt") + assert.True(t, names["file2.txt"], "should list file2.txt") + assert.True(t, names["sub"], "should list sub directory") + assert.Len(t, entries, 3) + + // Check that sub is a directory + for _, e := range entries { + if e.Name() == "sub" { + assert.True(t, e.IsDir()) + info, err := e.Info() + require.NoError(t, err) + assert.True(t, info.IsDir()) + } + } +} + +func TestList_Good_Root(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("root.txt", "content")) + require.NoError(t, m.Write("dir/nested.txt", "nested")) + + entries, err := m.List("") + require.NoError(t, err) + + names := make(map[string]bool) + for _, e := range entries { + names[e.Name()] = true + } + + assert.True(t, names["root.txt"]) + assert.True(t, names["dir"]) +} + +func TestStat_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "hello world")) + + info, err := m.Stat("file.txt") + require.NoError(t, err) + assert.Equal(t, "file.txt", info.Name()) + assert.Equal(t, int64(11), info.Size()) + assert.False(t, info.IsDir()) +} + +func TestStat_Bad_NotFound(t *testing.T) { + m, _ := newTestMedium(t) + + _, err := m.Stat("nonexistent.txt") + assert.Error(t, err) +} + +func TestStat_Bad_EmptyPath(t *testing.T) { + m, _ := newTestMedium(t) + _, err := m.Stat("") + assert.Error(t, err) +} + +func TestOpen_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "open me")) + + f, err := m.Open("file.txt") + require.NoError(t, err) + defer f.Close() + + data, err := goio.ReadAll(f.(goio.Reader)) + require.NoError(t, err) + assert.Equal(t, "open me", string(data)) + + stat, err := f.Stat() + require.NoError(t, err) + assert.Equal(t, "file.txt", stat.Name()) +} + +func TestOpen_Bad_NotFound(t *testing.T) { + m, _ := newTestMedium(t) + + _, err := m.Open("nonexistent.txt") + assert.Error(t, err) +} + +func TestCreate_Good(t *testing.T) { + m, _ := newTestMedium(t) + + w, err := m.Create("new.txt") + require.NoError(t, err) + + n, err := w.Write([]byte("created")) + require.NoError(t, err) + assert.Equal(t, 7, n) + + err = w.Close() + require.NoError(t, err) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "created", content) +} + +func TestAppend_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("append.txt", "hello")) + + w, err := m.Append("append.txt") + require.NoError(t, err) + + _, err = w.Write([]byte(" world")) + require.NoError(t, err) + err = w.Close() + require.NoError(t, err) + + content, err := m.Read("append.txt") + require.NoError(t, err) + assert.Equal(t, "hello world", content) +} + +func TestAppend_Good_NewFile(t *testing.T) { + m, _ := newTestMedium(t) + + w, err := m.Append("new.txt") + require.NoError(t, err) + + _, err = w.Write([]byte("fresh")) + require.NoError(t, err) + err = w.Close() + require.NoError(t, err) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "fresh", content) +} + +func TestReadStream_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("stream.txt", "streaming content")) + + reader, err := m.ReadStream("stream.txt") + require.NoError(t, err) + defer reader.Close() + + data, err := goio.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "streaming content", string(data)) +} + +func TestReadStream_Bad_NotFound(t *testing.T) { + m, _ := newTestMedium(t) + _, err := m.ReadStream("nonexistent.txt") + assert.Error(t, err) +} + +func TestWriteStream_Good(t *testing.T) { + m, _ := newTestMedium(t) + + writer, err := m.WriteStream("output.txt") + require.NoError(t, err) + + _, err = goio.Copy(writer, strings.NewReader("piped data")) + require.NoError(t, err) + err = writer.Close() + require.NoError(t, err) + + content, err := m.Read("output.txt") + require.NoError(t, err) + assert.Equal(t, "piped data", content) +} + +func TestExists_Good(t *testing.T) { + m, _ := newTestMedium(t) + + assert.False(t, m.Exists("nonexistent.txt")) + + require.NoError(t, m.Write("file.txt", "content")) + assert.True(t, m.Exists("file.txt")) +} + +func TestExists_Good_DirectoryPrefix(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("dir/file.txt", "content")) + // "dir" should exist as a directory prefix + assert.True(t, m.Exists("dir")) +} + +func TestIsDir_Good(t *testing.T) { + m, _ := newTestMedium(t) + + require.NoError(t, m.Write("dir/file.txt", "content")) + + assert.True(t, m.IsDir("dir")) + assert.False(t, m.IsDir("dir/file.txt")) + assert.False(t, m.IsDir("nonexistent")) + assert.False(t, m.IsDir("")) +} + +func TestKey_Good(t *testing.T) { + mock := newMockS3() + + // No prefix + m, _ := New("bucket", withAPI(mock)) + assert.Equal(t, "file.txt", m.key("file.txt")) + assert.Equal(t, "dir/file.txt", m.key("dir/file.txt")) + assert.Equal(t, "", m.key("")) + assert.Equal(t, "file.txt", m.key("/file.txt")) + assert.Equal(t, "file.txt", m.key("../file.txt")) + + // With prefix + m2, _ := New("bucket", withAPI(mock), WithPrefix("pfx")) + assert.Equal(t, "pfx/file.txt", m2.key("file.txt")) + assert.Equal(t, "pfx/dir/file.txt", m2.key("dir/file.txt")) + assert.Equal(t, "pfx/", m2.key("")) +} + +// Ugly: verify the Medium interface is satisfied at compile time. +func TestInterfaceCompliance_Ugly(t *testing.T) { + mock := newMockS3() + m, err := New("bucket", withAPI(mock)) + require.NoError(t, err) + + // Verify all methods exist by calling them in a way that + // proves compile-time satisfaction of the interface. + var _ interface { + Read(string) (string, error) + Write(string, string) error + EnsureDir(string) error + IsFile(string) bool + FileGet(string) (string, error) + FileSet(string, string) error + Delete(string) error + DeleteAll(string) error + Rename(string, string) error + List(string) ([]fs.DirEntry, error) + Stat(string) (fs.FileInfo, error) + Open(string) (fs.File, error) + Create(string) (goio.WriteCloser, error) + Append(string) (goio.WriteCloser, error) + ReadStream(string) (goio.ReadCloser, error) + WriteStream(string) (goio.WriteCloser, error) + Exists(string) bool + IsDir(string) bool + } = m +} diff --git a/pkg/io/sqlite/sqlite.go b/pkg/io/sqlite/sqlite.go new file mode 100644 index 0000000..734a749 --- /dev/null +++ b/pkg/io/sqlite/sqlite.go @@ -0,0 +1,669 @@ +// Package sqlite provides a SQLite-backed implementation of the io.Medium interface. +package sqlite + +import ( + "bytes" + "database/sql" + goio "io" + "io/fs" + "os" + "path" + "strings" + "time" + + coreerr "github.com/host-uk/core/pkg/framework/core" + + _ "modernc.org/sqlite" // Pure Go SQLite driver +) + +// Medium is a SQLite-backed storage backend implementing the io.Medium interface. +type Medium struct { + db *sql.DB + table string +} + +// Option configures a Medium. +type Option func(*Medium) + +// WithTable sets the table name (default: "files"). +func WithTable(table string) Option { + return func(m *Medium) { + m.table = table + } +} + +// New creates a new SQLite Medium at the given database path. +// Use ":memory:" for an in-memory database. +func New(dbPath string, opts ...Option) (*Medium, error) { + if dbPath == "" { + return nil, coreerr.E("sqlite.New", "database path is required", nil) + } + + m := &Medium{table: "files"} + for _, opt := range opts { + opt(m) + } + + db, err := sql.Open("sqlite", dbPath) + if err != nil { + return nil, coreerr.E("sqlite.New", "failed to open database", err) + } + + // Enable WAL mode for better concurrency + if _, err := db.Exec("PRAGMA journal_mode=WAL"); err != nil { + db.Close() + return nil, coreerr.E("sqlite.New", "failed to set WAL mode", err) + } + + // Create the schema + createSQL := `CREATE TABLE IF NOT EXISTS ` + m.table + ` ( + path TEXT PRIMARY KEY, + content BLOB NOT NULL, + mode INTEGER DEFAULT 420, + is_dir BOOLEAN DEFAULT FALSE, + mtime DATETIME DEFAULT CURRENT_TIMESTAMP + )` + if _, err := db.Exec(createSQL); err != nil { + db.Close() + return nil, coreerr.E("sqlite.New", "failed to create table", err) + } + + m.db = db + return m, nil +} + +// Close closes the underlying database connection. +func (m *Medium) Close() error { + if m.db != nil { + return m.db.Close() + } + return nil +} + +// cleanPath normalizes a path for consistent storage. +// Uses a leading "/" before Clean to sandbox traversal attempts. +func cleanPath(p string) string { + clean := path.Clean("/" + p) + if clean == "/" { + return "" + } + return strings.TrimPrefix(clean, "/") +} + +// Read retrieves the content of a file as a string. +func (m *Medium) Read(p string) (string, error) { + key := cleanPath(p) + if key == "" { + return "", coreerr.E("sqlite.Read", "path is required", os.ErrInvalid) + } + + var content []byte + var isDir bool + err := m.db.QueryRow( + `SELECT content, is_dir FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&content, &isDir) + if err == sql.ErrNoRows { + return "", coreerr.E("sqlite.Read", "file not found: "+key, os.ErrNotExist) + } + if err != nil { + return "", coreerr.E("sqlite.Read", "query failed: "+key, err) + } + if isDir { + return "", coreerr.E("sqlite.Read", "path is a directory: "+key, os.ErrInvalid) + } + return string(content), nil +} + +// Write saves the given content to a file, overwriting it if it exists. +func (m *Medium) Write(p, content string) error { + key := cleanPath(p) + if key == "" { + return coreerr.E("sqlite.Write", "path is required", os.ErrInvalid) + } + + _, err := m.db.Exec( + `INSERT INTO `+m.table+` (path, content, mode, is_dir, mtime) VALUES (?, ?, 420, FALSE, ?) + ON CONFLICT(path) DO UPDATE SET content = excluded.content, is_dir = FALSE, mtime = excluded.mtime`, + key, []byte(content), time.Now().UTC(), + ) + if err != nil { + return coreerr.E("sqlite.Write", "insert failed: "+key, err) + } + return nil +} + +// EnsureDir makes sure a directory exists, creating it if necessary. +func (m *Medium) EnsureDir(p string) error { + key := cleanPath(p) + if key == "" { + // Root always "exists" + return nil + } + + _, err := m.db.Exec( + `INSERT INTO `+m.table+` (path, content, mode, is_dir, mtime) VALUES (?, '', 493, TRUE, ?) + ON CONFLICT(path) DO NOTHING`, + key, time.Now().UTC(), + ) + if err != nil { + return coreerr.E("sqlite.EnsureDir", "insert failed: "+key, err) + } + return nil +} + +// IsFile checks if a path exists and is a regular file. +func (m *Medium) IsFile(p string) bool { + key := cleanPath(p) + if key == "" { + return false + } + + var isDir bool + err := m.db.QueryRow( + `SELECT is_dir FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&isDir) + if err != nil { + return false + } + return !isDir +} + +// FileGet is a convenience function that reads a file from the medium. +func (m *Medium) FileGet(p string) (string, error) { + return m.Read(p) +} + +// FileSet is a convenience function that writes a file to the medium. +func (m *Medium) FileSet(p, content string) error { + return m.Write(p, content) +} + +// Delete removes a file or empty directory. +func (m *Medium) Delete(p string) error { + key := cleanPath(p) + if key == "" { + return coreerr.E("sqlite.Delete", "path is required", os.ErrInvalid) + } + + // Check if it's a directory with children + var isDir bool + err := m.db.QueryRow( + `SELECT is_dir FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&isDir) + if err == sql.ErrNoRows { + return coreerr.E("sqlite.Delete", "path not found: "+key, os.ErrNotExist) + } + if err != nil { + return coreerr.E("sqlite.Delete", "query failed: "+key, err) + } + + if isDir { + // Check for children + prefix := key + "/" + var count int + err := m.db.QueryRow( + `SELECT COUNT(*) FROM `+m.table+` WHERE path LIKE ? AND path != ?`, prefix+"%", key, + ).Scan(&count) + if err != nil { + return coreerr.E("sqlite.Delete", "count failed: "+key, err) + } + if count > 0 { + return coreerr.E("sqlite.Delete", "directory not empty: "+key, os.ErrExist) + } + } + + res, err := m.db.Exec(`DELETE FROM `+m.table+` WHERE path = ?`, key) + if err != nil { + return coreerr.E("sqlite.Delete", "delete failed: "+key, err) + } + n, _ := res.RowsAffected() + if n == 0 { + return coreerr.E("sqlite.Delete", "path not found: "+key, os.ErrNotExist) + } + return nil +} + +// DeleteAll removes a file or directory and all its contents recursively. +func (m *Medium) DeleteAll(p string) error { + key := cleanPath(p) + if key == "" { + return coreerr.E("sqlite.DeleteAll", "path is required", os.ErrInvalid) + } + + prefix := key + "/" + + // Delete the exact path and all children + res, err := m.db.Exec( + `DELETE FROM `+m.table+` WHERE path = ? OR path LIKE ?`, + key, prefix+"%", + ) + if err != nil { + return coreerr.E("sqlite.DeleteAll", "delete failed: "+key, err) + } + n, _ := res.RowsAffected() + if n == 0 { + return coreerr.E("sqlite.DeleteAll", "path not found: "+key, os.ErrNotExist) + } + return nil +} + +// Rename moves a file or directory from oldPath to newPath. +func (m *Medium) Rename(oldPath, newPath string) error { + oldKey := cleanPath(oldPath) + newKey := cleanPath(newPath) + if oldKey == "" || newKey == "" { + return coreerr.E("sqlite.Rename", "both old and new paths are required", os.ErrInvalid) + } + + tx, err := m.db.Begin() + if err != nil { + return coreerr.E("sqlite.Rename", "begin tx failed", err) + } + defer tx.Rollback() + + // Check if source exists + var content []byte + var mode int + var isDir bool + var mtime time.Time + err = tx.QueryRow( + `SELECT content, mode, is_dir, mtime FROM `+m.table+` WHERE path = ?`, oldKey, + ).Scan(&content, &mode, &isDir, &mtime) + if err == sql.ErrNoRows { + return coreerr.E("sqlite.Rename", "source not found: "+oldKey, os.ErrNotExist) + } + if err != nil { + return coreerr.E("sqlite.Rename", "query failed: "+oldKey, err) + } + + // Insert or replace at new path + _, err = tx.Exec( + `INSERT INTO `+m.table+` (path, content, mode, is_dir, mtime) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET content = excluded.content, mode = excluded.mode, is_dir = excluded.is_dir, mtime = excluded.mtime`, + newKey, content, mode, isDir, mtime, + ) + if err != nil { + return coreerr.E("sqlite.Rename", "insert at new path failed: "+newKey, err) + } + + // Delete old path + _, err = tx.Exec(`DELETE FROM `+m.table+` WHERE path = ?`, oldKey) + if err != nil { + return coreerr.E("sqlite.Rename", "delete old path failed: "+oldKey, err) + } + + // If it's a directory, move all children + if isDir { + oldPrefix := oldKey + "/" + newPrefix := newKey + "/" + + rows, err := tx.Query( + `SELECT path, content, mode, is_dir, mtime FROM `+m.table+` WHERE path LIKE ?`, + oldPrefix+"%", + ) + if err != nil { + return coreerr.E("sqlite.Rename", "query children failed", err) + } + + type child struct { + path string + content []byte + mode int + isDir bool + mtime time.Time + } + var children []child + for rows.Next() { + var c child + if err := rows.Scan(&c.path, &c.content, &c.mode, &c.isDir, &c.mtime); err != nil { + rows.Close() + return coreerr.E("sqlite.Rename", "scan child failed", err) + } + children = append(children, c) + } + rows.Close() + + for _, c := range children { + newChildPath := newPrefix + strings.TrimPrefix(c.path, oldPrefix) + _, err = tx.Exec( + `INSERT INTO `+m.table+` (path, content, mode, is_dir, mtime) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(path) DO UPDATE SET content = excluded.content, mode = excluded.mode, is_dir = excluded.is_dir, mtime = excluded.mtime`, + newChildPath, c.content, c.mode, c.isDir, c.mtime, + ) + if err != nil { + return coreerr.E("sqlite.Rename", "insert child failed", err) + } + } + + // Delete old children + _, err = tx.Exec(`DELETE FROM `+m.table+` WHERE path LIKE ?`, oldPrefix+"%") + if err != nil { + return coreerr.E("sqlite.Rename", "delete old children failed", err) + } + } + + return tx.Commit() +} + +// List returns the directory entries for the given path. +func (m *Medium) List(p string) ([]fs.DirEntry, error) { + prefix := cleanPath(p) + if prefix != "" { + prefix += "/" + } + + // Query all paths under the prefix + rows, err := m.db.Query( + `SELECT path, content, mode, is_dir, mtime FROM `+m.table+` WHERE path LIKE ? OR path LIKE ?`, + prefix+"%", prefix+"%", + ) + if err != nil { + return nil, coreerr.E("sqlite.List", "query failed", err) + } + defer rows.Close() + + seen := make(map[string]bool) + var entries []fs.DirEntry + + for rows.Next() { + var rowPath string + var content []byte + var mode int + var isDir bool + var mtime time.Time + if err := rows.Scan(&rowPath, &content, &mode, &isDir, &mtime); err != nil { + return nil, coreerr.E("sqlite.List", "scan failed", err) + } + + rest := strings.TrimPrefix(rowPath, prefix) + if rest == "" { + continue + } + + // Check if this is a direct child or nested + if idx := strings.Index(rest, "/"); idx >= 0 { + // Nested - register as a directory + dirName := rest[:idx] + if !seen[dirName] { + seen[dirName] = true + entries = append(entries, &dirEntry{ + name: dirName, + isDir: true, + mode: fs.ModeDir | 0755, + info: &fileInfo{ + name: dirName, + isDir: true, + mode: fs.ModeDir | 0755, + }, + }) + } + } else { + // Direct child + if !seen[rest] { + seen[rest] = true + entries = append(entries, &dirEntry{ + name: rest, + isDir: isDir, + mode: fs.FileMode(mode), + info: &fileInfo{ + name: rest, + size: int64(len(content)), + mode: fs.FileMode(mode), + modTime: mtime, + isDir: isDir, + }, + }) + } + } + } + + return entries, rows.Err() +} + +// Stat returns file information for the given path. +func (m *Medium) Stat(p string) (fs.FileInfo, error) { + key := cleanPath(p) + if key == "" { + return nil, coreerr.E("sqlite.Stat", "path is required", os.ErrInvalid) + } + + var content []byte + var mode int + var isDir bool + var mtime time.Time + err := m.db.QueryRow( + `SELECT content, mode, is_dir, mtime FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&content, &mode, &isDir, &mtime) + if err == sql.ErrNoRows { + return nil, coreerr.E("sqlite.Stat", "path not found: "+key, os.ErrNotExist) + } + if err != nil { + return nil, coreerr.E("sqlite.Stat", "query failed: "+key, err) + } + + name := path.Base(key) + return &fileInfo{ + name: name, + size: int64(len(content)), + mode: fs.FileMode(mode), + modTime: mtime, + isDir: isDir, + }, nil +} + +// Open opens the named file for reading. +func (m *Medium) Open(p string) (fs.File, error) { + key := cleanPath(p) + if key == "" { + return nil, coreerr.E("sqlite.Open", "path is required", os.ErrInvalid) + } + + var content []byte + var mode int + var isDir bool + var mtime time.Time + err := m.db.QueryRow( + `SELECT content, mode, is_dir, mtime FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&content, &mode, &isDir, &mtime) + if err == sql.ErrNoRows { + return nil, coreerr.E("sqlite.Open", "file not found: "+key, os.ErrNotExist) + } + if err != nil { + return nil, coreerr.E("sqlite.Open", "query failed: "+key, err) + } + if isDir { + return nil, coreerr.E("sqlite.Open", "path is a directory: "+key, os.ErrInvalid) + } + + return &sqliteFile{ + name: path.Base(key), + content: content, + mode: fs.FileMode(mode), + modTime: mtime, + }, nil +} + +// Create creates or truncates the named file. +func (m *Medium) Create(p string) (goio.WriteCloser, error) { + key := cleanPath(p) + if key == "" { + return nil, coreerr.E("sqlite.Create", "path is required", os.ErrInvalid) + } + return &sqliteWriteCloser{ + medium: m, + path: key, + }, nil +} + +// Append opens the named file for appending, creating it if it doesn't exist. +func (m *Medium) Append(p string) (goio.WriteCloser, error) { + key := cleanPath(p) + if key == "" { + return nil, coreerr.E("sqlite.Append", "path is required", os.ErrInvalid) + } + + var existing []byte + err := m.db.QueryRow( + `SELECT content FROM `+m.table+` WHERE path = ? AND is_dir = FALSE`, key, + ).Scan(&existing) + if err != nil && err != sql.ErrNoRows { + return nil, coreerr.E("sqlite.Append", "query failed: "+key, err) + } + + return &sqliteWriteCloser{ + medium: m, + path: key, + data: existing, + }, nil +} + +// ReadStream returns a reader for the file content. +func (m *Medium) ReadStream(p string) (goio.ReadCloser, error) { + key := cleanPath(p) + if key == "" { + return nil, coreerr.E("sqlite.ReadStream", "path is required", os.ErrInvalid) + } + + var content []byte + var isDir bool + err := m.db.QueryRow( + `SELECT content, is_dir FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&content, &isDir) + if err == sql.ErrNoRows { + return nil, coreerr.E("sqlite.ReadStream", "file not found: "+key, os.ErrNotExist) + } + if err != nil { + return nil, coreerr.E("sqlite.ReadStream", "query failed: "+key, err) + } + if isDir { + return nil, coreerr.E("sqlite.ReadStream", "path is a directory: "+key, os.ErrInvalid) + } + + return goio.NopCloser(bytes.NewReader(content)), nil +} + +// WriteStream returns a writer for the file content. Content is stored on Close. +func (m *Medium) WriteStream(p string) (goio.WriteCloser, error) { + return m.Create(p) +} + +// Exists checks if a path exists (file or directory). +func (m *Medium) Exists(p string) bool { + key := cleanPath(p) + if key == "" { + // Root always exists + return true + } + + var count int + err := m.db.QueryRow( + `SELECT COUNT(*) FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&count) + if err != nil { + return false + } + return count > 0 +} + +// IsDir checks if a path exists and is a directory. +func (m *Medium) IsDir(p string) bool { + key := cleanPath(p) + if key == "" { + return false + } + + var isDir bool + err := m.db.QueryRow( + `SELECT is_dir FROM `+m.table+` WHERE path = ?`, key, + ).Scan(&isDir) + if err != nil { + return false + } + return isDir +} + +// --- Internal types --- + +// fileInfo implements fs.FileInfo for SQLite entries. +type fileInfo struct { + name string + size int64 + mode fs.FileMode + modTime time.Time + isDir bool +} + +func (fi *fileInfo) Name() string { return fi.name } +func (fi *fileInfo) Size() int64 { return fi.size } +func (fi *fileInfo) Mode() fs.FileMode { return fi.mode } +func (fi *fileInfo) ModTime() time.Time { return fi.modTime } +func (fi *fileInfo) IsDir() bool { return fi.isDir } +func (fi *fileInfo) Sys() any { return nil } + +// dirEntry implements fs.DirEntry for SQLite listings. +type dirEntry struct { + name string + isDir bool + mode fs.FileMode + info fs.FileInfo +} + +func (de *dirEntry) Name() string { return de.name } +func (de *dirEntry) IsDir() bool { return de.isDir } +func (de *dirEntry) Type() fs.FileMode { return de.mode.Type() } +func (de *dirEntry) Info() (fs.FileInfo, error) { return de.info, nil } + +// sqliteFile implements fs.File for SQLite entries. +type sqliteFile struct { + name string + content []byte + offset int64 + mode fs.FileMode + modTime time.Time +} + +func (f *sqliteFile) Stat() (fs.FileInfo, error) { + return &fileInfo{ + name: f.name, + size: int64(len(f.content)), + mode: f.mode, + modTime: f.modTime, + }, nil +} + +func (f *sqliteFile) Read(b []byte) (int, error) { + if f.offset >= int64(len(f.content)) { + return 0, goio.EOF + } + n := copy(b, f.content[f.offset:]) + f.offset += int64(n) + return n, nil +} + +func (f *sqliteFile) Close() error { + return nil +} + +// sqliteWriteCloser buffers writes and stores to SQLite on Close. +type sqliteWriteCloser struct { + medium *Medium + path string + data []byte +} + +func (w *sqliteWriteCloser) Write(p []byte) (int, error) { + w.data = append(w.data, p...) + return len(p), nil +} + +func (w *sqliteWriteCloser) Close() error { + _, err := w.medium.db.Exec( + `INSERT INTO `+w.medium.table+` (path, content, mode, is_dir, mtime) VALUES (?, ?, 420, FALSE, ?) + ON CONFLICT(path) DO UPDATE SET content = excluded.content, is_dir = FALSE, mtime = excluded.mtime`, + w.path, w.data, time.Now().UTC(), + ) + if err != nil { + return coreerr.E("sqlite.WriteCloser.Close", "store failed: "+w.path, err) + } + return nil +} diff --git a/pkg/io/sqlite/sqlite_test.go b/pkg/io/sqlite/sqlite_test.go new file mode 100644 index 0000000..97d6304 --- /dev/null +++ b/pkg/io/sqlite/sqlite_test.go @@ -0,0 +1,653 @@ +package sqlite + +import ( + goio "io" + "io/fs" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestMedium(t *testing.T) *Medium { + t.Helper() + m, err := New(":memory:") + require.NoError(t, err) + t.Cleanup(func() { m.Close() }) + return m +} + +// --- Constructor Tests --- + +func TestNew_Good(t *testing.T) { + m, err := New(":memory:") + require.NoError(t, err) + defer m.Close() + assert.Equal(t, "files", m.table) +} + +func TestNew_Good_WithTable(t *testing.T) { + m, err := New(":memory:", WithTable("custom")) + require.NoError(t, err) + defer m.Close() + assert.Equal(t, "custom", m.table) +} + +func TestNew_Bad_EmptyPath(t *testing.T) { + _, err := New("") + assert.Error(t, err) + assert.Contains(t, err.Error(), "database path is required") +} + +// --- Read/Write Tests --- + +func TestReadWrite_Good(t *testing.T) { + m := newTestMedium(t) + + err := m.Write("hello.txt", "world") + require.NoError(t, err) + + content, err := m.Read("hello.txt") + require.NoError(t, err) + assert.Equal(t, "world", content) +} + +func TestReadWrite_Good_Overwrite(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "first")) + require.NoError(t, m.Write("file.txt", "second")) + + content, err := m.Read("file.txt") + require.NoError(t, err) + assert.Equal(t, "second", content) +} + +func TestReadWrite_Good_NestedPath(t *testing.T) { + m := newTestMedium(t) + + err := m.Write("a/b/c.txt", "nested") + require.NoError(t, err) + + content, err := m.Read("a/b/c.txt") + require.NoError(t, err) + assert.Equal(t, "nested", content) +} + +func TestRead_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Read("nonexistent.txt") + assert.Error(t, err) +} + +func TestRead_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Read("") + assert.Error(t, err) +} + +func TestWrite_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + err := m.Write("", "content") + assert.Error(t, err) +} + +func TestRead_Bad_IsDirectory(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + _, err := m.Read("mydir") + assert.Error(t, err) +} + +// --- EnsureDir Tests --- + +func TestEnsureDir_Good(t *testing.T) { + m := newTestMedium(t) + + err := m.EnsureDir("mydir") + require.NoError(t, err) + assert.True(t, m.IsDir("mydir")) +} + +func TestEnsureDir_Good_EmptyPath(t *testing.T) { + m := newTestMedium(t) + // Root always exists, no-op + err := m.EnsureDir("") + assert.NoError(t, err) +} + +func TestEnsureDir_Good_Idempotent(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + require.NoError(t, m.EnsureDir("mydir")) + assert.True(t, m.IsDir("mydir")) +} + +// --- IsFile Tests --- + +func TestIsFile_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "content")) + require.NoError(t, m.EnsureDir("mydir")) + + assert.True(t, m.IsFile("file.txt")) + assert.False(t, m.IsFile("mydir")) + assert.False(t, m.IsFile("nonexistent")) + assert.False(t, m.IsFile("")) +} + +// --- FileGet/FileSet Tests --- + +func TestFileGetFileSet_Good(t *testing.T) { + m := newTestMedium(t) + + err := m.FileSet("key.txt", "value") + require.NoError(t, err) + + val, err := m.FileGet("key.txt") + require.NoError(t, err) + assert.Equal(t, "value", val) +} + +// --- Delete Tests --- + +func TestDelete_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("to-delete.txt", "content")) + assert.True(t, m.Exists("to-delete.txt")) + + err := m.Delete("to-delete.txt") + require.NoError(t, err) + assert.False(t, m.Exists("to-delete.txt")) +} + +func TestDelete_Good_EmptyDir(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("emptydir")) + assert.True(t, m.IsDir("emptydir")) + + err := m.Delete("emptydir") + require.NoError(t, err) + assert.False(t, m.IsDir("emptydir")) +} + +func TestDelete_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + err := m.Delete("nonexistent") + assert.Error(t, err) +} + +func TestDelete_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + err := m.Delete("") + assert.Error(t, err) +} + +func TestDelete_Bad_NotEmpty(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + require.NoError(t, m.Write("mydir/file.txt", "content")) + + err := m.Delete("mydir") + assert.Error(t, err) +} + +// --- DeleteAll Tests --- + +func TestDeleteAll_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("dir/file1.txt", "a")) + require.NoError(t, m.Write("dir/sub/file2.txt", "b")) + require.NoError(t, m.Write("other.txt", "c")) + + err := m.DeleteAll("dir") + require.NoError(t, err) + + assert.False(t, m.Exists("dir/file1.txt")) + assert.False(t, m.Exists("dir/sub/file2.txt")) + assert.True(t, m.Exists("other.txt")) +} + +func TestDeleteAll_Good_SingleFile(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "content")) + + err := m.DeleteAll("file.txt") + require.NoError(t, err) + assert.False(t, m.Exists("file.txt")) +} + +func TestDeleteAll_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + err := m.DeleteAll("nonexistent") + assert.Error(t, err) +} + +func TestDeleteAll_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + err := m.DeleteAll("") + assert.Error(t, err) +} + +// --- Rename Tests --- + +func TestRename_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("old.txt", "content")) + + err := m.Rename("old.txt", "new.txt") + require.NoError(t, err) + + assert.False(t, m.Exists("old.txt")) + assert.True(t, m.IsFile("new.txt")) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "content", content) +} + +func TestRename_Good_Directory(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("olddir")) + require.NoError(t, m.Write("olddir/file.txt", "content")) + + err := m.Rename("olddir", "newdir") + require.NoError(t, err) + + assert.False(t, m.Exists("olddir")) + assert.False(t, m.Exists("olddir/file.txt")) + assert.True(t, m.IsDir("newdir")) + assert.True(t, m.IsFile("newdir/file.txt")) + + content, err := m.Read("newdir/file.txt") + require.NoError(t, err) + assert.Equal(t, "content", content) +} + +func TestRename_Bad_SourceNotFound(t *testing.T) { + m := newTestMedium(t) + + err := m.Rename("nonexistent", "new") + assert.Error(t, err) +} + +func TestRename_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + err := m.Rename("", "new") + assert.Error(t, err) + + err = m.Rename("old", "") + assert.Error(t, err) +} + +// --- List Tests --- + +func TestList_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("dir/file1.txt", "a")) + require.NoError(t, m.Write("dir/file2.txt", "b")) + require.NoError(t, m.Write("dir/sub/file3.txt", "c")) + + entries, err := m.List("dir") + require.NoError(t, err) + + names := make(map[string]bool) + for _, e := range entries { + names[e.Name()] = true + } + + assert.True(t, names["file1.txt"]) + assert.True(t, names["file2.txt"]) + assert.True(t, names["sub"]) + assert.Len(t, entries, 3) +} + +func TestList_Good_Root(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("root.txt", "content")) + require.NoError(t, m.Write("dir/nested.txt", "nested")) + + entries, err := m.List("") + require.NoError(t, err) + + names := make(map[string]bool) + for _, e := range entries { + names[e.Name()] = true + } + + assert.True(t, names["root.txt"]) + assert.True(t, names["dir"]) +} + +func TestList_Good_DirectoryEntry(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("dir/sub/file.txt", "content")) + + entries, err := m.List("dir") + require.NoError(t, err) + + require.Len(t, entries, 1) + assert.Equal(t, "sub", entries[0].Name()) + assert.True(t, entries[0].IsDir()) + + info, err := entries[0].Info() + require.NoError(t, err) + assert.True(t, info.IsDir()) +} + +// --- Stat Tests --- + +func TestStat_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "hello world")) + + info, err := m.Stat("file.txt") + require.NoError(t, err) + assert.Equal(t, "file.txt", info.Name()) + assert.Equal(t, int64(11), info.Size()) + assert.False(t, info.IsDir()) +} + +func TestStat_Good_Directory(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + + info, err := m.Stat("mydir") + require.NoError(t, err) + assert.Equal(t, "mydir", info.Name()) + assert.True(t, info.IsDir()) +} + +func TestStat_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Stat("nonexistent") + assert.Error(t, err) +} + +func TestStat_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Stat("") + assert.Error(t, err) +} + +// --- Open Tests --- + +func TestOpen_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "open me")) + + f, err := m.Open("file.txt") + require.NoError(t, err) + defer f.Close() + + data, err := goio.ReadAll(f.(goio.Reader)) + require.NoError(t, err) + assert.Equal(t, "open me", string(data)) + + stat, err := f.Stat() + require.NoError(t, err) + assert.Equal(t, "file.txt", stat.Name()) +} + +func TestOpen_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Open("nonexistent.txt") + assert.Error(t, err) +} + +func TestOpen_Bad_IsDirectory(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + _, err := m.Open("mydir") + assert.Error(t, err) +} + +// --- Create Tests --- + +func TestCreate_Good(t *testing.T) { + m := newTestMedium(t) + + w, err := m.Create("new.txt") + require.NoError(t, err) + + n, err := w.Write([]byte("created")) + require.NoError(t, err) + assert.Equal(t, 7, n) + + err = w.Close() + require.NoError(t, err) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "created", content) +} + +func TestCreate_Good_Overwrite(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "old content")) + + w, err := m.Create("file.txt") + require.NoError(t, err) + _, err = w.Write([]byte("new")) + require.NoError(t, err) + require.NoError(t, w.Close()) + + content, err := m.Read("file.txt") + require.NoError(t, err) + assert.Equal(t, "new", content) +} + +func TestCreate_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Create("") + assert.Error(t, err) +} + +// --- Append Tests --- + +func TestAppend_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("append.txt", "hello")) + + w, err := m.Append("append.txt") + require.NoError(t, err) + + _, err = w.Write([]byte(" world")) + require.NoError(t, err) + require.NoError(t, w.Close()) + + content, err := m.Read("append.txt") + require.NoError(t, err) + assert.Equal(t, "hello world", content) +} + +func TestAppend_Good_NewFile(t *testing.T) { + m := newTestMedium(t) + + w, err := m.Append("new.txt") + require.NoError(t, err) + + _, err = w.Write([]byte("fresh")) + require.NoError(t, err) + require.NoError(t, w.Close()) + + content, err := m.Read("new.txt") + require.NoError(t, err) + assert.Equal(t, "fresh", content) +} + +func TestAppend_Bad_EmptyPath(t *testing.T) { + m := newTestMedium(t) + + _, err := m.Append("") + assert.Error(t, err) +} + +// --- ReadStream Tests --- + +func TestReadStream_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("stream.txt", "streaming content")) + + reader, err := m.ReadStream("stream.txt") + require.NoError(t, err) + defer reader.Close() + + data, err := goio.ReadAll(reader) + require.NoError(t, err) + assert.Equal(t, "streaming content", string(data)) +} + +func TestReadStream_Bad_NotFound(t *testing.T) { + m := newTestMedium(t) + + _, err := m.ReadStream("nonexistent.txt") + assert.Error(t, err) +} + +func TestReadStream_Bad_IsDirectory(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.EnsureDir("mydir")) + _, err := m.ReadStream("mydir") + assert.Error(t, err) +} + +// --- WriteStream Tests --- + +func TestWriteStream_Good(t *testing.T) { + m := newTestMedium(t) + + writer, err := m.WriteStream("output.txt") + require.NoError(t, err) + + _, err = goio.Copy(writer, strings.NewReader("piped data")) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + content, err := m.Read("output.txt") + require.NoError(t, err) + assert.Equal(t, "piped data", content) +} + +// --- Exists Tests --- + +func TestExists_Good(t *testing.T) { + m := newTestMedium(t) + + assert.False(t, m.Exists("nonexistent")) + + require.NoError(t, m.Write("file.txt", "content")) + assert.True(t, m.Exists("file.txt")) + + require.NoError(t, m.EnsureDir("mydir")) + assert.True(t, m.Exists("mydir")) +} + +func TestExists_Good_EmptyPath(t *testing.T) { + m := newTestMedium(t) + // Root always exists + assert.True(t, m.Exists("")) +} + +// --- IsDir Tests --- + +func TestIsDir_Good(t *testing.T) { + m := newTestMedium(t) + + require.NoError(t, m.Write("file.txt", "content")) + require.NoError(t, m.EnsureDir("mydir")) + + assert.True(t, m.IsDir("mydir")) + assert.False(t, m.IsDir("file.txt")) + assert.False(t, m.IsDir("nonexistent")) + assert.False(t, m.IsDir("")) +} + +// --- cleanPath Tests --- + +func TestCleanPath_Good(t *testing.T) { + assert.Equal(t, "file.txt", cleanPath("file.txt")) + assert.Equal(t, "dir/file.txt", cleanPath("dir/file.txt")) + assert.Equal(t, "file.txt", cleanPath("/file.txt")) + assert.Equal(t, "file.txt", cleanPath("../file.txt")) + assert.Equal(t, "file.txt", cleanPath("dir/../file.txt")) + assert.Equal(t, "", cleanPath("")) + assert.Equal(t, "", cleanPath(".")) + assert.Equal(t, "", cleanPath("/")) +} + +// --- Interface Compliance --- + +func TestInterfaceCompliance_Ugly(t *testing.T) { + m := newTestMedium(t) + + // Verify all methods exist by asserting the interface shape. + var _ interface { + Read(string) (string, error) + Write(string, string) error + EnsureDir(string) error + IsFile(string) bool + FileGet(string) (string, error) + FileSet(string, string) error + Delete(string) error + DeleteAll(string) error + Rename(string, string) error + List(string) ([]fs.DirEntry, error) + Stat(string) (fs.FileInfo, error) + Open(string) (fs.File, error) + Create(string) (goio.WriteCloser, error) + Append(string) (goio.WriteCloser, error) + ReadStream(string) (goio.ReadCloser, error) + WriteStream(string) (goio.WriteCloser, error) + Exists(string) bool + IsDir(string) bool + } = m +} + +// --- Custom Table --- + +func TestCustomTable_Good(t *testing.T) { + m, err := New(":memory:", WithTable("my_files")) + require.NoError(t, err) + defer m.Close() + + require.NoError(t, m.Write("file.txt", "content")) + + content, err := m.Read("file.txt") + require.NoError(t, err) + assert.Equal(t, "content", content) +}