From dc1790f12b167f50841a771ff0c9585d8408ec5f Mon Sep 17 00:00:00 2001 From: Snider Date: Sat, 14 Mar 2026 12:22:37 +0000 Subject: [PATCH] feat(marketplace): add provider discovery and registry Extend Manifest with provider fields (namespace, port, binary, args, element, spec) and add IsProvider() helper. New DiscoverProviders() scans directories for runtime provider manifests. ProviderRegistryFile handles registry.yaml read/write for tracking installed providers. Includes 20 tests. Co-Authored-By: Virgil --- manifest/manifest.go | 25 ++++ manifest/manifest_test.go | 63 ++++++++ marketplace/discovery.go | 161 ++++++++++++++++++++ marketplace/discovery_test.go | 273 ++++++++++++++++++++++++++++++++++ 4 files changed, 522 insertions(+) create mode 100644 marketplace/discovery.go create mode 100644 marketplace/discovery_test.go diff --git a/manifest/manifest.go b/manifest/manifest.go index 2d40e8e..c9e344d 100644 --- a/manifest/manifest.go +++ b/manifest/manifest.go @@ -12,15 +12,40 @@ type Manifest struct { Name string `yaml:"name" json:"name"` Description string `yaml:"description,omitempty" json:"description,omitempty"` Version string `yaml:"version" json:"version"` + Author string `yaml:"author,omitempty" json:"author,omitempty"` + Licence string `yaml:"licence,omitempty" json:"licence,omitempty"` Sign string `yaml:"sign,omitempty" json:"sign,omitempty"` Layout string `yaml:"layout,omitempty" json:"layout,omitempty"` Slots map[string]string `yaml:"slots,omitempty" json:"slots,omitempty"` + // Provider fields — used by runtime provider loading. + Namespace string `yaml:"namespace,omitempty" json:"namespace,omitempty"` // API route prefix, e.g. /api/v1/cool-widget + Port int `yaml:"port,omitempty" json:"port,omitempty"` // Listen port (0 = auto-assign) + Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` // Path to provider binary (relative to provider dir) + Args []string `yaml:"args,omitempty" json:"args,omitempty"` // Additional CLI args for the binary + Element *ElementSpec `yaml:"element,omitempty" json:"element,omitempty"` // Custom element for GUI rendering + Spec string `yaml:"spec,omitempty" json:"spec,omitempty"` // Path to OpenAPI spec file + Permissions Permissions `yaml:"permissions,omitempty" json:"permissions,omitempty"` Modules []string `yaml:"modules,omitempty" json:"modules,omitempty"` Daemons map[string]DaemonSpec `yaml:"daemons,omitempty" json:"daemons,omitempty"` } +// ElementSpec describes a web component for GUI rendering. +type ElementSpec struct { + // Tag is the custom element tag name, e.g. "core-cool-widget". + Tag string `yaml:"tag" json:"tag"` + + // Source is the path to the JS bundle (relative to provider dir). + Source string `yaml:"source" json:"source"` +} + +// IsProvider returns true if this manifest declares provider fields +// (namespace and binary), indicating it is a runtime provider. +func (m *Manifest) IsProvider() bool { + return m.Namespace != "" && m.Binary != "" +} + // Permissions declares the I/O capabilities a module requires. type Permissions struct { Read []string `yaml:"read" json:"read"` diff --git a/manifest/manifest_test.go b/manifest/manifest_test.go index 04a61fb..38c3e1d 100644 --- a/manifest/manifest_test.go +++ b/manifest/manifest_test.go @@ -154,6 +154,69 @@ func TestManifest_DefaultDaemon_Bad_MultipleNoneDefault(t *testing.T) { assert.False(t, ok) } +func TestParse_Good_WithProviderFields(t *testing.T) { + raw := ` +code: cool-widget +name: Cool Widget Dashboard +version: 1.0.0 +author: someone +licence: EUPL-1.2 + +namespace: /api/v1/cool-widget +port: 0 +binary: ./cool-widget +args: ["--verbose"] + +element: + tag: core-cool-widget + source: ./assets/core-cool-widget.js + +spec: ./openapi.json + +layout: HCF +slots: + H: toolbar + C: dashboard + F: status +` + m, err := Parse([]byte(raw)) + require.NoError(t, err) + assert.Equal(t, "cool-widget", m.Code) + assert.Equal(t, "Cool Widget Dashboard", m.Name) + assert.Equal(t, "1.0.0", m.Version) + assert.Equal(t, "someone", m.Author) + assert.Equal(t, "EUPL-1.2", m.Licence) + assert.Equal(t, "/api/v1/cool-widget", m.Namespace) + assert.Equal(t, 0, m.Port) + assert.Equal(t, "./cool-widget", m.Binary) + assert.Equal(t, []string{"--verbose"}, m.Args) + assert.Equal(t, "./openapi.json", m.Spec) + require.NotNil(t, m.Element) + assert.Equal(t, "core-cool-widget", m.Element.Tag) + assert.Equal(t, "./assets/core-cool-widget.js", m.Element.Source) + assert.True(t, m.IsProvider()) +} + +func TestManifest_IsProvider_Good(t *testing.T) { + m := Manifest{Namespace: "/api/v1/test", Binary: "./test"} + assert.True(t, m.IsProvider()) +} + +func TestManifest_IsProvider_Bad_NoNamespace(t *testing.T) { + m := Manifest{Binary: "./test"} + assert.False(t, m.IsProvider()) +} + +func TestManifest_IsProvider_Bad_NoBinary(t *testing.T) { + m := Manifest{Namespace: "/api/v1/test"} + assert.False(t, m.IsProvider()) +} + +func TestManifest_IsProvider_Bad_Empty(t *testing.T) { + m := Manifest{} + assert.False(t, m.IsProvider()) +} + func TestManifest_DefaultDaemon_Good_SingleImplicit(t *testing.T) { m := Manifest{ Daemons: map[string]DaemonSpec{ diff --git a/marketplace/discovery.go b/marketplace/discovery.go new file mode 100644 index 0000000..49588c5 --- /dev/null +++ b/marketplace/discovery.go @@ -0,0 +1,161 @@ +package marketplace + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "forge.lthn.ai/core/go-scm/manifest" + "gopkg.in/yaml.v3" +) + +// DiscoveredProvider represents a runtime provider found on disk. +type DiscoveredProvider struct { + // Dir is the absolute path to the provider directory. + Dir string + + // Manifest is the parsed manifest from the provider directory. + Manifest *manifest.Manifest +} + +// DiscoverProviders scans the given directory for runtime provider manifests. +// Each subdirectory is checked for a .core/manifest.yaml file. Directories +// without a valid manifest are skipped with a log warning. +// Only manifests with provider fields (namespace + binary) are returned. +func DiscoverProviders(dir string) ([]DiscoveredProvider, error) { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil // No providers directory — not an error. + } + return nil, fmt.Errorf("marketplace.DiscoverProviders: %w", err) + } + + var providers []DiscoveredProvider + for _, e := range entries { + if !e.IsDir() { + continue + } + + providerDir := filepath.Join(dir, e.Name()) + manifestPath := filepath.Join(providerDir, ".core", "manifest.yaml") + + data, err := os.ReadFile(manifestPath) + if err != nil { + log.Printf("marketplace: skipping %s: %v", e.Name(), err) + continue + } + + m, err := manifest.Parse(data) + if err != nil { + log.Printf("marketplace: skipping %s: invalid manifest: %v", e.Name(), err) + continue + } + + if !m.IsProvider() { + log.Printf("marketplace: skipping %s: not a provider (missing namespace or binary)", e.Name()) + continue + } + + providers = append(providers, DiscoveredProvider{ + Dir: providerDir, + Manifest: m, + }) + } + + return providers, nil +} + +// ProviderRegistryEntry records metadata about an installed provider. +type ProviderRegistryEntry struct { + Installed string `yaml:"installed" json:"installed"` + Version string `yaml:"version" json:"version"` + Source string `yaml:"source" json:"source"` + AutoStart bool `yaml:"auto_start" json:"auto_start"` +} + +// ProviderRegistryFile represents the registry.yaml file tracking installed providers. +type ProviderRegistryFile struct { + Version int `yaml:"version" json:"version"` + Providers map[string]ProviderRegistryEntry `yaml:"providers" json:"providers"` +} + +// LoadProviderRegistry reads a registry.yaml file from the given path. +// Returns an empty registry if the file does not exist. +func LoadProviderRegistry(path string) (*ProviderRegistryFile, error) { + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return &ProviderRegistryFile{ + Version: 1, + Providers: make(map[string]ProviderRegistryEntry), + }, nil + } + return nil, fmt.Errorf("marketplace.LoadProviderRegistry: %w", err) + } + + var reg ProviderRegistryFile + if err := yaml.Unmarshal(data, ®); err != nil { + return nil, fmt.Errorf("marketplace.LoadProviderRegistry: %w", err) + } + + if reg.Providers == nil { + reg.Providers = make(map[string]ProviderRegistryEntry) + } + + return ®, nil +} + +// SaveProviderRegistry writes the registry to the given path. +func SaveProviderRegistry(path string, reg *ProviderRegistryFile) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return fmt.Errorf("marketplace.SaveProviderRegistry: %w", err) + } + + data, err := yaml.Marshal(reg) + if err != nil { + return fmt.Errorf("marketplace.SaveProviderRegistry: %w", err) + } + + return os.WriteFile(path, data, 0644) +} + +// Add adds or updates a provider entry in the registry. +func (r *ProviderRegistryFile) Add(code string, entry ProviderRegistryEntry) { + if r.Providers == nil { + r.Providers = make(map[string]ProviderRegistryEntry) + } + r.Providers[code] = entry +} + +// Remove removes a provider entry from the registry. +func (r *ProviderRegistryFile) Remove(code string) { + delete(r.Providers, code) +} + +// Get returns a provider entry and true if found, or zero value and false. +func (r *ProviderRegistryFile) Get(code string) (ProviderRegistryEntry, bool) { + entry, ok := r.Providers[code] + return entry, ok +} + +// List returns all provider codes in the registry. +func (r *ProviderRegistryFile) List() []string { + codes := make([]string, 0, len(r.Providers)) + for code := range r.Providers { + codes = append(codes, code) + } + return codes +} + +// AutoStartProviders returns codes of providers with auto_start enabled. +func (r *ProviderRegistryFile) AutoStartProviders() []string { + var codes []string + for code, entry := range r.Providers { + if entry.AutoStart { + codes = append(codes, code) + } + } + return codes +} diff --git a/marketplace/discovery_test.go b/marketplace/discovery_test.go new file mode 100644 index 0000000..c96c9c8 --- /dev/null +++ b/marketplace/discovery_test.go @@ -0,0 +1,273 @@ +package marketplace + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createProviderDir creates a provider directory with a .core/manifest.yaml. +func createProviderDir(t *testing.T, baseDir, code string, manifestYAML string) string { + t.Helper() + provDir := filepath.Join(baseDir, code) + coreDir := filepath.Join(provDir, ".core") + require.NoError(t, os.MkdirAll(coreDir, 0755)) + require.NoError(t, os.WriteFile( + filepath.Join(coreDir, "manifest.yaml"), + []byte(manifestYAML), 0644, + )) + return provDir +} + +func TestDiscoverProviders_Good(t *testing.T) { + dir := t.TempDir() + + createProviderDir(t, dir, "cool-widget", ` +code: cool-widget +name: Cool Widget +version: 1.0.0 +namespace: /api/v1/cool-widget +binary: ./cool-widget +element: + tag: core-cool-widget + source: ./assets/core-cool-widget.js +`) + + createProviderDir(t, dir, "data-viz", ` +code: data-viz +name: Data Visualiser +version: 0.2.0 +namespace: /api/v1/data-viz +binary: ./data-viz +`) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Len(t, providers, 2) + + codes := map[string]bool{} + for _, p := range providers { + codes[p.Manifest.Code] = true + } + assert.True(t, codes["cool-widget"]) + assert.True(t, codes["data-viz"]) +} + +func TestDiscoverProviders_Good_SkipNonProvider(t *testing.T) { + dir := t.TempDir() + + // This has a valid manifest but no namespace/binary — not a provider. + createProviderDir(t, dir, "plain-module", ` +code: plain-module +name: Plain Module +version: 1.0.0 +`) + + // This IS a provider. + createProviderDir(t, dir, "real-provider", ` +code: real-provider +name: Real Provider +version: 1.0.0 +namespace: /api/v1/real +binary: ./real-provider +`) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Len(t, providers, 1) + assert.Equal(t, "real-provider", providers[0].Manifest.Code) +} + +func TestDiscoverProviders_Good_SkipNoManifest(t *testing.T) { + dir := t.TempDir() + + // Directory with no manifest. + require.NoError(t, os.MkdirAll(filepath.Join(dir, "no-manifest"), 0755)) + + // Directory with a valid provider manifest. + createProviderDir(t, dir, "good-provider", ` +code: good-provider +name: Good Provider +version: 1.0.0 +namespace: /api/v1/good +binary: ./good-provider +`) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Len(t, providers, 1) + assert.Equal(t, "good-provider", providers[0].Manifest.Code) +} + +func TestDiscoverProviders_Good_SkipInvalidManifest(t *testing.T) { + dir := t.TempDir() + + // Directory with invalid YAML. + provDir := filepath.Join(dir, "bad-yaml") + coreDir := filepath.Join(provDir, ".core") + require.NoError(t, os.MkdirAll(coreDir, 0755)) + require.NoError(t, os.WriteFile( + filepath.Join(coreDir, "manifest.yaml"), + []byte("not: valid: yaml: ["), 0644, + )) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Empty(t, providers) +} + +func TestDiscoverProviders_Good_EmptyDir(t *testing.T) { + dir := t.TempDir() + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Empty(t, providers) +} + +func TestDiscoverProviders_Good_NonexistentDir(t *testing.T) { + providers, err := DiscoverProviders("/tmp/nonexistent-discovery-test-dir") + require.NoError(t, err) + assert.Nil(t, providers) +} + +func TestDiscoverProviders_Good_SkipFiles(t *testing.T) { + dir := t.TempDir() + + // Create a regular file (not a directory). + require.NoError(t, os.WriteFile(filepath.Join(dir, "readme.md"), []byte("# readme"), 0644)) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + assert.Empty(t, providers) +} + +func TestDiscoverProviders_Good_ProviderDir(t *testing.T) { + dir := t.TempDir() + + createProviderDir(t, dir, "test-prov", ` +code: test-prov +name: Test Provider +version: 1.0.0 +namespace: /api/v1/test-prov +binary: ./test-prov +`) + + providers, err := DiscoverProviders(dir) + require.NoError(t, err) + require.Len(t, providers, 1) + assert.Equal(t, filepath.Join(dir, "test-prov"), providers[0].Dir) +} + +// -- ProviderRegistryFile tests ----------------------------------------------- + +func TestProviderRegistry_LoadSave_Good(t *testing.T) { + path := filepath.Join(t.TempDir(), "registry.yaml") + + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{}, + } + reg.Add("cool-widget", ProviderRegistryEntry{ + Installed: "2026-03-14T12:00:00Z", + Version: "1.0.0", + Source: "forge.lthn.ai/someone/cool-widget", + AutoStart: true, + }) + + err := SaveProviderRegistry(path, reg) + require.NoError(t, err) + + loaded, err := LoadProviderRegistry(path) + require.NoError(t, err) + assert.Equal(t, 1, loaded.Version) + assert.Len(t, loaded.Providers, 1) + + entry, ok := loaded.Get("cool-widget") + require.True(t, ok) + assert.Equal(t, "1.0.0", entry.Version) + assert.Equal(t, "forge.lthn.ai/someone/cool-widget", entry.Source) + assert.True(t, entry.AutoStart) +} + +func TestProviderRegistry_Load_Good_NonexistentFile(t *testing.T) { + reg, err := LoadProviderRegistry("/tmp/nonexistent-registry-test.yaml") + require.NoError(t, err) + assert.Equal(t, 1, reg.Version) + assert.Empty(t, reg.Providers) +} + +func TestProviderRegistry_Add_Good(t *testing.T) { + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{}, + } + + reg.Add("widget-a", ProviderRegistryEntry{Version: "1.0.0", AutoStart: true}) + reg.Add("widget-b", ProviderRegistryEntry{Version: "2.0.0", AutoStart: false}) + + assert.Len(t, reg.Providers, 2) + + a, ok := reg.Get("widget-a") + require.True(t, ok) + assert.Equal(t, "1.0.0", a.Version) +} + +func TestProviderRegistry_Remove_Good(t *testing.T) { + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{ + "widget-a": {Version: "1.0.0"}, + "widget-b": {Version: "2.0.0"}, + }, + } + + reg.Remove("widget-a") + assert.Len(t, reg.Providers, 1) + + _, ok := reg.Get("widget-a") + assert.False(t, ok) +} + +func TestProviderRegistry_Get_Bad_NotFound(t *testing.T) { + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{}, + } + + _, ok := reg.Get("nonexistent") + assert.False(t, ok) +} + +func TestProviderRegistry_List_Good(t *testing.T) { + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{ + "a": {Version: "1.0"}, + "b": {Version: "2.0"}, + }, + } + + codes := reg.List() + assert.Len(t, codes, 2) + assert.Contains(t, codes, "a") + assert.Contains(t, codes, "b") +} + +func TestProviderRegistry_AutoStartProviders_Good(t *testing.T) { + reg := &ProviderRegistryFile{ + Version: 1, + Providers: map[string]ProviderRegistryEntry{ + "auto-a": {Version: "1.0", AutoStart: true}, + "manual-b": {Version: "2.0", AutoStart: false}, + "auto-c": {Version: "3.0", AutoStart: true}, + }, + } + + auto := reg.AutoStartProviders() + assert.Len(t, auto, 2) + assert.Contains(t, auto, "auto-a") + assert.Contains(t, auto, "auto-c") +}