feat(marketplace): use compiled manifests in index build

Co-Authored-By: Virgil <virgil@lethean.io>

cmd/scm/cmd_index.go

@@ -5,6 +5,7 @@ package scm
 import (
 	filepath "dappco.re/go/core/scm/internal/ax/filepathx"
 	fmt "dappco.re/go/core/scm/internal/ax/fmtx"
+	os "dappco.re/go/core/scm/internal/ax/osx"
 
 	"dappco.re/go/core/io"
 	"dappco.re/go/core/scm/marketplace"
@@ -41,12 +42,15 @@ func addIndexCommand(parent *cli.Command) {
 }
 
 func runIndex(dirs []string, output, forgeURL, org string) error {
-	b := &marketplace.Builder{
-		BaseURL: forgeURL,
-		Org:     org,
+	repoPaths, err := expandIndexRepoPaths(dirs)
+	if err != nil {
+		return err
 	}
 
-	idx, err := b.BuildFromDirs(dirs...)
+	idx, err := marketplace.BuildIndex(io.Local, repoPaths, marketplace.IndexOptions{
+		ForgeURL: forgeURL,
+		Org:      org,
+	})
 	if err != nil {
 		return cli.WrapVerb(err, "build", "index")
 	}
@@ -66,3 +70,28 @@ func runIndex(dirs []string, output, forgeURL, org string) error {
 
 	return nil
 }
+
+func expandIndexRepoPaths(dirs []string) ([]string, error) {
+	var repoPaths []string
+
+	for _, dir := range dirs {
+		repoPaths = append(repoPaths, dir)
+
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, cli.WrapVerb(err, "read", dir)
+		}
+
+		for _, entry := range entries {
+			if !entry.IsDir() {
+				continue
+			}
+			repoPaths = append(repoPaths, filepath.Join(dir, entry.Name()))
+		}
+	}
+
+	return repoPaths, nil
+}

cmd/scm/cmd_index_test.go

@@ -4,10 +4,12 @@ package scm
 
 import (
 	filepath "dappco.re/go/core/scm/internal/ax/filepathx"
+	json "dappco.re/go/core/scm/internal/ax/jsonx"
 	os "dappco.re/go/core/scm/internal/ax/osx"
 	"testing"
 
 	"dappco.re/go/core/io"
+	"dappco.re/go/core/scm/manifest"
 	"dappco.re/go/core/scm/marketplace"
 	"forge.lthn.ai/core/cli/pkg/cli"
 	"github.com/stretchr/testify/assert"
@@ -37,6 +39,42 @@ sign: key-a
 	assert.Equal(t, "https://forge.example.com/core/mod-a.git", idx.Modules[0].Repo)
 }
 
+func TestRunIndex_Good_PrefersCompiledManifest_Good(t *testing.T) {
+	root := t.TempDir()
+
+	modDir := filepath.Join(root, "mod-a")
+	require.NoError(t, os.MkdirAll(filepath.Join(modDir, ".core"), 0755))
+
+	cm := &manifest.CompiledManifest{
+		Manifest: manifest.Manifest{
+			Code:    "compiled-mod",
+			Name:    "Compiled Module",
+			Version: "2.0.0",
+			Sign:    "compiled-key",
+		},
+		Commit: "abc123",
+	}
+	data, err := json.Marshal(cm)
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(filepath.Join(modDir, "core.json"), data, 0644))
+	require.NoError(t, os.WriteFile(filepath.Join(modDir, ".core", "manifest.yaml"), []byte(`
+code: source-mod
+name: Source Module
+version: 1.0.0
+sign: source-key
+`), 0644))
+
+	output := filepath.Join(root, "index.json")
+	err = runIndex([]string{root}, output, "https://forge.example.com", "core")
+	require.NoError(t, err)
+
+	idx, err := marketplace.LoadIndex(io.Local, output)
+	require.NoError(t, err)
+	require.Len(t, idx.Modules, 1)
+	assert.Equal(t, "compiled-mod", idx.Modules[0].Code)
+	assert.Equal(t, "compiled-key", idx.Modules[0].SignKey)
+}
+
 func TestAddScmCommands_Good_IndexForgeURLFlagAlias_Good(t *testing.T) {
 	root := &cli.Command{Use: "root"}
 

marketplace/indexer.go

@@ -3,6 +3,7 @@
 package marketplace
 
 import (
+	filepath "dappco.re/go/core/scm/internal/ax/filepathx"
 	"fmt"
 	"sort"
 	"strings"
@@ -44,11 +45,8 @@ func BuildIndex(medium io.Medium, repoPaths []string, opts IndexOptions) (*Index
 	categories := make(map[string]bool)
 
 	for _, repoPath := range repoPaths {
-		m, err := manifest.Load(medium, repoPath)
-		if err != nil {
-			continue
-		}
-		if m == nil || m.Code == "" {
+		m, err := loadIndexManifest(medium, repoPath)
+		if err != nil || m == nil || m.Code == "" {
 			continue
 		}
 		if seen[m.Code] {
@@ -86,3 +84,16 @@ func BuildIndex(medium io.Medium, repoPaths []string, opts IndexOptions) (*Index
 
 	return idx, nil
 }
+
+func loadIndexManifest(medium io.Medium, repoPath string) (*manifest.Manifest, error) {
+	coreJSON := filepath.Join(repoPath, "core.json")
+	if raw, err := medium.Read(coreJSON); err == nil {
+		cm, parseErr := manifest.ParseCompiled([]byte(raw))
+		if parseErr != nil {
+			return nil, parseErr
+		}
+		return &cm.Manifest, nil
+	}
+
+	return manifest.Load(medium, repoPath)
+}

marketplace/indexer_test.go

@@ -3,9 +3,12 @@
 package marketplace
 
 import (
+	filepath "dappco.re/go/core/scm/internal/ax/filepathx"
 	"testing"
 
 	"dappco.re/go/core/io"
+	json "dappco.re/go/core/scm/internal/ax/jsonx"
+	"dappco.re/go/core/scm/manifest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -71,3 +74,36 @@ sign: key-one
 	assert.Equal(t, "one", idx.Modules[0].Code)
 	assert.Empty(t, idx.Categories)
 }
+
+func TestBuildIndex_Good_PrefersCompiledManifest_Good(t *testing.T) {
+	medium := io.NewMockMedium()
+
+	repoDir := "/repos/compiled"
+	require.NoError(t, medium.EnsureDir(filepath.Join(repoDir, ".core")))
+
+	cm := &manifest.CompiledManifest{
+		Manifest: manifest.Manifest{
+			Code:    "compiled",
+			Name:    "Compiled Module",
+			Version: "2.0.0",
+			Sign:    "key-compiled",
+		},
+		Commit: "abc123",
+	}
+	raw, err := json.Marshal(cm)
+	require.NoError(t, err)
+	require.NoError(t, medium.Write(filepath.Join(repoDir, "core.json"), string(raw)))
+	require.NoError(t, medium.Write(filepath.Join(repoDir, ".core", "manifest.yaml"), `
+code: source
+name: Source Module
+version: 1.0.0
+sign: key-source
+`))
+
+	idx, err := BuildIndex(medium, []string{repoDir}, IndexOptions{})
+	require.NoError(t, err)
+
+	require.Len(t, idx.Modules, 1)
+	assert.Equal(t, "compiled", idx.Modules[0].Code)
+	assert.Equal(t, "key-compiled", idx.Modules[0].SignKey)
+}