Compare commits

...

52 commits
v0.4.0 ... dev

Author SHA1 Message Date
Snider
aa82451444 fix: migrate module paths from forge.lthn.ai to dappco.re
Some checks failed
Security Scan / security (push) Has been cancelled
Test / test (push) Has been cancelled
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-04 16:21:13 +01:00
Virgil
de94350f13 feat(manifest): allow version override during compile
Some checks failed
Security Scan / security (push) Failing after 16s
Test / test (push) Failing after 55s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:51:41 +00:00
Virgil
2f599eb6d5 refactor(marketplace): inject mediums into SCM helpers
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Failing after 1m0s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:47:49 +00:00
Virgil
905889a9f8 feat(marketplace): use compiled manifests in index build
Some checks failed
Security Scan / security (push) Failing after 19s
Test / test (push) Failing after 1m37s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:41:52 +00:00
Virgil
0fd4386e20 fix(manifest): reject invalid public keys in verify
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Successful in 1m10s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:36:38 +00:00
Virgil
dd71070a9d fix(manifest): validate signing inputs
Some checks failed
Security Scan / security (push) Failing after 17s
Test / test (push) Failing after 3m0s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:30:04 +00:00
Virgil
e73809cf8d feat(cmd/scm): add manifest sign and verify commands
Some checks failed
Security Scan / security (push) Failing after 16s
Test / test (push) Successful in 2m18s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:24:55 +00:00
Virgil
48d1eb22b0 refactor(ax): dedupe sync repo parsing
Some checks failed
Security Scan / security (push) Failing after 15s
Test / test (push) Failing after 1m37s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:20:14 +00:00
Virgil
a14feec8ab feat(gitea): add current user helper
Some checks failed
Security Scan / security (push) Failing after 17s
Test / test (push) Successful in 1m38s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:16:01 +00:00
Virgil
8a269fa107 feat(cmd/scm): add forge-url alias for index links
Some checks failed
Security Scan / security (push) Failing after 18s
Test / test (push) Successful in 2m25s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:10:58 +00:00
Virgil
6dbb70d626 feat(cmd/scm): add custom compile output path
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Successful in 1m50s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:05:22 +00:00
Virgil
25667064ca fix(pkg/api): emit installed change events
Some checks failed
Security Scan / security (push) Failing after 19s
Test / test (push) Failing after 2m0s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 14:00:53 +00:00
Virgil
fe8c7e5982 feat(forge): add issue label getter
Some checks failed
Security Scan / security (push) Failing after 16s
Test / test (push) Failing after 54s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 13:56:22 +00:00
Virgil
e0ff9d2c28 feat(forge): add undismiss review helper
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Has been cancelled
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 13:51:08 +00:00
Virgil
c394ef2a9c feat(gitea): add pull request helpers
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Successful in 2m24s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 13:41:08 +00:00
Virgil
b65ec9f052 feat(agentci): honour validation threshold in weave
Some checks failed
Security Scan / security (push) Failing after 13s
Test / test (push) Successful in 2m31s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 13:26:35 +00:00
Virgil
c303abbd95 refactor(marketplace): use medium for index writes
Some checks failed
Security Scan / security (push) Failing after 16s
Test / test (push) Successful in 2m24s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 13:22:14 +00:00
Virgil
8292f3ae79 fix(cmd/scm): avoid masking invalid core.json
Some checks failed
Security Scan / security (push) Failing after 13s
Test / test (push) Successful in 2m12s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 07:20:33 +00:00
Virgil
1bccde8828 refactor(collect): make verbose mode emit progress
Some checks failed
Security Scan / security (push) Failing after 13s
Test / test (push) Successful in 2m29s
Verbose is now a real AX-facing behaviour instead of dead documentation. Excavator emits additional progress telemetry when verbose mode is enabled, and the new regression test protects that path.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 07:12:56 +00:00
Virgil
9a0a1f4435 feat(gitea): add issue mutation helpers
Some checks failed
Security Scan / security (push) Failing after 14s
Test / test (push) Has been cancelled
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 07:08:43 +00:00
Virgil
5a561690be feat(ui): polish scm agent views
Some checks failed
Test / test (push) Waiting to run
Security Scan / security (push) Failing after 14s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 07:04:25 +00:00
Virgil
32e65b8b43 feat(ui): refresh scm views from live events
Some checks failed
Security Scan / security (push) Failing after 13s
Test / test (push) Successful in 2m20s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 06:58:47 +00:00
Virgil
676130ab84 refactor(repos): stabilise registry ordering
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Successful in 2m21s
Sort registry and provider-registry listings for deterministic output and add coverage for the stable ordering guarantees.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-02 06:52:45 +00:00
Virgil
697bfde215 feat(forge): add org label iterator
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 2m17s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 09:42:41 +00:00
Virgil
f27a01d3c5 feat(forge): add repo label iterator
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 2m12s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 09:28:20 +00:00
Virgil
0d80388d18 feat(gitea): add issue comment list API
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 2m8s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 09:07:55 +00:00
Virgil
5bb8e61708 feat(scm): add issue comment iterators
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m17s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 08:58:12 +00:00
Virgil
d852087c45 feat(forge): add org iterator
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Successful in 2m16s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 08:47:55 +00:00
Virgil
b94caf0a9d feat(forge): add PR review iterator
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Failing after 51s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 08:18:36 +00:00
Virgil
94c5870c46 feat(forge): add repo webhook iterator
Some checks failed
Security Scan / security (push) Failing after 13s
Test / test (push) Successful in 2m27s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 07:35:26 +00:00
Virgil
0193bd50ea feat(scm): add issue iterators
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Successful in 2m18s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 07:23:21 +00:00
Virgil
64042ac8a6 feat(gitea): generalise mirror creation
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m20s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 07:15:44 +00:00
Virgil
f2c9cb39d0 fix(agentci): split path sanitising and validation
Some checks are pending
Security Scan / security (push) Waiting to run
Test / test (push) Waiting to run
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 07:09:27 +00:00
Virgil
ec8149efbe feat(marketplace): index root directories
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 2m14s
Include the directories passed to BuildFromDirs before scanning children so scm index . handles a manifest at the root of the scanned directory.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:59:57 +00:00
Virgil
2e188e346a feat(agentci): add context-aware ssh command helper
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Successful in 2m15s
Thread dispatch SSH subprocesses through the caller context so cancellation applies to ticket transfer, remote cleanup, and existence checks.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:53:33 +00:00
Virgil
8021e5e2cb fix(scm): paginate issue listings
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m13s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:47:34 +00:00
Virgil
6233664c5d fix(pkg/api): combine marketplace query and category filters
Some checks failed
Security Scan / security (push) Failing after 9s
Test / test (push) Successful in 2m9s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:40:27 +00:00
Virgil
369103f8dc feat(pkg/api): list registry repos in dependency order
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m8s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:32:16 +00:00
Virgil
89925a0e83 feat(marketplace): emit clone URLs in indexes
Some checks failed
Security Scan / security (push) Failing after 9s
Test / test (push) Successful in 2m8s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:24:22 +00:00
Virgil
2f4d2e5811 feat(jobrunner): add journal replay query
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 5m2s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:14:28 +00:00
Virgil
b2bbc11746 feat(marketplace): propagate signing keys in indexes
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 2m11s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:09:07 +00:00
Virgil
1f98d7ab8a feat(marketplace): add category-aware index builder
Some checks failed
Security Scan / security (push) Failing after 11s
Test / test (push) Successful in 2m11s
Propagate category metadata while building marketplace indexes and deduplicate the category list for consumers such as the UI.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 06:03:23 +00:00
Virgil
82c25469e8 feat(pkg/api): refresh marketplace index
Some checks failed
Security Scan / security (push) Failing after 12s
Test / test (push) Successful in 1m55s
Add marketplace index loading and a provider endpoint to refresh the in-memory catalogue from index.json.

Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 05:41:05 +00:00
Virgil
5a53d244cc feat(forge): aggregate org labels across repos
Some checks failed
Security Scan / security (push) Failing after 9s
Test / test (push) Successful in 2m17s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 05:11:34 +00:00
Virgil
976d20c02f feat(jobrunner): resolve dispatch target branch
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m12s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-04-01 04:45:40 +00:00
Virgil
a0fac1341b chore(ax): add usage docs to exported APIs
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m11s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-03-30 14:11:15 +00:00
Virgil
dd59b177c6 chore(ax): normalise test naming and usage annotations
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m2s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-03-30 06:37:20 +00:00
Claude
a6c15980a3 chore: update dependencies to dappco.re tagged versions
Some checks failed
Security Scan / security (push) Failing after 9s
Test / test (push) Successful in 2m0s
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-30 06:19:54 +00:00
Virgil
c42cc4a6ce chore(ax): gofmt exported declaration comments
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Successful in 2m4s
Co-Authored-By: Virgil <virgil@lethean.io>
2026-03-30 05:44:09 +00:00
Virgil
305aa0da6f chore(ax): normalize SPDX header identifier
Some checks failed
Security Scan / security (push) Failing after 8s
Test / test (push) Successful in 4m12s
2026-03-30 00:54:20 +00:00
Virgil
5f73d41184 chore(ax): add SPDX headers to remaining Go files
Some checks failed
Security Scan / security (push) Failing after 15s
Test / test (push) Failing after 29s
2026-03-30 00:19:43 +00:00
Virgil
d5f98c1341 refactor(ax): align code with AX principles
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Failing after 25s
2026-03-29 23:59:48 +00:00
198 changed files with 10301 additions and 1455 deletions

View file

@ -1,8 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"context"
"strings"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"math"
"dappco.re/go/core/scm/jobrunner"
)
@ -11,8 +14,10 @@ import (
type RunMode string
const (
//
ModeStandard RunMode = "standard"
ModeDual RunMode = "dual" // The Clotho Protocol — dual-run verification
//
ModeDual RunMode = "dual" // The Clotho Protocol — dual-run verification
)
// Spinner is the Clotho orchestrator that determines the fate of each task.
@ -22,6 +27,7 @@ type Spinner struct {
}
// NewSpinner creates a new Clotho orchestrator.
// Usage: NewSpinner(...)
func NewSpinner(cfg ClothoConfig, agents map[string]AgentConfig) *Spinner {
return &Spinner{
Config: cfg,
@ -31,6 +37,7 @@ func NewSpinner(cfg ClothoConfig, agents map[string]AgentConfig) *Spinner {
// DeterminePlan decides if a signal requires dual-run verification based on
// the global strategy, agent configuration, and repository criticality.
// Usage: DeterminePlan(...)
func (s *Spinner) DeterminePlan(signal *jobrunner.PipelineSignal, agentName string) RunMode {
if s.Config.Strategy != "clotho-verified" {
return ModeStandard
@ -53,6 +60,7 @@ func (s *Spinner) DeterminePlan(signal *jobrunner.PipelineSignal, agentName stri
}
// GetVerifierModel returns the model for the secondary "signed" verification run.
// Usage: GetVerifierModel(...)
func (s *Spinner) GetVerifierModel(agentName string) string {
agent, ok := s.Agents[agentName]
if !ok || agent.VerifyModel == "" {
@ -63,6 +71,7 @@ func (s *Spinner) GetVerifierModel(agentName string) string {
// FindByForgejoUser resolves a Forgejo username to the agent config key and config.
// This decouples agent naming (mythological roles) from Forgejo identity.
// Usage: FindByForgejoUser(...)
func (s *Spinner) FindByForgejoUser(forgejoUser string) (string, AgentConfig, bool) {
if forgejoUser == "" {
return "", AgentConfig{}, false
@ -81,7 +90,61 @@ func (s *Spinner) FindByForgejoUser(forgejoUser string) (string, AgentConfig, bo
}
// Weave compares primary and verifier outputs. Returns true if they converge.
// This is a placeholder for future semantic diff logic.
// The comparison is a coarse token-overlap check controlled by the configured
// validation threshold. It is intentionally deterministic and fast; richer
// semantic diffing can replace it later without changing the signature.
// Usage: Weave(...)
func (s *Spinner) Weave(ctx context.Context, primaryOutput, signedOutput []byte) (bool, error) {
return string(primaryOutput) == string(signedOutput), nil
if ctx != nil {
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
}
primary := tokenizeWeaveOutput(primaryOutput)
signed := tokenizeWeaveOutput(signedOutput)
if len(primary) == 0 && len(signed) == 0 {
return true, nil
}
threshold := s.Config.ValidationThreshold
if threshold <= 0 || threshold > 1 {
threshold = 0.85
}
similarity := weaveDiceSimilarity(primary, signed)
return similarity >= threshold, nil
}
func tokenizeWeaveOutput(output []byte) []string {
fields := strings.Fields(strings.ReplaceAll(string(output), "\r\n", "\n"))
if len(fields) == 0 {
return nil
}
return fields
}
func weaveDiceSimilarity(primary, signed []string) float64 {
if len(primary) == 0 || len(signed) == 0 {
return 0
}
counts := make(map[string]int, len(primary))
for _, token := range primary {
counts[token]++
}
common := 0
for _, token := range signed {
if counts[token] == 0 {
continue
}
counts[token]--
common++
}
return math.Min(1, (2*float64(common))/float64(len(primary)+len(signed)))
}

73
agentci/clotho_test.go Normal file
View file

@ -0,0 +1,73 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"context"
"errors"
"testing"
"dappco.re/go/core/scm/jobrunner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSpinner_Weave_Good_ExactMatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.85}, nil)
ok, err := spinner.Weave(context.Background(), []byte("alpha beta gamma"), []byte("alpha beta gamma"))
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Good_ThresholdMatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.8}, nil)
ok, err := spinner.Weave(
context.Background(),
[]byte("alpha beta gamma delta epsilon zeta"),
[]byte("alpha beta gamma delta epsilon eta"),
)
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Bad_ThresholdMismatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.9}, nil)
ok, err := spinner.Weave(
context.Background(),
[]byte("alpha beta gamma delta epsilon zeta"),
[]byte("alpha beta gamma delta epsilon eta"),
)
require.NoError(t, err)
assert.False(t, ok)
}
func TestSpinner_Weave_Good_EmptyOutputs(t *testing.T) {
spinner := NewSpinner(ClothoConfig{}, nil)
ok, err := spinner.Weave(context.Background(), nil, nil)
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Bad_ContextCancelled(t *testing.T) {
spinner := NewSpinner(ClothoConfig{}, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
ok, err := spinner.Weave(ctx, []byte("alpha"), []byte("alpha"))
assert.False(t, ok)
require.Error(t, err)
assert.True(t, errors.Is(err, context.Canceled))
}
func TestSpinner_DeterminePlan_Good(t *testing.T) {
spinner := NewSpinner(ClothoConfig{Strategy: "clotho-verified"}, map[string]AgentConfig{
"charon": {DualRun: true},
})
ok := spinner.DeterminePlan(&jobrunner.PipelineSignal{RepoName: "docs"}, "charon")
assert.Equal(t, ModeDual, ok)
}

View file

@ -1,11 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
// Package agentci provides configuration, security, and orchestration for AgentCI dispatch targets.
package agentci
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/config"
coreerr "dappco.re/go/core/log"
"forge.lthn.ai/core/config"
)
// AgentConfig represents a single agent machine in the config file.
@ -31,6 +33,7 @@ type ClothoConfig struct {
// LoadAgents reads agent targets from config and returns a map of AgentConfig.
// Returns an empty map (not an error) if no agents are configured.
// Usage: LoadAgents(...)
func LoadAgents(cfg *config.Config) (map[string]AgentConfig, error) {
var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil {
@ -61,6 +64,7 @@ func LoadAgents(cfg *config.Config) (map[string]AgentConfig, error) {
}
// LoadActiveAgents returns only active agents.
// Usage: LoadActiveAgents(...)
func LoadActiveAgents(cfg *config.Config) (map[string]AgentConfig, error) {
all, err := LoadAgents(cfg)
if err != nil {
@ -77,6 +81,7 @@ func LoadActiveAgents(cfg *config.Config) (map[string]AgentConfig, error) {
// LoadClothoConfig loads the Clotho orchestrator settings.
// Returns sensible defaults if no config is present.
// Usage: LoadClothoConfig(...)
func LoadClothoConfig(cfg *config.Config) (ClothoConfig, error) {
var cc ClothoConfig
if err := cfg.Get("agentci.clotho", &cc); err != nil {
@ -95,6 +100,7 @@ func LoadClothoConfig(cfg *config.Config) (ClothoConfig, error) {
}
// SaveAgent writes an agent config entry to the config file.
// Usage: SaveAgent(...)
func SaveAgent(cfg *config.Config, name string, ac AgentConfig) error {
key := fmt.Sprintf("agentci.agents.%s", name)
data := map[string]any{
@ -123,6 +129,7 @@ func SaveAgent(cfg *config.Config, name string, ac AgentConfig) error {
}
// RemoveAgent removes an agent from the config file.
// Usage: RemoveAgent(...)
func RemoveAgent(cfg *config.Config, name string) error {
var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil {
@ -136,6 +143,7 @@ func RemoveAgent(cfg *config.Config, name string) error {
}
// ListAgents returns all configured agents (active and inactive).
// Usage: ListAgents(...)
func ListAgents(cfg *config.Config) (map[string]AgentConfig, error) {
var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil {

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"testing"
"forge.lthn.ai/core/config"
"dappco.re/go/core/io"
"forge.lthn.ai/core/config"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -43,7 +45,7 @@ agentci:
assert.Equal(t, "claude", agent.Runner)
}
func TestLoadAgents_Good_MultipleAgents(t *testing.T) {
func TestLoadAgents_Good_MultipleAgents_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -64,7 +66,7 @@ agentci:
assert.Contains(t, agents, "local-codex")
}
func TestLoadAgents_Good_SkipsInactive(t *testing.T) {
func TestLoadAgents_Good_SkipsInactive_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -99,7 +101,7 @@ agentci:
assert.Contains(t, active, "active-agent")
}
func TestLoadAgents_Good_Defaults(t *testing.T) {
func TestLoadAgents_Good_Defaults_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -117,14 +119,14 @@ agentci:
assert.Equal(t, "claude", agent.Runner)
}
func TestLoadAgents_Good_NoConfig(t *testing.T) {
func TestLoadAgents_Good_NoConfig_Good(t *testing.T) {
cfg := newTestConfig(t, "")
agents, err := LoadAgents(cfg)
require.NoError(t, err)
assert.Empty(t, agents)
}
func TestLoadAgents_Bad_MissingHost(t *testing.T) {
func TestLoadAgents_Bad_MissingHost_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -137,7 +139,7 @@ agentci:
assert.Contains(t, err.Error(), "host is required")
}
func TestLoadAgents_Good_WithDualRun(t *testing.T) {
func TestLoadAgents_Good_WithDualRun_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -174,7 +176,7 @@ agentci:
assert.Equal(t, "/etc/core/keys/clotho.pub", cc.SigningKeyPath)
}
func TestLoadClothoConfig_Good_Defaults(t *testing.T) {
func TestLoadClothoConfig_Good_Defaults_Good(t *testing.T) {
cfg := newTestConfig(t, "")
cc, err := LoadClothoConfig(cfg)
require.NoError(t, err)
@ -202,7 +204,7 @@ func TestSaveAgent_Good(t *testing.T) {
assert.Equal(t, "haiku", agents["new-agent"].Model)
}
func TestSaveAgent_Good_WithDualRun(t *testing.T) {
func TestSaveAgent_Good_WithDualRun_Good(t *testing.T) {
cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "verified-agent", AgentConfig{
@ -220,7 +222,7 @@ func TestSaveAgent_Good_WithDualRun(t *testing.T) {
assert.True(t, agents["verified-agent"].DualRun)
}
func TestSaveAgent_Good_OmitsEmptyOptionals(t *testing.T) {
func TestSaveAgent_Good_OmitsEmptyOptionals_Good(t *testing.T) {
cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "minimal", AgentConfig{
@ -254,7 +256,7 @@ agentci:
assert.Contains(t, agents, "to-keep")
}
func TestRemoveAgent_Bad_NotFound(t *testing.T) {
func TestRemoveAgent_Bad_NotFound_Good(t *testing.T) {
cfg := newTestConfig(t, `
agentci:
agents:
@ -267,7 +269,7 @@ agentci:
assert.Contains(t, err.Error(), "not found")
}
func TestRemoveAgent_Bad_NoAgents(t *testing.T) {
func TestRemoveAgent_Bad_NoAgents_Good(t *testing.T) {
cfg := newTestConfig(t, "")
err := RemoveAgent(cfg, "anything")
assert.Error(t, err)
@ -292,14 +294,14 @@ agentci:
assert.False(t, agents["agent-b"].Active)
}
func TestListAgents_Good_Empty(t *testing.T) {
func TestListAgents_Good_Empty_Good(t *testing.T) {
cfg := newTestConfig(t, "")
agents, err := ListAgents(cfg)
require.NoError(t, err)
assert.Empty(t, agents)
}
func TestRoundTrip_SaveThenLoad(t *testing.T) {
func TestRoundTrip_Good_SaveThenLoad_Good(t *testing.T) {
cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "alpha", AgentConfig{

View file

@ -1,38 +1,171 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"os/exec"
"path/filepath"
"context"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"path"
"regexp"
"strings"
coreerr "dappco.re/go/core/log"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
)
var safeNameRegex = regexp.MustCompile(`^[a-zA-Z0-9\-\_\.]+$`)
// SanitizePath ensures a filename or directory name is safe and prevents path traversal.
// Returns filepath.Base of the input after validation.
// Returns the validated basename.
// Usage: SanitizePath(...)
func SanitizePath(input string) (string, error) {
base := filepath.Base(input)
if !safeNameRegex.MatchString(base) {
if input == "" {
return "", coreerr.E("agentci.SanitizePath", "path element is required", nil)
}
safeName := filepath.Base(input)
if safeName == "." || safeName == ".." {
return "", coreerr.E("agentci.SanitizePath", "invalid path element: "+input, nil)
}
if strings.ContainsAny(safeName, `/\`) {
return "", coreerr.E("agentci.SanitizePath", "path separators are not allowed: "+input, nil)
}
if !safeNameRegex.MatchString(safeName) {
return "", coreerr.E("agentci.SanitizePath", "invalid characters in path element: "+input, nil)
}
if base == "." || base == ".." || base == "/" {
return "", coreerr.E("agentci.SanitizePath", "invalid path element: "+base, nil)
return safeName, nil
}
// ValidatePathElement validates a single local path element and returns its safe form.
// Usage: ValidatePathElement(...)
func ValidatePathElement(input string) (string, error) {
safeName, err := SanitizePath(input)
if err != nil {
return "", err
}
return base, nil
if safeName != input {
return "", coreerr.E("agentci.ValidatePathElement", "path separators are not allowed: "+input, nil)
}
return safeName, nil
}
// ResolvePathWithinRoot resolves a validated path element beneath a root directory.
// Usage: ResolvePathWithinRoot(...)
func ResolvePathWithinRoot(root string, input string) (string, string, error) {
safeName, err := ValidatePathElement(input)
if err != nil {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "invalid path element", err)
}
absRoot, err := filepath.Abs(root)
if err != nil {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "resolve root", err)
}
resolved := filepath.Clean(filepath.Join(absRoot, safeName))
cleanRoot := filepath.Clean(absRoot)
rootPrefix := cleanRoot + string(filepath.Separator)
if resolved != cleanRoot && !strings.HasPrefix(resolved, rootPrefix) {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "resolved path escaped root", nil)
}
return safeName, resolved, nil
}
// ValidateRemoteDir validates a remote directory path used over SSH.
// Usage: ValidateRemoteDir(...)
func ValidateRemoteDir(dir string) (string, error) {
if strings.TrimSpace(dir) == "" {
return "", coreerr.E("agentci.ValidateRemoteDir", "directory is required", nil)
}
if strings.ContainsAny(dir, `\`) {
return "", coreerr.E("agentci.ValidateRemoteDir", "backslashes are not allowed", nil)
}
switch dir {
case "/", "~":
return dir, nil
}
cleaned := path.Clean(dir)
prefix := ""
rest := cleaned
if strings.HasPrefix(dir, "~/") {
prefix = "~/"
rest = strings.TrimPrefix(cleaned, "~/")
}
if strings.HasPrefix(dir, "/") {
prefix = "/"
rest = strings.TrimPrefix(cleaned, "/")
}
if rest == "." || rest == ".." || strings.HasPrefix(rest, "../") {
return "", coreerr.E("agentci.ValidateRemoteDir", "directory escaped root", nil)
}
for _, part := range strings.Split(rest, "/") {
if part == "" {
continue
}
if _, err := ValidatePathElement(part); err != nil {
return "", coreerr.E("agentci.ValidateRemoteDir", "invalid directory segment", err)
}
}
if rest == "" || rest == "." {
return prefix, nil
}
return prefix + rest, nil
}
// JoinRemotePath joins validated remote path elements using forward slashes.
// Usage: JoinRemotePath(...)
func JoinRemotePath(base string, parts ...string) (string, error) {
safeBase, err := ValidateRemoteDir(base)
if err != nil {
return "", coreerr.E("agentci.JoinRemotePath", "invalid base directory", err)
}
cleanParts := make([]string, 0, len(parts))
for _, part := range parts {
safePart, partErr := ValidatePathElement(part)
if partErr != nil {
return "", coreerr.E("agentci.JoinRemotePath", "invalid path element", partErr)
}
cleanParts = append(cleanParts, safePart)
}
if safeBase == "~" {
return path.Join("~", path.Join(cleanParts...)), nil
}
if strings.HasPrefix(safeBase, "~/") {
return "~/" + path.Join(strings.TrimPrefix(safeBase, "~/"), path.Join(cleanParts...)), nil
}
return path.Join(append([]string{safeBase}, cleanParts...)...), nil
}
// EscapeShellArg wraps a string in single quotes for safe remote shell insertion.
// Prefer exec.Command arguments over constructing shell strings where possible.
// Usage: EscapeShellArg(...)
func EscapeShellArg(arg string) string {
return "'" + strings.ReplaceAll(arg, "'", "'\\''") + "'"
}
// SecureSSHCommand creates an SSH exec.Cmd with strict host key checking and batch mode.
// Usage: SecureSSHCommand(...)
func SecureSSHCommand(host string, remoteCmd string) *exec.Cmd {
return exec.Command("ssh",
return SecureSSHCommandContext(context.Background(), host, remoteCmd)
}
// SecureSSHCommandContext creates an SSH exec.Cmd with strict host key checking and batch mode.
// Usage: SecureSSHCommandContext(...)
func SecureSSHCommandContext(ctx context.Context, host string, remoteCmd string) *exec.Cmd {
if ctx == nil {
ctx = context.Background()
}
return exec.CommandContext(ctx, "ssh",
"-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes",
"-o", "ConnectTimeout=10",
@ -42,6 +175,7 @@ func SecureSSHCommand(host string, remoteCmd string) *exec.Cmd {
}
// MaskToken returns a masked version of a token for safe logging.
// Usage: MaskToken(...)
func MaskToken(token string) string {
if len(token) < 8 {
return "*****"

View file

@ -1,8 +1,9 @@
// SPDX-Licence-Identifier: EUPL-1.2
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
@ -20,7 +21,9 @@ func TestSanitizePath_Good(t *testing.T) {
{"with.dot", "with.dot"},
{"CamelCase", "CamelCase"},
{"123", "123"},
{"path/to/file.txt", "file.txt"},
{"../secret", "secret"},
{"/var/tmp/report.txt", "report.txt"},
{"nested/path/file", "file"},
}
for _, tt := range tests {
@ -44,8 +47,11 @@ func TestSanitizePath_Bad(t *testing.T) {
{"pipe", "file|name"},
{"ampersand", "file&name"},
{"dollar", "file$name"},
{"backslash", `path\to\file.txt`},
{"current dir", "."},
{"parent traversal base", ".."},
{"root", "/"},
{"empty", ""},
}
for _, tt := range tests {
@ -87,6 +93,19 @@ func TestSecureSSHCommand_Good(t *testing.T) {
assert.Equal(t, "ls -la", args[len(args)-1])
}
func TestSecureSSHCommandContext_Good(t *testing.T) {
cmd := SecureSSHCommandContext(context.Background(), "host.example.com", "ls -la")
args := cmd.Args
assert.Equal(t, "ssh", args[0])
assert.Contains(t, args, "-o")
assert.Contains(t, args, "StrictHostKeyChecking=yes")
assert.Contains(t, args, "BatchMode=yes")
assert.Contains(t, args, "ConnectTimeout=10")
assert.Equal(t, "host.example.com", args[len(args)-2])
assert.Equal(t, "ls -la", args[len(args)-1])
}
func TestMaskToken_Good(t *testing.T) {
tests := []struct {
name string

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
func init() {
@ -28,6 +30,7 @@ var (
)
// AddCollectCommands registers the 'collect' command and all subcommands.
// Usage: AddCollectCommands(...)
func AddCollectCommands(root *cli.Command) {
collectCmd := &cli.Command{
Use: "collect",

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"strings"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// BitcoinTalk command flags

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"time"
"forge.lthn.ai/core/cli/pkg/cli"
collectpkg "dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
collectpkg "dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// addDispatchCommand adds the 'dispatch' subcommand to the collect parent.

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Excavate command flags

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"strings"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// GitHub command flags

View file

@ -1,11 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Market command flags

View file

@ -1,11 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Papers command flags

View file

@ -1,11 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
)
// addProcessCommand adds the 'process' subcommand to the collect parent.

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Auth command flags.

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Config command flags.

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package forge provides CLI commands for managing a Forgejo instance.
//
// Commands:
@ -33,6 +35,7 @@ var (
)
// AddForgeCommands registers the 'forge' command and all subcommands.
// Usage: AddForgeCommands(...)
func AddForgeCommands(root *cli.Command) {
forgeCmd := &cli.Command{
Use: "forge",

View file

@ -1,13 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
"strings"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Issues command flags.

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Labels command flags.

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Migrate command flags.

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// addOrgsCommand adds the 'orgs' subcommand for listing organisations.

View file

@ -1,13 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
"strings"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// PRs command flags.

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Repos command flags.

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli"
)
// addStatusCommand adds the 'status' subcommand for instance info.

View file

@ -1,17 +1,21 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
os "dappco.re/go/core/scm/internal/ax/osx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci"
"dappco.re/go/core/scm/cmd/internal/syncutil"
fg "dappco.re/go/core/scm/forge"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"forge.lthn.ai/core/cli/pkg/cli"
coreerr "dappco.re/go/core/log"
fg "dappco.re/go/core/scm/forge"
)
// Sync command flags.
@ -95,11 +99,14 @@ func buildSyncRepoList(client *fg.Client, args []string, basePath string) ([]syn
if len(args) > 0 {
for _, arg := range args {
name := arg
if parts := strings.SplitN(arg, "/", 2); len(parts) == 2 {
name = parts[1]
name, err := syncutil.ParseRepoName(arg)
if err != nil {
return nil, coreerr.E("forge.buildSyncRepoList", "invalid repo argument", err)
}
_, localPath, err := agentci.ResolvePathWithinRoot(basePath, name)
if err != nil {
return nil, coreerr.E("forge.buildSyncRepoList", "resolve local path", err)
}
localPath := filepath.Join(basePath, name)
branch := syncDetectDefaultBranch(localPath)
repos = append(repos, syncRepoEntry{
name: name,
@ -113,10 +120,17 @@ func buildSyncRepoList(client *fg.Client, args []string, basePath string) ([]syn
return nil, err
}
for _, r := range orgRepos {
localPath := filepath.Join(basePath, r.Name)
name, err := agentci.ValidatePathElement(r.Name)
if err != nil {
return nil, coreerr.E("forge.buildSyncRepoList", "invalid repo name from org list", err)
}
_, localPath, err := agentci.ResolvePathWithinRoot(basePath, name)
if err != nil {
return nil, coreerr.E("forge.buildSyncRepoList", "resolve local path", err)
}
branch := syncDetectDefaultBranch(localPath)
repos = append(repos, syncRepoEntry{
name: r.Name,
name: name,
localPath: localPath,
defaultBranch: branch,
})

View file

@ -0,0 +1,55 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildSyncRepoList_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildSyncRepoList(nil, []string{"host-uk/core"}, basePath)
require.NoError(t, err)
require.Len(t, repos, 1)
assert.Equal(t, "core", repos[0].name)
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
}
func TestBuildSyncRepoList_Bad_PathTraversal_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"../escape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}
func TestBuildSyncRepoList_Good_OwnerRepo_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildSyncRepoList(nil, []string{"Host-UK/core"}, basePath)
require.NoError(t, err)
require.Len(t, repos, 1)
assert.Equal(t, "core", repos[0].name)
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
}
func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"host-uk/../escape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}
func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepoEncoded_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"path"
"strings"
"forge.lthn.ai/core/cli/pkg/cli"
)

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Config command flags.

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package gitea provides CLI commands for managing a Gitea instance.
//
// Commands:
@ -30,6 +32,7 @@ var (
)
// AddGiteaCommands registers the 'gitea' command and all subcommands.
// Usage: AddGiteaCommands(...)
func AddGiteaCommands(root *cli.Command) {
giteaCmd := &cli.Command{
Use: "gitea",

View file

@ -1,13 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
"strings"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"code.gitea.io/sdk/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Issues command flags.

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
"os/exec"
"strings"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Mirror command flags.

View file

@ -1,13 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
"strings"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
sdk "code.gitea.io/sdk/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
)
// PRs command flags.

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
)
// Repos command flags.

View file

@ -1,17 +1,21 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
os "dappco.re/go/core/scm/internal/ax/osx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci"
"dappco.re/go/core/scm/cmd/internal/syncutil"
gt "dappco.re/go/core/scm/gitea"
"code.gitea.io/sdk/gitea"
"forge.lthn.ai/core/cli/pkg/cli"
coreerr "dappco.re/go/core/log"
gt "dappco.re/go/core/scm/gitea"
)
// Sync command flags.
@ -96,12 +100,14 @@ func buildRepoList(client *gt.Client, args []string, basePath string) ([]repoEnt
if len(args) > 0 {
// Specific repos from args
for _, arg := range args {
name := arg
// Strip owner/ prefix if given
if parts := strings.SplitN(arg, "/", 2); len(parts) == 2 {
name = parts[1]
name, err := syncutil.ParseRepoName(arg)
if err != nil {
return nil, coreerr.E("gitea.buildRepoList", "invalid repo argument", err)
}
_, localPath, err := agentci.ResolvePathWithinRoot(basePath, name)
if err != nil {
return nil, coreerr.E("gitea.buildRepoList", "resolve local path", err)
}
localPath := filepath.Join(basePath, name)
branch := detectDefaultBranch(localPath)
repos = append(repos, repoEntry{
name: name,
@ -116,10 +122,17 @@ func buildRepoList(client *gt.Client, args []string, basePath string) ([]repoEnt
return nil, err
}
for _, r := range orgRepos {
localPath := filepath.Join(basePath, r.Name)
name, err := agentci.ValidatePathElement(r.Name)
if err != nil {
return nil, coreerr.E("gitea.buildRepoList", "invalid repo name from org list", err)
}
_, localPath, err := agentci.ResolvePathWithinRoot(basePath, name)
if err != nil {
return nil, coreerr.E("gitea.buildRepoList", "resolve local path", err)
}
branch := detectDefaultBranch(localPath)
repos = append(repos, repoEntry{
name: r.Name,
name: name,
localPath: localPath,
defaultBranch: branch,
})

View file

@ -0,0 +1,55 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildRepoList_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildRepoList(nil, []string{"host-uk/core"}, basePath)
require.NoError(t, err)
require.Len(t, repos, 1)
assert.Equal(t, "core", repos[0].name)
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
}
func TestBuildRepoList_Bad_PathTraversal_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"../escape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}
func TestBuildRepoList_Good_OwnerRepo_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildRepoList(nil, []string{"Host-UK/core"}, basePath)
require.NoError(t, err)
require.Len(t, repos, 1)
assert.Equal(t, "core", repos[0].name)
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
}
func TestBuildRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"host-uk/../escape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}
func TestBuildRepoList_Bad_PathTraversal_OwnerRepoEncoded_Good(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath)
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo argument")
}

View file

@ -0,0 +1,37 @@
// SPDX-License-Identifier: EUPL-1.2
package syncutil
import (
"net/url"
coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
)
// ParseRepoName normalises a sync argument into a validated repo name.
// Usage: ParseRepoName(...)
func ParseRepoName(arg string) (string, error) {
decoded, err := url.PathUnescape(arg)
if err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "decode repo argument", err)
}
parts := strings.Split(decoded, "/")
switch len(parts) {
case 1:
return agentci.ValidatePathElement(parts[0])
case 2:
if _, err := agentci.ValidatePathElement(parts[0]); err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "invalid repo owner", err)
}
name, err := agentci.ValidatePathElement(parts[1])
if err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "invalid repo name", err)
}
return name, nil
default:
return "", coreerr.E("syncutil.ParseRepoName", "repo argument must be repo or owner/repo", nil)
}
}

View file

@ -0,0 +1,34 @@
// SPDX-License-Identifier: EUPL-1.2
package syncutil
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseRepoName_Good(t *testing.T) {
name, err := ParseRepoName("core")
require.NoError(t, err)
assert.Equal(t, "core", name)
}
func TestParseRepoName_Good_OwnerRepo(t *testing.T) {
name, err := ParseRepoName("host-uk/core")
require.NoError(t, err)
assert.Equal(t, "core", name)
}
func TestParseRepoName_Bad_PathTraversal(t *testing.T) {
_, err := ParseRepoName("../escape")
require.Error(t, err)
assert.Contains(t, err.Error(), "syncutil.ParseRepoName")
}
func TestParseRepoName_Bad_PathTraversalEncoded(t *testing.T) {
_, err := ParseRepoName("host-uk%2F..%2Fescape")
require.Error(t, err)
assert.Contains(t, err.Error(), "syncutil.ParseRepoName")
}

View file

@ -1,40 +1,47 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"crypto/ed25519"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"encoding/hex"
"os/exec"
"strings"
exec "golang.org/x/sys/execabs"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
)
func addCompileCommand(parent *cli.Command) {
var (
version string
dir string
signKey string
builtBy string
output string
)
cmd := &cli.Command{
Use: "compile",
Short: "Compile manifest.yaml into core.json",
Long: "Read .core/manifest.yaml, attach build metadata (commit, tag), and write core.json to the project root.",
Long: "Read .core/manifest.yaml, attach build metadata (commit, tag), and write core.json to the project root or a custom output path.",
RunE: func(cmd *cli.Command, args []string) error {
return runCompile(dir, signKey, builtBy)
return runCompile(dir, version, signKey, builtBy, output)
},
}
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&version, "version", "", "Override the manifest version")
cmd.Flags().StringVar(&signKey, "sign-key", "", "Hex-encoded ed25519 private key for signing")
cmd.Flags().StringVar(&builtBy, "built-by", "core scm compile", "Builder identity")
cmd.Flags().StringVarP(&output, "output", "o", "core.json", "Output path for the compiled manifest")
parent.AddCommand(cmd)
}
func runCompile(dir, signKeyHex, builtBy string) error {
func runCompile(dir, version, signKeyHex, builtBy, output string) error {
medium, err := io.NewSandboxed(dir)
if err != nil {
return cli.WrapVerb(err, "open", dir)
@ -46,6 +53,7 @@ func runCompile(dir, signKeyHex, builtBy string) error {
}
opts := manifest.CompileOptions{
Version: version,
Commit: gitCommit(dir),
Tag: gitTag(dir),
BuiltBy: builtBy,
@ -64,20 +72,28 @@ func runCompile(dir, signKeyHex, builtBy string) error {
return err
}
if err := manifest.WriteCompiled(medium, ".", cm); err != nil {
return err
data, err := manifest.MarshalJSON(cm)
if err != nil {
return cli.WrapVerb(err, "marshal", "manifest")
}
if err := medium.EnsureDir(filepath.Dir(output)); err != nil {
return cli.WrapVerb(err, "create", filepath.Dir(output))
}
if err := medium.Write(output, string(data)); err != nil {
return cli.WrapVerb(err, "write", output)
}
cli.Blank()
cli.Print(" %s %s\n", successStyle.Render("compiled"), valueStyle.Render(m.Code))
cli.Print(" %s %s\n", dimStyle.Render("version:"), valueStyle.Render(m.Version))
cli.Print(" %s %s\n", dimStyle.Render("version:"), valueStyle.Render(cm.Version))
if opts.Commit != "" {
cli.Print(" %s %s\n", dimStyle.Render("commit:"), valueStyle.Render(opts.Commit))
}
if opts.Tag != "" {
cli.Print(" %s %s\n", dimStyle.Render("tag:"), valueStyle.Render(opts.Tag))
}
cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render("core.json"))
cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render(output))
cli.Blank()
return nil

122
cmd/scm/cmd_compile_test.go Normal file
View file

@ -0,0 +1,122 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"encoding/hex"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunCompile_Good_DefaultOutput_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-default
name: Compile Default
version: 1.0.0
`), 0644))
err := runCompile(dir, "", "", "core scm compile", "core.json")
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, "core.json"))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "compile-default", cm.Code)
assert.Equal(t, "core scm compile", cm.BuiltBy)
}
func TestRunCompile_Good_CustomOutput_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-custom
name: Compile Custom
version: 2.0.0
`), 0644))
output := filepath.Join("dist", "core.json")
err := runCompile(dir, "", "", "custom builder", output)
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, output))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "compile-custom", cm.Code)
assert.Equal(t, "custom builder", cm.BuiltBy)
}
func TestRunCompile_Bad_InvalidSignKey_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-invalid-key
name: Compile Invalid Key
version: 1.0.0
`), 0644))
err := runCompile(dir, "", hex.EncodeToString([]byte("short")), "core scm compile", "core.json")
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid private key length")
}
func TestRunCompile_Good_VersionOverride_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-version
name: Compile Version
version: 1.0.0
`), 0644))
err := runCompile(dir, "3.2.1", "", "core scm compile", "core.json")
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, "core.json"))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "3.2.1", cm.Version)
}
func TestAddScmCommands_Good_CompileVersionFlagRegistered_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var compileCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
if cmd.Name() == "compile" {
compileCmd = cmd
break
}
}
require.NotNil(t, compileCmd)
assert.NotNil(t, compileCmd.Flags().Lookup("version"))
}

View file

@ -1,11 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
os "dappco.re/go/core/scm/internal/ax/osx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
)
func addExportCommand(parent *cli.Command) {
@ -14,7 +17,7 @@ func addExportCommand(parent *cli.Command) {
cmd := &cli.Command{
Use: "export",
Short: "Export compiled manifest as JSON",
Long: "Read core.json from the project root and print it to stdout. Falls back to compiling .core/manifest.yaml if core.json is not found.",
Long: "Read core.json from the project root and print it to stdout. Falls back to compiling .core/manifest.yaml only when core.json is missing.",
RunE: func(cmd *cli.Command, args []string) error {
return runExport(dir)
},
@ -31,10 +34,18 @@ func runExport(dir string) error {
return cli.WrapVerb(err, "open", dir)
}
// Try core.json first.
cm, err := manifest.LoadCompiled(medium, ".")
if err != nil {
// Fall back to compiling from source.
var cm *manifest.CompiledManifest
// Prefer core.json if it exists and is valid.
if raw, readErr := medium.Read("core.json"); readErr == nil {
cm, err = manifest.ParseCompiled([]byte(raw))
if err != nil {
return err
}
} else if !os.IsNotExist(readErr) {
return cli.WrapVerb(readErr, "read", "core.json")
} else {
// Fall back to compiling from source only when the compiled artifact is absent.
m, loadErr := manifest.Load(medium, ".")
if loadErr != nil {
return cli.WrapVerb(loadErr, "load", "manifest")

View file

@ -0,0 +1,64 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"testing"
"dappco.re/go/core/scm/manifest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunExport_Good_CompiledManifest_Good(t *testing.T) {
dir := t.TempDir()
cm := &manifest.CompiledManifest{
Manifest: manifest.Manifest{
Code: "compiled-mod",
Name: "Compiled Module",
Version: "1.0.0",
},
Commit: "abc123",
}
data, err := manifest.MarshalJSON(cm)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(dir, "core.json"), data, 0644))
err = runExport(dir)
require.NoError(t, err)
}
func TestRunExport_Good_FallsBackToSource_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
`), 0644))
err := runExport(dir)
require.NoError(t, err)
}
func TestRunExport_Bad_InvalidCompiledManifest_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
`), 0644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "core.json"), []byte("{not-json"), 0644))
err := runExport(dir)
require.Error(t, err)
assert.Contains(t, err.Error(), "manifest.ParseCompiled")
}

View file

@ -1,19 +1,23 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"fmt"
"path/filepath"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
os "dappco.re/go/core/scm/internal/ax/osx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/marketplace"
"forge.lthn.ai/core/cli/pkg/cli"
)
func addIndexCommand(parent *cli.Command) {
var (
dirs []string
output string
baseURL string
org string
dirs []string
output string
forgeURL string
org string
)
cmd := &cli.Command{
@ -24,31 +28,38 @@ func addIndexCommand(parent *cli.Command) {
if len(dirs) == 0 {
dirs = []string{"."}
}
return runIndex(dirs, output, baseURL, org)
return runIndex(dirs, output, forgeURL, org)
},
}
cmd.Flags().StringArrayVarP(&dirs, "dir", "d", nil, "Directories to scan (repeatable, default: current directory)")
cmd.Flags().StringVarP(&output, "output", "o", "index.json", "Output path for the index file")
cmd.Flags().StringVar(&baseURL, "base-url", "", "Base URL for repo links (e.g. https://forge.lthn.ai)")
cmd.Flags().StringVar(&forgeURL, "forge-url", "", "Forge base URL for repo links (e.g. https://forge.lthn.ai)")
cmd.Flags().StringVar(&forgeURL, "base-url", "", "Deprecated alias for --forge-url")
cmd.Flags().StringVar(&org, "org", "", "Organisation for repo links")
parent.AddCommand(cmd)
}
func runIndex(dirs []string, output, baseURL, org string) error {
b := &marketplace.Builder{
BaseURL: baseURL,
Org: org,
func runIndex(dirs []string, output, forgeURL, org string) error {
repoPaths, err := expandIndexRepoPaths(dirs)
if err != nil {
return err
}
idx, err := b.BuildFromDirs(dirs...)
idx, err := marketplace.BuildIndex(io.Local, repoPaths, marketplace.IndexOptions{
ForgeURL: forgeURL,
Org: org,
})
if err != nil {
return cli.WrapVerb(err, "build", "index")
}
absOutput, _ := filepath.Abs(output)
if err := marketplace.WriteIndex(absOutput, idx); err != nil {
absOutput, err := filepath.Abs(output)
if err != nil {
return cli.WrapVerb(err, "resolve", output)
}
if err := marketplace.WriteIndex(io.Local, absOutput, idx); err != nil {
return err
}
@ -59,3 +70,28 @@ func runIndex(dirs []string, output, baseURL, org string) error {
return nil
}
func expandIndexRepoPaths(dirs []string) ([]string, error) {
var repoPaths []string
for _, dir := range dirs {
repoPaths = append(repoPaths, dir)
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, cli.WrapVerb(err, "read", dir)
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
repoPaths = append(repoPaths, filepath.Join(dir, entry.Name()))
}
}
return repoPaths, nil
}

102
cmd/scm/cmd_index_test.go Normal file
View file

@ -0,0 +1,102 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
os "dappco.re/go/core/scm/internal/ax/osx"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"dappco.re/go/core/scm/marketplace"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunIndex_Good_WritesIndex_Good(t *testing.T) {
root := t.TempDir()
modDir := filepath.Join(root, "mod-a")
require.NoError(t, os.MkdirAll(filepath.Join(modDir, ".core"), 0755))
require.NoError(t, os.WriteFile(filepath.Join(modDir, ".core", "manifest.yaml"), []byte(`
code: mod-a
name: Module A
version: 1.0.0
sign: key-a
`), 0644))
output := filepath.Join(root, "index.json")
err := runIndex([]string{root}, output, "https://forge.example.com", "core")
require.NoError(t, err)
idx, err := marketplace.LoadIndex(io.Local, output)
require.NoError(t, err)
require.Len(t, idx.Modules, 1)
assert.Equal(t, "mod-a", idx.Modules[0].Code)
assert.Equal(t, "https://forge.example.com/core/mod-a.git", idx.Modules[0].Repo)
}
func TestRunIndex_Good_PrefersCompiledManifest_Good(t *testing.T) {
root := t.TempDir()
modDir := filepath.Join(root, "mod-a")
require.NoError(t, os.MkdirAll(filepath.Join(modDir, ".core"), 0755))
cm := &manifest.CompiledManifest{
Manifest: manifest.Manifest{
Code: "compiled-mod",
Name: "Compiled Module",
Version: "2.0.0",
Sign: "compiled-key",
},
Commit: "abc123",
}
data, err := json.Marshal(cm)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(modDir, "core.json"), data, 0644))
require.NoError(t, os.WriteFile(filepath.Join(modDir, ".core", "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
sign: source-key
`), 0644))
output := filepath.Join(root, "index.json")
err = runIndex([]string{root}, output, "https://forge.example.com", "core")
require.NoError(t, err)
idx, err := marketplace.LoadIndex(io.Local, output)
require.NoError(t, err)
require.Len(t, idx.Modules, 1)
assert.Equal(t, "compiled-mod", idx.Modules[0].Code)
assert.Equal(t, "compiled-key", idx.Modules[0].SignKey)
}
func TestAddScmCommands_Good_IndexForgeURLFlagAlias_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var indexCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
if cmd.Name() == "index" {
indexCmd = cmd
break
}
}
require.NotNil(t, indexCmd)
assert.NotNil(t, indexCmd.Flags().Lookup("forge-url"))
assert.NotNil(t, indexCmd.Flags().Lookup("base-url"))
}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package scm provides CLI commands for manifest compilation and marketplace
// index generation.
//
@ -5,6 +7,8 @@
// - compile: Compile .core/manifest.yaml into core.json
// - index: Build marketplace index from repository directories
// - export: Export a compiled manifest as JSON to stdout
// - sign: Sign .core/manifest.yaml with an ed25519 private key
// - verify: Verify a manifest signature with an ed25519 public key
package scm
import (
@ -25,6 +29,7 @@ var (
)
// AddScmCommands registers the 'scm' command and all subcommands.
// Usage: AddScmCommands(...)
func AddScmCommands(root *cli.Command) {
scmCmd := &cli.Command{
Use: "scm",
@ -36,4 +41,6 @@ func AddScmCommands(root *cli.Command) {
addCompileCommand(scmCmd)
addIndexCommand(scmCmd)
addExportCommand(scmCmd)
addSignCommand(scmCmd)
addVerifyCommand(scmCmd)
}

137
cmd/scm/cmd_sign_verify.go Normal file
View file

@ -0,0 +1,137 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"crypto/ed25519"
"encoding/hex"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
)
func addSignCommand(parent *cli.Command) {
var (
dir string
signKey string
)
cmd := &cli.Command{
Use: "sign",
Short: "Sign manifest.yaml with a private key",
Long: "Read .core/manifest.yaml, attach an ed25519 signature, and write the signed manifest back to disk.",
RunE: func(cmd *cli.Command, args []string) error {
return runSign(dir, signKey)
},
}
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&signKey, "sign-key", "", "Hex-encoded ed25519 private key")
parent.AddCommand(cmd)
}
func runSign(dir, signKeyHex string) error {
if signKeyHex == "" {
return cli.Err("sign key is required")
}
medium, err := io.NewSandboxed(dir)
if err != nil {
return cli.WrapVerb(err, "open", dir)
}
m, err := manifest.Load(medium, ".")
if err != nil {
return cli.WrapVerb(err, "load", "manifest")
}
keyBytes, err := hex.DecodeString(signKeyHex)
if err != nil {
return cli.WrapVerb(err, "decode", "sign key")
}
if len(keyBytes) != ed25519.PrivateKeySize {
return cli.Err("sign key must be %d bytes when decoded", ed25519.PrivateKeySize)
}
if err := manifest.Sign(m, ed25519.PrivateKey(keyBytes)); err != nil {
return err
}
data, err := manifest.MarshalYAML(m)
if err != nil {
return cli.WrapVerb(err, "marshal", "manifest")
}
if err := medium.Write(".core/manifest.yaml", string(data)); err != nil {
return cli.WrapVerb(err, "write", ".core/manifest.yaml")
}
cli.Blank()
cli.Print(" %s %s\n", successStyle.Render("signed"), valueStyle.Render(m.Code))
cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render(".core/manifest.yaml"))
cli.Blank()
return nil
}
func addVerifyCommand(parent *cli.Command) {
var (
dir string
publicKey string
)
cmd := &cli.Command{
Use: "verify",
Short: "Verify manifest signature with a public key",
Long: "Read .core/manifest.yaml and verify its ed25519 signature against a public key.",
RunE: func(cmd *cli.Command, args []string) error {
return runVerify(dir, publicKey)
},
}
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&publicKey, "public-key", "", "Hex-encoded ed25519 public key")
parent.AddCommand(cmd)
}
func runVerify(dir, publicKeyHex string) error {
if publicKeyHex == "" {
return cli.Err("public key is required")
}
medium, err := io.NewSandboxed(dir)
if err != nil {
return cli.WrapVerb(err, "open", dir)
}
m, err := manifest.Load(medium, ".")
if err != nil {
return cli.WrapVerb(err, "load", "manifest")
}
keyBytes, err := hex.DecodeString(publicKeyHex)
if err != nil {
return cli.WrapVerb(err, "decode", "public key")
}
if len(keyBytes) != ed25519.PublicKeySize {
return cli.Err("public key must be %d bytes when decoded", ed25519.PublicKeySize)
}
valid, err := manifest.Verify(m, ed25519.PublicKey(keyBytes))
if err != nil {
return cli.WrapVerb(err, "verify", "manifest")
}
if !valid {
return cli.Err("signature verification failed for %s", m.Code)
}
cli.Blank()
cli.Success("Signature verified")
cli.Print(" %s %s\n", dimStyle.Render("code:"), valueStyle.Render(m.Code))
cli.Blank()
return nil
}

View file

@ -0,0 +1,99 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"crypto/ed25519"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"encoding/hex"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunSign_Good_WritesSignedManifest_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: signed-cli
name: Signed CLI
version: 1.0.0
`), 0644))
pub, priv, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
err = runSign(dir, hex.EncodeToString(priv))
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, ".core", "manifest.yaml"))
require.NoError(t, err)
m, err := manifest.Parse([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "signed-cli", m.Code)
assert.NotEmpty(t, m.Sign)
valid, err := manifest.Verify(m, pub)
require.NoError(t, err)
assert.True(t, valid)
}
func TestRunVerify_Good_ValidSignature_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
pub, priv, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
m := &manifest.Manifest{
Code: "verified-cli",
Name: "Verified CLI",
Version: "1.0.0",
}
require.NoError(t, manifest.Sign(m, priv))
data, err := manifest.MarshalYAML(m)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), data, 0644))
err = runVerify(dir, hex.EncodeToString(pub))
require.NoError(t, err)
}
func TestAddScmCommands_Good_SignAndVerifyRegistered_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var signCmd *cli.Command
var verifyCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
switch cmd.Name() {
case "sign":
signCmd = cmd
case "verify":
verifyCmd = cmd
}
}
require.NotNil(t, signCmd)
require.NotNil(t, verifyCmd)
assert.NotNil(t, signCmd.Flags().Lookup("sign-key"))
assert.NotNil(t, verifyCmd.Flags().Lookup("public-key"))
}

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"iter"
"net/http"
"path/filepath"
"strings"
"time"
core "dappco.re/go/core/log"
@ -33,6 +35,7 @@ type BitcoinTalkCollector struct {
}
// Name returns the collector name.
// Usage: Name(...)
func (b *BitcoinTalkCollector) Name() string {
id := b.TopicID
if id == "" && b.URL != "" {
@ -42,6 +45,7 @@ func (b *BitcoinTalkCollector) Name() string {
}
// Collect gathers posts from a BitcoinTalk topic.
// Usage: Collect(...)
func (b *BitcoinTalkCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: b.Name()}
@ -281,6 +285,7 @@ func formatPostMarkdown(num int, post btPost) string {
// ParsePostsFromHTML parses BitcoinTalk posts from raw HTML content.
// This is exported for testing purposes.
// Usage: ParsePostsFromHTML(...)
func ParsePostsFromHTML(htmlContent string) ([]btPost, error) {
doc, err := html.Parse(strings.NewReader(htmlContent))
if err != nil {
@ -290,6 +295,7 @@ func ParsePostsFromHTML(htmlContent string) ([]btPost, error) {
}
// FormatPostMarkdown is exported for testing purposes.
// Usage: FormatPostMarkdown(...)
func FormatPostMarkdown(num int, author, date, content string) string {
return formatPostMarkdown(num, btPost{Author: author, Date: date, Content: content})
}
@ -305,6 +311,7 @@ type BitcoinTalkCollectorWithFetcher struct {
// SetHTTPClient replaces the package-level HTTP client.
// Use this in tests to inject a custom transport or timeout.
// Usage: SetHTTPClient(...)
func SetHTTPClient(c *http.Client) {
httpClient = c
}

View file

@ -1,11 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http"
"net/http/httptest"
"strings"
"testing"
"dappco.re/go/core/io"
@ -33,7 +35,7 @@ func sampleBTCTalkPage(count int) string {
return page.String()
}
func TestBitcoinTalkCollector_Collect_Good_OnePage(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_OnePage_Good(t *testing.T) {
// Serve a single page with 5 posts (< 20, so collection stops after one page).
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
@ -73,7 +75,7 @@ func TestBitcoinTalkCollector_Collect_Good_OnePage(t *testing.T) {
}
}
func TestBitcoinTalkCollector_Collect_Good_PageLimit(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_PageLimit_Good(t *testing.T) {
pageCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
pageCount++
@ -101,7 +103,7 @@ func TestBitcoinTalkCollector_Collect_Good_PageLimit(t *testing.T) {
assert.Equal(t, 2, pageCount)
}
func TestBitcoinTalkCollector_Collect_Good_CancelledContext(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_CancelledContext_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(5)))
@ -125,7 +127,7 @@ func TestBitcoinTalkCollector_Collect_Good_CancelledContext(t *testing.T) {
assert.Error(t, err)
}
func TestBitcoinTalkCollector_Collect_Bad_ServerError(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_ServerError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
@ -149,7 +151,7 @@ func TestBitcoinTalkCollector_Collect_Bad_ServerError(t *testing.T) {
assert.Equal(t, 1, result.Errors)
}
func TestBitcoinTalkCollector_Collect_Good_EmitsEvents(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_EmitsEvents_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(2)))
@ -207,7 +209,7 @@ func TestFetchPage_Good(t *testing.T) {
assert.Len(t, posts, 3)
}
func TestFetchPage_Bad_StatusCode(t *testing.T) {
func TestFetchPage_Bad_StatusCode_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}))
@ -222,7 +224,7 @@ func TestFetchPage_Bad_StatusCode(t *testing.T) {
assert.Error(t, err)
}
func TestFetchPage_Bad_InvalidHTML(t *testing.T) {
func TestFetchPage_Bad_InvalidHTML_Good(t *testing.T) {
// html.Parse is very forgiving, so serve an empty page.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -13,12 +15,12 @@ func TestBitcoinTalkCollector_Name_Good(t *testing.T) {
assert.Equal(t, "bitcointalk:12345", b.Name())
}
func TestBitcoinTalkCollector_Name_Good_URL(t *testing.T) {
func TestBitcoinTalkCollector_Name_Good_URL_Good(t *testing.T) {
b := &BitcoinTalkCollector{URL: "https://bitcointalk.org/index.php?topic=12345.0"}
assert.Equal(t, "bitcointalk:url", b.Name())
}
func TestBitcoinTalkCollector_Collect_Bad_NoTopicID(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_NoTopicID_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -27,7 +29,7 @@ func TestBitcoinTalkCollector_Collect_Bad_NoTopicID(t *testing.T) {
assert.Error(t, err)
}
func TestBitcoinTalkCollector_Collect_Good_DryRun(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_DryRun_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -70,7 +72,7 @@ func TestParsePostsFromHTML_Good(t *testing.T) {
assert.Contains(t, posts[1].Content, "Running bitcoin!")
}
func TestParsePostsFromHTML_Good_Empty(t *testing.T) {
func TestParsePostsFromHTML_Good_Empty_Good(t *testing.T) {
posts, err := ParsePostsFromHTML("<html><body></body></html>")
assert.NoError(t, err)
assert.Empty(t, posts)
@ -84,7 +86,7 @@ func TestFormatPostMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "Hello, world!")
}
func TestFormatPostMarkdown_Good_NoDate(t *testing.T) {
func TestFormatPostMarkdown_Good_NoDate_Good(t *testing.T) {
md := FormatPostMarkdown(5, "user", "", "Content here")
assert.Contains(t, md, "# Post 5 by user")

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package collect provides a data collection subsystem for gathering information
// from multiple sources including GitHub, BitcoinTalk, CoinGecko, and academic
// paper repositories. It supports rate limiting, incremental state tracking,
@ -6,9 +8,11 @@ package collect
import (
"context"
"path/filepath"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"dappco.re/go/core/io"
core "dappco.re/go/core/log"
)
// Collector is the interface all collection sources implement.
@ -65,6 +69,7 @@ type Result struct {
// NewConfig creates a Config with sensible defaults.
// It initialises a MockMedium for output if none is provided,
// sets up a rate limiter, state tracker, and event dispatcher.
// Usage: NewConfig(...)
func NewConfig(outputDir string) *Config {
m := io.NewMockMedium()
return &Config{
@ -77,6 +82,7 @@ func NewConfig(outputDir string) *Config {
}
// NewConfigWithMedium creates a Config using the specified storage medium.
// Usage: NewConfigWithMedium(...)
func NewConfigWithMedium(m io.Medium, outputDir string) *Config {
return &Config{
Output: m,
@ -87,7 +93,21 @@ func NewConfigWithMedium(m io.Medium, outputDir string) *Config {
}
}
// verboseProgress emits a progress event when verbose mode is enabled.
// Usage: verboseProgress(cfg, "excavator", "loading state")
func verboseProgress(cfg *Config, source, message string) {
if cfg == nil || !cfg.Verbose {
return
}
if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitProgress(source, message, nil)
return
}
core.Warn(fmt.Sprintf("%s: %s", source, message))
}
// MergeResults combines multiple results into a single aggregated result.
// Usage: MergeResults(...)
func MergeResults(source string, results ...*Result) *Result {
merged := &Result{Source: source}
for _, r := range results {

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -54,13 +56,13 @@ func TestMergeResults_Good(t *testing.T) {
assert.Len(t, merged.Files, 3)
}
func TestMergeResults_Good_NilResults(t *testing.T) {
func TestMergeResults_Good_NilResults_Good(t *testing.T) {
r1 := &Result{Items: 3}
merged := MergeResults("test", r1, nil, nil)
assert.Equal(t, 3, merged.Items)
}
func TestMergeResults_Good_Empty(t *testing.T) {
func TestMergeResults_Good_Empty_Good(t *testing.T) {
merged := MergeResults("empty")
assert.Equal(t, 0, merged.Items)
assert.Equal(t, 0, merged.Errors)

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"net/http"
"net/http/httptest"
"testing"
@ -15,7 +17,7 @@ import (
// --- GitHub collector: context cancellation and orchestration ---
func TestGitHubCollector_Collect_Good_ContextCancelledInLoop(t *testing.T) {
func TestGitHubCollector_Collect_Good_ContextCancelledInLoop_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = false
@ -31,7 +33,7 @@ func TestGitHubCollector_Collect_Good_ContextCancelledInLoop(t *testing.T) {
assert.NotNil(t, result)
}
func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress(t *testing.T) {
func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -47,7 +49,7 @@ func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress(t *testing.T) {
assert.GreaterOrEqual(t, progressCount, 1)
}
func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues(t *testing.T) {
func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -59,7 +61,7 @@ func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestGitHubCollector_Collect_Good_EmitsStartAndComplete(t *testing.T) {
func TestGitHubCollector_Collect_Good_EmitsStartAndComplete_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -76,7 +78,7 @@ func TestGitHubCollector_Collect_Good_EmitsStartAndComplete(t *testing.T) {
assert.Equal(t, 1, completes)
}
func TestGitHubCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
func TestGitHubCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -89,7 +91,7 @@ func TestGitHubCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestFormatIssueMarkdown_Good_NoBodyNoURL(t *testing.T) {
func TestFormatIssueMarkdown_Good_NoBodyNoURL_Good(t *testing.T) {
issue := ghIssue{
Number: 1,
Title: "No Body Issue",
@ -106,7 +108,7 @@ func TestFormatIssueMarkdown_Good_NoBodyNoURL(t *testing.T) {
// --- Market collector: fetchJSON edge cases ---
func TestFetchJSON_Bad_NonJSONBody(t *testing.T) {
func TestFetchJSON_Bad_NonJSONBody_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(`<html>not json</html>`))
@ -117,17 +119,17 @@ func TestFetchJSON_Bad_NonJSONBody(t *testing.T) {
assert.Error(t, err)
}
func TestFetchJSON_Bad_MalformedURL(t *testing.T) {
func TestFetchJSON_Bad_MalformedURL_Good(t *testing.T) {
_, err := fetchJSON[coinData](context.Background(), "://bad-url")
assert.Error(t, err)
}
func TestFetchJSON_Bad_ServerUnavailable(t *testing.T) {
func TestFetchJSON_Bad_ServerUnavailable_Good(t *testing.T) {
_, err := fetchJSON[coinData](context.Background(), "http://127.0.0.1:1")
assert.Error(t, err)
}
func TestFetchJSON_Bad_Non200StatusCode(t *testing.T) {
func TestFetchJSON_Bad_Non200StatusCode_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
@ -138,7 +140,7 @@ func TestFetchJSON_Bad_Non200StatusCode(t *testing.T) {
assert.Contains(t, err.Error(), "unexpected status code")
}
func TestMarketCollector_Collect_Bad_MissingCoinID(t *testing.T) {
func TestMarketCollector_Collect_Bad_MissingCoinID_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -148,7 +150,7 @@ func TestMarketCollector_Collect_Bad_MissingCoinID(t *testing.T) {
assert.Contains(t, err.Error(), "coin ID is required")
}
func TestMarketCollector_Collect_Good_NoDispatcher(t *testing.T) {
func TestMarketCollector_Collect_Good_NoDispatcher_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
data := coinData{ID: "test", Symbol: "tst", Name: "Test",
@ -173,7 +175,7 @@ func TestMarketCollector_Collect_Good_NoDispatcher(t *testing.T) {
assert.Equal(t, 2, result.Items)
}
func TestMarketCollector_Collect_Bad_CurrentFetchFails(t *testing.T) {
func TestMarketCollector_Collect_Bad_CurrentFetchFails_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
@ -195,7 +197,7 @@ func TestMarketCollector_Collect_Bad_CurrentFetchFails(t *testing.T) {
assert.Equal(t, 1, result.Errors)
}
func TestMarketCollector_CollectHistorical_Good_DefaultDays(t *testing.T) {
func TestMarketCollector_CollectHistorical_Good_DefaultDays_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -227,7 +229,7 @@ func TestMarketCollector_CollectHistorical_Good_DefaultDays(t *testing.T) {
assert.Equal(t, 3, result.Items)
}
func TestMarketCollector_CollectHistorical_Good_WithRateLimiter(t *testing.T) {
func TestMarketCollector_CollectHistorical_Good_WithRateLimiter_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -261,7 +263,7 @@ func TestMarketCollector_CollectHistorical_Good_WithRateLimiter(t *testing.T) {
// --- State: error paths ---
func TestState_Load_Bad_MalformedJSON(t *testing.T) {
func TestState_Load_Bad_MalformedJSON_Good(t *testing.T) {
m := io.NewMockMedium()
m.Files["/state.json"] = `{invalid json`
@ -272,7 +274,7 @@ func TestState_Load_Bad_MalformedJSON(t *testing.T) {
// --- Process: additional coverage for uncovered branches ---
func TestHTMLToMarkdown_Good_PreCodeBlock(t *testing.T) {
func TestHTMLToMarkdown_Good_PreCodeBlock_Good(t *testing.T) {
input := `<pre>some code here</pre>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -280,7 +282,7 @@ func TestHTMLToMarkdown_Good_PreCodeBlock(t *testing.T) {
assert.Contains(t, result, "some code here")
}
func TestHTMLToMarkdown_Good_StrongAndEmElements(t *testing.T) {
func TestHTMLToMarkdown_Good_StrongAndEmElements_Good(t *testing.T) {
input := `<strong>bold</strong> and <em>italic</em>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -288,21 +290,21 @@ func TestHTMLToMarkdown_Good_StrongAndEmElements(t *testing.T) {
assert.Contains(t, result, "*italic*")
}
func TestHTMLToMarkdown_Good_InlineCode(t *testing.T) {
func TestHTMLToMarkdown_Good_InlineCode_Good(t *testing.T) {
input := `<code>var x = 1</code>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "`var x = 1`")
}
func TestHTMLToMarkdown_Good_AnchorWithHref(t *testing.T) {
func TestHTMLToMarkdown_Good_AnchorWithHref_Good(t *testing.T) {
input := `<a href="https://example.com">Click here</a>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "[Click here](https://example.com)")
}
func TestHTMLToMarkdown_Good_ScriptTagRemoved(t *testing.T) {
func TestHTMLToMarkdown_Good_ScriptTagRemoved_Good(t *testing.T) {
input := `<html><body><script>alert('xss')</script><p>Safe text</p></body></html>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -310,7 +312,7 @@ func TestHTMLToMarkdown_Good_ScriptTagRemoved(t *testing.T) {
assert.NotContains(t, result, "alert")
}
func TestHTMLToMarkdown_Good_H1H2H3Headers(t *testing.T) {
func TestHTMLToMarkdown_Good_H1H2H3Headers_Good(t *testing.T) {
input := `<h1>One</h1><h2>Two</h2><h3>Three</h3>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -319,7 +321,7 @@ func TestHTMLToMarkdown_Good_H1H2H3Headers(t *testing.T) {
assert.Contains(t, result, "### Three")
}
func TestHTMLToMarkdown_Good_MultiParagraph(t *testing.T) {
func TestHTMLToMarkdown_Good_MultiParagraph_Good(t *testing.T) {
input := `<p>First paragraph</p><p>Second paragraph</p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -327,12 +329,12 @@ func TestHTMLToMarkdown_Good_MultiParagraph(t *testing.T) {
assert.Contains(t, result, "Second paragraph")
}
func TestJSONToMarkdown_Bad_Malformed(t *testing.T) {
func TestJSONToMarkdown_Bad_Malformed_Good(t *testing.T) {
_, err := JSONToMarkdown(`{invalid}`)
assert.Error(t, err)
}
func TestJSONToMarkdown_Good_FlatObject(t *testing.T) {
func TestJSONToMarkdown_Good_FlatObject_Good(t *testing.T) {
input := `{"name": "Alice", "age": 30}`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
@ -340,7 +342,7 @@ func TestJSONToMarkdown_Good_FlatObject(t *testing.T) {
assert.Contains(t, result, "**age:** 30")
}
func TestJSONToMarkdown_Good_ScalarList(t *testing.T) {
func TestJSONToMarkdown_Good_ScalarList_Good(t *testing.T) {
input := `["hello", "world"]`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
@ -348,14 +350,14 @@ func TestJSONToMarkdown_Good_ScalarList(t *testing.T) {
assert.Contains(t, result, "- world")
}
func TestJSONToMarkdown_Good_ObjectContainingArray(t *testing.T) {
func TestJSONToMarkdown_Good_ObjectContainingArray_Good(t *testing.T) {
input := `{"items": [1, 2, 3]}`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "**items:**")
}
func TestProcessor_Process_Bad_MissingDir(t *testing.T) {
func TestProcessor_Process_Bad_MissingDir_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -365,7 +367,7 @@ func TestProcessor_Process_Bad_MissingDir(t *testing.T) {
assert.Contains(t, err.Error(), "directory is required")
}
func TestProcessor_Process_Good_DryRunEmitsProgress(t *testing.T) {
func TestProcessor_Process_Good_DryRunEmitsProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -381,7 +383,7 @@ func TestProcessor_Process_Good_DryRunEmitsProgress(t *testing.T) {
assert.Equal(t, 1, progressCount)
}
func TestProcessor_Process_Good_SkipsUnsupportedExtension(t *testing.T) {
func TestProcessor_Process_Good_SkipsUnsupportedExtension_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/data.csv"] = `a,b,c`
@ -397,7 +399,7 @@ func TestProcessor_Process_Good_SkipsUnsupportedExtension(t *testing.T) {
assert.Equal(t, 1, result.Skipped)
}
func TestProcessor_Process_Good_MarkdownPassthroughTrimmed(t *testing.T) {
func TestProcessor_Process_Good_MarkdownPassthroughTrimmed_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/readme.md"] = `# Hello World `
@ -416,7 +418,7 @@ func TestProcessor_Process_Good_MarkdownPassthroughTrimmed(t *testing.T) {
assert.Equal(t, "# Hello World", content)
}
func TestProcessor_Process_Good_HTMExtensionHandled(t *testing.T) {
func TestProcessor_Process_Good_HTMExtensionHandled_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/page.htm"] = `<h1>HTM File</h1>`
@ -431,7 +433,7 @@ func TestProcessor_Process_Good_HTMExtensionHandled(t *testing.T) {
assert.Equal(t, 1, result.Items)
}
func TestProcessor_Process_Good_NilDispatcherHandled(t *testing.T) {
func TestProcessor_Process_Good_NilDispatcherHandled_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/test.html"] = `<p>Text</p>`
@ -449,12 +451,12 @@ func TestProcessor_Process_Good_NilDispatcherHandled(t *testing.T) {
// --- BitcoinTalk: additional edge cases ---
func TestBitcoinTalkCollector_Name_Good_EmptyTopicAndURL(t *testing.T) {
func TestBitcoinTalkCollector_Name_Good_EmptyTopicAndURL_Good(t *testing.T) {
b := &BitcoinTalkCollector{}
assert.Equal(t, "bitcointalk:", b.Name())
}
func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(2)))
@ -478,7 +480,7 @@ func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
assert.Equal(t, 2, result.Items)
}
func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -494,7 +496,7 @@ func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
assert.True(t, progressEmitted)
}
func TestParsePostsFromHTML_Good_PostWithNoInnerContent(t *testing.T) {
func TestParsePostsFromHTML_Good_PostWithNoInnerContent_Good(t *testing.T) {
htmlContent := `<html><body>
<div class="post">
<div class="poster_info">user1</div>
@ -505,7 +507,7 @@ func TestParsePostsFromHTML_Good_PostWithNoInnerContent(t *testing.T) {
assert.Empty(t, posts)
}
func TestFormatPostMarkdown_Good_WithDateContent(t *testing.T) {
func TestFormatPostMarkdown_Good_WithDateContent_Good(t *testing.T) {
md := FormatPostMarkdown(1, "alice", "2025-01-15", "Hello world")
assert.Contains(t, md, "# Post 1 by alice")
assert.Contains(t, md, "**Date:** 2025-01-15")
@ -514,7 +516,7 @@ func TestFormatPostMarkdown_Good_WithDateContent(t *testing.T) {
// --- Papers collector: edge cases ---
func TestPapersCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
func TestPapersCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -530,7 +532,7 @@ func TestPapersCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
assert.True(t, progressEmitted)
}
func TestPapersCollector_Collect_Good_NilDispatcherIACR(t *testing.T) {
func TestPapersCollector_Collect_Good_NilDispatcherIACR_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML))
@ -554,7 +556,7 @@ func TestPapersCollector_Collect_Good_NilDispatcherIACR(t *testing.T) {
assert.Equal(t, 2, result.Items)
}
func TestArXivEntryToPaper_Good_NoAlternateLink(t *testing.T) {
func TestArXivEntryToPaper_Good_NoAlternateLink_Good(t *testing.T) {
entry := arxivEntry{
ID: "http://arxiv.org/abs/2501.99999v1",
Title: "No Alternate",
@ -569,7 +571,7 @@ func TestArXivEntryToPaper_Good_NoAlternateLink(t *testing.T) {
// --- Excavator: additional edge cases ---
func TestExcavator_Run_Good_ResumeLoadError(t *testing.T) {
func TestExcavator_Run_Good_ResumeLoadError_Good(t *testing.T) {
m := io.NewMockMedium()
m.Files["/output/.collect-state.json"] = `{invalid`
@ -589,7 +591,7 @@ func TestExcavator_Run_Good_ResumeLoadError(t *testing.T) {
// --- RateLimiter: additional edge cases ---
func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay(t *testing.T) {
func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay_Good(t *testing.T) {
rl := NewRateLimiter()
rl.SetDelay("fast", 1*time.Millisecond)
@ -608,7 +610,7 @@ func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay(t *testing.T) {
// --- FormatMarketSummary: with empty market data values ---
func TestFormatMarketSummary_Good_ZeroRank(t *testing.T) {
func TestFormatMarketSummary_Good_ZeroRank_Good(t *testing.T) {
data := &coinData{
Name: "Tiny Token",
Symbol: "tiny",
@ -622,7 +624,7 @@ func TestFormatMarketSummary_Good_ZeroRank(t *testing.T) {
assert.NotContains(t, summary, "Market Cap Rank")
}
func TestFormatMarketSummary_Good_ZeroSupply(t *testing.T) {
func TestFormatMarketSummary_Good_ZeroSupply_Good(t *testing.T) {
data := &coinData{
Name: "Zero Supply",
Symbol: "zs",
@ -636,7 +638,7 @@ func TestFormatMarketSummary_Good_ZeroSupply(t *testing.T) {
assert.NotContains(t, summary, "Total Supply")
}
func TestFormatMarketSummary_Good_NoLastUpdated(t *testing.T) {
func TestFormatMarketSummary_Good_NoLastUpdated_Good(t *testing.T) {
data := &coinData{
Name: "No Update",
Symbol: "nu",

View file

@ -1,14 +1,17 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
"fmt"
core "dappco.re/go/core"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
goio "io"
"io/fs"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@ -17,6 +20,14 @@ import (
"github.com/stretchr/testify/require"
)
func testErr(msg string) error {
return core.E("collect.test", msg, nil)
}
func testErrf(format string, args ...any) error {
return core.E("collect.test", fmt.Sprintf(format, args...), nil)
}
// errorMedium wraps MockMedium and injects errors on specific operations.
type errorMedium struct {
*io.MockMedium
@ -50,16 +61,18 @@ func (e *errorMedium) Read(path string) (string, error) {
}
return e.MockMedium.Read(path)
}
func (e *errorMedium) FileGet(path string) (string, error) { return e.MockMedium.FileGet(path) }
func (e *errorMedium) FileSet(path, content string) error { return e.MockMedium.FileSet(path, content) }
func (e *errorMedium) Delete(path string) error { return e.MockMedium.Delete(path) }
func (e *errorMedium) DeleteAll(path string) error { return e.MockMedium.DeleteAll(path) }
func (e *errorMedium) Rename(old, new string) error { return e.MockMedium.Rename(old, new) }
func (e *errorMedium) Stat(path string) (fs.FileInfo, error) { return e.MockMedium.Stat(path) }
func (e *errorMedium) Open(path string) (fs.File, error) { return e.MockMedium.Open(path) }
func (e *errorMedium) Create(path string) (goio.WriteCloser, error) { return e.MockMedium.Create(path) }
func (e *errorMedium) Append(path string) (goio.WriteCloser, error) { return e.MockMedium.Append(path) }
func (e *errorMedium) ReadStream(path string) (goio.ReadCloser, error) { return e.MockMedium.ReadStream(path) }
func (e *errorMedium) FileGet(path string) (string, error) { return e.MockMedium.FileGet(path) }
func (e *errorMedium) FileSet(path, content string) error { return e.MockMedium.FileSet(path, content) }
func (e *errorMedium) Delete(path string) error { return e.MockMedium.Delete(path) }
func (e *errorMedium) DeleteAll(path string) error { return e.MockMedium.DeleteAll(path) }
func (e *errorMedium) Rename(old, new string) error { return e.MockMedium.Rename(old, new) }
func (e *errorMedium) Stat(path string) (fs.FileInfo, error) { return e.MockMedium.Stat(path) }
func (e *errorMedium) Open(path string) (fs.File, error) { return e.MockMedium.Open(path) }
func (e *errorMedium) Create(path string) (goio.WriteCloser, error) { return e.MockMedium.Create(path) }
func (e *errorMedium) Append(path string) (goio.WriteCloser, error) { return e.MockMedium.Append(path) }
func (e *errorMedium) ReadStream(path string) (goio.ReadCloser, error) {
return e.MockMedium.ReadStream(path)
}
func (e *errorMedium) WriteStream(path string) (goio.WriteCloser, error) {
return e.MockMedium.WriteStream(path)
}
@ -73,8 +86,8 @@ type errorLimiterWaiter struct{}
// --- Processor: list error ---
func TestProcessor_Process_Bad_ListError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), listErr: fmt.Errorf("list denied")}
func TestProcessor_Process_Bad_ListError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), listErr: testErr("list denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
p := &Processor{Source: "test", Dir: "/input"}
@ -85,8 +98,8 @@ func TestProcessor_Process_Bad_ListError(t *testing.T) {
// --- Processor: ensureDir error ---
func TestProcessor_Process_Bad_EnsureDirError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
func TestProcessor_Process_Bad_EnsureDirError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")}
// Need to ensure List returns entries
em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>"
@ -101,7 +114,7 @@ func TestProcessor_Process_Bad_EnsureDirError(t *testing.T) {
// --- Processor: context cancellation during processing ---
func TestProcessor_Process_Bad_ContextCancelledDuringLoop(t *testing.T) {
func TestProcessor_Process_Bad_ContextCancelledDuringLoop_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/a.html"] = "<h1>Test</h1>"
@ -120,8 +133,8 @@ func TestProcessor_Process_Bad_ContextCancelledDuringLoop(t *testing.T) {
// --- Processor: read error during file processing ---
func TestProcessor_Process_Bad_ReadError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: fmt.Errorf("read denied")}
func TestProcessor_Process_Bad_ReadError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: testErr("read denied")}
em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>"
@ -135,7 +148,7 @@ func TestProcessor_Process_Bad_ReadError(t *testing.T) {
// --- Processor: JSON conversion error ---
func TestProcessor_Process_Bad_InvalidJSONFile(t *testing.T) {
func TestProcessor_Process_Bad_InvalidJSONFile_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/bad.json"] = "not valid json {"
@ -153,8 +166,8 @@ func TestProcessor_Process_Bad_InvalidJSONFile(t *testing.T) {
// --- Processor: write error during output ---
func TestProcessor_Process_Bad_WriteError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
func TestProcessor_Process_Bad_WriteError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/page.html"] = "<h1>Title</h1>"
@ -168,7 +181,7 @@ func TestProcessor_Process_Bad_WriteError(t *testing.T) {
// --- Processor: successful processing with events ---
func TestProcessor_Process_Good_EmitsItemAndComplete(t *testing.T) {
func TestProcessor_Process_Good_EmitsItemAndComplete_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/page.html"] = "<h1>Title</h1><p>Body</p>"
@ -188,7 +201,7 @@ func TestProcessor_Process_Good_EmitsItemAndComplete(t *testing.T) {
// --- Papers: with rate limiter that fails ---
func TestPapersCollector_CollectIACR_Bad_LimiterError(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_LimiterError_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -202,7 +215,7 @@ func TestPapersCollector_CollectIACR_Bad_LimiterError(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_CollectArXiv_Bad_LimiterError(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_LimiterError_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -218,7 +231,7 @@ func TestPapersCollector_CollectArXiv_Bad_LimiterError(t *testing.T) {
// --- Papers: IACR with bad HTML response ---
func TestPapersCollector_CollectIACR_Bad_InvalidHTML(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_InvalidHTML_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
// Serve valid-ish HTML but with no papers - the parse succeeds but returns empty.
@ -243,7 +256,7 @@ func TestPapersCollector_CollectIACR_Bad_InvalidHTML(t *testing.T) {
// --- Papers: IACR write error ---
func TestPapersCollector_CollectIACR_Bad_WriteError(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_WriteError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML))
@ -255,19 +268,19 @@ func TestPapersCollector_CollectIACR_Bad_WriteError(t *testing.T) {
httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
p := &PapersCollector{Source: PaperSourceIACR, Query: "test"}
result, err := p.Collect(context.Background(), cfg)
require.NoError(t, err) // Write errors increment Errors, not returned
require.NoError(t, err) // Write errors increment Errors, not returned
assert.Equal(t, 2, result.Errors) // 2 papers both fail to write
}
// --- Papers: IACR EnsureDir error ---
func TestPapersCollector_CollectIACR_Bad_EnsureDirError(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_EnsureDirError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML))
@ -279,7 +292,7 @@ func TestPapersCollector_CollectIACR_Bad_EnsureDirError(t *testing.T) {
httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -291,7 +304,7 @@ func TestPapersCollector_CollectIACR_Bad_EnsureDirError(t *testing.T) {
// --- Papers: arXiv write error ---
func TestPapersCollector_CollectArXiv_Bad_WriteError(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_WriteError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML))
@ -303,7 +316,7 @@ func TestPapersCollector_CollectArXiv_Bad_WriteError(t *testing.T) {
httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -315,7 +328,7 @@ func TestPapersCollector_CollectArXiv_Bad_WriteError(t *testing.T) {
// --- Papers: arXiv EnsureDir error ---
func TestPapersCollector_CollectArXiv_Bad_EnsureDirError(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_EnsureDirError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML))
@ -327,7 +340,7 @@ func TestPapersCollector_CollectArXiv_Bad_EnsureDirError(t *testing.T) {
httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -339,7 +352,7 @@ func TestPapersCollector_CollectArXiv_Bad_EnsureDirError(t *testing.T) {
// --- Papers: collectAll with dispatcher events ---
func TestPapersCollector_CollectAll_Good_WithDispatcher(t *testing.T) {
func TestPapersCollector_CollectAll_Good_WithDispatcher_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -374,7 +387,7 @@ func TestPapersCollector_CollectAll_Good_WithDispatcher(t *testing.T) {
// --- Papers: IACR with events on item emit ---
func TestPapersCollector_CollectIACR_Good_EmitsItemEvents(t *testing.T) {
func TestPapersCollector_CollectIACR_Good_EmitsItemEvents_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML))
@ -402,7 +415,7 @@ func TestPapersCollector_CollectIACR_Good_EmitsItemEvents(t *testing.T) {
// --- Papers: arXiv with events on item emit ---
func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents(t *testing.T) {
func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML))
@ -430,7 +443,7 @@ func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents(t *testing.T) {
// --- Market: collectCurrent write error (summary path) ---
func TestMarketCollector_Collect_Bad_WriteError(t *testing.T) {
func TestMarketCollector_Collect_Bad_WriteError_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") {
@ -453,7 +466,7 @@ func TestMarketCollector_Collect_Bad_WriteError(t *testing.T) {
coinGeckoBaseURL = server.URL
defer func() { coinGeckoBaseURL = oldURL }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -466,7 +479,7 @@ func TestMarketCollector_Collect_Bad_WriteError(t *testing.T) {
// --- Market: EnsureDir error ---
func TestMarketCollector_Collect_Bad_EnsureDirError(t *testing.T) {
func TestMarketCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"})
@ -477,7 +490,7 @@ func TestMarketCollector_Collect_Bad_EnsureDirError(t *testing.T) {
coinGeckoBaseURL = server.URL
defer func() { coinGeckoBaseURL = oldURL }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -489,7 +502,7 @@ func TestMarketCollector_Collect_Bad_EnsureDirError(t *testing.T) {
// --- Market: collectCurrent with limiter wait error ---
func TestMarketCollector_Collect_Bad_LimiterError(t *testing.T) {
func TestMarketCollector_Collect_Bad_LimiterError_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"})
@ -516,7 +529,7 @@ func TestMarketCollector_Collect_Bad_LimiterError(t *testing.T) {
// --- Market: collectHistorical with custom FromDate ---
func TestMarketCollector_Collect_Good_HistoricalCustomDate(t *testing.T) {
func TestMarketCollector_Collect_Good_HistoricalCustomDate_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") {
@ -551,8 +564,8 @@ func TestMarketCollector_Collect_Good_HistoricalCustomDate(t *testing.T) {
// --- BitcoinTalk: EnsureDir error ---
func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
@ -564,7 +577,7 @@ func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError(t *testing.T) {
// --- BitcoinTalk: limiter error ---
func TestBitcoinTalkCollector_Collect_Bad_LimiterError(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_LimiterError_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -580,7 +593,7 @@ func TestBitcoinTalkCollector_Collect_Bad_LimiterError(t *testing.T) {
// --- BitcoinTalk: write error during post saving ---
func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(3)))
@ -592,20 +605,20 @@ func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts(t *testing.T) {
httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil
b := &BitcoinTalkCollector{TopicID: "12345"}
result, err := b.Collect(context.Background(), cfg)
require.NoError(t, err) // write errors are counted
require.NoError(t, err) // write errors are counted
assert.Equal(t, 3, result.Errors) // 3 posts all fail to write
assert.Equal(t, 0, result.Items)
}
// --- BitcoinTalk: fetchPage with bad HTTP status ---
func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus(t *testing.T) {
func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
}))
@ -619,7 +632,7 @@ func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus(t *testing.T) {
// --- BitcoinTalk: fetchPage with request error ---
func TestBitcoinTalkCollector_FetchPage_Bad_RequestError(t *testing.T) {
func TestBitcoinTalkCollector_FetchPage_Bad_RequestError_Good(t *testing.T) {
old := httpClient
httpClient = &http.Client{Transport: &rewriteTransport{target: "http://127.0.0.1:1"}} // Connection refused
defer func() { httpClient = old }()
@ -632,7 +645,7 @@ func TestBitcoinTalkCollector_FetchPage_Bad_RequestError(t *testing.T) {
// --- BitcoinTalk: fetchPage with valid but empty page ---
func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage(t *testing.T) {
func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html><body></body></html>"))
@ -651,7 +664,7 @@ func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage(t *testing.T) {
// --- BitcoinTalk: Collect with fetch error + dispatcher ---
func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
@ -678,7 +691,7 @@ func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher(t *testing.T)
// --- State: Save with a populated state ---
func TestState_Save_Good_RoundTrip(t *testing.T) {
func TestState_Save_Good_RoundTrip_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/data/state.json")
@ -704,7 +717,7 @@ func TestState_Save_Good_RoundTrip(t *testing.T) {
// --- GitHub: Collect with Repo set triggers collectIssues/collectPRs (which fail via gh) ---
func TestGitHubCollector_Collect_Bad_GhNotAuthenticated(t *testing.T) {
func TestGitHubCollector_Collect_Bad_GhNotAuthenticated_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -724,7 +737,7 @@ func TestGitHubCollector_Collect_Bad_GhNotAuthenticated(t *testing.T) {
// --- GitHub: Collect IssuesOnly triggers only issues, not PRs ---
func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails(t *testing.T) {
func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -737,7 +750,7 @@ func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails(t *testing.T) {
// --- GitHub: Collect PRsOnly triggers only PRs, not issues ---
func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails(t *testing.T) {
func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -750,7 +763,7 @@ func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails(t *testing.T) {
// --- extractText: text before a br/p/div element adds newline ---
func TestExtractText_Good_TextBeforeBR(t *testing.T) {
func TestExtractText_Good_TextBeforeBR_Good(t *testing.T) {
htmlStr := `<div class="inner">Hello<br>World<p>End</p></div>`
posts, err := ParsePostsFromHTML(fmt.Sprintf(`<html><body><div class="post"><div class="inner">%s</div></div></body></html>`,
"First text<br>Second text<div>Third text</div>"))
@ -764,7 +777,7 @@ func TestExtractText_Good_TextBeforeBR(t *testing.T) {
// --- ParsePostsFromHTML: posts with full structure ---
func TestParsePostsFromHTML_Good_FullStructure(t *testing.T) {
func TestParsePostsFromHTML_Good_FullStructure_Good(t *testing.T) {
htmlContent := `<html><body>
<div class="post">
<div class="poster_info">TestAuthor</div>
@ -783,7 +796,7 @@ func TestParsePostsFromHTML_Good_FullStructure(t *testing.T) {
// --- getChildrenText: nested element node path ---
func TestHTMLToMarkdown_Good_NestedElements(t *testing.T) {
func TestHTMLToMarkdown_Good_NestedElements_Good(t *testing.T) {
// <a> with nested <span> triggers getChildrenText with non-text child nodes
input := `<p><a href="https://example.com"><span>Nested</span> Link</a></p>`
result, err := HTMLToMarkdown(input)
@ -793,7 +806,7 @@ func TestHTMLToMarkdown_Good_NestedElements(t *testing.T) {
// --- HTML: ordered list ---
func TestHTMLToMarkdown_Good_OL(t *testing.T) {
func TestHTMLToMarkdown_Good_OL_Good(t *testing.T) {
input := `<ol><li>First</li><li>Second</li></ol>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -803,7 +816,7 @@ func TestHTMLToMarkdown_Good_OL(t *testing.T) {
// --- HTML: blockquote ---
func TestHTMLToMarkdown_Good_BlockquoteElement(t *testing.T) {
func TestHTMLToMarkdown_Good_BlockquoteElement_Good(t *testing.T) {
input := `<blockquote>Quoted text</blockquote>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -812,7 +825,7 @@ func TestHTMLToMarkdown_Good_BlockquoteElement(t *testing.T) {
// --- HTML: hr ---
func TestHTMLToMarkdown_Good_HR(t *testing.T) {
func TestHTMLToMarkdown_Good_HR_Good(t *testing.T) {
input := `<p>Before</p><hr><p>After</p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -821,7 +834,7 @@ func TestHTMLToMarkdown_Good_HR(t *testing.T) {
// --- HTML: h4, h5, h6 ---
func TestHTMLToMarkdown_Good_AllHeadingLevels(t *testing.T) {
func TestHTMLToMarkdown_Good_AllHeadingLevels_Good(t *testing.T) {
input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -832,7 +845,7 @@ func TestHTMLToMarkdown_Good_AllHeadingLevels(t *testing.T) {
// --- HTML: link without href ---
func TestHTMLToMarkdown_Good_LinkNoHref(t *testing.T) {
func TestHTMLToMarkdown_Good_LinkNoHref_Good(t *testing.T) {
input := `<a>bare link text</a>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -842,7 +855,7 @@ func TestHTMLToMarkdown_Good_LinkNoHref(t *testing.T) {
// --- HTML: unordered list ---
func TestHTMLToMarkdown_Good_UL(t *testing.T) {
func TestHTMLToMarkdown_Good_UL_Good(t *testing.T) {
input := `<ul><li>Item A</li><li>Item B</li></ul>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -852,7 +865,7 @@ func TestHTMLToMarkdown_Good_UL(t *testing.T) {
// --- HTML: br tag ---
func TestHTMLToMarkdown_Good_BRTag(t *testing.T) {
func TestHTMLToMarkdown_Good_BRTag_Good(t *testing.T) {
input := `<p>Line one<br>Line two</p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -862,7 +875,7 @@ func TestHTMLToMarkdown_Good_BRTag(t *testing.T) {
// --- HTML: style tag stripped ---
func TestHTMLToMarkdown_Good_StyleStripped(t *testing.T) {
func TestHTMLToMarkdown_Good_StyleStripped_Good(t *testing.T) {
input := `<html><head><style>body{color:red}</style></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -872,7 +885,7 @@ func TestHTMLToMarkdown_Good_StyleStripped(t *testing.T) {
// --- HTML: i and b tags ---
func TestHTMLToMarkdown_Good_AlternateBoldItalic(t *testing.T) {
func TestHTMLToMarkdown_Good_AlternateBoldItalic_Good(t *testing.T) {
input := `<p><b>bold</b> and <i>italic</i></p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -882,7 +895,7 @@ func TestHTMLToMarkdown_Good_AlternateBoldItalic(t *testing.T) {
// --- Market: collectCurrent with limiter that actually blocks ---
func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled(t *testing.T) {
func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin", Symbol: "btc", Name: "Bitcoin",
@ -914,7 +927,7 @@ func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled(t *testing.T) {
// --- Papers: IACR with limiter that blocks ---
func TestPapersCollector_CollectIACR_Bad_LimiterBlocks(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_LimiterBlocks_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -931,7 +944,7 @@ func TestPapersCollector_CollectIACR_Bad_LimiterBlocks(t *testing.T) {
// --- Papers: arXiv with limiter that blocks ---
func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -948,7 +961,7 @@ func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks(t *testing.T) {
// --- BitcoinTalk: limiter that blocks ---
func TestBitcoinTalkCollector_Collect_Bad_LimiterBlocks(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Bad_LimiterBlocks_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter()
@ -968,37 +981,47 @@ func TestBitcoinTalkCollector_Collect_Bad_LimiterBlocks(t *testing.T) {
// writeCountMedium fails after N successful writes.
type writeCountMedium struct {
*io.MockMedium
writeCount int
failAfterN int
writeCount int
failAfterN int
}
func (w *writeCountMedium) Write(path, content string) error {
w.writeCount++
if w.writeCount > w.failAfterN {
return fmt.Errorf("write %d: disk full", w.writeCount)
return testErrf("write %d: disk full", w.writeCount)
}
return w.MockMedium.Write(path, content)
}
func (w *writeCountMedium) EnsureDir(path string) error { return w.MockMedium.EnsureDir(path) }
func (w *writeCountMedium) Read(path string) (string, error) { return w.MockMedium.Read(path) }
func (w *writeCountMedium) List(path string) ([]fs.DirEntry, error) { return w.MockMedium.List(path) }
func (w *writeCountMedium) IsFile(path string) bool { return w.MockMedium.IsFile(path) }
func (w *writeCountMedium) FileGet(path string) (string, error) { return w.MockMedium.FileGet(path) }
func (w *writeCountMedium) FileSet(path, content string) error { return w.MockMedium.FileSet(path, content) }
func (w *writeCountMedium) Delete(path string) error { return w.MockMedium.Delete(path) }
func (w *writeCountMedium) DeleteAll(path string) error { return w.MockMedium.DeleteAll(path) }
func (w *writeCountMedium) Rename(old, new string) error { return w.MockMedium.Rename(old, new) }
func (w *writeCountMedium) Stat(path string) (fs.FileInfo, error) { return w.MockMedium.Stat(path) }
func (w *writeCountMedium) Open(path string) (fs.File, error) { return w.MockMedium.Open(path) }
func (w *writeCountMedium) Create(path string) (goio.WriteCloser, error) { return w.MockMedium.Create(path) }
func (w *writeCountMedium) Append(path string) (goio.WriteCloser, error) { return w.MockMedium.Append(path) }
func (w *writeCountMedium) ReadStream(path string) (goio.ReadCloser, error) { return w.MockMedium.ReadStream(path) }
func (w *writeCountMedium) WriteStream(path string) (goio.WriteCloser, error) { return w.MockMedium.WriteStream(path) }
func (w *writeCountMedium) Exists(path string) bool { return w.MockMedium.Exists(path) }
func (w *writeCountMedium) IsDir(path string) bool { return w.MockMedium.IsDir(path) }
func (w *writeCountMedium) EnsureDir(path string) error { return w.MockMedium.EnsureDir(path) }
func (w *writeCountMedium) Read(path string) (string, error) { return w.MockMedium.Read(path) }
func (w *writeCountMedium) List(path string) ([]fs.DirEntry, error) { return w.MockMedium.List(path) }
func (w *writeCountMedium) IsFile(path string) bool { return w.MockMedium.IsFile(path) }
func (w *writeCountMedium) FileGet(path string) (string, error) { return w.MockMedium.FileGet(path) }
func (w *writeCountMedium) FileSet(path, content string) error {
return w.MockMedium.FileSet(path, content)
}
func (w *writeCountMedium) Delete(path string) error { return w.MockMedium.Delete(path) }
func (w *writeCountMedium) DeleteAll(path string) error { return w.MockMedium.DeleteAll(path) }
func (w *writeCountMedium) Rename(old, new string) error { return w.MockMedium.Rename(old, new) }
func (w *writeCountMedium) Stat(path string) (fs.FileInfo, error) { return w.MockMedium.Stat(path) }
func (w *writeCountMedium) Open(path string) (fs.File, error) { return w.MockMedium.Open(path) }
func (w *writeCountMedium) Create(path string) (goio.WriteCloser, error) {
return w.MockMedium.Create(path)
}
func (w *writeCountMedium) Append(path string) (goio.WriteCloser, error) {
return w.MockMedium.Append(path)
}
func (w *writeCountMedium) ReadStream(path string) (goio.ReadCloser, error) {
return w.MockMedium.ReadStream(path)
}
func (w *writeCountMedium) WriteStream(path string) (goio.WriteCloser, error) {
return w.MockMedium.WriteStream(path)
}
func (w *writeCountMedium) Exists(path string) bool { return w.MockMedium.Exists(path) }
func (w *writeCountMedium) IsDir(path string) bool { return w.MockMedium.IsDir(path) }
// Test that the summary.md write error in collectCurrent is handled.
func TestMarketCollector_Collect_Bad_SummaryWriteError(t *testing.T) {
func TestMarketCollector_Collect_Bad_SummaryWriteError_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") {
@ -1035,7 +1058,7 @@ func TestMarketCollector_Collect_Bad_SummaryWriteError(t *testing.T) {
// --- Market: collectHistorical write error ---
func TestMarketCollector_Collect_Bad_HistoricalWriteError(t *testing.T) {
func TestMarketCollector_Collect_Bad_HistoricalWriteError_Good(t *testing.T) {
callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -1074,8 +1097,8 @@ func TestMarketCollector_Collect_Bad_HistoricalWriteError(t *testing.T) {
// --- State: Save write error ---
func TestState_Save_Bad_WriteError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
func TestState_Save_Bad_WriteError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")}
s := NewState(em, "/state.json")
s.Set("test", &StateEntry{Source: "test", Items: 1})
@ -1086,7 +1109,7 @@ func TestState_Save_Bad_WriteError(t *testing.T) {
// --- Excavator: collector with state error ---
func TestExcavator_Run_Bad_CollectorStateError(t *testing.T) {
func TestExcavator_Run_Bad_CollectorStateError_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.State = NewState(m, "/state.json")
@ -1108,7 +1131,7 @@ func TestExcavator_Run_Bad_CollectorStateError(t *testing.T) {
// --- BitcoinTalk: page returns zero posts (empty content) ---
func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage(t *testing.T) {
func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
// Valid HTML with no post divs at all
@ -1133,8 +1156,8 @@ func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage(t *testing.T) {
// --- Excavator: state save error after collection ---
func TestExcavator_Run_Bad_StateSaveError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("state write failed")}
func TestExcavator_Run_Bad_StateSaveError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("state write failed")}
cfg := &Config{
Output: io.NewMockMedium(), // Use regular medium for output
OutputDir: "/output",
@ -1157,8 +1180,8 @@ func TestExcavator_Run_Bad_StateSaveError(t *testing.T) {
// --- State: Load with read error ---
func TestState_Load_Bad_ReadError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: fmt.Errorf("read denied")}
func TestState_Load_Bad_ReadError_Good(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: testErr("read denied")}
em.MockMedium.Files["/state.json"] = "{}" // File exists but read will fail
s := NewState(em, "/state.json")
@ -1169,7 +1192,7 @@ func TestState_Load_Bad_ReadError(t *testing.T) {
// --- Papers: PaperSourceAll emits complete ---
func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR(t *testing.T) {
func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -1206,7 +1229,7 @@ func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR(t *testing.T) {
// --- Papers: IACR with cancelled context (request creation fails) ---
func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails_Good(t *testing.T) {
// Don't set up any server - the request should fail because context is cancelled.
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -1222,7 +1245,7 @@ func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails(t *testing
// --- Papers: arXiv with cancelled context ---
func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -1237,7 +1260,7 @@ func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails(t *testin
// --- Market: collectHistorical limiter blocks ---
func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks(t *testing.T) {
func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{
@ -1276,7 +1299,7 @@ func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks(t *testing.T) {
// --- BitcoinTalk: fetchPage with invalid URL ---
func TestBitcoinTalkCollector_FetchPage_Bad_InvalidURL(t *testing.T) {
func TestBitcoinTalkCollector_FetchPage_Bad_InvalidURL_Good(t *testing.T) {
b := &BitcoinTalkCollector{TopicID: "12345"}
// Use a URL with control character that will fail NewRequestWithContext
_, err := b.fetchPage(context.Background(), "http://\x7f/invalid")

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -8,18 +10,23 @@ import (
// Event types used by the collection subsystem.
const (
// EventStart is emitted when a collector begins its run.
//
EventStart = "start"
// EventProgress is emitted to report incremental progress.
//
EventProgress = "progress"
// EventItem is emitted when a single item is collected.
//
EventItem = "item"
// EventError is emitted when an error occurs during collection.
//
EventError = "error"
// EventComplete is emitted when a collector finishes its run.
//
EventComplete = "complete"
)
@ -52,6 +59,7 @@ type Dispatcher struct {
}
// NewDispatcher creates a new event dispatcher.
// Usage: NewDispatcher(...)
func NewDispatcher() *Dispatcher {
return &Dispatcher{
handlers: make(map[string][]EventHandler),
@ -60,6 +68,7 @@ func NewDispatcher() *Dispatcher {
// On registers a handler for an event type. Multiple handlers can be
// registered for the same event type and will be called in order.
// Usage: On(...)
func (d *Dispatcher) On(eventType string, handler EventHandler) {
d.mu.Lock()
defer d.mu.Unlock()
@ -69,6 +78,7 @@ func (d *Dispatcher) On(eventType string, handler EventHandler) {
// Emit dispatches an event to all registered handlers for that event type.
// If no handlers are registered for the event type, the event is silently dropped.
// The event's Time field is set to now if it is zero.
// Usage: Emit(...)
func (d *Dispatcher) Emit(event Event) {
if event.Time.IsZero() {
event.Time = time.Now()
@ -84,6 +94,7 @@ func (d *Dispatcher) Emit(event Event) {
}
// EmitStart emits a start event for the given source.
// Usage: EmitStart(...)
func (d *Dispatcher) EmitStart(source, message string) {
d.Emit(Event{
Type: EventStart,
@ -93,6 +104,7 @@ func (d *Dispatcher) EmitStart(source, message string) {
}
// EmitProgress emits a progress event.
// Usage: EmitProgress(...)
func (d *Dispatcher) EmitProgress(source, message string, data any) {
d.Emit(Event{
Type: EventProgress,
@ -103,6 +115,7 @@ func (d *Dispatcher) EmitProgress(source, message string, data any) {
}
// EmitItem emits an item event.
// Usage: EmitItem(...)
func (d *Dispatcher) EmitItem(source, message string, data any) {
d.Emit(Event{
Type: EventItem,
@ -113,6 +126,7 @@ func (d *Dispatcher) EmitItem(source, message string, data any) {
}
// EmitError emits an error event.
// Usage: EmitError(...)
func (d *Dispatcher) EmitError(source, message string, data any) {
d.Emit(Event{
Type: EventError,
@ -123,6 +137,7 @@ func (d *Dispatcher) EmitError(source, message string, data any) {
}
// EmitComplete emits a complete event.
// Usage: EmitComplete(...)
func (d *Dispatcher) EmitComplete(source, message string, data any) {
d.Emit(Event{
Type: EventComplete,

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -41,7 +43,7 @@ func TestDispatcher_On_Good(t *testing.T) {
assert.Equal(t, 3, count, "All three handlers should be called")
}
func TestDispatcher_Emit_Good_NoHandlers(t *testing.T) {
func TestDispatcher_Emit_Good_NoHandlers_Good(t *testing.T) {
d := NewDispatcher()
// Should not panic when emitting an event with no handlers
@ -54,7 +56,7 @@ func TestDispatcher_Emit_Good_NoHandlers(t *testing.T) {
})
}
func TestDispatcher_Emit_Good_MultipleEventTypes(t *testing.T) {
func TestDispatcher_Emit_Good_MultipleEventTypes_Good(t *testing.T) {
d := NewDispatcher()
var starts, errors int
@ -69,7 +71,7 @@ func TestDispatcher_Emit_Good_MultipleEventTypes(t *testing.T) {
assert.Equal(t, 1, errors)
}
func TestDispatcher_Emit_Good_SetsTime(t *testing.T) {
func TestDispatcher_Emit_Good_SetsTime_Good(t *testing.T) {
d := NewDispatcher()
var received Event
@ -85,7 +87,7 @@ func TestDispatcher_Emit_Good_SetsTime(t *testing.T) {
assert.True(t, received.Time.Before(after) || received.Time.Equal(after))
}
func TestDispatcher_Emit_Good_PreservesExistingTime(t *testing.T) {
func TestDispatcher_Emit_Good_PreservesExistingTime_Good(t *testing.T) {
d := NewDispatcher()
customTime := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"time"
core "dappco.re/go/core/log"
@ -23,12 +25,14 @@ type Excavator struct {
}
// Name returns the orchestrator name.
// Usage: Name(...)
func (e *Excavator) Name() string {
return "excavator"
}
// Run executes all collectors sequentially, respecting rate limits and
// using state for resume support. Results are aggregated from all collectors.
// Usage: Run(...)
func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: e.Name()}
@ -39,9 +43,11 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitStart(e.Name(), fmt.Sprintf("Starting excavation with %d collectors", len(e.Collectors)))
}
verboseProgress(cfg, e.Name(), fmt.Sprintf("queueing %d collectors", len(e.Collectors)))
// Load state if resuming
if e.Resume && cfg.State != nil {
verboseProgress(cfg, e.Name(), "loading resume state")
if err := cfg.State.Load(); err != nil {
return result, core.E("collect.Excavator.Run", "failed to load state", err)
}
@ -53,6 +59,7 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitProgress(e.Name(), fmt.Sprintf("[scan] Would run collector: %s", c.Name()), nil)
}
verboseProgress(cfg, e.Name(), fmt.Sprintf("scan-only collector: %s", c.Name()))
}
return result, nil
}
@ -66,6 +73,7 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
cfg.Dispatcher.EmitProgress(e.Name(),
fmt.Sprintf("Running collector %d/%d: %s", i+1, len(e.Collectors), c.Name()), nil)
}
verboseProgress(cfg, e.Name(), fmt.Sprintf("dispatching collector %d/%d: %s", i+1, len(e.Collectors), c.Name()))
// Check if we should skip (already completed in a previous run)
if e.Resume && cfg.State != nil {
@ -76,6 +84,7 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
fmt.Sprintf("Skipping %s (already collected %d items on %s)",
c.Name(), entry.Items, entry.LastRun.Format(time.RFC3339)), nil)
}
verboseProgress(cfg, e.Name(), fmt.Sprintf("resume skip: %s", c.Name()))
result.Skipped++
continue
}
@ -111,6 +120,7 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
// Save state
if cfg.State != nil {
verboseProgress(cfg, e.Name(), "saving resume state")
if err := cfg.State.Save(); err != nil {
if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitError(e.Name(), fmt.Sprintf("Failed to save state: %v", err), nil)

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -10,7 +12,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestExcavator_Run_Good_ResumeSkipsCompleted(t *testing.T) {
func TestExcavator_Run_Good_ResumeSkipsCompleted_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -39,7 +41,7 @@ func TestExcavator_Run_Good_ResumeSkipsCompleted(t *testing.T) {
assert.Equal(t, 1, result.Skipped)
}
func TestExcavator_Run_Good_ResumeRunsIncomplete(t *testing.T) {
func TestExcavator_Run_Good_ResumeRunsIncomplete_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -65,7 +67,7 @@ func TestExcavator_Run_Good_ResumeRunsIncomplete(t *testing.T) {
assert.Equal(t, 5, result.Items)
}
func TestExcavator_Run_Good_NilState(t *testing.T) {
func TestExcavator_Run_Good_NilState_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.State = nil
@ -83,7 +85,7 @@ func TestExcavator_Run_Good_NilState(t *testing.T) {
assert.Equal(t, 3, result.Items)
}
func TestExcavator_Run_Good_NilDispatcher(t *testing.T) {
func TestExcavator_Run_Good_NilDispatcher_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Dispatcher = nil
@ -101,7 +103,7 @@ func TestExcavator_Run_Good_NilDispatcher(t *testing.T) {
assert.Equal(t, 2, result.Items)
}
func TestExcavator_Run_Good_ProgressEvents(t *testing.T) {
func TestExcavator_Run_Good_ProgressEvents_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil

View file

@ -1,8 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
core "dappco.re/go/core"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"testing"
"dappco.re/go/core/io"
@ -63,7 +66,7 @@ func TestExcavator_Run_Good(t *testing.T) {
assert.Len(t, result.Files, 8)
}
func TestExcavator_Run_Good_Empty(t *testing.T) {
func TestExcavator_Run_Good_Empty_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -74,7 +77,7 @@ func TestExcavator_Run_Good_Empty(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestExcavator_Run_Good_DryRun(t *testing.T) {
func TestExcavator_Run_Good_DryRun_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -95,7 +98,7 @@ func TestExcavator_Run_Good_DryRun(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestExcavator_Run_Good_ScanOnly(t *testing.T) {
func TestExcavator_Run_Good_ScanOnly_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -120,13 +123,13 @@ func TestExcavator_Run_Good_ScanOnly(t *testing.T) {
assert.Contains(t, progressMessages[0], "source-a")
}
func TestExcavator_Run_Good_WithErrors(t *testing.T) {
func TestExcavator_Run_Good_WithErrors_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
c1 := &mockCollector{name: "good", items: 5}
c2 := &mockCollector{name: "bad", err: fmt.Errorf("network error")}
c2 := &mockCollector{name: "bad", err: core.E("collect.mockCollector.Collect", "network error", nil)}
c3 := &mockCollector{name: "also-good", items: 3}
e := &Excavator{
@ -143,7 +146,7 @@ func TestExcavator_Run_Good_WithErrors(t *testing.T) {
assert.True(t, c3.called)
}
func TestExcavator_Run_Good_CancelledContext(t *testing.T) {
func TestExcavator_Run_Good_CancelledContext_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -160,7 +163,7 @@ func TestExcavator_Run_Good_CancelledContext(t *testing.T) {
assert.Error(t, err)
}
func TestExcavator_Run_Good_SavesState(t *testing.T) {
func TestExcavator_Run_Good_SavesState_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -181,7 +184,7 @@ func TestExcavator_Run_Good_SavesState(t *testing.T) {
assert.Equal(t, "source-a", entry.Source)
}
func TestExcavator_Run_Good_Events(t *testing.T) {
func TestExcavator_Run_Good_Events_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
@ -200,3 +203,24 @@ func TestExcavator_Run_Good_Events(t *testing.T) {
assert.Equal(t, 1, startCount)
assert.Equal(t, 1, completeCount)
}
func TestExcavator_Run_Good_VerboseProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
cfg.Verbose = true
var progressCount int
cfg.Dispatcher.On(EventProgress, func(e Event) {
progressCount++
})
c1 := &mockCollector{name: "source-a", items: 1}
e := &Excavator{
Collectors: []Collector{c1},
}
_, err := e.Run(context.Background(), cfg)
assert.NoError(t, err)
assert.GreaterOrEqual(t, progressCount, 2)
}

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
"fmt"
"os/exec"
"path/filepath"
"strings"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"time"
core "dappco.re/go/core/log"
@ -53,6 +55,7 @@ type GitHubCollector struct {
}
// Name returns the collector name.
// Usage: Name(...)
func (g *GitHubCollector) Name() string {
if g.Repo != "" {
return fmt.Sprintf("github:%s/%s", g.Org, g.Repo)
@ -61,6 +64,7 @@ func (g *GitHubCollector) Name() string {
}
// Collect gathers issues and/or PRs from GitHub repositories.
// Usage: Collect(...)
func (g *GitHubCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: g.Name()}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -14,12 +16,12 @@ func TestGitHubCollector_Name_Good(t *testing.T) {
assert.Equal(t, "github:host-uk/core", g.Name())
}
func TestGitHubCollector_Name_Good_OrgOnly(t *testing.T) {
func TestGitHubCollector_Name_Good_OrgOnly_Good(t *testing.T) {
g := &GitHubCollector{Org: "host-uk"}
assert.Equal(t, "github:host-uk", g.Name())
}
func TestGitHubCollector_Collect_Good_DryRun(t *testing.T) {
func TestGitHubCollector_Collect_Good_DryRun_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -38,7 +40,7 @@ func TestGitHubCollector_Collect_Good_DryRun(t *testing.T) {
assert.True(t, progressEmitted, "Should emit progress event in dry-run mode")
}
func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly(t *testing.T) {
func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -50,7 +52,7 @@ func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestGitHubCollector_Collect_Good_DryRun_PRsOnly(t *testing.T) {
func TestGitHubCollector_Collect_Good_DryRun_PRsOnly_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -88,7 +90,7 @@ func TestFormatIssueMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "**URL:** https://github.com/test/repo/issues/42")
}
func TestFormatIssueMarkdown_Good_NoLabels(t *testing.T) {
func TestFormatIssueMarkdown_Good_NoLabels_Good(t *testing.T) {
issue := ghIssue{
Number: 1,
Title: "Simple",

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
"fmt"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http"
"path/filepath"
"strings"
"time"
core "dappco.re/go/core/log"
@ -29,6 +31,7 @@ type MarketCollector struct {
}
// Name returns the collector name.
// Usage: Name(...)
func (m *MarketCollector) Name() string {
return fmt.Sprintf("market:%s", m.CoinID)
}
@ -63,6 +66,7 @@ type historicalData struct {
}
// Collect gathers market data from CoinGecko.
// Usage: Collect(...)
func (m *MarketCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: m.Name()}
@ -272,6 +276,7 @@ func formatMarketSummary(data *coinData) string {
}
// FormatMarketSummary is exported for testing.
// Usage: FormatMarketSummary(...)
func FormatMarketSummary(data *coinData) string {
return formatMarketSummary(data)
}

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"net/http"
"net/http/httptest"
"testing"
@ -12,7 +14,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestMarketCollector_Collect_Good_HistoricalWithFromDate(t *testing.T) {
func TestMarketCollector_Collect_Good_HistoricalWithFromDate_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -56,7 +58,7 @@ func TestMarketCollector_Collect_Good_HistoricalWithFromDate(t *testing.T) {
assert.Equal(t, 3, result.Items)
}
func TestMarketCollector_Collect_Good_HistoricalInvalidDate(t *testing.T) {
func TestMarketCollector_Collect_Good_HistoricalInvalidDate_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -98,7 +100,7 @@ func TestMarketCollector_Collect_Good_HistoricalInvalidDate(t *testing.T) {
assert.Equal(t, 3, result.Items)
}
func TestMarketCollector_Collect_Bad_HistoricalServerError(t *testing.T) {
func TestMarketCollector_Collect_Bad_HistoricalServerError_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -137,7 +139,7 @@ func TestMarketCollector_Collect_Bad_HistoricalServerError(t *testing.T) {
assert.Equal(t, 1, result.Errors) // historical failed
}
func TestMarketCollector_Collect_Good_EmitsEvents(t *testing.T) {
func TestMarketCollector_Collect_Good_EmitsEvents_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
data := coinData{
@ -172,7 +174,7 @@ func TestMarketCollector_Collect_Good_EmitsEvents(t *testing.T) {
assert.Equal(t, 1, completes)
}
func TestMarketCollector_Collect_Good_CancelledContext(t *testing.T) {
func TestMarketCollector_Collect_Good_CancelledContext_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
@ -197,7 +199,7 @@ func TestMarketCollector_Collect_Good_CancelledContext(t *testing.T) {
assert.Equal(t, 1, result.Errors)
}
func TestFormatMarketSummary_Good_AllFields(t *testing.T) {
func TestFormatMarketSummary_Good_AllFields_Good(t *testing.T) {
data := &coinData{
Name: "Lethean",
Symbol: "lthn",
@ -229,7 +231,7 @@ func TestFormatMarketSummary_Good_AllFields(t *testing.T) {
assert.Contains(t, summary, "Last updated")
}
func TestFormatMarketSummary_Good_Minimal(t *testing.T) {
func TestFormatMarketSummary_Good_Minimal_Good(t *testing.T) {
data := &coinData{
Name: "Unknown",
Symbol: "ukn",

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"net/http"
"net/http/httptest"
"testing"
@ -16,7 +18,7 @@ func TestMarketCollector_Name_Good(t *testing.T) {
assert.Equal(t, "market:bitcoin", m.Name())
}
func TestMarketCollector_Collect_Bad_NoCoinID(t *testing.T) {
func TestMarketCollector_Collect_Bad_NoCoinID_Good(t *testing.T) {
mock := io.NewMockMedium()
cfg := NewConfigWithMedium(mock, "/output")
@ -25,7 +27,7 @@ func TestMarketCollector_Collect_Bad_NoCoinID(t *testing.T) {
assert.Error(t, err)
}
func TestMarketCollector_Collect_Good_DryRun(t *testing.T) {
func TestMarketCollector_Collect_Good_DryRun_Good(t *testing.T) {
mock := io.NewMockMedium()
cfg := NewConfigWithMedium(mock, "/output")
cfg.DryRun = true
@ -37,7 +39,7 @@ func TestMarketCollector_Collect_Good_DryRun(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestMarketCollector_Collect_Good_CurrentData(t *testing.T) {
func TestMarketCollector_Collect_Good_CurrentData_Good(t *testing.T) {
// Set up a mock CoinGecko server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
data := coinData{
@ -92,7 +94,7 @@ func TestMarketCollector_Collect_Good_CurrentData(t *testing.T) {
assert.Contains(t, summary, "42000.50")
}
func TestMarketCollector_Collect_Good_Historical(t *testing.T) {
func TestMarketCollector_Collect_Good_Historical_Good(t *testing.T) {
callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -164,7 +166,7 @@ func TestFormatMarketSummary_Good(t *testing.T) {
assert.Contains(t, summary, "Total Supply")
}
func TestMarketCollector_Collect_Bad_ServerError(t *testing.T) {
func TestMarketCollector_Collect_Bad_ServerError_Good(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))

View file

@ -1,14 +1,16 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"encoding/xml"
"fmt"
"iter"
"net/http"
"net/url"
"path/filepath"
"strings"
core "dappco.re/go/core/log"
"golang.org/x/net/html"
@ -16,9 +18,12 @@ import (
// Paper source identifiers.
const (
PaperSourceIACR = "iacr"
//
PaperSourceIACR = "iacr"
//
PaperSourceArXiv = "arxiv"
PaperSourceAll = "all"
//
PaperSourceAll = "all"
)
// PapersCollector collects papers from IACR and arXiv.
@ -34,6 +39,7 @@ type PapersCollector struct {
}
// Name returns the collector name.
// Usage: Name(...)
func (p *PapersCollector) Name() string {
return fmt.Sprintf("papers:%s", p.Source)
}
@ -50,6 +56,7 @@ type paper struct {
}
// Collect gathers papers from the configured sources.
// Usage: Collect(...)
func (p *PapersCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: p.Name()}
@ -403,6 +410,7 @@ func formatPaperMarkdown(ppr paper) string {
}
// FormatPaperMarkdown is exported for testing.
// Usage: FormatPaperMarkdown(...)
func FormatPaperMarkdown(title string, authors []string, date, paperURL, source, abstract string) string {
return formatPaperMarkdown(paper{
Title: title,

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http"
"net/http/httptest"
"strings"
"testing"
"dappco.re/go/core/io"
@ -109,7 +111,7 @@ func TestPapersCollector_CollectArXiv_Good(t *testing.T) {
assert.Contains(t, content, "Alice")
}
func TestPapersCollector_CollectArXiv_Good_WithCategory(t *testing.T) {
func TestPapersCollector_CollectArXiv_Good_WithCategory_Good(t *testing.T) {
var capturedQuery string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
capturedQuery = r.URL.RawQuery
@ -165,7 +167,7 @@ func TestPapersCollector_CollectAll_Good(t *testing.T) {
assert.Equal(t, 4, result.Items) // 2 IACR + 2 arXiv
}
func TestPapersCollector_CollectIACR_Bad_ServerError(t *testing.T) {
func TestPapersCollector_CollectIACR_Bad_ServerError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
@ -185,7 +187,7 @@ func TestPapersCollector_CollectIACR_Bad_ServerError(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_CollectArXiv_Bad_ServerError(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_ServerError_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
}))
@ -205,7 +207,7 @@ func TestPapersCollector_CollectArXiv_Bad_ServerError(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_CollectArXiv_Bad_InvalidXML(t *testing.T) {
func TestPapersCollector_CollectArXiv_Bad_InvalidXML_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(`not xml at all`))
@ -226,7 +228,7 @@ func TestPapersCollector_CollectArXiv_Bad_InvalidXML(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_CollectAll_Bad_BothFail(t *testing.T) {
func TestPapersCollector_CollectAll_Bad_BothFail_Good(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
}))
@ -246,7 +248,7 @@ func TestPapersCollector_CollectAll_Bad_BothFail(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_CollectAll_Good_OneFails(t *testing.T) {
func TestPapersCollector_CollectAll_Good_OneFails_Good(t *testing.T) {
callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++
@ -295,7 +297,7 @@ func TestExtractIACRPapers_Good(t *testing.T) {
assert.Equal(t, "Lattice Cryptography", papers[1].Title)
}
func TestExtractIACRPapers_Good_Empty(t *testing.T) {
func TestExtractIACRPapers_Good_Empty_Good(t *testing.T) {
doc, err := html.Parse(strings.NewReader(`<html><body></body></html>`))
require.NoError(t, err)
@ -303,7 +305,7 @@ func TestExtractIACRPapers_Good_Empty(t *testing.T) {
assert.Empty(t, papers)
}
func TestExtractIACRPapers_Good_NoTitle(t *testing.T) {
func TestExtractIACRPapers_Good_NoTitle_Good(t *testing.T) {
doc, err := html.Parse(strings.NewReader(`<html><body><div class="paperentry"></div></body></html>`))
require.NoError(t, err)

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -13,17 +15,17 @@ func TestPapersCollector_Name_Good(t *testing.T) {
assert.Equal(t, "papers:iacr", p.Name())
}
func TestPapersCollector_Name_Good_ArXiv(t *testing.T) {
func TestPapersCollector_Name_Good_ArXiv_Good(t *testing.T) {
p := &PapersCollector{Source: PaperSourceArXiv}
assert.Equal(t, "papers:arxiv", p.Name())
}
func TestPapersCollector_Name_Good_All(t *testing.T) {
func TestPapersCollector_Name_Good_All_Good(t *testing.T) {
p := &PapersCollector{Source: PaperSourceAll}
assert.Equal(t, "papers:all", p.Name())
}
func TestPapersCollector_Collect_Bad_NoQuery(t *testing.T) {
func TestPapersCollector_Collect_Bad_NoQuery_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -32,7 +34,7 @@ func TestPapersCollector_Collect_Bad_NoQuery(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_Collect_Bad_UnknownSource(t *testing.T) {
func TestPapersCollector_Collect_Bad_UnknownSource_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -41,7 +43,7 @@ func TestPapersCollector_Collect_Bad_UnknownSource(t *testing.T) {
assert.Error(t, err)
}
func TestPapersCollector_Collect_Good_DryRun(t *testing.T) {
func TestPapersCollector_Collect_Good_DryRun_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -72,7 +74,7 @@ func TestFormatPaperMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "zero-knowledge proofs")
}
func TestFormatPaperMarkdown_Good_Minimal(t *testing.T) {
func TestFormatPaperMarkdown_Good_Minimal_Good(t *testing.T) {
md := FormatPaperMarkdown("Title Only", nil, "", "", "", "")
assert.Contains(t, md, "# Title Only")

View file

@ -1,13 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"encoding/json"
"fmt"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"maps"
"path/filepath"
"slices"
"strings"
core "dappco.re/go/core/log"
"golang.org/x/net/html"
@ -23,12 +25,14 @@ type Processor struct {
}
// Name returns the processor name.
// Usage: Name(...)
func (p *Processor) Name() string {
return fmt.Sprintf("process:%s", p.Source)
}
// Process reads files from the source directory, converts HTML or JSON
// to clean markdown, and writes the results to the output directory.
// Usage: Process(...)
func (p *Processor) Process(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: p.Name()}
@ -331,11 +335,13 @@ func jsonValueToMarkdown(b *strings.Builder, data any, depth int) {
}
// HTMLToMarkdown is exported for testing.
// Usage: HTMLToMarkdown(...)
func HTMLToMarkdown(content string) (string, error) {
return htmlToMarkdown(content)
}
// JSONToMarkdown is exported for testing.
// Usage: JSONToMarkdown(...)
func JSONToMarkdown(content string) (string, error) {
return jsonToMarkdown(content)
}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -9,7 +11,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestHTMLToMarkdown_Good_OrderedList(t *testing.T) {
func TestHTMLToMarkdown_Good_OrderedList_Good(t *testing.T) {
input := `<ol><li>First</li><li>Second</li><li>Third</li></ol>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -18,7 +20,7 @@ func TestHTMLToMarkdown_Good_OrderedList(t *testing.T) {
assert.Contains(t, result, "3. Third")
}
func TestHTMLToMarkdown_Good_UnorderedList(t *testing.T) {
func TestHTMLToMarkdown_Good_UnorderedList_Good(t *testing.T) {
input := `<ul><li>Alpha</li><li>Beta</li></ul>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -26,21 +28,21 @@ func TestHTMLToMarkdown_Good_UnorderedList(t *testing.T) {
assert.Contains(t, result, "- Beta")
}
func TestHTMLToMarkdown_Good_Blockquote(t *testing.T) {
func TestHTMLToMarkdown_Good_Blockquote_Good(t *testing.T) {
input := `<blockquote>A wise quote</blockquote>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "> A wise quote")
}
func TestHTMLToMarkdown_Good_HorizontalRule(t *testing.T) {
func TestHTMLToMarkdown_Good_HorizontalRule_Good(t *testing.T) {
input := `<p>Before</p><hr/><p>After</p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "---")
}
func TestHTMLToMarkdown_Good_LinkWithoutHref(t *testing.T) {
func TestHTMLToMarkdown_Good_LinkWithoutHref_Good(t *testing.T) {
input := `<a>bare link text</a>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -48,7 +50,7 @@ func TestHTMLToMarkdown_Good_LinkWithoutHref(t *testing.T) {
assert.NotContains(t, result, "[")
}
func TestHTMLToMarkdown_Good_H4H5H6(t *testing.T) {
func TestHTMLToMarkdown_Good_H4H5H6_Good(t *testing.T) {
input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -57,7 +59,7 @@ func TestHTMLToMarkdown_Good_H4H5H6(t *testing.T) {
assert.Contains(t, result, "###### H6")
}
func TestHTMLToMarkdown_Good_StripsStyle(t *testing.T) {
func TestHTMLToMarkdown_Good_StripsStyle_Good(t *testing.T) {
input := `<html><head><style>.foo{color:red}</style></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -65,7 +67,7 @@ func TestHTMLToMarkdown_Good_StripsStyle(t *testing.T) {
assert.NotContains(t, result, "color")
}
func TestHTMLToMarkdown_Good_LineBreak(t *testing.T) {
func TestHTMLToMarkdown_Good_LineBreak_Good(t *testing.T) {
input := `<p>Line one<br/>Line two</p>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -73,7 +75,7 @@ func TestHTMLToMarkdown_Good_LineBreak(t *testing.T) {
assert.Contains(t, result, "Line two")
}
func TestHTMLToMarkdown_Good_NestedBoldItalic(t *testing.T) {
func TestHTMLToMarkdown_Good_NestedBoldItalic_Good(t *testing.T) {
input := `<b>bold text</b> and <i>italic text</i>`
result, err := HTMLToMarkdown(input)
require.NoError(t, err)
@ -81,7 +83,7 @@ func TestHTMLToMarkdown_Good_NestedBoldItalic(t *testing.T) {
assert.Contains(t, result, "*italic text*")
}
func TestJSONToMarkdown_Good_NestedObject(t *testing.T) {
func TestJSONToMarkdown_Good_NestedObject_Good(t *testing.T) {
input := `{"outer": {"inner_key": "inner_value"}}`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
@ -89,7 +91,7 @@ func TestJSONToMarkdown_Good_NestedObject(t *testing.T) {
assert.Contains(t, result, "**inner_key:** inner_value")
}
func TestJSONToMarkdown_Good_NestedArray(t *testing.T) {
func TestJSONToMarkdown_Good_NestedArray_Good(t *testing.T) {
input := `[["a", "b"], ["c"]]`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
@ -98,14 +100,14 @@ func TestJSONToMarkdown_Good_NestedArray(t *testing.T) {
assert.Contains(t, result, "b")
}
func TestJSONToMarkdown_Good_ScalarValue(t *testing.T) {
func TestJSONToMarkdown_Good_ScalarValue_Good(t *testing.T) {
input := `42`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
assert.Contains(t, result, "42")
}
func TestJSONToMarkdown_Good_ArrayOfObjects(t *testing.T) {
func TestJSONToMarkdown_Good_ArrayOfObjects_Good(t *testing.T) {
input := `[{"name": "Alice"}, {"name": "Bob"}]`
result, err := JSONToMarkdown(input)
require.NoError(t, err)
@ -115,7 +117,7 @@ func TestJSONToMarkdown_Good_ArrayOfObjects(t *testing.T) {
assert.Contains(t, result, "Bob")
}
func TestProcessor_Process_Good_CancelledContext(t *testing.T) {
func TestProcessor_Process_Good_CancelledContext_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/file.html"] = `<h1>Test</h1>`
@ -131,7 +133,7 @@ func TestProcessor_Process_Good_CancelledContext(t *testing.T) {
assert.Error(t, err)
}
func TestProcessor_Process_Good_EmitsEvents(t *testing.T) {
func TestProcessor_Process_Good_EmitsEvents_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/a.html"] = `<h1>Title</h1>`
@ -155,7 +157,7 @@ func TestProcessor_Process_Good_EmitsEvents(t *testing.T) {
assert.Equal(t, 1, completes)
}
func TestProcessor_Process_Good_BadHTML(t *testing.T) {
func TestProcessor_Process_Good_BadHTML_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
// html.Parse is very tolerant, so even bad HTML will parse. But we test
@ -172,7 +174,7 @@ func TestProcessor_Process_Good_BadHTML(t *testing.T) {
assert.Equal(t, 1, result.Items)
}
func TestProcessor_Process_Good_BadJSON(t *testing.T) {
func TestProcessor_Process_Good_BadJSON_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/bad.json"] = `not valid json`

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -13,7 +15,7 @@ func TestProcessor_Name_Good(t *testing.T) {
assert.Equal(t, "process:github", p.Name())
}
func TestProcessor_Process_Bad_NoDir(t *testing.T) {
func TestProcessor_Process_Bad_NoDir_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
@ -22,7 +24,7 @@ func TestProcessor_Process_Bad_NoDir(t *testing.T) {
assert.Error(t, err)
}
func TestProcessor_Process_Good_DryRun(t *testing.T) {
func TestProcessor_Process_Good_DryRun_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true
@ -34,7 +36,7 @@ func TestProcessor_Process_Good_DryRun(t *testing.T) {
assert.Equal(t, 0, result.Items)
}
func TestProcessor_Process_Good_HTMLFiles(t *testing.T) {
func TestProcessor_Process_Good_HTMLFiles_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/page.html"] = `<html><body><h1>Hello</h1><p>World</p></body></html>`
@ -55,7 +57,7 @@ func TestProcessor_Process_Good_HTMLFiles(t *testing.T) {
assert.Contains(t, content, "World")
}
func TestProcessor_Process_Good_JSONFiles(t *testing.T) {
func TestProcessor_Process_Good_JSONFiles_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/data.json"] = `{"name": "Bitcoin", "price": 42000}`
@ -75,7 +77,7 @@ func TestProcessor_Process_Good_JSONFiles(t *testing.T) {
assert.Contains(t, content, "Bitcoin")
}
func TestProcessor_Process_Good_MarkdownPassthrough(t *testing.T) {
func TestProcessor_Process_Good_MarkdownPassthrough_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/readme.md"] = "# Already Markdown\n\nThis is already formatted."
@ -94,7 +96,7 @@ func TestProcessor_Process_Good_MarkdownPassthrough(t *testing.T) {
assert.Contains(t, content, "# Already Markdown")
}
func TestProcessor_Process_Good_SkipUnknownTypes(t *testing.T) {
func TestProcessor_Process_Good_SkipUnknownTypes_Good(t *testing.T) {
m := io.NewMockMedium()
m.Dirs["/input"] = true
m.Files["/input/image.png"] = "binary data"
@ -170,7 +172,7 @@ func TestHTMLToMarkdown_Good(t *testing.T) {
}
}
func TestHTMLToMarkdown_Good_StripsScripts(t *testing.T) {
func TestHTMLToMarkdown_Good_StripsScripts_Good(t *testing.T) {
input := `<html><head><script>alert('xss')</script></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input)
assert.NoError(t, err)
@ -188,14 +190,14 @@ func TestJSONToMarkdown_Good(t *testing.T) {
assert.Contains(t, result, "42")
}
func TestJSONToMarkdown_Good_Array(t *testing.T) {
func TestJSONToMarkdown_Good_Array_Good(t *testing.T) {
input := `[{"id": 1}, {"id": 2}]`
result, err := JSONToMarkdown(input)
assert.NoError(t, err)
assert.Contains(t, result, "# Data")
}
func TestJSONToMarkdown_Bad_InvalidJSON(t *testing.T) {
func TestJSONToMarkdown_Bad_InvalidJSON_Good(t *testing.T) {
_, err := JSONToMarkdown("not json")
assert.Error(t, err)
}

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"context"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"maps"
"os/exec"
"strconv"
"strings"
"sync"
"time"
@ -30,6 +32,7 @@ var defaultDelays = map[string]time.Duration{
}
// NewRateLimiter creates a limiter with default delays.
// Usage: NewRateLimiter(...)
func NewRateLimiter() *RateLimiter {
delays := make(map[string]time.Duration, len(defaultDelays))
maps.Copy(delays, defaultDelays)
@ -41,6 +44,7 @@ func NewRateLimiter() *RateLimiter {
// Wait blocks until the rate limit allows the next request for the given source.
// It respects context cancellation.
// Usage: Wait(...)
func (r *RateLimiter) Wait(ctx context.Context, source string) error {
r.mu.Lock()
delay, ok := r.delays[source]
@ -75,6 +79,7 @@ func (r *RateLimiter) Wait(ctx context.Context, source string) error {
}
// SetDelay sets the delay for a source.
// Usage: SetDelay(...)
func (r *RateLimiter) SetDelay(source string, d time.Duration) {
r.mu.Lock()
defer r.mu.Unlock()
@ -82,6 +87,7 @@ func (r *RateLimiter) SetDelay(source string, d time.Duration) {
}
// GetDelay returns the delay configured for a source.
// Usage: GetDelay(...)
func (r *RateLimiter) GetDelay(source string) time.Duration {
r.mu.Lock()
defer r.mu.Unlock()
@ -95,6 +101,7 @@ func (r *RateLimiter) GetDelay(source string) time.Duration {
// Returns used and limit counts. Auto-pauses at 75% usage by increasing
// the GitHub rate limit delay.
// Deprecated: Use CheckGitHubRateLimitCtx for context-aware cancellation.
// Usage: CheckGitHubRateLimit(...)
func (r *RateLimiter) CheckGitHubRateLimit() (used, limit int, err error) {
return r.CheckGitHubRateLimitCtx(context.Background())
}
@ -102,6 +109,7 @@ func (r *RateLimiter) CheckGitHubRateLimit() (used, limit int, err error) {
// CheckGitHubRateLimitCtx checks GitHub API rate limit status via gh api with context support.
// Returns used and limit counts. Auto-pauses at 75% usage by increasing
// the GitHub rate limit delay.
// Usage: CheckGitHubRateLimitCtx(...)
func (r *RateLimiter) CheckGitHubRateLimitCtx(ctx context.Context) (used, limit int, err error) {
cmd := exec.CommandContext(ctx, "gh", "api", "rate_limit", "--jq", ".rate | \"\\(.used) \\(.limit)\"")
out, err := cmd.Output()

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -27,7 +29,7 @@ func TestRateLimiter_Wait_Good(t *testing.T) {
assert.GreaterOrEqual(t, time.Since(start), 40*time.Millisecond) // allow small timing variance
}
func TestRateLimiter_Wait_Bad_ContextCancelled(t *testing.T) {
func TestRateLimiter_Wait_Bad_ContextCancelled_Good(t *testing.T) {
rl := NewRateLimiter()
rl.SetDelay("test", 5*time.Second)
@ -51,7 +53,7 @@ func TestRateLimiter_SetDelay_Good(t *testing.T) {
assert.Equal(t, 3*time.Second, rl.GetDelay("custom"))
}
func TestRateLimiter_GetDelay_Good_Defaults(t *testing.T) {
func TestRateLimiter_GetDelay_Good_Defaults_Good(t *testing.T) {
rl := NewRateLimiter()
assert.Equal(t, 500*time.Millisecond, rl.GetDelay("github"))
@ -60,13 +62,13 @@ func TestRateLimiter_GetDelay_Good_Defaults(t *testing.T) {
assert.Equal(t, 1*time.Second, rl.GetDelay("iacr"))
}
func TestRateLimiter_GetDelay_Good_UnknownSource(t *testing.T) {
func TestRateLimiter_GetDelay_Good_UnknownSource_Good(t *testing.T) {
rl := NewRateLimiter()
// Unknown sources should get the default 500ms delay
assert.Equal(t, 500*time.Millisecond, rl.GetDelay("unknown"))
}
func TestRateLimiter_Wait_Good_UnknownSource(t *testing.T) {
func TestRateLimiter_Wait_Good_UnknownSource_Good(t *testing.T) {
rl := NewRateLimiter()
ctx := context.Background()

View file

@ -1,12 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
"encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"sync"
"time"
core "dappco.re/go/core/log"
"dappco.re/go/core/io"
core "dappco.re/go/core/log"
)
// State tracks collection progress for incremental runs.
@ -39,6 +41,7 @@ type StateEntry struct {
// NewState creates a state tracker that persists to the given path
// using the provided storage medium.
// Usage: NewState(...)
func NewState(m io.Medium, path string) *State {
return &State{
medium: m,
@ -49,6 +52,7 @@ func NewState(m io.Medium, path string) *State {
// Load reads state from disk. If the file does not exist, the state
// is initialised as empty without error.
// Usage: Load(...)
func (s *State) Load() error {
s.mu.Lock()
defer s.mu.Unlock()
@ -75,6 +79,7 @@ func (s *State) Load() error {
}
// Save writes state to disk.
// Usage: Save(...)
func (s *State) Save() error {
s.mu.Lock()
defer s.mu.Unlock()
@ -93,6 +98,7 @@ func (s *State) Save() error {
// Get returns a copy of the state for a source. The second return value
// indicates whether the entry was found.
// Usage: Get(...)
func (s *State) Get(source string) (*StateEntry, bool) {
s.mu.Lock()
defer s.mu.Unlock()
@ -106,6 +112,7 @@ func (s *State) Get(source string) (*StateEntry, bool) {
}
// Set updates state for a source.
// Usage: Set(...)
func (s *State) Set(source string, entry *StateEntry) {
s.mu.Lock()
defer s.mu.Unlock()

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -8,7 +10,7 @@ import (
"github.com/stretchr/testify/require"
)
func TestState_Get_Good_ReturnsCopy(t *testing.T) {
func TestState_Get_Good_ReturnsCopy_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/state.json")
@ -24,7 +26,7 @@ func TestState_Get_Good_ReturnsCopy(t *testing.T) {
assert.Equal(t, 5, again.Items, "internal state should not be mutated")
}
func TestState_Save_Good_WritesJSON(t *testing.T) {
func TestState_Save_Good_WritesJSON_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/data/state.json")
@ -40,7 +42,7 @@ func TestState_Save_Good_WritesJSON(t *testing.T) {
assert.Contains(t, content, `"abc"`)
}
func TestState_Load_Good_NullJSON(t *testing.T) {
func TestState_Load_Good_NullJSON_Good(t *testing.T) {
m := io.NewMockMedium()
m.Files["/state.json"] = "null"
@ -53,7 +55,7 @@ func TestState_Load_Good_NullJSON(t *testing.T) {
assert.False(t, ok)
}
func TestState_SaveLoad_Good_WithCursor(t *testing.T) {
func TestState_SaveLoad_Good_WithCursor_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/state.json")

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package collect
import (
@ -73,7 +75,7 @@ func TestState_SaveLoad_Good(t *testing.T) {
assert.True(t, now.Equal(got.LastRun))
}
func TestState_Load_Good_NoFile(t *testing.T) {
func TestState_Load_Good_NoFile_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/nonexistent.json")
@ -86,7 +88,7 @@ func TestState_Load_Good_NoFile(t *testing.T) {
assert.False(t, ok)
}
func TestState_Load_Bad_InvalidJSON(t *testing.T) {
func TestState_Load_Bad_InvalidJSON_Good(t *testing.T) {
m := io.NewMockMedium()
m.Files["/state.json"] = "not valid json"
@ -95,7 +97,7 @@ func TestState_Load_Bad_InvalidJSON(t *testing.T) {
assert.Error(t, err)
}
func TestState_SaveLoad_Good_MultipleEntries(t *testing.T) {
func TestState_SaveLoad_Good_MultipleEntries_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/state.json")
@ -123,7 +125,7 @@ func TestState_SaveLoad_Good_MultipleEntries(t *testing.T) {
assert.Equal(t, 30, c.Items)
}
func TestState_Set_Good_Overwrite(t *testing.T) {
func TestState_Set_Good_Overwrite_Good(t *testing.T) {
m := io.NewMockMedium()
s := NewState(m, "/state.json")

View file

@ -88,9 +88,9 @@ The `gitea/` package mirrors this using `GITEA_URL`/`GITEA_TOKEN` and `gitea.*`
|------|-----------|
| `client.go` | `New`, `NewFromConfig`, `GetCurrentUser`, `ForkRepo`, `CreatePullRequest` |
| `repos.go` | `ListOrgRepos`, `ListOrgReposIter`, `ListUserRepos`, `ListUserReposIter`, `GetRepo`, `CreateOrgRepo`, `DeleteRepo`, `MigrateRepo` |
| `issues.go` | `ListIssues`, `GetIssue`, `CreateIssue`, `EditIssue`, `AssignIssue`, `ListPullRequests`, `ListPullRequestsIter`, `GetPullRequest`, `CreateIssueComment`, `ListIssueComments`, `CloseIssue` |
| `labels.go` | `ListOrgLabels`, `ListRepoLabels`, `CreateRepoLabel`, `GetLabelByName`, `EnsureLabel`, `AddIssueLabels`, `RemoveIssueLabel` |
| `prs.go` | `MergePullRequest`, `SetPRDraft`, `ListPRReviews`, `GetCombinedStatus`, `DismissReview` |
| `issues.go` | `ListIssues`, `ListIssuesIter`, `GetIssue`, `CreateIssue`, `EditIssue`, `AssignIssue`, `ListPullRequests`, `ListPullRequestsIter`, `GetPullRequest`, `CreateIssueComment`, `GetIssueLabels`, `ListIssueComments`, `ListIssueCommentsIter`, `CloseIssue` |
| `labels.go` | `ListOrgLabels`, `ListOrgLabelsIter`, `ListRepoLabels`, `ListRepoLabelsIter`, `CreateRepoLabel`, `GetLabelByName`, `EnsureLabel`, `AddIssueLabels`, `RemoveIssueLabel` |
| `prs.go` | `MergePullRequest`, `SetPRDraft`, `ListPRReviews`, `GetCombinedStatus`, `DismissReview`, `UndismissReview` |
| `webhooks.go` | `CreateRepoWebhook`, `ListRepoWebhooks` |
| `orgs.go` | `ListMyOrgs`, `GetOrg`, `CreateOrg` |
| `meta.go` | `GetPRMeta`, `GetCommentBodies`, `GetIssueBody` |
@ -119,7 +119,7 @@ The two packages are structurally parallel but intentionally not unified behind
- PR merge, draft status, reviews, combined status, review dismissal
- Repository migration (full import with issues/labels/PRs)
The Gitea client has a `CreateMirror` method for setting up pull mirrors from GitHub -- a capability specific to the public mirror workflow.
The Gitea client has a `GetCurrentUser` helper and a `CreateMirror` method for setting up pull mirrors from GitHub -- a capability specific to the public mirror workflow.
**SDK limitation:** The Forgejo SDK v2 does not accept `context.Context` on API methods. All SDK calls are synchronous. Context propagation through the wrapper layer is nominal -- contexts are accepted at the boundary but cannot be forwarded.
@ -350,7 +350,7 @@ agentci:
3. If the repository name is `core` or contains `security`, dual (Axiom 1: critical repos always verified).
4. Otherwise, standard.
In dual-run mode, `DispatchHandler` populates `DispatchTicket.VerifyModel` and `DispatchTicket.DualRun=true`. The `Weave` method compares primary and verifier outputs for convergence (currently byte-equal; semantic diff reserved for a future phase).
In dual-run mode, `DispatchHandler` populates `DispatchTicket.VerifyModel` and `DispatchTicket.DualRun=true`. The `Weave` method compares primary and verifier outputs for convergence using a deterministic token-overlap score against `validation_threshold`; richer semantic diffing remains a future phase.
### Dispatch Ticket Transfer

View file

@ -122,17 +122,17 @@ Full signal-to-result flow tested for all five handlers via a mock Forgejo serve
The Forgejo SDK v2 and Gitea SDK do not accept `context.Context`. All Forgejo/Gitea API calls are blocking with no cancellation path. When the SDK is updated to support context (v3 or later), a follow-up task should thread `ctx` through all forge/ and gitea/ wrapper signatures.
**Clotho Weave — byte-equal only**
**Clotho Weave — thresholded token overlap**
`Spinner.Weave(ctx, primary, signed)` currently returns `string(primaryOutput) == string(signedOutput)`. This is a placeholder. Meaningful dual-run verification requires semantic diff logic (e.g., normalised AST comparison, embedding cosine similarity, or LLM-assisted diffing). The interface signature is stable; the implementation is not production-ready for divergent outputs.
`Spinner.Weave(ctx, primary, signed)` now uses the configured `validation_threshold` to decide convergence from a deterministic token-overlap score. This is still a lightweight approximation rather than full semantic diffing, but it now honours the config knob already exposed by `ClothoConfig`.
**collect/ HTTP collectors — no retry**
None of the HTTP-dependent collectors (`bitcointalk.go`, `github.go`, `market.go`, `papers.go`) implement retry on transient failures. A single HTTP error causes the collector to return an error and increment the `Errors` count in the result. The `Excavator` continues to the next collector. For long-running collection runs, transient network errors cause silent data gaps.
**Journal replay — no public API**
**Journal replay**
The journal can be replayed by scanning the JSONL files directly, but there is no exported `Query` or `Filter` function. Replay filtering patterns exist only in tests. A future phase should export a query interface.
The journal now exposes `Journal.Query(...)` for replay and filtering over the JSONL archive. It supports repo, action, and time-range filters while preserving the date-partitioned storage layout used by `Append(...)`.
**git.Service framework integration**

View file

@ -0,0 +1,141 @@
# Verification Pass 2026-03-27
- Repository note: `CODEX.md` was not present under `/workspace`; conventions were taken from `CLAUDE.md`.
- Commands run: `go build ./...`, `go vet ./...`, `go test ./...`
- Command status: all passed
## Banned imports
ZERO FINDINGS for banned imports: `os`, `os/exec`, `encoding/json`, `fmt`, `errors`, `strings`, `path/filepath`.
## Test names not matching `TestFile_Function_{Good,Bad,Ugly}`
597 findings across 67 files:
```text
agentci/config_test.go:23,46,67,85,102,120,127,140,162,177,185,205,223,237,257,270,277,295,302
agentci/security_test.go:12,34,61,79,92,109
cmd/forge/cmd_sync_test.go:11,21,29,39,47
cmd/gitea/cmd_sync_test.go:11,21,29,39,47
collect/bitcointalk_http_test.go:36,76,104,128,152,183,192,210,225
collect/bitcointalk_test.go:16,21,30,42,73,79,87
collect/collect_test.go:10,23,35,57,63
collect/coverage_boost_test.go:18,34,50,62,79,92,109,120,125,130,141,151,176,198,230,264,275,283,291,298,305,313,322,330,335,343,351,358,368,384,400,419,434,452,457,481,497,508,517,533,557,572,592,611,625,639
collect/coverage_phase2_test.go:87,99,115,134,149,167,182,202,216,232,257,281,305,329,353,388,416,444,480,503,530,565,578,594,619,633,646,665,692,718,738,751,764,778,797,807,817,826,835,846,856,866,876,886,896,928,945,962,1022,1059,1098,1110,1132,1157,1181,1193,1230,1246,1261,1300
collect/events_test.go:44,57,72,88,129
collect/excavate_extra_test.go:13,42,68,86,104
collect/excavate_test.go:67,78,99,124,147,164,185
collect/github_test.go:17,22,41,53,65,91
collect/market_extra_test.go:15,59,101,140,175,200,232
collect/market_test.go:19,28,40,95,145,167
collect/papers_http_test.go:112,168,188,208,229,249,281,298,306
collect/papers_test.go:16,21,26,35,44,56,75,83
collect/process_extra_test.go:12,21,29,36,43,51,60,68,76,84,92,101,108,118,134,158,175
collect/process_test.go:16,25,37,58,78,97,114,173,182,191,198
collect/ratelimit_test.go:30,54,63,69,78
collect/state_extra_test.go:11,27,43,56
collect/state_test.go:76,89,98,126,138
forge/client_test.go:16,28,64,92,100,124,135,159,185,211,225,263,326,356,387,409,435,441
forge/config_test.go:19,28,39,50,61,71,77,85,100
forge/issues_test.go:22,44,55,73,94,116,135,154,176,194,211,230,247
forge/labels_test.go:27,45,63,72,81,91,111,127,144
forge/meta_test.go:26,48,66
forge/orgs_test.go:22,40,62
forge/prs_test.go:22,30,38,61,79,96,105,133,142
forge/repos_test.go:22,42,60,81,100,122
forge/webhooks_test.go:26,46
gitea/client_test.go:10,21
gitea/config_test.go:18,27,38,49,59,69,75,83,95
gitea/coverage_boost_test.go:15,26,35,44,90,124,176,220,239,264,294,306
gitea/issues_test.go:22,44,55,73,94,115,137,155,166
gitea/meta_test.go:26,48,66,77,103,115
gitea/repos_test.go:22,42,60,69,79,89,106,127
jobrunner/forgejo/source_test.go:38,109,155,162,166
jobrunner/handlers/dispatch_test.go:38,50,63,76,88,100,120,145,210,232,255,276,302,339
jobrunner/handlers/enable_auto_merge_test.go:29,42,84
jobrunner/handlers/publish_draft_test.go:26,36
jobrunner/handlers/resolve_threads_test.go:26
jobrunner/handlers/send_fix_command_test.go:16,25,37,49
jobrunner/handlers/tick_parent_test.go:26
jobrunner/journal_test.go:116,198,233,249
jobrunner/poller_test.go:121,152,193
jobrunner/types_test.go:28
manifest/compile_test.go:14,38,61,67,74,81,106,123,128,152,163,169
manifest/loader_test.go:12,28,34,51
manifest/manifest_test.go:10,49,67,127,135,146,157,205,210,215,220
manifest/sign_test.go:11,32,44
marketplace/builder_test.go:39,56,71,87,102,121,138,147,154,166,178,189,198,221
marketplace/discovery_test.go:25,59,84,105,122,130,136,147,195,234
marketplace/installer_test.go:78,104,125,142,166,189,212,223,243,270,281,309
marketplace/marketplace_test.go:10,26,38,50,61
pkg/api/provider_handlers_test.go:20,47,66,79,90,105,118,131,142,155,169,183
pkg/api/provider_security_test.go:12,18,23
pkg/api/provider_test.go:92,116,167,181,200,230
plugin/installer_test.go:14,26,36,47,60,81,95,105,120,132,140,148,156,162,168,174,180,186,192,198,204
plugin/loader_test.go:46,71,92,123,132
plugin/manifest_test.go:10,33,50,58,78,89,100
plugin/plugin_test.go:10,25,37
plugin/registry_test.go:28,69,78,129,138,149,160,173
repos/gitstate_test.go:14,50,61,110,132,158,178,189,199,204,210
repos/kbconfig_test.go:13,48,57,76,93
repos/registry_test.go:13,40,72,93,120,127,181,201,209,243,266,286,309,334,350,363,373,382,390,398,403,411,421,427,474,481
repos/workconfig_test.go:14,51,60,79,97,102
```
## Exported functions missing usage-example comments
199 findings across 51 files:
```text
agentci/clotho.go:39,61,71,90
collect/bitcointalk.go:37,46
collect/events.go:72,81,96,105,115,125,135
collect/excavate.go:27,33
collect/github.go:57,65
collect/market.go:33,67
collect/papers.go:41,57
collect/process.go:27,33
collect/ratelimit.go:46,80,87,100,107
collect/state.go:55,81,99,112
forge/client.go:37,40,43,46,55,69
forge/issues.go:21,56,66,76,86,97,130,164,174,185,209
forge/labels.go:15,31,55,65,81,94,105
forge/meta.go:43,100,139
forge/orgs.go:10,34,44
forge/prs.go:18,43,84,108,117
forge/repos.go:12,36,61,85,110,120,130,141
forge/webhooks.go:10,20
git/git.go:32,37,42,283,293
git/service.go:75,113,116,121,132,145,156
gitea/client.go:36,39
gitea/issues.go:20,52,62,72,105,139
gitea/meta.go:43,101,141
gitea/repos.go:12,36,61,85,110,122,145,155
internal/ax/stdio/stdio.go:12,24
jobrunner/forgejo/source.go:36,42,65
jobrunner/handlers/completion.go:33,38,43
jobrunner/handlers/dispatch.go:76,82,91
jobrunner/handlers/enable_auto_merge.go:25,31,40
jobrunner/handlers/publish_draft.go:25,30,37
jobrunner/handlers/resolve_threads.go:30,35,40
jobrunner/handlers/send_fix_command.go:26,32,46
jobrunner/handlers/tick_parent.go:30,35,41
jobrunner/journal.go:98
jobrunner/poller.go:51,58,65,72,79,88,110
jobrunner/types.go:37,42
manifest/manifest.go:46,79,95
marketplace/builder.go:35
marketplace/discovery.go:132,140,145,151,160
marketplace/installer.go:53,115,133,179
marketplace/marketplace.go:39,53,64
pkg/api/provider.go:61,64,67,75,86,108
plugin/installer.go:35,99,133
plugin/loader.go:28,53
plugin/manifest.go:41
plugin/plugin.go:44,47,50,53,56
plugin/registry.go:35,48,54,63,78,105
repos/gitstate.go:101,106,111,120,131,144,162
repos/kbconfig.go:116,121
repos/registry.go:255,265,271,283,325,330
repos/workconfig.go:104
```

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package forge provides a thin wrapper around the Forgejo Go SDK
// for managing repositories, issues, and pull requests on a Forgejo instance.
//
@ -22,6 +24,7 @@ type Client struct {
}
// New creates a new Forgejo API client for the given URL and token.
// Usage: New(...)
func New(url, token string) (*Client, error) {
api, err := forgejo.NewClient(url, forgejo.SetToken(token))
if err != nil {
@ -32,15 +35,19 @@ func New(url, token string) (*Client, error) {
}
// API exposes the underlying SDK client for direct access.
// Usage: API(...)
func (c *Client) API() *forgejo.Client { return c.api }
// URL returns the Forgejo instance URL.
// Usage: URL(...)
func (c *Client) URL() string { return c.url }
// Token returns the Forgejo API token.
// Usage: Token(...)
func (c *Client) Token() string { return c.token }
// GetCurrentUser returns the authenticated user's information.
// Usage: GetCurrentUser(...)
func (c *Client) GetCurrentUser() (*forgejo.User, error) {
user, _, err := c.api.GetMyUserInfo()
if err != nil {
@ -50,6 +57,7 @@ func (c *Client) GetCurrentUser() (*forgejo.User, error) {
}
// ForkRepo forks a repository. If org is non-empty, forks into that organisation.
// Usage: ForkRepo(...)
func (c *Client) ForkRepo(owner, repo string, org string) (*forgejo.Repository, error) {
opts := forgejo.CreateForkOption{}
if org != "" {
@ -64,6 +72,7 @@ func (c *Client) ForkRepo(owner, repo string, org string) (*forgejo.Repository,
}
// CreatePullRequest creates a pull request on the given repository.
// Usage: CreatePullRequest(...)
func (c *Client) CreatePullRequest(owner, repo string, opts forgejo.CreatePullRequestOption) (*forgejo.PullRequest, error) {
pr, _, err := c.api.CreatePullRequest(owner, repo, opts)
if err != nil {

View file

@ -1,8 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"encoding/json"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"net/http"
"net/http/httptest"
"testing"
@ -25,7 +27,7 @@ func TestNew_Good(t *testing.T) {
assert.Equal(t, "test-token-123", client.Token())
}
func TestNew_Bad_InvalidURL(t *testing.T) {
func TestNew_Bad_InvalidURL_Good(t *testing.T) {
// The Forgejo SDK may reject certain URL formats.
_, err := New("://invalid-url", "token")
assert.Error(t, err)
@ -61,7 +63,7 @@ func TestClient_GetCurrentUser_Good(t *testing.T) {
assert.Equal(t, "test-user", user.UserName)
}
func TestClient_GetCurrentUser_Bad_ServerError(t *testing.T) {
func TestClient_GetCurrentUser_Bad_ServerError_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -89,7 +91,7 @@ func TestClient_SetPRDraft_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_SetPRDraft_Good_Undraft(t *testing.T) {
func TestClient_SetPRDraft_Good_Undraft_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -97,7 +99,7 @@ func TestClient_SetPRDraft_Good_Undraft(t *testing.T) {
require.NoError(t, err)
}
func TestClient_SetPRDraft_Bad_ServerError(t *testing.T) {
func TestClient_SetPRDraft_Bad_ServerError_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -121,7 +123,7 @@ func TestClient_SetPRDraft_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "unexpected status 403")
}
func TestClient_SetPRDraft_Bad_ConnectionRefused(t *testing.T) {
func TestClient_SetPRDraft_Bad_ConnectionRefused_Good(t *testing.T) {
// Use a closed server to simulate connection errors.
srv := newMockForgejoServer(t)
client, err := New(srv.URL, "token")
@ -132,7 +134,7 @@ func TestClient_SetPRDraft_Bad_ConnectionRefused(t *testing.T) {
assert.Error(t, err)
}
func TestClient_SetPRDraft_URLConstruction(t *testing.T) {
func TestClient_SetPRDraft_Good_URLConstruction_Good(t *testing.T) {
// Verify the URL is constructed correctly by checking the request path.
var capturedPath string
mux := http.NewServeMux()
@ -156,7 +158,7 @@ func TestClient_SetPRDraft_URLConstruction(t *testing.T) {
assert.Equal(t, "/api/v1/repos/my-org/my-repo/pulls/42", capturedPath)
}
func TestClient_SetPRDraft_AuthHeader(t *testing.T) {
func TestClient_SetPRDraft_Good_AuthHeader_Good(t *testing.T) {
// Verify the authorisation header is set correctly.
var capturedAuth string
mux := http.NewServeMux()
@ -182,7 +184,7 @@ func TestClient_SetPRDraft_AuthHeader(t *testing.T) {
// --- PRMeta and Comment struct tests ---
func TestPRMeta_Fields(t *testing.T) {
func TestPRMeta_Good_Fields_Good(t *testing.T) {
meta := &PRMeta{
Number: 42,
Title: "Test PR",
@ -208,7 +210,7 @@ func TestPRMeta_Fields(t *testing.T) {
assert.Equal(t, 5, meta.CommentCount)
}
func TestComment_Fields(t *testing.T) {
func TestComment_Good_Fields_Good(t *testing.T) {
comment := Comment{
ID: 123,
Author: "reviewer",
@ -222,7 +224,7 @@ func TestComment_Fields(t *testing.T) {
// --- MergePullRequest merge style mapping ---
func TestMergePullRequest_StyleMapping(t *testing.T) {
func TestMergePullRequest_Good_StyleMapping_Good(t *testing.T) {
// We can't easily test the SDK call, but we can verify the method
// errors when the server returns failure. This exercises the style mapping code.
tests := []struct {
@ -260,7 +262,7 @@ func TestMergePullRequest_StyleMapping(t *testing.T) {
// --- ListIssuesOpts defaulting ---
func TestListIssuesOpts_Defaults(t *testing.T) {
func TestListIssuesOpts_Good_Defaults_Good(t *testing.T) {
tests := []struct {
name string
opts ListIssuesOpts
@ -323,7 +325,7 @@ func TestListIssuesOpts_Defaults(t *testing.T) {
// --- ForkRepo error handling ---
func TestClient_ForkRepo_Good_WithOrg(t *testing.T) {
func TestClient_ForkRepo_Good_WithOrg_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -353,7 +355,7 @@ func TestClient_ForkRepo_Good_WithOrg(t *testing.T) {
assert.Equal(t, "target-org", capturedBody["organization"])
}
func TestClient_ForkRepo_Good_WithoutOrg(t *testing.T) {
func TestClient_ForkRepo_Good_WithoutOrg_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -384,7 +386,7 @@ func TestClient_ForkRepo_Good_WithoutOrg(t *testing.T) {
// The SDK may or may not include it in the JSON; just verify the fork succeeded.
}
func TestClient_ForkRepo_Bad_ServerError(t *testing.T) {
func TestClient_ForkRepo_Bad_ServerError_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -406,7 +408,7 @@ func TestClient_ForkRepo_Bad_ServerError(t *testing.T) {
// --- CreatePullRequest error handling ---
func TestClient_CreatePullRequest_Bad_ServerError(t *testing.T) {
func TestClient_CreatePullRequest_Bad_ServerError_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
@ -432,13 +434,13 @@ func TestClient_CreatePullRequest_Bad_ServerError(t *testing.T) {
// --- commentPageSize constant test ---
func TestCommentPageSize(t *testing.T) {
func TestCommentPageSize_Good(t *testing.T) {
assert.Equal(t, 50, commentPageSize, "comment page size should be 50")
}
// --- ListPullRequests state mapping ---
func TestListPullRequests_StateMapping(t *testing.T) {
func TestListPullRequests_Good_StateMapping_Good(t *testing.T) {
// Verify state mapping via error path (server returns error).
tests := []struct {
name string

View file

@ -1,19 +1,24 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"os"
os "dappco.re/go/core/scm/internal/ax/osx"
"forge.lthn.ai/core/config"
"dappco.re/go/core/log"
"forge.lthn.ai/core/config"
)
const (
// ConfigKeyURL is the config key for the Forgejo instance URL.
//
ConfigKeyURL = "forge.url"
// ConfigKeyToken is the config key for the Forgejo API token.
//
ConfigKeyToken = "forge.token"
// DefaultURL is the default Forgejo instance URL.
//
DefaultURL = "http://localhost:4000"
)
@ -22,6 +27,8 @@ const (
// 1. ~/.core/config.yaml keys: forge.token, forge.url
// 2. FORGE_TOKEN + FORGE_URL environment variables (override config file)
// 3. Provided flag overrides (highest priority; pass empty to skip)
//
// Usage: NewFromConfig(...)
func NewFromConfig(flagURL, flagToken string) (*Client, error) {
url, token, err := ResolveConfig(flagURL, flagToken)
if err != nil {
@ -37,6 +44,7 @@ func NewFromConfig(flagURL, flagToken string) (*Client, error) {
// ResolveConfig resolves the Forgejo URL and token from all config sources.
// Flag values take highest priority, then env vars, then config file.
// Usage: ResolveConfig(...)
func ResolveConfig(flagURL, flagToken string) (url, token string, err error) {
// Start with config file values
cfg, cfgErr := config.New()
@ -70,6 +78,7 @@ func ResolveConfig(flagURL, flagToken string) (url, token string, err error) {
}
// SaveConfig persists the Forgejo URL and/or token to the config file.
// Usage: SaveConfig(...)
func SaveConfig(url, token string) error {
cfg, err := config.New()
if err != nil {

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
@ -16,7 +18,7 @@ func isolateConfigEnv(t *testing.T) {
t.Setenv("HOME", t.TempDir())
}
func TestResolveConfig_Good_Defaults(t *testing.T) {
func TestResolveConfig_Good_Defaults_Good(t *testing.T) {
isolateConfigEnv(t)
url, token, err := ResolveConfig("", "")
@ -25,7 +27,7 @@ func TestResolveConfig_Good_Defaults(t *testing.T) {
assert.Empty(t, token, "token should be empty when nothing configured")
}
func TestResolveConfig_Good_FlagsOverrideAll(t *testing.T) {
func TestResolveConfig_Good_FlagsOverrideAll_Good(t *testing.T) {
isolateConfigEnv(t)
t.Setenv("FORGE_URL", "https://env-url.example.com")
t.Setenv("FORGE_TOKEN", "env-token-abc")
@ -36,7 +38,7 @@ func TestResolveConfig_Good_FlagsOverrideAll(t *testing.T) {
assert.Equal(t, "flag-token-xyz", token, "flag token should override env")
}
func TestResolveConfig_Good_EnvVarsOverrideConfig(t *testing.T) {
func TestResolveConfig_Good_EnvVarsOverrideConfig_Good(t *testing.T) {
isolateConfigEnv(t)
t.Setenv("FORGE_URL", "https://env-url.example.com")
t.Setenv("FORGE_TOKEN", "env-token-123")
@ -47,7 +49,7 @@ func TestResolveConfig_Good_EnvVarsOverrideConfig(t *testing.T) {
assert.Equal(t, "env-token-123", token)
}
func TestResolveConfig_Good_PartialOverrides(t *testing.T) {
func TestResolveConfig_Good_PartialOverrides_Good(t *testing.T) {
isolateConfigEnv(t)
// Set only env URL, flag token.
t.Setenv("FORGE_URL", "https://env-only.example.com")
@ -58,7 +60,7 @@ func TestResolveConfig_Good_PartialOverrides(t *testing.T) {
assert.Equal(t, "flag-only-token", token, "flag token should be used")
}
func TestResolveConfig_Good_URLDefaultsWhenEmpty(t *testing.T) {
func TestResolveConfig_Good_URLDefaultsWhenEmpty_Good(t *testing.T) {
isolateConfigEnv(t)
t.Setenv("FORGE_TOKEN", "some-token")
@ -68,13 +70,13 @@ func TestResolveConfig_Good_URLDefaultsWhenEmpty(t *testing.T) {
assert.Equal(t, "some-token", token)
}
func TestConstants(t *testing.T) {
func TestConstants_Good(t *testing.T) {
assert.Equal(t, "forge.url", ConfigKeyURL)
assert.Equal(t, "forge.token", ConfigKeyToken)
assert.Equal(t, "http://localhost:4000", DefaultURL)
}
func TestNewFromConfig_Bad_NoToken(t *testing.T) {
func TestNewFromConfig_Bad_NoToken_Good(t *testing.T) {
isolateConfigEnv(t)
_, err := NewFromConfig("", "")
@ -82,7 +84,7 @@ func TestNewFromConfig_Bad_NoToken(t *testing.T) {
assert.Contains(t, err.Error(), "no API token configured")
}
func TestNewFromConfig_Good_WithFlagToken(t *testing.T) {
func TestNewFromConfig_Good_WithFlagToken_Good(t *testing.T) {
isolateConfigEnv(t)
// The Forgejo SDK NewClient validates the token by calling /api/v1/version,
@ -97,7 +99,7 @@ func TestNewFromConfig_Good_WithFlagToken(t *testing.T) {
assert.Equal(t, "test-token", client.Token())
}
func TestNewFromConfig_Good_EnvToken(t *testing.T) {
func TestNewFromConfig_Good_EnvToken_Good(t *testing.T) {
isolateConfigEnv(t)
srv := newMockForgejoServer(t)

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
@ -17,6 +19,7 @@ type ListIssuesOpts struct {
}
// ListIssues returns issues for the given repository.
// Usage: ListIssues(...)
func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo.Issue, error) {
state := forgejo.StateOpen
switch opts.State {
@ -36,22 +39,85 @@ func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo
page = 1
}
listOpt := forgejo.ListIssueOption{
ListOptions: forgejo.ListOptions{Page: page, PageSize: limit},
State: state,
Type: forgejo.IssueTypeIssue,
Labels: opts.Labels,
var all []*forgejo.Issue
for {
listOpt := forgejo.ListIssueOption{
ListOptions: forgejo.ListOptions{Page: page, PageSize: limit},
State: state,
Type: forgejo.IssueTypeIssue,
Labels: opts.Labels,
}
issues, resp, err := c.api.ListRepoIssues(owner, repo, listOpt)
if err != nil {
return nil, log.E("forge.ListIssues", "failed to list issues", err)
}
all = append(all, issues...)
if len(issues) < limit || len(issues) == 0 {
break
}
if resp != nil && resp.LastPage > 0 && page >= resp.LastPage {
break
}
page++
}
issues, _, err := c.api.ListRepoIssues(owner, repo, listOpt)
if err != nil {
return nil, log.E("forge.ListIssues", "failed to list issues", err)
return all, nil
}
// ListIssuesIter returns an iterator over issues for the given repository.
// Usage: ListIssuesIter(...)
func (c *Client) ListIssuesIter(owner, repo string, opts ListIssuesOpts) iter.Seq2[*forgejo.Issue, error] {
state := forgejo.StateOpen
switch opts.State {
case "closed":
state = forgejo.StateClosed
case "all":
state = forgejo.StateAll
}
return issues, nil
limit := opts.Limit
if limit == 0 {
limit = 50
}
page := opts.Page
if page == 0 {
page = 1
}
return func(yield func(*forgejo.Issue, error) bool) {
for {
issues, resp, err := c.api.ListRepoIssues(owner, repo, forgejo.ListIssueOption{
ListOptions: forgejo.ListOptions{Page: page, PageSize: limit},
State: state,
Type: forgejo.IssueTypeIssue,
Labels: opts.Labels,
})
if err != nil {
yield(nil, log.E("forge.ListIssues", "failed to list issues", err))
return
}
for _, issue := range issues {
if !yield(issue, nil) {
return
}
}
if len(issues) < limit || len(issues) == 0 {
break
}
if resp != nil && resp.LastPage > 0 && page >= resp.LastPage {
break
}
page++
}
}
}
// GetIssue returns a single issue by number.
// Usage: GetIssue(...)
func (c *Client) GetIssue(owner, repo string, number int64) (*forgejo.Issue, error) {
issue, _, err := c.api.GetIssue(owner, repo, number)
if err != nil {
@ -62,6 +128,7 @@ func (c *Client) GetIssue(owner, repo string, number int64) (*forgejo.Issue, err
}
// CreateIssue creates a new issue in the given repository.
// Usage: CreateIssue(...)
func (c *Client) CreateIssue(owner, repo string, opts forgejo.CreateIssueOption) (*forgejo.Issue, error) {
issue, _, err := c.api.CreateIssue(owner, repo, opts)
if err != nil {
@ -72,6 +139,7 @@ func (c *Client) CreateIssue(owner, repo string, opts forgejo.CreateIssueOption)
}
// EditIssue edits an existing issue.
// Usage: EditIssue(...)
func (c *Client) EditIssue(owner, repo string, number int64, opts forgejo.EditIssueOption) (*forgejo.Issue, error) {
issue, _, err := c.api.EditIssue(owner, repo, number, opts)
if err != nil {
@ -82,6 +150,7 @@ func (c *Client) EditIssue(owner, repo string, number int64, opts forgejo.EditIs
}
// AssignIssue assigns an issue to the specified users.
// Usage: AssignIssue(...)
func (c *Client) AssignIssue(owner, repo string, number int64, assignees []string) error {
_, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{
Assignees: assignees,
@ -93,6 +162,7 @@ func (c *Client) AssignIssue(owner, repo string, number int64, assignees []strin
}
// ListPullRequests returns pull requests for the given repository.
// Usage: ListPullRequests(...)
func (c *Client) ListPullRequests(owner, repo string, state string) ([]*forgejo.PullRequest, error) {
st := forgejo.StateOpen
switch state {
@ -126,6 +196,7 @@ func (c *Client) ListPullRequests(owner, repo string, state string) ([]*forgejo.
}
// ListPullRequestsIter returns an iterator over pull requests for the given repository.
// Usage: ListPullRequestsIter(...)
func (c *Client) ListPullRequestsIter(owner, repo string, state string) iter.Seq2[*forgejo.PullRequest, error] {
st := forgejo.StateOpen
switch state {
@ -160,6 +231,7 @@ func (c *Client) ListPullRequestsIter(owner, repo string, state string) iter.Seq
}
// GetPullRequest returns a single pull request by number.
// Usage: GetPullRequest(...)
func (c *Client) GetPullRequest(owner, repo string, number int64) (*forgejo.PullRequest, error) {
pr, _, err := c.api.GetPullRequest(owner, repo, number)
if err != nil {
@ -170,6 +242,7 @@ func (c *Client) GetPullRequest(owner, repo string, number int64) (*forgejo.Pull
}
// CreateIssueComment posts a comment on an issue or pull request.
// Usage: CreateIssueComment(...)
func (c *Client) CreateIssueComment(owner, repo string, issue int64, body string) error {
_, _, err := c.api.CreateIssueComment(owner, repo, issue, forgejo.CreateIssueCommentOption{
Body: body,
@ -180,7 +253,19 @@ func (c *Client) CreateIssueComment(owner, repo string, issue int64, body string
return nil
}
// GetIssueLabels returns the labels currently attached to an issue.
// Usage: GetIssueLabels(...)
func (c *Client) GetIssueLabels(owner, repo string, number int64) ([]*forgejo.Label, error) {
labels, _, err := c.api.GetIssueLabels(owner, repo, number, forgejo.ListLabelsOptions{})
if err != nil {
return nil, log.E("forge.GetIssueLabels", "failed to get issue labels", err)
}
return labels, nil
}
// ListIssueComments returns comments for an issue.
// Usage: ListIssueComments(...)
func (c *Client) ListIssueComments(owner, repo string, number int64) ([]*forgejo.Comment, error) {
var all []*forgejo.Comment
page := 1
@ -204,7 +289,34 @@ func (c *Client) ListIssueComments(owner, repo string, number int64) ([]*forgejo
return all, nil
}
// ListIssueCommentsIter returns an iterator over comments for an issue.
// Usage: ListIssueCommentsIter(...)
func (c *Client) ListIssueCommentsIter(owner, repo string, number int64) iter.Seq2[*forgejo.Comment, error] {
return func(yield func(*forgejo.Comment, error) bool) {
page := 1
for {
comments, resp, err := c.api.ListIssueComments(owner, repo, number, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
if err != nil {
yield(nil, log.E("forge.ListIssueComments", "failed to list comments", err))
return
}
for _, comment := range comments {
if !yield(comment, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// CloseIssue closes an issue by setting its state to closed.
// Usage: CloseIssue(...)
func (c *Client) CloseIssue(owner, repo string, number int64) error {
closed := forgejo.StateClosed
_, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{

View file

@ -1,6 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"net/http"
"net/http/httptest"
"strconv"
"testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -9,6 +14,71 @@ import (
"github.com/stretchr/testify/require"
)
func newPaginatedIssuesClient(t *testing.T) (*Client, *httptest.Server) {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/issues", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "number": 2, "title": "Issue 2", "state": "open", "body": "Second issue"},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
jsonResponse(w, []map[string]any{
{"id": 1, "number": 1, "title": "Issue 1", "state": "open", "body": "First issue"},
})
}
})
srv := httptest.NewServer(mux)
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
return client, srv
}
func newPaginatedCommentsClient(t *testing.T) (*Client, *httptest.Server) {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 150, "body": "comment 51", "user": map[string]any{"login": "user51"}, "created_at": "2026-01-02T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", `</api/v1/repos/test-org/org-repo/issues/1/comments?page=2>; rel="next", </api/v1/repos/test-org/org-repo/issues/1/comments?page=2>; rel="last"`)
comments := make([]map[string]any, 0, 50)
for i := 1; i <= 50; i++ {
comments = append(comments, map[string]any{
"id": 99 + i,
"body": "comment " + strconv.Itoa(i),
"user": map[string]any{"login": "user" + strconv.Itoa(i)},
"created_at": "2026-01-01T00:00:00Z",
"updated_at": "2026-01-01T00:00:00Z",
})
}
jsonResponse(w, comments)
}
})
srv := httptest.NewServer(mux)
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
return client, srv
}
func TestClient_ListIssues_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -19,7 +89,32 @@ func TestClient_ListIssues_Good(t *testing.T) {
assert.Equal(t, "Issue 1", issues[0].Title)
}
func TestClient_ListIssues_Good_StateMapping(t *testing.T) {
func TestClient_ListIssues_Good_Paginates_Good(t *testing.T) {
client, srv := newPaginatedIssuesClient(t)
defer srv.Close()
issues, err := client.ListIssues("test-org", "org-repo", ListIssuesOpts{Limit: 1})
require.NoError(t, err)
require.Len(t, issues, 2)
assert.Equal(t, "Issue 1", issues[0].Title)
assert.Equal(t, "Issue 2", issues[1].Title)
}
func TestClient_ListIssuesIter_Good_Paginates_Good(t *testing.T) {
client, srv := newPaginatedIssuesClient(t)
defer srv.Close()
var titles []string
for issue, err := range client.ListIssuesIter("test-org", "org-repo", ListIssuesOpts{Limit: 1}) {
require.NoError(t, err)
titles = append(titles, issue.Title)
}
require.Len(t, titles, 2)
assert.Equal(t, []string{"Issue 1", "Issue 2"}, titles)
}
func TestClient_ListIssues_Good_StateMapping_Good(t *testing.T) {
tests := []struct {
name string
state string
@ -41,7 +136,7 @@ func TestClient_ListIssues_Good_StateMapping(t *testing.T) {
}
}
func TestClient_ListIssues_Good_CustomPageAndLimit(t *testing.T) {
func TestClient_ListIssues_Good_CustomPageAndLimit_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -52,7 +147,7 @@ func TestClient_ListIssues_Good_CustomPageAndLimit(t *testing.T) {
require.NoError(t, err)
}
func TestClient_ListIssues_Bad_ServerError(t *testing.T) {
func TestClient_ListIssues_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -70,7 +165,7 @@ func TestClient_GetIssue_Good(t *testing.T) {
assert.Equal(t, "Issue 1", issue.Title)
}
func TestClient_GetIssue_Bad_ServerError(t *testing.T) {
func TestClient_GetIssue_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -91,7 +186,7 @@ func TestClient_CreateIssue_Good(t *testing.T) {
assert.NotNil(t, issue)
}
func TestClient_CreateIssue_Bad_ServerError(t *testing.T) {
func TestClient_CreateIssue_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -113,7 +208,7 @@ func TestClient_EditIssue_Good(t *testing.T) {
assert.NotNil(t, issue)
}
func TestClient_EditIssue_Bad_ServerError(t *testing.T) {
func TestClient_EditIssue_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -132,7 +227,7 @@ func TestClient_AssignIssue_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_AssignIssue_Bad_ServerError(t *testing.T) {
func TestClient_AssignIssue_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -151,7 +246,7 @@ func TestClient_ListPullRequests_Good(t *testing.T) {
assert.Equal(t, "PR 1", prs[0].Title)
}
func TestClient_ListPullRequests_Good_StateMapping(t *testing.T) {
func TestClient_ListPullRequests_Good_StateMapping_Good(t *testing.T) {
tests := []struct {
name string
state string
@ -173,7 +268,7 @@ func TestClient_ListPullRequests_Good_StateMapping(t *testing.T) {
}
}
func TestClient_ListPullRequests_Bad_ServerError(t *testing.T) {
func TestClient_ListPullRequests_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -191,7 +286,7 @@ func TestClient_GetPullRequest_Good(t *testing.T) {
assert.Equal(t, "PR 1", pr.Title)
}
func TestClient_GetPullRequest_Bad_ServerError(t *testing.T) {
func TestClient_GetPullRequest_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -208,7 +303,7 @@ func TestClient_CreateIssueComment_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_CreateIssueComment_Bad_ServerError(t *testing.T) {
func TestClient_CreateIssueComment_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -217,6 +312,25 @@ func TestClient_CreateIssueComment_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "failed to create comment")
}
func TestClient_GetIssueLabels_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
labels, err := client.GetIssueLabels("test-org", "org-repo", 1)
require.NoError(t, err)
require.Len(t, labels, 1)
assert.Equal(t, "bug", labels[0].Name)
}
func TestClient_GetIssueLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
_, err := client.GetIssueLabels("test-org", "org-repo", 1)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to get issue labels")
}
func TestClient_ListIssueComments_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -227,7 +341,7 @@ func TestClient_ListIssueComments_Good(t *testing.T) {
assert.Equal(t, "comment 1", comments[0].Body)
}
func TestClient_ListIssueComments_Bad_ServerError(t *testing.T) {
func TestClient_ListIssueComments_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -236,6 +350,22 @@ func TestClient_ListIssueComments_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list comments")
}
func TestClient_ListIssueCommentsIter_Good_Paginates_Good(t *testing.T) {
client, srv := newPaginatedCommentsClient(t)
defer srv.Close()
var bodies []string
for comment, err := range client.ListIssueCommentsIter("test-org", "org-repo", 1) {
require.NoError(t, err)
bodies = append(bodies, comment.Body)
}
require.Len(t, bodies, 51)
assert.Equal(t, "comment 1", bodies[0])
assert.Equal(t, "comment 50", bodies[49])
assert.Equal(t, "comment 51", bodies[50])
}
func TestClient_CloseIssue_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -244,7 +374,7 @@ func TestClient_CloseIssue_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_CloseIssue_Bad_ServerError(t *testing.T) {
func TestClient_CloseIssue_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()

View file

@ -1,19 +1,23 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"strings"
"iter"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log"
)
// ListOrgLabels returns all labels for repos in the given organisation.
// ListOrgLabels returns all unique labels across repos in the given organisation.
// Note: The Forgejo SDK does not have a dedicated org-level labels endpoint.
// This lists labels from the first repo found, which works when orgs use shared label sets.
// For org-wide label management, use ListRepoLabels with a specific repo.
// We aggregate labels from each repo and deduplicate them by name, preserving
// the first seen label metadata.
// Usage: ListOrgLabels(...)
func (c *Client) ListOrgLabels(org string) ([]*forgejo.Label, error) {
// Forgejo doesn't expose org-level labels via SDK — list repos and aggregate unique labels.
repos, err := c.ListOrgRepos(org)
if err != nil {
return nil, err
@ -23,11 +27,63 @@ func (c *Client) ListOrgLabels(org string) ([]*forgejo.Label, error) {
return nil, nil
}
// Use the first repo's labels as representative of the org's label set.
return c.ListRepoLabels(repos[0].Owner.UserName, repos[0].Name)
seen := make(map[string]struct{}, len(repos))
var all []*forgejo.Label
for _, repo := range repos {
labels, err := c.ListRepoLabels(repo.Owner.UserName, repo.Name)
if err != nil {
return nil, err
}
for _, label := range labels {
key := strings.ToLower(label.Name)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
all = append(all, label)
}
}
return all, nil
}
// ListOrgLabelsIter returns an iterator over unique labels across repos in the given organisation.
// Note: The Forgejo SDK does not have a dedicated org-level labels endpoint.
// Labels are yielded in first-seen order across repositories and deduplicated by name.
// Usage: ListOrgLabelsIter(...)
func (c *Client) ListOrgLabelsIter(org string) iter.Seq2[*forgejo.Label, error] {
return func(yield func(*forgejo.Label, error) bool) {
seen := make(map[string]struct{})
for repo, err := range c.ListOrgReposIter(org) {
if err != nil {
yield(nil, log.E("forge.ListOrgLabels", "failed to list org repos", err))
return
}
for label, err := range c.ListRepoLabelsIter(repo.Owner.UserName, repo.Name) {
if err != nil {
yield(nil, log.E("forge.ListOrgLabels", "failed to list repo labels", err))
return
}
key := strings.ToLower(label.Name)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
if !yield(label, nil) {
return
}
}
}
}
}
// ListRepoLabels returns all labels for a repository.
// Usage: ListRepoLabels(...)
func (c *Client) ListRepoLabels(owner, repo string) ([]*forgejo.Label, error) {
var all []*forgejo.Label
page := 1
@ -51,7 +107,37 @@ func (c *Client) ListRepoLabels(owner, repo string) ([]*forgejo.Label, error) {
return all, nil
}
// ListRepoLabelsIter returns an iterator over labels for a repository.
// Usage: ListRepoLabelsIter(...)
func (c *Client) ListRepoLabelsIter(owner, repo string) iter.Seq2[*forgejo.Label, error] {
return func(yield func(*forgejo.Label, error) bool) {
page := 1
for {
labels, resp, err := c.api.ListRepoLabels(owner, repo, forgejo.ListLabelsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListRepoLabels", "failed to list repo labels", err))
return
}
for _, label := range labels {
if !yield(label, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// CreateRepoLabel creates a label on a repository.
// Usage: CreateRepoLabel(...)
func (c *Client) CreateRepoLabel(owner, repo string, opts forgejo.CreateLabelOption) (*forgejo.Label, error) {
label, _, err := c.api.CreateLabel(owner, repo, opts)
if err != nil {
@ -62,6 +148,7 @@ func (c *Client) CreateRepoLabel(owner, repo string, opts forgejo.CreateLabelOpt
}
// GetLabelByName retrieves a specific label by name from a repository.
// Usage: GetLabelByName(...)
func (c *Client) GetLabelByName(owner, repo, name string) (*forgejo.Label, error) {
labels, err := c.ListRepoLabels(owner, repo)
if err != nil {
@ -78,6 +165,7 @@ func (c *Client) GetLabelByName(owner, repo, name string) (*forgejo.Label, error
}
// EnsureLabel checks if a label exists, and creates it if it doesn't.
// Usage: EnsureLabel(...)
func (c *Client) EnsureLabel(owner, repo, name, color string) (*forgejo.Label, error) {
label, err := c.GetLabelByName(owner, repo, name)
if err == nil {
@ -91,6 +179,7 @@ func (c *Client) EnsureLabel(owner, repo, name, color string) (*forgejo.Label, e
}
// AddIssueLabels adds labels to an issue.
// Usage: AddIssueLabels(...)
func (c *Client) AddIssueLabels(owner, repo string, number int64, labelIDs []int64) error {
_, _, err := c.api.AddIssueLabels(owner, repo, number, forgejo.IssueLabelsOption{
Labels: labelIDs,
@ -102,6 +191,7 @@ func (c *Client) AddIssueLabels(owner, repo string, number int64, labelIDs []int
}
// RemoveIssueLabel removes a label from an issue.
// Usage: RemoveIssueLabel(...)
func (c *Client) RemoveIssueLabel(owner, repo string, number int64, labelID int64) error {
_, err := c.api.DeleteIssueLabel(owner, repo, number, labelID)
if err != nil {

View file

@ -1,6 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"net/http"
"net/http/httptest"
"testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -24,7 +28,7 @@ func TestClient_ListRepoLabels_Good(t *testing.T) {
assert.Equal(t, "feature", labels[1].Name)
}
func TestClient_ListRepoLabels_Bad_ServerError(t *testing.T) {
func TestClient_ListRepoLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -33,6 +37,53 @@ func TestClient_ListRepoLabels_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list repo labels")
}
func TestClient_ListRepoLabelsIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/labels", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 3, "name": "documentation", "color": "#00aa00"},
})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/labels?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/labels?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "name": "bug", "color": "#ff0000"},
{"id": 2, "name": "feature", "color": "#0000ff"},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var names []string
for label, err := range client.ListRepoLabelsIter("test-org", "org-repo") {
require.NoError(t, err)
names = append(names, label.Name)
}
require.Len(t, names, 3)
assert.Equal(t, []string{"bug", "feature", "documentation"}, names)
}
func TestClient_ListRepoLabelsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListRepoLabelsIter("test-org", "org-repo") {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list repo labels")
break
}
}
func TestClient_CreateRepoLabel_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -42,7 +93,7 @@ func TestClient_CreateRepoLabel_Good(t *testing.T) {
assert.NotNil(t, label)
}
func TestClient_CreateRepoLabel_Bad_ServerError(t *testing.T) {
func TestClient_CreateRepoLabel_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -60,7 +111,7 @@ func TestClient_GetLabelByName_Good(t *testing.T) {
assert.Equal(t, "bug", label.Name)
}
func TestClient_GetLabelByName_Good_CaseInsensitive(t *testing.T) {
func TestClient_GetLabelByName_Good_CaseInsensitive_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -69,7 +120,7 @@ func TestClient_GetLabelByName_Good_CaseInsensitive(t *testing.T) {
assert.Equal(t, "bug", label.Name)
}
func TestClient_GetLabelByName_Bad_NotFound(t *testing.T) {
func TestClient_GetLabelByName_Bad_NotFound_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -78,7 +129,7 @@ func TestClient_GetLabelByName_Bad_NotFound(t *testing.T) {
assert.Contains(t, err.Error(), "label nonexistent not found")
}
func TestClient_EnsureLabel_Good_Exists(t *testing.T) {
func TestClient_EnsureLabel_Good_Exists_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -88,7 +139,7 @@ func TestClient_EnsureLabel_Good_Exists(t *testing.T) {
assert.Equal(t, "bug", label.Name)
}
func TestClient_EnsureLabel_Good_Creates(t *testing.T) {
func TestClient_EnsureLabel_Good_Creates_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -104,11 +155,38 @@ func TestClient_ListOrgLabels_Good(t *testing.T) {
labels, err := client.ListOrgLabels("test-org")
require.NoError(t, err)
// Uses first repo's labels as representative.
assert.NotEmpty(t, labels)
require.Len(t, labels, 3)
assert.Equal(t, "bug", labels[0].Name)
assert.Equal(t, "feature", labels[1].Name)
assert.Equal(t, "documentation", labels[2].Name)
}
func TestClient_ListOrgLabels_Bad_ServerError(t *testing.T) {
func TestClient_ListOrgLabelsIter_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
var names []string
for label, err := range client.ListOrgLabelsIter("test-org") {
require.NoError(t, err)
names = append(names, label.Name)
}
require.Len(t, names, 3)
assert.Equal(t, []string{"bug", "feature", "documentation"}, names)
}
func TestClient_ListOrgLabelsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListOrgLabelsIter("test-org") {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list org repos")
break
}
}
func TestClient_ListOrgLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -124,7 +202,7 @@ func TestClient_AddIssueLabels_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_AddIssueLabels_Bad_ServerError(t *testing.T) {
func TestClient_AddIssueLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -141,7 +219,7 @@ func TestClient_RemoveIssueLabel_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_RemoveIssueLabel_Bad_ServerError(t *testing.T) {
func TestClient_RemoveIssueLabel_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()

View file

@ -1,10 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"time"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log"
)
@ -38,6 +38,7 @@ const commentPageSize = 50
// GetPRMeta returns structural signals for a pull request.
// This is the Forgejo side of the dual MetaReader described in the pipeline design.
// Usage: GetPRMeta(...)
func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
pull, _, err := c.api.GetPullRequest(owner, repo, pr)
if err != nil {
@ -75,19 +76,11 @@ func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
// Fetch comment count from the issue side (PRs are issues in Forgejo).
// Paginate to get an accurate count.
count := 0
page := 1
for {
comments, _, listErr := c.api.ListIssueComments(owner, repo, pr, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
if listErr != nil {
for _, err := range c.ListIssueCommentsIter(owner, repo, pr) {
if err != nil {
break
}
count += len(comments)
if len(comments) < commentPageSize {
break
}
page++
count++
}
meta.CommentCount = count
@ -95,45 +88,31 @@ func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
}
// GetCommentBodies returns all comment bodies for a pull request.
// Usage: GetCommentBodies(...)
func (c *Client) GetCommentBodies(owner, repo string, pr int64) ([]Comment, error) {
var comments []Comment
page := 1
for {
raw, _, err := c.api.ListIssueComments(owner, repo, pr, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
for raw, err := range c.ListIssueCommentsIter(owner, repo, pr) {
if err != nil {
return nil, log.E("forge.GetCommentBodies", "failed to get PR comments", err)
}
if len(raw) == 0 {
break
comment := Comment{
ID: raw.ID,
Body: raw.Body,
CreatedAt: raw.Created,
UpdatedAt: raw.Updated,
}
for _, rc := range raw {
comment := Comment{
ID: rc.ID,
Body: rc.Body,
CreatedAt: rc.Created,
UpdatedAt: rc.Updated,
}
if rc.Poster != nil {
comment.Author = rc.Poster.UserName
}
comments = append(comments, comment)
if raw.Poster != nil {
comment.Author = raw.Poster.UserName
}
if len(raw) < commentPageSize {
break
}
page++
comments = append(comments, comment)
}
return comments, nil
}
// GetIssueBody returns the body text of an issue.
// Usage: GetIssueBody(...)
func (c *Client) GetIssueBody(owner, repo string, issue int64) (string, error) {
iss, _, err := c.api.GetIssue(owner, repo, issue)
if err != nil {

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
@ -23,7 +25,7 @@ func TestClient_GetPRMeta_Good(t *testing.T) {
assert.False(t, meta.IsMerged)
}
func TestClient_GetPRMeta_Bad_ServerError(t *testing.T) {
func TestClient_GetPRMeta_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -45,7 +47,7 @@ func TestClient_GetCommentBodies_Good(t *testing.T) {
assert.Equal(t, "user2", comments[1].Author)
}
func TestClient_GetCommentBodies_Bad_ServerError(t *testing.T) {
func TestClient_GetCommentBodies_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -63,7 +65,7 @@ func TestClient_GetIssueBody_Good(t *testing.T) {
assert.Equal(t, "First issue body", body)
}
func TestClient_GetIssueBody_Bad_ServerError(t *testing.T) {
func TestClient_GetIssueBody_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()

View file

@ -1,12 +1,17 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"iter"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log"
)
// ListMyOrgs returns all organisations for the authenticated user.
// Usage: ListMyOrgs(...)
func (c *Client) ListMyOrgs() ([]*forgejo.Organization, error) {
var all []*forgejo.Organization
page := 1
@ -30,7 +35,37 @@ func (c *Client) ListMyOrgs() ([]*forgejo.Organization, error) {
return all, nil
}
// ListMyOrgsIter returns an iterator over organisations for the authenticated user.
// Usage: ListMyOrgsIter(...)
func (c *Client) ListMyOrgsIter() iter.Seq2[*forgejo.Organization, error] {
return func(yield func(*forgejo.Organization, error) bool) {
page := 1
for {
orgs, resp, err := c.api.ListMyOrgs(forgejo.ListOrgsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListMyOrgs", "failed to list orgs", err))
return
}
for _, org := range orgs {
if !yield(org, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// GetOrg returns a single organisation by name.
// Usage: GetOrg(...)
func (c *Client) GetOrg(name string) (*forgejo.Organization, error) {
org, _, err := c.api.GetOrg(name)
if err != nil {
@ -41,6 +76,7 @@ func (c *Client) GetOrg(name string) (*forgejo.Organization, error) {
}
// CreateOrg creates a new organisation.
// Usage: CreateOrg(...)
func (c *Client) CreateOrg(opts forgejo.CreateOrgOption) (*forgejo.Organization, error) {
org, _, err := c.api.CreateOrg(opts)
if err != nil {

View file

@ -1,6 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"net/http"
"net/http/httptest"
"testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -19,7 +23,42 @@ func TestClient_ListMyOrgs_Good(t *testing.T) {
assert.Equal(t, "test-org", orgs[0].UserName)
}
func TestClient_ListMyOrgs_Bad_ServerError(t *testing.T) {
func TestClient_ListMyOrgsIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/user/orgs", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 101, "login": "second-org", "username": "second-org", "full_name": "Second Organisation"},
})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/user/orgs?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/user/orgs?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 100, "login": "test-org", "username": "test-org", "full_name": "Test Organisation"},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var names []string
for org, err := range client.ListMyOrgsIter() {
require.NoError(t, err)
names = append(names, org.UserName)
}
require.Len(t, names, 2)
assert.Equal(t, []string{"test-org", "second-org"}, names)
}
func TestClient_ListMyOrgs_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -28,6 +67,17 @@ func TestClient_ListMyOrgs_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list orgs")
}
func TestClient_ListMyOrgsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListMyOrgsIter() {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list orgs")
break
}
}
func TestClient_GetOrg_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -37,7 +87,7 @@ func TestClient_GetOrg_Good(t *testing.T) {
assert.Equal(t, "test-org", org.UserName)
}
func TestClient_GetOrg_Bad_ServerError(t *testing.T) {
func TestClient_GetOrg_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -59,7 +109,7 @@ func TestClient_CreateOrg_Good(t *testing.T) {
assert.NotNil(t, org)
}
func TestClient_CreateOrg_Bad_ServerError(t *testing.T) {
func TestClient_CreateOrg_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()

View file

@ -1,17 +1,24 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"bytes"
"encoding/json"
"fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
"iter"
"net/http"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"net/url"
"strconv"
"dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
)
// MergePullRequest merges a pull request with the given method ("squash", "rebase", "merge").
// Usage: MergePullRequest(...)
func (c *Client) MergePullRequest(owner, repo string, index int64, method string) error {
style := forgejo.MergeStyleMerge
switch method {
@ -37,15 +44,29 @@ func (c *Client) MergePullRequest(owner, repo string, index int64, method string
// SetPRDraft sets or clears the draft status on a pull request.
// The Forgejo SDK v2.2.0 doesn't expose the draft field on EditPullRequestOption,
// so we use a raw HTTP PATCH request.
// Usage: SetPRDraft(...)
func (c *Client) SetPRDraft(owner, repo string, index int64, draft bool) error {
safeOwner, err := agentci.ValidatePathElement(owner)
if err != nil {
return log.E("forge.SetPRDraft", "invalid owner", err)
}
safeRepo, err := agentci.ValidatePathElement(repo)
if err != nil {
return log.E("forge.SetPRDraft", "invalid repo", err)
}
payload := map[string]bool{"draft": draft}
body, err := json.Marshal(payload)
if err != nil {
return log.E("forge.SetPRDraft", "marshal payload", err)
}
url := fmt.Sprintf("%s/api/v1/repos/%s/%s/pulls/%d", c.url, owner, repo, index)
req, err := http.NewRequest(http.MethodPatch, url, bytes.NewReader(body))
path, err := url.JoinPath(c.url, "api", "v1", "repos", safeOwner, safeRepo, "pulls", strconv.FormatInt(index, 10))
if err != nil {
return log.E("forge.SetPRDraft", "failed to build request path", err)
}
req, err := http.NewRequest(http.MethodPatch, path, bytes.NewReader(body))
if err != nil {
return log.E("forge.SetPRDraft", "create request", err)
}
@ -65,6 +86,7 @@ func (c *Client) SetPRDraft(owner, repo string, index int64, draft bool) error {
}
// ListPRReviews returns all reviews for a pull request.
// Usage: ListPRReviews(...)
func (c *Client) ListPRReviews(owner, repo string, index int64) ([]*forgejo.PullReview, error) {
var all []*forgejo.PullReview
page := 1
@ -88,7 +110,35 @@ func (c *Client) ListPRReviews(owner, repo string, index int64) ([]*forgejo.Pull
return all, nil
}
// ListPRReviewsIter returns an iterator over reviews for a pull request.
// Usage: ListPRReviewsIter(...)
func (c *Client) ListPRReviewsIter(owner, repo string, index int64) iter.Seq2[*forgejo.PullReview, error] {
return func(yield func(*forgejo.PullReview, error) bool) {
page := 1
for {
reviews, resp, err := c.api.ListPullReviews(owner, repo, index, forgejo.ListPullReviewsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListPRReviews", "failed to list reviews", err))
return
}
for _, review := range reviews {
if !yield(review, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// GetCombinedStatus returns the combined commit status for a ref (SHA or branch).
// Usage: GetCombinedStatus(...)
func (c *Client) GetCombinedStatus(owner, repo string, ref string) (*forgejo.CombinedStatus, error) {
status, _, err := c.api.GetCombinedStatus(owner, repo, ref)
if err != nil {
@ -98,6 +148,7 @@ func (c *Client) GetCombinedStatus(owner, repo string, ref string) (*forgejo.Com
}
// DismissReview dismisses a pull request review by ID.
// Usage: DismissReview(...)
func (c *Client) DismissReview(owner, repo string, index, reviewID int64, message string) error {
_, err := c.api.DismissPullReview(owner, repo, index, reviewID, forgejo.DismissPullReviewOptions{
Message: message,
@ -107,3 +158,13 @@ func (c *Client) DismissReview(owner, repo string, index, reviewID int64, messag
}
return nil
}
// UndismissReview removes a dismissal from a pull request review.
// Usage: UndismissReview(...)
func (c *Client) UndismissReview(owner, repo string, index, reviewID int64) error {
_, err := c.api.UnDismissPullReview(owner, repo, index, reviewID)
if err != nil {
return log.E("forge.UndismissReview", "failed to undismiss review", err)
}
return nil
}

View file

@ -1,7 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"strings"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http"
"net/http/httptest"
"testing"
"github.com/stretchr/testify/assert"
@ -16,7 +21,7 @@ func TestClient_MergePullRequest_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_MergePullRequest_Good_Squash(t *testing.T) {
func TestClient_MergePullRequest_Good_Squash_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -24,7 +29,7 @@ func TestClient_MergePullRequest_Good_Squash(t *testing.T) {
require.NoError(t, err)
}
func TestClient_MergePullRequest_Good_Rebase(t *testing.T) {
func TestClient_MergePullRequest_Good_Rebase_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -32,7 +37,7 @@ func TestClient_MergePullRequest_Good_Rebase(t *testing.T) {
require.NoError(t, err)
}
func TestClient_MergePullRequest_Bad_ServerError(t *testing.T) {
func TestClient_MergePullRequest_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -55,7 +60,43 @@ func TestClient_ListPRReviews_Good(t *testing.T) {
require.Len(t, reviews, 1)
}
func TestClient_ListPRReviews_Bad_ServerError(t *testing.T) {
func TestClient_ListPRReviewsIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/pulls/1/reviews", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "state": "REQUEST_CHANGES", "user": map[string]any{"login": "reviewer2"}},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/pulls/1/reviews?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/pulls/1/reviews?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "state": "APPROVED", "user": map[string]any{"login": "reviewer1"}},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var states []string
for review, err := range client.ListPRReviewsIter("test-org", "org-repo", 1) {
require.NoError(t, err)
states = append(states, string(review.State))
}
require.Equal(t, []string{"APPROVED", "REQUEST_CHANGES"}, states)
}
func TestClient_ListPRReviews_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -64,6 +105,19 @@ func TestClient_ListPRReviews_Bad_ServerError(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list reviews")
}
func TestClient_ListPRReviewsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
var got bool
for _, err := range client.ListPRReviewsIter("test-org", "org-repo", 1) {
assert.Error(t, err)
got = true
}
assert.True(t, got)
}
func TestClient_GetCombinedStatus_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
@ -73,7 +127,7 @@ func TestClient_GetCombinedStatus_Good(t *testing.T) {
assert.NotNil(t, status)
}
func TestClient_GetCombinedStatus_Bad_ServerError(t *testing.T) {
func TestClient_GetCombinedStatus_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -90,7 +144,7 @@ func TestClient_DismissReview_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_DismissReview_Bad_ServerError(t *testing.T) {
func TestClient_DismissReview_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -98,3 +152,66 @@ func TestClient_DismissReview_Bad_ServerError(t *testing.T) {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to dismiss review")
}
func TestClient_UndismissReview_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
err := client.UndismissReview("test-org", "org-repo", 1, 1)
require.NoError(t, err)
}
func TestClient_UndismissReview_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
err := client.UndismissReview("test-org", "org-repo", 1, 1)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to undismiss review")
}
func TestClient_SetPRDraft_Good_Request_Good(t *testing.T) {
var method, path string
var payload map[string]any
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/pulls/3", func(w http.ResponseWriter, r *http.Request) {
method = r.Method
path = r.URL.Path
require.NoError(t, json.NewDecoder(r.Body).Decode(&payload))
jsonResponse(w, map[string]any{"number": 3})
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
err = client.SetPRDraft("test-org", "org-repo", 3, false)
assert.NoError(t, err)
assert.Equal(t, http.MethodPatch, method)
assert.Equal(t, "/api/v1/repos/test-org/org-repo/pulls/3", path)
assert.Equal(t, false, payload["draft"])
}
func TestClient_SetPRDraft_Bad_PathTraversalOwner_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
err := client.SetPRDraft("../owner", "org-repo", 3, true)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid owner")
}
func TestClient_SetPRDraft_Bad_PathTraversalRepo_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
err := client.SetPRDraft("test-org", "..", 3, true)
assert.Error(t, err)
assert.Contains(t, err.Error(), "invalid repo")
}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
@ -9,6 +11,7 @@ import (
)
// ListOrgRepos returns all repositories for the given organisation.
// Usage: ListOrgRepos(...)
func (c *Client) ListOrgRepos(org string) ([]*forgejo.Repository, error) {
var all []*forgejo.Repository
page := 1
@ -33,6 +36,7 @@ func (c *Client) ListOrgRepos(org string) ([]*forgejo.Repository, error) {
}
// ListOrgReposIter returns an iterator over repositories for the given organisation.
// Usage: ListOrgReposIter(...)
func (c *Client) ListOrgReposIter(org string) iter.Seq2[*forgejo.Repository, error] {
return func(yield func(*forgejo.Repository, error) bool) {
page := 1
@ -58,6 +62,7 @@ func (c *Client) ListOrgReposIter(org string) iter.Seq2[*forgejo.Repository, err
}
// ListUserRepos returns all repositories for the authenticated user.
// Usage: ListUserRepos(...)
func (c *Client) ListUserRepos() ([]*forgejo.Repository, error) {
var all []*forgejo.Repository
page := 1
@ -82,6 +87,7 @@ func (c *Client) ListUserRepos() ([]*forgejo.Repository, error) {
}
// ListUserReposIter returns an iterator over repositories for the authenticated user.
// Usage: ListUserReposIter(...)
func (c *Client) ListUserReposIter() iter.Seq2[*forgejo.Repository, error] {
return func(yield func(*forgejo.Repository, error) bool) {
page := 1
@ -107,6 +113,7 @@ func (c *Client) ListUserReposIter() iter.Seq2[*forgejo.Repository, error] {
}
// GetRepo returns a single repository by owner and name.
// Usage: GetRepo(...)
func (c *Client) GetRepo(owner, name string) (*forgejo.Repository, error) {
repo, _, err := c.api.GetRepo(owner, name)
if err != nil {
@ -117,6 +124,7 @@ func (c *Client) GetRepo(owner, name string) (*forgejo.Repository, error) {
}
// CreateOrgRepo creates a new empty repository under an organisation.
// Usage: CreateOrgRepo(...)
func (c *Client) CreateOrgRepo(org string, opts forgejo.CreateRepoOption) (*forgejo.Repository, error) {
repo, _, err := c.api.CreateOrgRepo(org, opts)
if err != nil {
@ -127,6 +135,7 @@ func (c *Client) CreateOrgRepo(org string, opts forgejo.CreateRepoOption) (*forg
}
// DeleteRepo deletes a repository from Forgejo.
// Usage: DeleteRepo(...)
func (c *Client) DeleteRepo(owner, name string) error {
_, err := c.api.DeleteRepo(owner, name)
if err != nil {
@ -138,6 +147,7 @@ func (c *Client) DeleteRepo(owner, name string) error {
// MigrateRepo migrates a repository from an external service using the Forgejo migration API.
// Unlike CreateMirror, this supports importing issues, labels, PRs, and more.
// Usage: MigrateRepo(...)
func (c *Client) MigrateRepo(opts forgejo.MigrateRepoOption) (*forgejo.Repository, error) {
repo, _, err := c.api.MigrateRepo(opts)
if err != nil {

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
@ -15,11 +17,12 @@ func TestClient_ListOrgRepos_Good(t *testing.T) {
repos, err := client.ListOrgRepos("test-org")
require.NoError(t, err)
require.Len(t, repos, 1)
require.Len(t, repos, 2)
assert.Equal(t, "org-repo", repos[0].Name)
assert.Equal(t, "second-repo", repos[1].Name)
}
func TestClient_ListOrgRepos_Bad_ServerError(t *testing.T) {
func TestClient_ListOrgRepos_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -39,7 +42,7 @@ func TestClient_ListUserRepos_Good(t *testing.T) {
assert.Equal(t, "repo-b", repos[1].Name)
}
func TestClient_ListUserRepos_Bad_ServerError(t *testing.T) {
func TestClient_ListUserRepos_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -57,7 +60,7 @@ func TestClient_GetRepo_Good(t *testing.T) {
assert.Equal(t, "org-repo", repo.Name)
}
func TestClient_GetRepo_Bad_ServerError(t *testing.T) {
func TestClient_GetRepo_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -78,7 +81,7 @@ func TestClient_CreateOrgRepo_Good(t *testing.T) {
assert.NotNil(t, repo)
}
func TestClient_CreateOrgRepo_Bad_ServerError(t *testing.T) {
func TestClient_CreateOrgRepo_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -97,7 +100,7 @@ func TestClient_DeleteRepo_Good(t *testing.T) {
require.NoError(t, err)
}
func TestClient_DeleteRepo_Bad_ServerError(t *testing.T) {
func TestClient_DeleteRepo_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -119,7 +122,7 @@ func TestClient_MigrateRepo_Good(t *testing.T) {
assert.NotNil(t, repo)
}
func TestClient_MigrateRepo_Bad_ServerError(t *testing.T) {
func TestClient_MigrateRepo_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()

View file

@ -1,10 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
@ -56,6 +58,7 @@ func newForgejoMux() *http.ServeMux {
}
jsonResponse(w, []map[string]any{
{"id": 10, "name": "org-repo", "full_name": "test-org/org-repo", "owner": map[string]any{"login": "test-org", "id": 100}},
{"id": 11, "name": "second-repo", "full_name": "test-org/second-repo", "owner": map[string]any{"login": "test-org", "id": 100}},
})
})
@ -80,7 +83,8 @@ func newForgejoMux() *http.ServeMux {
}
jsonResponse(w, map[string]any{
"id": 10, "name": "org-repo", "full_name": "test-org/org-repo",
"owner": map[string]any{"login": "test-org"},
"owner": map[string]any{"login": "test-org"},
"default_branch": "main",
})
})
@ -226,6 +230,13 @@ func newForgejoMux() *http.ServeMux {
})
})
mux.HandleFunc("/api/v1/repos/test-org/second-repo/labels", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, []map[string]any{
{"id": 2, "name": "feature", "color": "#0000ff"},
{"id": 3, "name": "documentation", "color": "#00aa00"},
})
})
// Webhooks.
mux.HandleFunc("/api/v1/repos/test-org/org-repo/hooks", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
@ -293,6 +304,13 @@ func newForgejoMux() *http.ServeMux {
})
})
// Undismiss review.
mux.HandleFunc("/api/v1/repos/test-org/org-repo/pulls/1/reviews/1/undismissals", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]any{
"id": 1, "state": "open",
})
})
// Generic fallback — handles PATCH for SetPRDraft and other unmatched routes.
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// Handle PATCH requests (SetPRDraft).

View file

@ -1,12 +1,17 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"iter"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log"
)
// CreateRepoWebhook creates a webhook on a repository.
// Usage: CreateRepoWebhook(...)
func (c *Client) CreateRepoWebhook(owner, repo string, opts forgejo.CreateHookOption) (*forgejo.Hook, error) {
hook, _, err := c.api.CreateRepoHook(owner, repo, opts)
if err != nil {
@ -17,6 +22,7 @@ func (c *Client) CreateRepoWebhook(owner, repo string, opts forgejo.CreateHookOp
}
// ListRepoWebhooks returns all webhooks for a repository.
// Usage: ListRepoWebhooks(...)
func (c *Client) ListRepoWebhooks(owner, repo string) ([]*forgejo.Hook, error) {
var all []*forgejo.Hook
page := 1
@ -39,3 +45,29 @@ func (c *Client) ListRepoWebhooks(owner, repo string) ([]*forgejo.Hook, error) {
return all, nil
}
// ListRepoWebhooksIter returns an iterator over webhooks for a repository.
// Usage: ListRepoWebhooksIter(...)
func (c *Client) ListRepoWebhooksIter(owner, repo string) iter.Seq2[*forgejo.Hook, error] {
return func(yield func(*forgejo.Hook, error) bool) {
page := 1
for {
hooks, resp, err := c.api.ListRepoHooks(owner, repo, forgejo.ListHooksOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListRepoWebhooks", "failed to list repo webhooks", err))
return
}
for _, hook := range hooks {
if !yield(hook, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}

View file

@ -1,6 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge
import (
"net/http"
"net/http/httptest"
"testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -23,7 +27,7 @@ func TestClient_CreateRepoWebhook_Good(t *testing.T) {
assert.NotNil(t, hook)
}
func TestClient_CreateRepoWebhook_Bad_ServerError(t *testing.T) {
func TestClient_CreateRepoWebhook_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -43,7 +47,7 @@ func TestClient_ListRepoWebhooks_Good(t *testing.T) {
require.Len(t, hooks, 1)
}
func TestClient_ListRepoWebhooks_Bad_ServerError(t *testing.T) {
func TestClient_ListRepoWebhooks_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
@ -51,3 +55,40 @@ func TestClient_ListRepoWebhooks_Bad_ServerError(t *testing.T) {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list repo webhooks")
}
func TestClient_ListRepoWebhooksIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/hooks", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "type": "forgejo", "active": true, "config": map[string]any{"url": "https://example.com/second"}},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/hooks?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/hooks?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "type": "forgejo", "active": true, "config": map[string]any{"url": "https://example.com/hook"}},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var urls []string
for hook, err := range client.ListRepoWebhooksIter("test-org", "org-repo") {
require.NoError(t, err)
urls = append(urls, hook.Config["url"])
}
require.Len(t, urls, 2)
assert.Equal(t, []string{"https://example.com/hook", "https://example.com/second"}, urls)
}

View file

@ -1,16 +1,18 @@
// SPDX-License-Identifier: EUPL-1.2
// Package git provides utilities for git operations across multiple repositories.
package git
import (
"bytes"
"context"
os "dappco.re/go/core/scm/internal/ax/osx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"io"
"iter"
"os"
"os/exec"
"slices"
"strconv"
"strings"
"sync"
)
@ -28,16 +30,19 @@ type RepoStatus struct {
}
// IsDirty returns true if there are uncommitted changes.
// Usage: IsDirty(...)
func (s *RepoStatus) IsDirty() bool {
return s.Modified > 0 || s.Untracked > 0 || s.Staged > 0
}
// HasUnpushed returns true if there are commits to push.
// Usage: HasUnpushed(...)
func (s *RepoStatus) HasUnpushed() bool {
return s.Ahead > 0
}
// HasUnpulled returns true if there are commits to pull.
// Usage: HasUnpulled(...)
func (s *RepoStatus) HasUnpulled() bool {
return s.Behind > 0
}
@ -51,6 +56,7 @@ type StatusOptions struct {
}
// Status checks git status for multiple repositories in parallel.
// Usage: Status(...)
func Status(ctx context.Context, opts StatusOptions) []RepoStatus {
var wg sync.WaitGroup
results := make([]RepoStatus, len(opts.Paths))
@ -72,6 +78,7 @@ func Status(ctx context.Context, opts StatusOptions) []RepoStatus {
}
// StatusIter returns an iterator over git status for multiple repositories.
// Usage: StatusIter(...)
func StatusIter(ctx context.Context, opts StatusOptions) iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) {
results := Status(ctx, opts)
@ -156,17 +163,20 @@ func getAheadBehind(ctx context.Context, path string) (ahead, behind int) {
// Push pushes commits for a single repository.
// Uses interactive mode to support SSH passphrase prompts.
// Usage: Push(...)
func Push(ctx context.Context, path string) error {
return gitInteractive(ctx, path, "push")
}
// Pull pulls changes for a single repository.
// Uses interactive mode to support SSH passphrase prompts.
// Usage: Pull(...)
func Pull(ctx context.Context, path string) error {
return gitInteractive(ctx, path, "pull", "--rebase")
}
// IsNonFastForward checks if an error is a non-fast-forward rejection.
// Usage: IsNonFastForward(...)
func IsNonFastForward(err error) bool {
if err == nil {
return false
@ -210,11 +220,13 @@ type PushResult struct {
// PushMultiple pushes multiple repositories sequentially.
// Sequential because SSH passphrase prompts need user interaction.
// Usage: PushMultiple(...)
func PushMultiple(ctx context.Context, paths []string, names map[string]string) []PushResult {
return slices.Collect(PushMultipleIter(ctx, paths, names))
}
// PushMultipleIter returns an iterator that pushes repositories sequentially and yields results.
// Usage: PushMultipleIter(...)
func PushMultipleIter(ctx context.Context, paths []string, names map[string]string) iter.Seq[PushResult] {
return func(yield func(PushResult) bool) {
for _, path := range paths {
@ -269,6 +281,7 @@ type GitError struct {
}
// Error returns the git error message, preferring stderr output.
// Usage: Error(...)
func (e *GitError) Error() string {
// Return just the stderr message, trimmed
msg := strings.TrimSpace(e.Stderr)
@ -279,6 +292,7 @@ func (e *GitError) Error() string {
}
// Unwrap returns the underlying error for error chain inspection.
// Usage: Unwrap(...)
func (e *GitError) Unwrap() error {
return e.Err
}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package git
import (
@ -54,6 +56,7 @@ type Service struct {
}
// NewService creates a git service factory.
// Usage: NewService(...)
func NewService(opts ServiceOptions) func(*core.Core) (any, error) {
return func(c *core.Core) (any, error) {
return &Service{
@ -63,6 +66,7 @@ func NewService(opts ServiceOptions) func(*core.Core) (any, error) {
}
// OnStartup registers query and task handlers.
// Usage: OnStartup(...)
func (s *Service) OnStartup(ctx context.Context) error {
s.Core().RegisterQuery(s.handleQuery)
s.Core().RegisterTask(s.handleTask)
@ -101,14 +105,17 @@ func (s *Service) handleTask(c *core.Core, t core.Task) core.Result {
}
// Status returns last status result.
// Usage: Status(...)
func (s *Service) Status() []RepoStatus { return s.lastStatus }
// StatusIter returns an iterator over last status result.
// Usage: StatusIter(...)
func (s *Service) StatusIter() iter.Seq[RepoStatus] {
return slices.Values(s.lastStatus)
}
// DirtyRepos returns repos with uncommitted changes.
// Usage: DirtyRepos(...)
func (s *Service) DirtyRepos() []RepoStatus {
var dirty []RepoStatus
for _, st := range s.lastStatus {
@ -120,6 +127,7 @@ func (s *Service) DirtyRepos() []RepoStatus {
}
// DirtyReposIter returns an iterator over repos with uncommitted changes.
// Usage: DirtyReposIter(...)
func (s *Service) DirtyReposIter() iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) {
for _, st := range s.lastStatus {
@ -133,6 +141,7 @@ func (s *Service) DirtyReposIter() iter.Seq[RepoStatus] {
}
// AheadRepos returns repos with unpushed commits.
// Usage: AheadRepos(...)
func (s *Service) AheadRepos() []RepoStatus {
var ahead []RepoStatus
for _, st := range s.lastStatus {
@ -144,6 +153,7 @@ func (s *Service) AheadRepos() []RepoStatus {
}
// AheadReposIter returns an iterator over repos with unpushed commits.
// Usage: AheadReposIter(...)
func (s *Service) AheadReposIter() iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) {
for _, st := range s.lastStatus {

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
// Package gitea provides a thin wrapper around the Gitea Go SDK
// for managing repositories, issues, and pull requests on a Gitea instance.
//
@ -16,22 +18,41 @@ import (
// Client wraps the Gitea SDK client with config-based auth.
type Client struct {
api *gitea.Client
url string
api *gitea.Client
url string
token string
}
// New creates a new Gitea API client for the given URL and token.
// Usage: New(...)
func New(url, token string) (*Client, error) {
api, err := gitea.NewClient(url, gitea.SetToken(token))
if err != nil {
return nil, log.E("gitea.New", "failed to create client", err)
}
return &Client{api: api, url: url}, nil
return &Client{api: api, url: url, token: token}, nil
}
// API exposes the underlying SDK client for direct access.
// Usage: API(...)
func (c *Client) API() *gitea.Client { return c.api }
// URL returns the Gitea instance URL.
// Usage: URL(...)
func (c *Client) URL() string { return c.url }
// Token returns the Gitea API token.
// Usage: Token(...)
func (c *Client) Token() string { return c.token }
// GetCurrentUser returns the authenticated user's information.
// Usage: GetCurrentUser(...)
func (c *Client) GetCurrentUser() (*gitea.User, error) {
user, _, err := c.api.GetMyUserInfo()
if err != nil {
return nil, log.E("gitea.GetCurrentUser", "failed to get current user", err)
}
return user, nil
}

View file

@ -1,3 +1,5 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea
import (
@ -18,7 +20,7 @@ func TestNew_Good(t *testing.T) {
assert.Equal(t, srv.URL, client.URL())
}
func TestNew_Bad_InvalidURL(t *testing.T) {
func TestNew_Bad_InvalidURL_Good(t *testing.T) {
_, err := New("://invalid-url", "token")
assert.Error(t, err)
}
@ -36,3 +38,22 @@ func TestClient_URL_Good(t *testing.T) {
assert.Equal(t, srv.URL, client.URL())
}
func TestClient_GetCurrentUser_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
user, err := client.GetCurrentUser()
require.NoError(t, err)
require.NotNil(t, user)
assert.Equal(t, "test-user", user.UserName)
}
func TestClient_GetCurrentUser_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
_, err := client.GetCurrentUser()
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to get current user")
}

Some files were not shown because too many files have changed in this diff Show more