Compare commits

..

6 commits
dev ... main

Author SHA1 Message Date
476fbab546 Merge pull request 'dev' (#12) from dev into main
Some checks failed
Security Scan / security (push) Failing after 10s
Test / test (push) Failing after 42s
Reviewed-on: #12
2026-03-23 20:41:08 +00:00
e93f5673ef Merge pull request '[agent/codex] Convention drift check. Read CLAUDE.md. stdlib→core.*, UK ...' (#11) from agent/convention-drift-check--read-claude-md into dev
Some checks failed
Security Scan / security (pull_request) Failing after 9s
Test / test (pull_request) Failing after 37s
2026-03-23 15:11:46 +00:00
Virgil
8e2f02868a docs: add convention drift audit report
Co-Authored-By: Virgil <virgil@lethean.io>
2026-03-23 15:11:18 +00:00
7bdbd1301f Merge pull request '[agent/codex:gpt-5.3-codex-spark] Fix ALL security findings from issue #6. Read CLAUDE.md. Com...' (#10) from agent/fix-all-security-findings-in-issue--6--r into dev
Some checks failed
Security Scan / security (push) Failing after 8s
Test / test (push) Failing after 37s
2026-03-23 14:32:29 +00:00
Virgil
ae0677a046 fix(security): harden installer, marketplace, and sync path handling
Co-Authored-By: Virgil <virgil@lethean.io>
2026-03-23 14:32:11 +00:00
Claude
2d3d61c6f4
chore: update dependencies to dappco.re tagged versions
Some checks failed
Security Scan / security (push) Failing after 8s
Test / test (push) Successful in 4m54s
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-22 01:10:23 +00:00
199 changed files with 1729 additions and 9755 deletions

View file

@ -1,11 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci package agentci
import ( import (
"context" "context"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
"math"
"dappco.re/go/core/scm/jobrunner" "dappco.re/go/core/scm/jobrunner"
) )
@ -14,9 +11,7 @@ import (
type RunMode string type RunMode string
const ( const (
//
ModeStandard RunMode = "standard" ModeStandard RunMode = "standard"
//
ModeDual RunMode = "dual" // The Clotho Protocol — dual-run verification ModeDual RunMode = "dual" // The Clotho Protocol — dual-run verification
) )
@ -27,7 +22,6 @@ type Spinner struct {
} }
// NewSpinner creates a new Clotho orchestrator. // NewSpinner creates a new Clotho orchestrator.
// Usage: NewSpinner(...)
func NewSpinner(cfg ClothoConfig, agents map[string]AgentConfig) *Spinner { func NewSpinner(cfg ClothoConfig, agents map[string]AgentConfig) *Spinner {
return &Spinner{ return &Spinner{
Config: cfg, Config: cfg,
@ -37,7 +31,6 @@ func NewSpinner(cfg ClothoConfig, agents map[string]AgentConfig) *Spinner {
// DeterminePlan decides if a signal requires dual-run verification based on // DeterminePlan decides if a signal requires dual-run verification based on
// the global strategy, agent configuration, and repository criticality. // the global strategy, agent configuration, and repository criticality.
// Usage: DeterminePlan(...)
func (s *Spinner) DeterminePlan(signal *jobrunner.PipelineSignal, agentName string) RunMode { func (s *Spinner) DeterminePlan(signal *jobrunner.PipelineSignal, agentName string) RunMode {
if s.Config.Strategy != "clotho-verified" { if s.Config.Strategy != "clotho-verified" {
return ModeStandard return ModeStandard
@ -60,7 +53,6 @@ func (s *Spinner) DeterminePlan(signal *jobrunner.PipelineSignal, agentName stri
} }
// GetVerifierModel returns the model for the secondary "signed" verification run. // GetVerifierModel returns the model for the secondary "signed" verification run.
// Usage: GetVerifierModel(...)
func (s *Spinner) GetVerifierModel(agentName string) string { func (s *Spinner) GetVerifierModel(agentName string) string {
agent, ok := s.Agents[agentName] agent, ok := s.Agents[agentName]
if !ok || agent.VerifyModel == "" { if !ok || agent.VerifyModel == "" {
@ -71,7 +63,6 @@ func (s *Spinner) GetVerifierModel(agentName string) string {
// FindByForgejoUser resolves a Forgejo username to the agent config key and config. // FindByForgejoUser resolves a Forgejo username to the agent config key and config.
// This decouples agent naming (mythological roles) from Forgejo identity. // This decouples agent naming (mythological roles) from Forgejo identity.
// Usage: FindByForgejoUser(...)
func (s *Spinner) FindByForgejoUser(forgejoUser string) (string, AgentConfig, bool) { func (s *Spinner) FindByForgejoUser(forgejoUser string) (string, AgentConfig, bool) {
if forgejoUser == "" { if forgejoUser == "" {
return "", AgentConfig{}, false return "", AgentConfig{}, false
@ -90,61 +81,7 @@ func (s *Spinner) FindByForgejoUser(forgejoUser string) (string, AgentConfig, bo
} }
// Weave compares primary and verifier outputs. Returns true if they converge. // Weave compares primary and verifier outputs. Returns true if they converge.
// The comparison is a coarse token-overlap check controlled by the configured // This is a placeholder for future semantic diff logic.
// validation threshold. It is intentionally deterministic and fast; richer
// semantic diffing can replace it later without changing the signature.
// Usage: Weave(...)
func (s *Spinner) Weave(ctx context.Context, primaryOutput, signedOutput []byte) (bool, error) { func (s *Spinner) Weave(ctx context.Context, primaryOutput, signedOutput []byte) (bool, error) {
if ctx != nil { return string(primaryOutput) == string(signedOutput), nil
select {
case <-ctx.Done():
return false, ctx.Err()
default:
}
}
primary := tokenizeWeaveOutput(primaryOutput)
signed := tokenizeWeaveOutput(signedOutput)
if len(primary) == 0 && len(signed) == 0 {
return true, nil
}
threshold := s.Config.ValidationThreshold
if threshold <= 0 || threshold > 1 {
threshold = 0.85
}
similarity := weaveDiceSimilarity(primary, signed)
return similarity >= threshold, nil
}
func tokenizeWeaveOutput(output []byte) []string {
fields := strings.Fields(strings.ReplaceAll(string(output), "\r\n", "\n"))
if len(fields) == 0 {
return nil
}
return fields
}
func weaveDiceSimilarity(primary, signed []string) float64 {
if len(primary) == 0 || len(signed) == 0 {
return 0
}
counts := make(map[string]int, len(primary))
for _, token := range primary {
counts[token]++
}
common := 0
for _, token := range signed {
if counts[token] == 0 {
continue
}
counts[token]--
common++
}
return math.Min(1, (2*float64(common))/float64(len(primary)+len(signed)))
} }

View file

@ -1,73 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci
import (
"context"
"errors"
"testing"
"dappco.re/go/core/scm/jobrunner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSpinner_Weave_Good_ExactMatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.85}, nil)
ok, err := spinner.Weave(context.Background(), []byte("alpha beta gamma"), []byte("alpha beta gamma"))
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Good_ThresholdMatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.8}, nil)
ok, err := spinner.Weave(
context.Background(),
[]byte("alpha beta gamma delta epsilon zeta"),
[]byte("alpha beta gamma delta epsilon eta"),
)
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Bad_ThresholdMismatch(t *testing.T) {
spinner := NewSpinner(ClothoConfig{ValidationThreshold: 0.9}, nil)
ok, err := spinner.Weave(
context.Background(),
[]byte("alpha beta gamma delta epsilon zeta"),
[]byte("alpha beta gamma delta epsilon eta"),
)
require.NoError(t, err)
assert.False(t, ok)
}
func TestSpinner_Weave_Good_EmptyOutputs(t *testing.T) {
spinner := NewSpinner(ClothoConfig{}, nil)
ok, err := spinner.Weave(context.Background(), nil, nil)
require.NoError(t, err)
assert.True(t, ok)
}
func TestSpinner_Weave_Bad_ContextCancelled(t *testing.T) {
spinner := NewSpinner(ClothoConfig{}, nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
ok, err := spinner.Weave(ctx, []byte("alpha"), []byte("alpha"))
assert.False(t, ok)
require.Error(t, err)
assert.True(t, errors.Is(err, context.Canceled))
}
func TestSpinner_DeterminePlan_Good(t *testing.T) {
spinner := NewSpinner(ClothoConfig{Strategy: "clotho-verified"}, map[string]AgentConfig{
"charon": {DualRun: true},
})
ok := spinner.DeterminePlan(&jobrunner.PipelineSignal{RepoName: "docs"}, "charon")
assert.Equal(t, ModeDual, ok)
}

View file

@ -1,13 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
// Package agentci provides configuration, security, and orchestration for AgentCI dispatch targets. // Package agentci provides configuration, security, and orchestration for AgentCI dispatch targets.
package agentci package agentci
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
coreerr "dappco.re/go/core/log"
"forge.lthn.ai/core/config" "forge.lthn.ai/core/config"
coreerr "dappco.re/go/core/log"
) )
// AgentConfig represents a single agent machine in the config file. // AgentConfig represents a single agent machine in the config file.
@ -33,7 +31,6 @@ type ClothoConfig struct {
// LoadAgents reads agent targets from config and returns a map of AgentConfig. // LoadAgents reads agent targets from config and returns a map of AgentConfig.
// Returns an empty map (not an error) if no agents are configured. // Returns an empty map (not an error) if no agents are configured.
// Usage: LoadAgents(...)
func LoadAgents(cfg *config.Config) (map[string]AgentConfig, error) { func LoadAgents(cfg *config.Config) (map[string]AgentConfig, error) {
var agents map[string]AgentConfig var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil { if err := cfg.Get("agentci.agents", &agents); err != nil {
@ -64,7 +61,6 @@ func LoadAgents(cfg *config.Config) (map[string]AgentConfig, error) {
} }
// LoadActiveAgents returns only active agents. // LoadActiveAgents returns only active agents.
// Usage: LoadActiveAgents(...)
func LoadActiveAgents(cfg *config.Config) (map[string]AgentConfig, error) { func LoadActiveAgents(cfg *config.Config) (map[string]AgentConfig, error) {
all, err := LoadAgents(cfg) all, err := LoadAgents(cfg)
if err != nil { if err != nil {
@ -81,7 +77,6 @@ func LoadActiveAgents(cfg *config.Config) (map[string]AgentConfig, error) {
// LoadClothoConfig loads the Clotho orchestrator settings. // LoadClothoConfig loads the Clotho orchestrator settings.
// Returns sensible defaults if no config is present. // Returns sensible defaults if no config is present.
// Usage: LoadClothoConfig(...)
func LoadClothoConfig(cfg *config.Config) (ClothoConfig, error) { func LoadClothoConfig(cfg *config.Config) (ClothoConfig, error) {
var cc ClothoConfig var cc ClothoConfig
if err := cfg.Get("agentci.clotho", &cc); err != nil { if err := cfg.Get("agentci.clotho", &cc); err != nil {
@ -100,7 +95,6 @@ func LoadClothoConfig(cfg *config.Config) (ClothoConfig, error) {
} }
// SaveAgent writes an agent config entry to the config file. // SaveAgent writes an agent config entry to the config file.
// Usage: SaveAgent(...)
func SaveAgent(cfg *config.Config, name string, ac AgentConfig) error { func SaveAgent(cfg *config.Config, name string, ac AgentConfig) error {
key := fmt.Sprintf("agentci.agents.%s", name) key := fmt.Sprintf("agentci.agents.%s", name)
data := map[string]any{ data := map[string]any{
@ -129,7 +123,6 @@ func SaveAgent(cfg *config.Config, name string, ac AgentConfig) error {
} }
// RemoveAgent removes an agent from the config file. // RemoveAgent removes an agent from the config file.
// Usage: RemoveAgent(...)
func RemoveAgent(cfg *config.Config, name string) error { func RemoveAgent(cfg *config.Config, name string) error {
var agents map[string]AgentConfig var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil { if err := cfg.Get("agentci.agents", &agents); err != nil {
@ -143,7 +136,6 @@ func RemoveAgent(cfg *config.Config, name string) error {
} }
// ListAgents returns all configured agents (active and inactive). // ListAgents returns all configured agents (active and inactive).
// Usage: ListAgents(...)
func ListAgents(cfg *config.Config) (map[string]AgentConfig, error) { func ListAgents(cfg *config.Config) (map[string]AgentConfig, error) {
var agents map[string]AgentConfig var agents map[string]AgentConfig
if err := cfg.Get("agentci.agents", &agents); err != nil { if err := cfg.Get("agentci.agents", &agents); err != nil {

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci package agentci
import ( import (
"testing" "testing"
"dappco.re/go/core/io"
"forge.lthn.ai/core/config" "forge.lthn.ai/core/config"
"dappco.re/go/core/io"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@ -45,7 +43,7 @@ agentci:
assert.Equal(t, "claude", agent.Runner) assert.Equal(t, "claude", agent.Runner)
} }
func TestLoadAgents_Good_MultipleAgents_Good(t *testing.T) { func TestLoadAgents_Good_MultipleAgents(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -66,7 +64,7 @@ agentci:
assert.Contains(t, agents, "local-codex") assert.Contains(t, agents, "local-codex")
} }
func TestLoadAgents_Good_SkipsInactive_Good(t *testing.T) { func TestLoadAgents_Good_SkipsInactive(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -101,7 +99,7 @@ agentci:
assert.Contains(t, active, "active-agent") assert.Contains(t, active, "active-agent")
} }
func TestLoadAgents_Good_Defaults_Good(t *testing.T) { func TestLoadAgents_Good_Defaults(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -119,14 +117,14 @@ agentci:
assert.Equal(t, "claude", agent.Runner) assert.Equal(t, "claude", agent.Runner)
} }
func TestLoadAgents_Good_NoConfig_Good(t *testing.T) { func TestLoadAgents_Good_NoConfig(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
agents, err := LoadAgents(cfg) agents, err := LoadAgents(cfg)
require.NoError(t, err) require.NoError(t, err)
assert.Empty(t, agents) assert.Empty(t, agents)
} }
func TestLoadAgents_Bad_MissingHost_Good(t *testing.T) { func TestLoadAgents_Bad_MissingHost(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -139,7 +137,7 @@ agentci:
assert.Contains(t, err.Error(), "host is required") assert.Contains(t, err.Error(), "host is required")
} }
func TestLoadAgents_Good_WithDualRun_Good(t *testing.T) { func TestLoadAgents_Good_WithDualRun(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -176,7 +174,7 @@ agentci:
assert.Equal(t, "/etc/core/keys/clotho.pub", cc.SigningKeyPath) assert.Equal(t, "/etc/core/keys/clotho.pub", cc.SigningKeyPath)
} }
func TestLoadClothoConfig_Good_Defaults_Good(t *testing.T) { func TestLoadClothoConfig_Good_Defaults(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
cc, err := LoadClothoConfig(cfg) cc, err := LoadClothoConfig(cfg)
require.NoError(t, err) require.NoError(t, err)
@ -204,7 +202,7 @@ func TestSaveAgent_Good(t *testing.T) {
assert.Equal(t, "haiku", agents["new-agent"].Model) assert.Equal(t, "haiku", agents["new-agent"].Model)
} }
func TestSaveAgent_Good_WithDualRun_Good(t *testing.T) { func TestSaveAgent_Good_WithDualRun(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "verified-agent", AgentConfig{ err := SaveAgent(cfg, "verified-agent", AgentConfig{
@ -222,7 +220,7 @@ func TestSaveAgent_Good_WithDualRun_Good(t *testing.T) {
assert.True(t, agents["verified-agent"].DualRun) assert.True(t, agents["verified-agent"].DualRun)
} }
func TestSaveAgent_Good_OmitsEmptyOptionals_Good(t *testing.T) { func TestSaveAgent_Good_OmitsEmptyOptionals(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "minimal", AgentConfig{ err := SaveAgent(cfg, "minimal", AgentConfig{
@ -256,7 +254,7 @@ agentci:
assert.Contains(t, agents, "to-keep") assert.Contains(t, agents, "to-keep")
} }
func TestRemoveAgent_Bad_NotFound_Good(t *testing.T) { func TestRemoveAgent_Bad_NotFound(t *testing.T) {
cfg := newTestConfig(t, ` cfg := newTestConfig(t, `
agentci: agentci:
agents: agents:
@ -269,7 +267,7 @@ agentci:
assert.Contains(t, err.Error(), "not found") assert.Contains(t, err.Error(), "not found")
} }
func TestRemoveAgent_Bad_NoAgents_Good(t *testing.T) { func TestRemoveAgent_Bad_NoAgents(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
err := RemoveAgent(cfg, "anything") err := RemoveAgent(cfg, "anything")
assert.Error(t, err) assert.Error(t, err)
@ -294,14 +292,14 @@ agentci:
assert.False(t, agents["agent-b"].Active) assert.False(t, agents["agent-b"].Active)
} }
func TestListAgents_Good_Empty_Good(t *testing.T) { func TestListAgents_Good_Empty(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
agents, err := ListAgents(cfg) agents, err := ListAgents(cfg)
require.NoError(t, err) require.NoError(t, err)
assert.Empty(t, agents) assert.Empty(t, agents)
} }
func TestRoundTrip_Good_SaveThenLoad_Good(t *testing.T) { func TestRoundTrip_SaveThenLoad(t *testing.T) {
cfg := newTestConfig(t, "") cfg := newTestConfig(t, "")
err := SaveAgent(cfg, "alpha", AgentConfig{ err := SaveAgent(cfg, "alpha", AgentConfig{

View file

@ -1,171 +1,38 @@
// SPDX-License-Identifier: EUPL-1.2
package agentci package agentci
import ( import (
"context" "os/exec"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "path/filepath"
exec "golang.org/x/sys/execabs"
"path"
"regexp" "regexp"
"strings"
coreerr "dappco.re/go/core/log" coreerr "dappco.re/go/core/log"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
) )
var safeNameRegex = regexp.MustCompile(`^[a-zA-Z0-9\-\_\.]+$`) var safeNameRegex = regexp.MustCompile(`^[a-zA-Z0-9\-\_\.]+$`)
// SanitizePath ensures a filename or directory name is safe and prevents path traversal. // SanitizePath ensures a filename or directory name is safe and prevents path traversal.
// Returns the validated basename. // Returns filepath.Base of the input after validation.
// Usage: SanitizePath(...)
func SanitizePath(input string) (string, error) { func SanitizePath(input string) (string, error) {
if input == "" { base := filepath.Base(input)
return "", coreerr.E("agentci.SanitizePath", "path element is required", nil) if !safeNameRegex.MatchString(base) {
}
safeName := filepath.Base(input)
if safeName == "." || safeName == ".." {
return "", coreerr.E("agentci.SanitizePath", "invalid path element: "+input, nil)
}
if strings.ContainsAny(safeName, `/\`) {
return "", coreerr.E("agentci.SanitizePath", "path separators are not allowed: "+input, nil)
}
if !safeNameRegex.MatchString(safeName) {
return "", coreerr.E("agentci.SanitizePath", "invalid characters in path element: "+input, nil) return "", coreerr.E("agentci.SanitizePath", "invalid characters in path element: "+input, nil)
} }
return safeName, nil if base == "." || base == ".." || base == "/" {
return "", coreerr.E("agentci.SanitizePath", "invalid path element: "+base, nil)
} }
return base, nil
// ValidatePathElement validates a single local path element and returns its safe form.
// Usage: ValidatePathElement(...)
func ValidatePathElement(input string) (string, error) {
safeName, err := SanitizePath(input)
if err != nil {
return "", err
}
if safeName != input {
return "", coreerr.E("agentci.ValidatePathElement", "path separators are not allowed: "+input, nil)
}
return safeName, nil
}
// ResolvePathWithinRoot resolves a validated path element beneath a root directory.
// Usage: ResolvePathWithinRoot(...)
func ResolvePathWithinRoot(root string, input string) (string, string, error) {
safeName, err := ValidatePathElement(input)
if err != nil {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "invalid path element", err)
}
absRoot, err := filepath.Abs(root)
if err != nil {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "resolve root", err)
}
resolved := filepath.Clean(filepath.Join(absRoot, safeName))
cleanRoot := filepath.Clean(absRoot)
rootPrefix := cleanRoot + string(filepath.Separator)
if resolved != cleanRoot && !strings.HasPrefix(resolved, rootPrefix) {
return "", "", coreerr.E("agentci.ResolvePathWithinRoot", "resolved path escaped root", nil)
}
return safeName, resolved, nil
}
// ValidateRemoteDir validates a remote directory path used over SSH.
// Usage: ValidateRemoteDir(...)
func ValidateRemoteDir(dir string) (string, error) {
if strings.TrimSpace(dir) == "" {
return "", coreerr.E("agentci.ValidateRemoteDir", "directory is required", nil)
}
if strings.ContainsAny(dir, `\`) {
return "", coreerr.E("agentci.ValidateRemoteDir", "backslashes are not allowed", nil)
}
switch dir {
case "/", "~":
return dir, nil
}
cleaned := path.Clean(dir)
prefix := ""
rest := cleaned
if strings.HasPrefix(dir, "~/") {
prefix = "~/"
rest = strings.TrimPrefix(cleaned, "~/")
}
if strings.HasPrefix(dir, "/") {
prefix = "/"
rest = strings.TrimPrefix(cleaned, "/")
}
if rest == "." || rest == ".." || strings.HasPrefix(rest, "../") {
return "", coreerr.E("agentci.ValidateRemoteDir", "directory escaped root", nil)
}
for _, part := range strings.Split(rest, "/") {
if part == "" {
continue
}
if _, err := ValidatePathElement(part); err != nil {
return "", coreerr.E("agentci.ValidateRemoteDir", "invalid directory segment", err)
}
}
if rest == "" || rest == "." {
return prefix, nil
}
return prefix + rest, nil
}
// JoinRemotePath joins validated remote path elements using forward slashes.
// Usage: JoinRemotePath(...)
func JoinRemotePath(base string, parts ...string) (string, error) {
safeBase, err := ValidateRemoteDir(base)
if err != nil {
return "", coreerr.E("agentci.JoinRemotePath", "invalid base directory", err)
}
cleanParts := make([]string, 0, len(parts))
for _, part := range parts {
safePart, partErr := ValidatePathElement(part)
if partErr != nil {
return "", coreerr.E("agentci.JoinRemotePath", "invalid path element", partErr)
}
cleanParts = append(cleanParts, safePart)
}
if safeBase == "~" {
return path.Join("~", path.Join(cleanParts...)), nil
}
if strings.HasPrefix(safeBase, "~/") {
return "~/" + path.Join(strings.TrimPrefix(safeBase, "~/"), path.Join(cleanParts...)), nil
}
return path.Join(append([]string{safeBase}, cleanParts...)...), nil
} }
// EscapeShellArg wraps a string in single quotes for safe remote shell insertion. // EscapeShellArg wraps a string in single quotes for safe remote shell insertion.
// Prefer exec.Command arguments over constructing shell strings where possible. // Prefer exec.Command arguments over constructing shell strings where possible.
// Usage: EscapeShellArg(...)
func EscapeShellArg(arg string) string { func EscapeShellArg(arg string) string {
return "'" + strings.ReplaceAll(arg, "'", "'\\''") + "'" return "'" + strings.ReplaceAll(arg, "'", "'\\''") + "'"
} }
// SecureSSHCommand creates an SSH exec.Cmd with strict host key checking and batch mode. // SecureSSHCommand creates an SSH exec.Cmd with strict host key checking and batch mode.
// Usage: SecureSSHCommand(...)
func SecureSSHCommand(host string, remoteCmd string) *exec.Cmd { func SecureSSHCommand(host string, remoteCmd string) *exec.Cmd {
return SecureSSHCommandContext(context.Background(), host, remoteCmd) return exec.Command("ssh",
}
// SecureSSHCommandContext creates an SSH exec.Cmd with strict host key checking and batch mode.
// Usage: SecureSSHCommandContext(...)
func SecureSSHCommandContext(ctx context.Context, host string, remoteCmd string) *exec.Cmd {
if ctx == nil {
ctx = context.Background()
}
return exec.CommandContext(ctx, "ssh",
"-o", "StrictHostKeyChecking=yes", "-o", "StrictHostKeyChecking=yes",
"-o", "BatchMode=yes", "-o", "BatchMode=yes",
"-o", "ConnectTimeout=10", "-o", "ConnectTimeout=10",
@ -175,7 +42,6 @@ func SecureSSHCommandContext(ctx context.Context, host string, remoteCmd string)
} }
// MaskToken returns a masked version of a token for safe logging. // MaskToken returns a masked version of a token for safe logging.
// Usage: MaskToken(...)
func MaskToken(token string) string { func MaskToken(token string) string {
if len(token) < 8 { if len(token) < 8 {
return "*****" return "*****"

View file

@ -1,9 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2 // SPDX-Licence-Identifier: EUPL-1.2
package agentci package agentci
import ( import (
"context"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -21,9 +20,7 @@ func TestSanitizePath_Good(t *testing.T) {
{"with.dot", "with.dot"}, {"with.dot", "with.dot"},
{"CamelCase", "CamelCase"}, {"CamelCase", "CamelCase"},
{"123", "123"}, {"123", "123"},
{"../secret", "secret"}, {"path/to/file.txt", "file.txt"},
{"/var/tmp/report.txt", "report.txt"},
{"nested/path/file", "file"},
} }
for _, tt := range tests { for _, tt := range tests {
@ -47,11 +44,8 @@ func TestSanitizePath_Bad(t *testing.T) {
{"pipe", "file|name"}, {"pipe", "file|name"},
{"ampersand", "file&name"}, {"ampersand", "file&name"},
{"dollar", "file$name"}, {"dollar", "file$name"},
{"backslash", `path\to\file.txt`},
{"current dir", "."},
{"parent traversal base", ".."}, {"parent traversal base", ".."},
{"root", "/"}, {"root", "/"},
{"empty", ""},
} }
for _, tt := range tests { for _, tt := range tests {
@ -93,19 +87,6 @@ func TestSecureSSHCommand_Good(t *testing.T) {
assert.Equal(t, "ls -la", args[len(args)-1]) assert.Equal(t, "ls -la", args[len(args)-1])
} }
func TestSecureSSHCommandContext_Good(t *testing.T) {
cmd := SecureSSHCommandContext(context.Background(), "host.example.com", "ls -la")
args := cmd.Args
assert.Equal(t, "ssh", args[0])
assert.Contains(t, args, "-o")
assert.Contains(t, args, "StrictHostKeyChecking=yes")
assert.Contains(t, args, "BatchMode=yes")
assert.Contains(t, args, "ConnectTimeout=10")
assert.Equal(t, "host.example.com", args[len(args)-2])
assert.Equal(t, "ls -la", args[len(args)-1])
}
func TestMaskToken_Good(t *testing.T) { func TestMaskToken_Good(t *testing.T) {
tests := []struct { tests := []struct {
name string name string

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n" "dappco.re/go/core/i18n"
"dappco.re/go/core/io" "dappco.re/go/core/io"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli"
) )
func init() { func init() {
@ -30,7 +28,6 @@ var (
) )
// AddCollectCommands registers the 'collect' command and all subcommands. // AddCollectCommands registers the 'collect' command and all subcommands.
// Usage: AddCollectCommands(...)
func AddCollectCommands(root *cli.Command) { func AddCollectCommands(root *cli.Command) {
collectCmd := &cli.Command{ collectCmd := &cli.Command{
Use: "collect", Use: "collect",

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// BitcoinTalk command flags // BitcoinTalk command flags

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
"time" "time"
"dappco.re/go/core/i18n"
collectpkg "dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
collectpkg "dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// addDispatchCommand adds the 'dispatch' subcommand to the collect parent. // addDispatchCommand adds the 'dispatch' subcommand to the collect parent.

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// Excavate command flags // Excavate command flags

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// GitHub command flags // GitHub command flags

View file

@ -1,13 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// Market command flags // Market command flags

View file

@ -1,13 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// Papers command flags // Papers command flags

View file

@ -1,13 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
"dappco.re/go/core/i18n"
"dappco.re/go/core/scm/collect"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/collect"
"dappco.re/go/core/i18n"
) )
// addProcessCommand adds the 'process' subcommand to the collect parent. // addProcessCommand adds the 'process' subcommand to the collect parent.

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Auth command flags. // Auth command flags.

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Config command flags. // Config command flags.

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package forge provides CLI commands for managing a Forgejo instance. // Package forge provides CLI commands for managing a Forgejo instance.
// //
// Commands: // Commands:
@ -35,7 +33,6 @@ var (
) )
// AddForgeCommands registers the 'forge' command and all subcommands. // AddForgeCommands registers the 'forge' command and all subcommands.
// Usage: AddForgeCommands(...)
func AddForgeCommands(root *cli.Command) { func AddForgeCommands(root *cli.Command) {
forgeCmd := &cli.Command{ forgeCmd := &cli.Command{
Use: "forge", Use: "forge",

View file

@ -1,15 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Issues command flags. // Issues command flags.

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Labels command flags. // Labels command flags.

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Migrate command flags. // Migrate command flags.

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// addOrgsCommand adds the 'orgs' subcommand for listing organisations. // addOrgsCommand adds the 'orgs' subcommand for listing organisations.

View file

@ -1,15 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// PRs command flags. // PRs command flags.

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// Repos command flags. // Repos command flags.

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
fg "dappco.re/go/core/scm/forge"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
fg "dappco.re/go/core/scm/forge"
) )
// addStatusCommand adds the 'status' subcommand for instance info. // addStatusCommand adds the 'status' subcommand for instance info.

View file

@ -1,17 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "net/url"
os "dappco.re/go/core/scm/internal/ax/osx" "os"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "os/exec"
exec "golang.org/x/sys/execabs" "path/filepath"
"strings"
coreerr "dappco.re/go/core/log" coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci" "dappco.re/go/core/scm/agentci"
"dappco.re/go/core/scm/cmd/internal/syncutil"
fg "dappco.re/go/core/scm/forge" fg "dappco.re/go/core/scm/forge"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -99,7 +97,7 @@ func buildSyncRepoList(client *fg.Client, args []string, basePath string) ([]syn
if len(args) > 0 { if len(args) > 0 {
for _, arg := range args { for _, arg := range args {
name, err := syncutil.ParseRepoName(arg) name, err := syncRepoNameFromArg(arg)
if err != nil { if err != nil {
return nil, coreerr.E("forge.buildSyncRepoList", "invalid repo argument", err) return nil, coreerr.E("forge.buildSyncRepoList", "invalid repo argument", err)
} }
@ -347,3 +345,27 @@ func syncCreateMainFromUpstream(client *fg.Client, org, repo string) error {
return nil return nil
} }
func syncRepoNameFromArg(arg string) (string, error) {
decoded, err := url.PathUnescape(arg)
if err != nil {
return "", coreerr.E("forge.syncRepoNameFromArg", "decode repo argument", err)
}
parts := strings.Split(decoded, "/")
switch len(parts) {
case 1:
return agentci.ValidatePathElement(parts[0])
case 2:
if _, err := agentci.ValidatePathElement(parts[0]); err != nil {
return "", coreerr.E("forge.syncRepoNameFromArg", "invalid repo owner", err)
}
name, err := agentci.ValidatePathElement(parts[1])
if err != nil {
return "", coreerr.E("forge.syncRepoNameFromArg", "invalid repo name", err)
}
return name, nil
default:
return "", coreerr.E("forge.syncRepoNameFromArg", "repo argument must be repo or owner/repo", nil)
}
}

View file

@ -1,9 +1,7 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -20,7 +18,7 @@ func TestBuildSyncRepoList_Good(t *testing.T) {
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath) assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
} }
func TestBuildSyncRepoList_Bad_PathTraversal_Good(t *testing.T) { func TestBuildSyncRepoList_Bad_PathTraversal(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"../escape"}, basePath) _, err := buildSyncRepoList(nil, []string{"../escape"}, basePath)
@ -28,7 +26,7 @@ func TestBuildSyncRepoList_Bad_PathTraversal_Good(t *testing.T) {
assert.Contains(t, err.Error(), "invalid repo argument") assert.Contains(t, err.Error(), "invalid repo argument")
} }
func TestBuildSyncRepoList_Good_OwnerRepo_Good(t *testing.T) { func TestBuildSyncRepoList_Good_OwnerRepo(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildSyncRepoList(nil, []string{"Host-UK/core"}, basePath) repos, err := buildSyncRepoList(nil, []string{"Host-UK/core"}, basePath)
@ -38,7 +36,7 @@ func TestBuildSyncRepoList_Good_OwnerRepo_Good(t *testing.T) {
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath) assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
} }
func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) { func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepo(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"host-uk/../escape"}, basePath) _, err := buildSyncRepoList(nil, []string{"host-uk/../escape"}, basePath)
@ -46,7 +44,7 @@ func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) {
assert.Contains(t, err.Error(), "invalid repo argument") assert.Contains(t, err.Error(), "invalid repo argument")
} }
func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepoEncoded_Good(t *testing.T) { func TestBuildSyncRepoList_Bad_PathTraversal_OwnerRepoEncoded(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildSyncRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath) _, err := buildSyncRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath)

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"path" "path"
"strings"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
) )

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
) )
// Config command flags. // Config command flags.

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package gitea provides CLI commands for managing a Gitea instance. // Package gitea provides CLI commands for managing a Gitea instance.
// //
// Commands: // Commands:
@ -32,7 +30,6 @@ var (
) )
// AddGiteaCommands registers the 'gitea' command and all subcommands. // AddGiteaCommands registers the 'gitea' command and all subcommands.
// Usage: AddGiteaCommands(...)
func AddGiteaCommands(root *cli.Command) { func AddGiteaCommands(root *cli.Command) {
giteaCmd := &cli.Command{ giteaCmd := &cli.Command{
Use: "gitea", Use: "gitea",

View file

@ -1,15 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
"code.gitea.io/sdk/gitea" "code.gitea.io/sdk/gitea"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
) )
// Issues command flags. // Issues command flags.

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "os/exec"
exec "golang.org/x/sys/execabs" "strings"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
) )
// Mirror command flags. // Mirror command flags.

View file

@ -1,15 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "strings"
sdk "code.gitea.io/sdk/gitea" sdk "code.gitea.io/sdk/gitea"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
) )
// PRs command flags. // PRs command flags.

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
gt "dappco.re/go/core/scm/gitea"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
gt "dappco.re/go/core/scm/gitea"
) )
// Repos command flags. // Repos command flags.

View file

@ -1,17 +1,15 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "net/url"
os "dappco.re/go/core/scm/internal/ax/osx" "os"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "os/exec"
exec "golang.org/x/sys/execabs" "path/filepath"
"strings"
coreerr "dappco.re/go/core/log" coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci" "dappco.re/go/core/scm/agentci"
"dappco.re/go/core/scm/cmd/internal/syncutil"
gt "dappco.re/go/core/scm/gitea" gt "dappco.re/go/core/scm/gitea"
"code.gitea.io/sdk/gitea" "code.gitea.io/sdk/gitea"
@ -100,7 +98,7 @@ func buildRepoList(client *gt.Client, args []string, basePath string) ([]repoEnt
if len(args) > 0 { if len(args) > 0 {
// Specific repos from args // Specific repos from args
for _, arg := range args { for _, arg := range args {
name, err := syncutil.ParseRepoName(arg) name, err := repoNameFromArg(arg)
if err != nil { if err != nil {
return nil, coreerr.E("gitea.buildRepoList", "invalid repo argument", err) return nil, coreerr.E("gitea.buildRepoList", "invalid repo argument", err)
} }
@ -365,3 +363,27 @@ func createMainFromUpstream(client *gt.Client, org, repo string) error {
} }
func strPtr(s string) *string { return &s } func strPtr(s string) *string { return &s }
func repoNameFromArg(arg string) (string, error) {
decoded, err := url.PathUnescape(arg)
if err != nil {
return "", coreerr.E("gitea.repoNameFromArg", "decode repo argument", err)
}
parts := strings.Split(decoded, "/")
switch len(parts) {
case 1:
return agentci.ValidatePathElement(parts[0])
case 2:
if _, err := agentci.ValidatePathElement(parts[0]); err != nil {
return "", coreerr.E("gitea.repoNameFromArg", "invalid repo owner", err)
}
name, err := agentci.ValidatePathElement(parts[1])
if err != nil {
return "", coreerr.E("gitea.repoNameFromArg", "invalid repo name", err)
}
return name, nil
default:
return "", coreerr.E("gitea.repoNameFromArg", "repo argument must be repo or owner/repo", nil)
}
}

View file

@ -1,9 +1,7 @@
// SPDX-License-Identifier: EUPL-1.2
package gitea package gitea
import ( import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -20,7 +18,7 @@ func TestBuildRepoList_Good(t *testing.T) {
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath) assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
} }
func TestBuildRepoList_Bad_PathTraversal_Good(t *testing.T) { func TestBuildRepoList_Bad_PathTraversal(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"../escape"}, basePath) _, err := buildRepoList(nil, []string{"../escape"}, basePath)
@ -28,7 +26,7 @@ func TestBuildRepoList_Bad_PathTraversal_Good(t *testing.T) {
assert.Contains(t, err.Error(), "invalid repo argument") assert.Contains(t, err.Error(), "invalid repo argument")
} }
func TestBuildRepoList_Good_OwnerRepo_Good(t *testing.T) { func TestBuildRepoList_Good_OwnerRepo(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
repos, err := buildRepoList(nil, []string{"Host-UK/core"}, basePath) repos, err := buildRepoList(nil, []string{"Host-UK/core"}, basePath)
@ -38,7 +36,7 @@ func TestBuildRepoList_Good_OwnerRepo_Good(t *testing.T) {
assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath) assert.Equal(t, filepath.Join(basePath, "core"), repos[0].localPath)
} }
func TestBuildRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) { func TestBuildRepoList_Bad_PathTraversal_OwnerRepo(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"host-uk/../escape"}, basePath) _, err := buildRepoList(nil, []string{"host-uk/../escape"}, basePath)
@ -46,7 +44,7 @@ func TestBuildRepoList_Bad_PathTraversal_OwnerRepo_Good(t *testing.T) {
assert.Contains(t, err.Error(), "invalid repo argument") assert.Contains(t, err.Error(), "invalid repo argument")
} }
func TestBuildRepoList_Bad_PathTraversal_OwnerRepoEncoded_Good(t *testing.T) { func TestBuildRepoList_Bad_PathTraversal_OwnerRepoEncoded(t *testing.T) {
basePath := filepath.Join(t.TempDir(), "repos") basePath := filepath.Join(t.TempDir(), "repos")
_, err := buildRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath) _, err := buildRepoList(nil, []string{"host-uk%2F..%2Fescape"}, basePath)

View file

@ -1,37 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package syncutil
import (
"net/url"
coreerr "dappco.re/go/core/log"
"dappco.re/go/core/scm/agentci"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
)
// ParseRepoName normalises a sync argument into a validated repo name.
// Usage: ParseRepoName(...)
func ParseRepoName(arg string) (string, error) {
decoded, err := url.PathUnescape(arg)
if err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "decode repo argument", err)
}
parts := strings.Split(decoded, "/")
switch len(parts) {
case 1:
return agentci.ValidatePathElement(parts[0])
case 2:
if _, err := agentci.ValidatePathElement(parts[0]); err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "invalid repo owner", err)
}
name, err := agentci.ValidatePathElement(parts[1])
if err != nil {
return "", coreerr.E("syncutil.ParseRepoName", "invalid repo name", err)
}
return name, nil
default:
return "", coreerr.E("syncutil.ParseRepoName", "repo argument must be repo or owner/repo", nil)
}
}

View file

@ -1,34 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package syncutil
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestParseRepoName_Good(t *testing.T) {
name, err := ParseRepoName("core")
require.NoError(t, err)
assert.Equal(t, "core", name)
}
func TestParseRepoName_Good_OwnerRepo(t *testing.T) {
name, err := ParseRepoName("host-uk/core")
require.NoError(t, err)
assert.Equal(t, "core", name)
}
func TestParseRepoName_Bad_PathTraversal(t *testing.T) {
_, err := ParseRepoName("../escape")
require.Error(t, err)
assert.Contains(t, err.Error(), "syncutil.ParseRepoName")
}
func TestParseRepoName_Bad_PathTraversalEncoded(t *testing.T) {
_, err := ParseRepoName("host-uk%2F..%2Fescape")
require.Error(t, err)
assert.Contains(t, err.Error(), "syncutil.ParseRepoName")
}

View file

@ -1,47 +1,40 @@
// SPDX-License-Identifier: EUPL-1.2
package scm package scm
import ( import (
"crypto/ed25519" "crypto/ed25519"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"encoding/hex" "encoding/hex"
exec "golang.org/x/sys/execabs" "os/exec"
"strings"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/io" "dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest" "dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
) )
func addCompileCommand(parent *cli.Command) { func addCompileCommand(parent *cli.Command) {
var ( var (
version string
dir string dir string
signKey string signKey string
builtBy string builtBy string
output string
) )
cmd := &cli.Command{ cmd := &cli.Command{
Use: "compile", Use: "compile",
Short: "Compile manifest.yaml into core.json", Short: "Compile manifest.yaml into core.json",
Long: "Read .core/manifest.yaml, attach build metadata (commit, tag), and write core.json to the project root or a custom output path.", Long: "Read .core/manifest.yaml, attach build metadata (commit, tag), and write core.json to the project root.",
RunE: func(cmd *cli.Command, args []string) error { RunE: func(cmd *cli.Command, args []string) error {
return runCompile(dir, version, signKey, builtBy, output) return runCompile(dir, signKey, builtBy)
}, },
} }
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory") cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&version, "version", "", "Override the manifest version")
cmd.Flags().StringVar(&signKey, "sign-key", "", "Hex-encoded ed25519 private key for signing") cmd.Flags().StringVar(&signKey, "sign-key", "", "Hex-encoded ed25519 private key for signing")
cmd.Flags().StringVar(&builtBy, "built-by", "core scm compile", "Builder identity") cmd.Flags().StringVar(&builtBy, "built-by", "core scm compile", "Builder identity")
cmd.Flags().StringVarP(&output, "output", "o", "core.json", "Output path for the compiled manifest")
parent.AddCommand(cmd) parent.AddCommand(cmd)
} }
func runCompile(dir, version, signKeyHex, builtBy, output string) error { func runCompile(dir, signKeyHex, builtBy string) error {
medium, err := io.NewSandboxed(dir) medium, err := io.NewSandboxed(dir)
if err != nil { if err != nil {
return cli.WrapVerb(err, "open", dir) return cli.WrapVerb(err, "open", dir)
@ -53,7 +46,6 @@ func runCompile(dir, version, signKeyHex, builtBy, output string) error {
} }
opts := manifest.CompileOptions{ opts := manifest.CompileOptions{
Version: version,
Commit: gitCommit(dir), Commit: gitCommit(dir),
Tag: gitTag(dir), Tag: gitTag(dir),
BuiltBy: builtBy, BuiltBy: builtBy,
@ -72,28 +64,20 @@ func runCompile(dir, version, signKeyHex, builtBy, output string) error {
return err return err
} }
data, err := manifest.MarshalJSON(cm) if err := manifest.WriteCompiled(medium, ".", cm); err != nil {
if err != nil { return err
return cli.WrapVerb(err, "marshal", "manifest")
}
if err := medium.EnsureDir(filepath.Dir(output)); err != nil {
return cli.WrapVerb(err, "create", filepath.Dir(output))
}
if err := medium.Write(output, string(data)); err != nil {
return cli.WrapVerb(err, "write", output)
} }
cli.Blank() cli.Blank()
cli.Print(" %s %s\n", successStyle.Render("compiled"), valueStyle.Render(m.Code)) cli.Print(" %s %s\n", successStyle.Render("compiled"), valueStyle.Render(m.Code))
cli.Print(" %s %s\n", dimStyle.Render("version:"), valueStyle.Render(cm.Version)) cli.Print(" %s %s\n", dimStyle.Render("version:"), valueStyle.Render(m.Version))
if opts.Commit != "" { if opts.Commit != "" {
cli.Print(" %s %s\n", dimStyle.Render("commit:"), valueStyle.Render(opts.Commit)) cli.Print(" %s %s\n", dimStyle.Render("commit:"), valueStyle.Render(opts.Commit))
} }
if opts.Tag != "" { if opts.Tag != "" {
cli.Print(" %s %s\n", dimStyle.Render("tag:"), valueStyle.Render(opts.Tag)) cli.Print(" %s %s\n", dimStyle.Render("tag:"), valueStyle.Render(opts.Tag))
} }
cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render(output)) cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render("core.json"))
cli.Blank() cli.Blank()
return nil return nil

View file

@ -1,122 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"encoding/hex"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunCompile_Good_DefaultOutput_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-default
name: Compile Default
version: 1.0.0
`), 0644))
err := runCompile(dir, "", "", "core scm compile", "core.json")
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, "core.json"))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "compile-default", cm.Code)
assert.Equal(t, "core scm compile", cm.BuiltBy)
}
func TestRunCompile_Good_CustomOutput_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-custom
name: Compile Custom
version: 2.0.0
`), 0644))
output := filepath.Join("dist", "core.json")
err := runCompile(dir, "", "", "custom builder", output)
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, output))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "compile-custom", cm.Code)
assert.Equal(t, "custom builder", cm.BuiltBy)
}
func TestRunCompile_Bad_InvalidSignKey_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-invalid-key
name: Compile Invalid Key
version: 1.0.0
`), 0644))
err := runCompile(dir, "", hex.EncodeToString([]byte("short")), "core scm compile", "core.json")
require.Error(t, err)
assert.Contains(t, err.Error(), "invalid private key length")
}
func TestRunCompile_Good_VersionOverride_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: compile-version
name: Compile Version
version: 1.0.0
`), 0644))
err := runCompile(dir, "3.2.1", "", "core scm compile", "core.json")
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, "core.json"))
require.NoError(t, err)
cm, err := manifest.ParseCompiled([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "3.2.1", cm.Version)
}
func TestAddScmCommands_Good_CompileVersionFlagRegistered_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var compileCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
if cmd.Name() == "compile" {
compileCmd = cmd
break
}
}
require.NotNil(t, compileCmd)
assert.NotNil(t, compileCmd.Flags().Lookup("version"))
}

View file

@ -1,14 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package scm package scm
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
os "dappco.re/go/core/scm/internal/ax/osx"
"forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/io" "dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest" "dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
) )
func addExportCommand(parent *cli.Command) { func addExportCommand(parent *cli.Command) {
@ -17,7 +14,7 @@ func addExportCommand(parent *cli.Command) {
cmd := &cli.Command{ cmd := &cli.Command{
Use: "export", Use: "export",
Short: "Export compiled manifest as JSON", Short: "Export compiled manifest as JSON",
Long: "Read core.json from the project root and print it to stdout. Falls back to compiling .core/manifest.yaml only when core.json is missing.", Long: "Read core.json from the project root and print it to stdout. Falls back to compiling .core/manifest.yaml if core.json is not found.",
RunE: func(cmd *cli.Command, args []string) error { RunE: func(cmd *cli.Command, args []string) error {
return runExport(dir) return runExport(dir)
}, },
@ -34,18 +31,10 @@ func runExport(dir string) error {
return cli.WrapVerb(err, "open", dir) return cli.WrapVerb(err, "open", dir)
} }
var cm *manifest.CompiledManifest // Try core.json first.
cm, err := manifest.LoadCompiled(medium, ".")
// Prefer core.json if it exists and is valid.
if raw, readErr := medium.Read("core.json"); readErr == nil {
cm, err = manifest.ParseCompiled([]byte(raw))
if err != nil { if err != nil {
return err // Fall back to compiling from source.
}
} else if !os.IsNotExist(readErr) {
return cli.WrapVerb(readErr, "read", "core.json")
} else {
// Fall back to compiling from source only when the compiled artifact is absent.
m, loadErr := manifest.Load(medium, ".") m, loadErr := manifest.Load(medium, ".")
if loadErr != nil { if loadErr != nil {
return cli.WrapVerb(loadErr, "load", "manifest") return cli.WrapVerb(loadErr, "load", "manifest")

View file

@ -1,64 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"testing"
"dappco.re/go/core/scm/manifest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunExport_Good_CompiledManifest_Good(t *testing.T) {
dir := t.TempDir()
cm := &manifest.CompiledManifest{
Manifest: manifest.Manifest{
Code: "compiled-mod",
Name: "Compiled Module",
Version: "1.0.0",
},
Commit: "abc123",
}
data, err := manifest.MarshalJSON(cm)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(dir, "core.json"), data, 0644))
err = runExport(dir)
require.NoError(t, err)
}
func TestRunExport_Good_FallsBackToSource_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
`), 0644))
err := runExport(dir)
require.NoError(t, err)
}
func TestRunExport_Bad_InvalidCompiledManifest_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
`), 0644))
require.NoError(t, os.WriteFile(filepath.Join(dir, "core.json"), []byte("{not-json"), 0644))
err := runExport(dir)
require.Error(t, err)
assert.Contains(t, err.Error(), "manifest.ParseCompiled")
}

View file

@ -1,22 +1,18 @@
// SPDX-License-Identifier: EUPL-1.2
package scm package scm
import ( import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "path/filepath"
os "dappco.re/go/core/scm/internal/ax/osx"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/marketplace"
"forge.lthn.ai/core/cli/pkg/cli" "forge.lthn.ai/core/cli/pkg/cli"
"dappco.re/go/core/scm/marketplace"
) )
func addIndexCommand(parent *cli.Command) { func addIndexCommand(parent *cli.Command) {
var ( var (
dirs []string dirs []string
output string output string
forgeURL string baseURL string
org string org string
) )
@ -28,38 +24,31 @@ func addIndexCommand(parent *cli.Command) {
if len(dirs) == 0 { if len(dirs) == 0 {
dirs = []string{"."} dirs = []string{"."}
} }
return runIndex(dirs, output, forgeURL, org) return runIndex(dirs, output, baseURL, org)
}, },
} }
cmd.Flags().StringArrayVarP(&dirs, "dir", "d", nil, "Directories to scan (repeatable, default: current directory)") cmd.Flags().StringArrayVarP(&dirs, "dir", "d", nil, "Directories to scan (repeatable, default: current directory)")
cmd.Flags().StringVarP(&output, "output", "o", "index.json", "Output path for the index file") cmd.Flags().StringVarP(&output, "output", "o", "index.json", "Output path for the index file")
cmd.Flags().StringVar(&forgeURL, "forge-url", "", "Forge base URL for repo links (e.g. https://forge.lthn.ai)") cmd.Flags().StringVar(&baseURL, "base-url", "", "Base URL for repo links (e.g. https://forge.lthn.ai)")
cmd.Flags().StringVar(&forgeURL, "base-url", "", "Deprecated alias for --forge-url")
cmd.Flags().StringVar(&org, "org", "", "Organisation for repo links") cmd.Flags().StringVar(&org, "org", "", "Organisation for repo links")
parent.AddCommand(cmd) parent.AddCommand(cmd)
} }
func runIndex(dirs []string, output, forgeURL, org string) error { func runIndex(dirs []string, output, baseURL, org string) error {
repoPaths, err := expandIndexRepoPaths(dirs) b := &marketplace.Builder{
if err != nil { BaseURL: baseURL,
return err Org: org,
} }
idx, err := marketplace.BuildIndex(io.Local, repoPaths, marketplace.IndexOptions{ idx, err := b.BuildFromDirs(dirs...)
ForgeURL: forgeURL,
Org: org,
})
if err != nil { if err != nil {
return cli.WrapVerb(err, "build", "index") return cli.WrapVerb(err, "build", "index")
} }
absOutput, err := filepath.Abs(output) absOutput, _ := filepath.Abs(output)
if err != nil { if err := marketplace.WriteIndex(absOutput, idx); err != nil {
return cli.WrapVerb(err, "resolve", output)
}
if err := marketplace.WriteIndex(io.Local, absOutput, idx); err != nil {
return err return err
} }
@ -70,28 +59,3 @@ func runIndex(dirs []string, output, forgeURL, org string) error {
return nil return nil
} }
func expandIndexRepoPaths(dirs []string) ([]string, error) {
var repoPaths []string
for _, dir := range dirs {
repoPaths = append(repoPaths, dir)
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, cli.WrapVerb(err, "read", dir)
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
repoPaths = append(repoPaths, filepath.Join(dir, entry.Name()))
}
}
return repoPaths, nil
}

View file

@ -1,102 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
json "dappco.re/go/core/scm/internal/ax/jsonx"
os "dappco.re/go/core/scm/internal/ax/osx"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"dappco.re/go/core/scm/marketplace"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunIndex_Good_WritesIndex_Good(t *testing.T) {
root := t.TempDir()
modDir := filepath.Join(root, "mod-a")
require.NoError(t, os.MkdirAll(filepath.Join(modDir, ".core"), 0755))
require.NoError(t, os.WriteFile(filepath.Join(modDir, ".core", "manifest.yaml"), []byte(`
code: mod-a
name: Module A
version: 1.0.0
sign: key-a
`), 0644))
output := filepath.Join(root, "index.json")
err := runIndex([]string{root}, output, "https://forge.example.com", "core")
require.NoError(t, err)
idx, err := marketplace.LoadIndex(io.Local, output)
require.NoError(t, err)
require.Len(t, idx.Modules, 1)
assert.Equal(t, "mod-a", idx.Modules[0].Code)
assert.Equal(t, "https://forge.example.com/core/mod-a.git", idx.Modules[0].Repo)
}
func TestRunIndex_Good_PrefersCompiledManifest_Good(t *testing.T) {
root := t.TempDir()
modDir := filepath.Join(root, "mod-a")
require.NoError(t, os.MkdirAll(filepath.Join(modDir, ".core"), 0755))
cm := &manifest.CompiledManifest{
Manifest: manifest.Manifest{
Code: "compiled-mod",
Name: "Compiled Module",
Version: "2.0.0",
Sign: "compiled-key",
},
Commit: "abc123",
}
data, err := json.Marshal(cm)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(modDir, "core.json"), data, 0644))
require.NoError(t, os.WriteFile(filepath.Join(modDir, ".core", "manifest.yaml"), []byte(`
code: source-mod
name: Source Module
version: 1.0.0
sign: source-key
`), 0644))
output := filepath.Join(root, "index.json")
err = runIndex([]string{root}, output, "https://forge.example.com", "core")
require.NoError(t, err)
idx, err := marketplace.LoadIndex(io.Local, output)
require.NoError(t, err)
require.Len(t, idx.Modules, 1)
assert.Equal(t, "compiled-mod", idx.Modules[0].Code)
assert.Equal(t, "compiled-key", idx.Modules[0].SignKey)
}
func TestAddScmCommands_Good_IndexForgeURLFlagAlias_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var indexCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
if cmd.Name() == "index" {
indexCmd = cmd
break
}
}
require.NotNil(t, indexCmd)
assert.NotNil(t, indexCmd.Flags().Lookup("forge-url"))
assert.NotNil(t, indexCmd.Flags().Lookup("base-url"))
}

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package scm provides CLI commands for manifest compilation and marketplace // Package scm provides CLI commands for manifest compilation and marketplace
// index generation. // index generation.
// //
@ -7,8 +5,6 @@
// - compile: Compile .core/manifest.yaml into core.json // - compile: Compile .core/manifest.yaml into core.json
// - index: Build marketplace index from repository directories // - index: Build marketplace index from repository directories
// - export: Export a compiled manifest as JSON to stdout // - export: Export a compiled manifest as JSON to stdout
// - sign: Sign .core/manifest.yaml with an ed25519 private key
// - verify: Verify a manifest signature with an ed25519 public key
package scm package scm
import ( import (
@ -29,7 +25,6 @@ var (
) )
// AddScmCommands registers the 'scm' command and all subcommands. // AddScmCommands registers the 'scm' command and all subcommands.
// Usage: AddScmCommands(...)
func AddScmCommands(root *cli.Command) { func AddScmCommands(root *cli.Command) {
scmCmd := &cli.Command{ scmCmd := &cli.Command{
Use: "scm", Use: "scm",
@ -41,6 +36,4 @@ func AddScmCommands(root *cli.Command) {
addCompileCommand(scmCmd) addCompileCommand(scmCmd)
addIndexCommand(scmCmd) addIndexCommand(scmCmd)
addExportCommand(scmCmd) addExportCommand(scmCmd)
addSignCommand(scmCmd)
addVerifyCommand(scmCmd)
} }

View file

@ -1,137 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"crypto/ed25519"
"encoding/hex"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
)
func addSignCommand(parent *cli.Command) {
var (
dir string
signKey string
)
cmd := &cli.Command{
Use: "sign",
Short: "Sign manifest.yaml with a private key",
Long: "Read .core/manifest.yaml, attach an ed25519 signature, and write the signed manifest back to disk.",
RunE: func(cmd *cli.Command, args []string) error {
return runSign(dir, signKey)
},
}
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&signKey, "sign-key", "", "Hex-encoded ed25519 private key")
parent.AddCommand(cmd)
}
func runSign(dir, signKeyHex string) error {
if signKeyHex == "" {
return cli.Err("sign key is required")
}
medium, err := io.NewSandboxed(dir)
if err != nil {
return cli.WrapVerb(err, "open", dir)
}
m, err := manifest.Load(medium, ".")
if err != nil {
return cli.WrapVerb(err, "load", "manifest")
}
keyBytes, err := hex.DecodeString(signKeyHex)
if err != nil {
return cli.WrapVerb(err, "decode", "sign key")
}
if len(keyBytes) != ed25519.PrivateKeySize {
return cli.Err("sign key must be %d bytes when decoded", ed25519.PrivateKeySize)
}
if err := manifest.Sign(m, ed25519.PrivateKey(keyBytes)); err != nil {
return err
}
data, err := manifest.MarshalYAML(m)
if err != nil {
return cli.WrapVerb(err, "marshal", "manifest")
}
if err := medium.Write(".core/manifest.yaml", string(data)); err != nil {
return cli.WrapVerb(err, "write", ".core/manifest.yaml")
}
cli.Blank()
cli.Print(" %s %s\n", successStyle.Render("signed"), valueStyle.Render(m.Code))
cli.Print(" %s %s\n", dimStyle.Render("output:"), valueStyle.Render(".core/manifest.yaml"))
cli.Blank()
return nil
}
func addVerifyCommand(parent *cli.Command) {
var (
dir string
publicKey string
)
cmd := &cli.Command{
Use: "verify",
Short: "Verify manifest signature with a public key",
Long: "Read .core/manifest.yaml and verify its ed25519 signature against a public key.",
RunE: func(cmd *cli.Command, args []string) error {
return runVerify(dir, publicKey)
},
}
cmd.Flags().StringVarP(&dir, "dir", "d", ".", "Project root directory")
cmd.Flags().StringVar(&publicKey, "public-key", "", "Hex-encoded ed25519 public key")
parent.AddCommand(cmd)
}
func runVerify(dir, publicKeyHex string) error {
if publicKeyHex == "" {
return cli.Err("public key is required")
}
medium, err := io.NewSandboxed(dir)
if err != nil {
return cli.WrapVerb(err, "open", dir)
}
m, err := manifest.Load(medium, ".")
if err != nil {
return cli.WrapVerb(err, "load", "manifest")
}
keyBytes, err := hex.DecodeString(publicKeyHex)
if err != nil {
return cli.WrapVerb(err, "decode", "public key")
}
if len(keyBytes) != ed25519.PublicKeySize {
return cli.Err("public key must be %d bytes when decoded", ed25519.PublicKeySize)
}
valid, err := manifest.Verify(m, ed25519.PublicKey(keyBytes))
if err != nil {
return cli.WrapVerb(err, "verify", "manifest")
}
if !valid {
return cli.Err("signature verification failed for %s", m.Code)
}
cli.Blank()
cli.Success("Signature verified")
cli.Print(" %s %s\n", dimStyle.Render("code:"), valueStyle.Render(m.Code))
cli.Blank()
return nil
}

View file

@ -1,99 +0,0 @@
// SPDX-License-Identifier: EUPL-1.2
package scm
import (
"crypto/ed25519"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
os "dappco.re/go/core/scm/internal/ax/osx"
"encoding/hex"
"testing"
"dappco.re/go/core/io"
"dappco.re/go/core/scm/manifest"
"forge.lthn.ai/core/cli/pkg/cli"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRunSign_Good_WritesSignedManifest_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), []byte(`
code: signed-cli
name: Signed CLI
version: 1.0.0
`), 0644))
pub, priv, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
err = runSign(dir, hex.EncodeToString(priv))
require.NoError(t, err)
raw, err := io.Local.Read(filepath.Join(dir, ".core", "manifest.yaml"))
require.NoError(t, err)
m, err := manifest.Parse([]byte(raw))
require.NoError(t, err)
assert.Equal(t, "signed-cli", m.Code)
assert.NotEmpty(t, m.Sign)
valid, err := manifest.Verify(m, pub)
require.NoError(t, err)
assert.True(t, valid)
}
func TestRunVerify_Good_ValidSignature_Good(t *testing.T) {
dir := t.TempDir()
coreDir := filepath.Join(dir, ".core")
require.NoError(t, os.MkdirAll(coreDir, 0755))
pub, priv, err := ed25519.GenerateKey(nil)
require.NoError(t, err)
m := &manifest.Manifest{
Code: "verified-cli",
Name: "Verified CLI",
Version: "1.0.0",
}
require.NoError(t, manifest.Sign(m, priv))
data, err := manifest.MarshalYAML(m)
require.NoError(t, err)
require.NoError(t, os.WriteFile(filepath.Join(coreDir, "manifest.yaml"), data, 0644))
err = runVerify(dir, hex.EncodeToString(pub))
require.NoError(t, err)
}
func TestAddScmCommands_Good_SignAndVerifyRegistered_Good(t *testing.T) {
root := &cli.Command{Use: "root"}
AddScmCommands(root)
var scmCmd *cli.Command
for _, cmd := range root.Commands() {
if cmd.Name() == "scm" {
scmCmd = cmd
break
}
}
require.NotNil(t, scmCmd)
var signCmd *cli.Command
var verifyCmd *cli.Command
for _, cmd := range scmCmd.Commands() {
switch cmd.Name() {
case "sign":
signCmd = cmd
case "verify":
verifyCmd = cmd
}
}
require.NotNil(t, signCmd)
require.NotNil(t, verifyCmd)
assert.NotNil(t, signCmd.Flags().Lookup("sign-key"))
assert.NotNil(t, verifyCmd.Flags().Lookup("public-key"))
}

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"iter" "iter"
"net/http" "net/http"
"path/filepath"
"strings"
"time" "time"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
@ -35,7 +33,6 @@ type BitcoinTalkCollector struct {
} }
// Name returns the collector name. // Name returns the collector name.
// Usage: Name(...)
func (b *BitcoinTalkCollector) Name() string { func (b *BitcoinTalkCollector) Name() string {
id := b.TopicID id := b.TopicID
if id == "" && b.URL != "" { if id == "" && b.URL != "" {
@ -45,7 +42,6 @@ func (b *BitcoinTalkCollector) Name() string {
} }
// Collect gathers posts from a BitcoinTalk topic. // Collect gathers posts from a BitcoinTalk topic.
// Usage: Collect(...)
func (b *BitcoinTalkCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) { func (b *BitcoinTalkCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: b.Name()} result := &Result{Source: b.Name()}
@ -285,7 +281,6 @@ func formatPostMarkdown(num int, post btPost) string {
// ParsePostsFromHTML parses BitcoinTalk posts from raw HTML content. // ParsePostsFromHTML parses BitcoinTalk posts from raw HTML content.
// This is exported for testing purposes. // This is exported for testing purposes.
// Usage: ParsePostsFromHTML(...)
func ParsePostsFromHTML(htmlContent string) ([]btPost, error) { func ParsePostsFromHTML(htmlContent string) ([]btPost, error) {
doc, err := html.Parse(strings.NewReader(htmlContent)) doc, err := html.Parse(strings.NewReader(htmlContent))
if err != nil { if err != nil {
@ -295,7 +290,6 @@ func ParsePostsFromHTML(htmlContent string) ([]btPost, error) {
} }
// FormatPostMarkdown is exported for testing purposes. // FormatPostMarkdown is exported for testing purposes.
// Usage: FormatPostMarkdown(...)
func FormatPostMarkdown(num int, author, date, content string) string { func FormatPostMarkdown(num int, author, date, content string) string {
return formatPostMarkdown(num, btPost{Author: author, Date: date, Content: content}) return formatPostMarkdown(num, btPost{Author: author, Date: date, Content: content})
} }
@ -311,7 +305,6 @@ type BitcoinTalkCollectorWithFetcher struct {
// SetHTTPClient replaces the package-level HTTP client. // SetHTTPClient replaces the package-level HTTP client.
// Use this in tests to inject a custom transport or timeout. // Use this in tests to inject a custom transport or timeout.
// Usage: SetHTTPClient(...)
func SetHTTPClient(c *http.Client) { func SetHTTPClient(c *http.Client) {
httpClient = c httpClient = c
} }

View file

@ -1,13 +1,11 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"dappco.re/go/core/io" "dappco.re/go/core/io"
@ -35,7 +33,7 @@ func sampleBTCTalkPage(count int) string {
return page.String() return page.String()
} }
func TestBitcoinTalkCollector_Collect_Good_OnePage_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_OnePage(t *testing.T) {
// Serve a single page with 5 posts (< 20, so collection stops after one page). // Serve a single page with 5 posts (< 20, so collection stops after one page).
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
@ -75,7 +73,7 @@ func TestBitcoinTalkCollector_Collect_Good_OnePage_Good(t *testing.T) {
} }
} }
func TestBitcoinTalkCollector_Collect_Good_PageLimit_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_PageLimit(t *testing.T) {
pageCount := 0 pageCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
pageCount++ pageCount++
@ -103,7 +101,7 @@ func TestBitcoinTalkCollector_Collect_Good_PageLimit_Good(t *testing.T) {
assert.Equal(t, 2, pageCount) assert.Equal(t, 2, pageCount)
} }
func TestBitcoinTalkCollector_Collect_Good_CancelledContext_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_CancelledContext(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(5))) _, _ = w.Write([]byte(sampleBTCTalkPage(5)))
@ -127,7 +125,7 @@ func TestBitcoinTalkCollector_Collect_Good_CancelledContext_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestBitcoinTalkCollector_Collect_Bad_ServerError_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_ServerError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))
@ -151,7 +149,7 @@ func TestBitcoinTalkCollector_Collect_Bad_ServerError_Good(t *testing.T) {
assert.Equal(t, 1, result.Errors) assert.Equal(t, 1, result.Errors)
} }
func TestBitcoinTalkCollector_Collect_Good_EmitsEvents_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_EmitsEvents(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(2))) _, _ = w.Write([]byte(sampleBTCTalkPage(2)))
@ -209,7 +207,7 @@ func TestFetchPage_Good(t *testing.T) {
assert.Len(t, posts, 3) assert.Len(t, posts, 3)
} }
func TestFetchPage_Bad_StatusCode_Good(t *testing.T) { func TestFetchPage_Bad_StatusCode(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden) w.WriteHeader(http.StatusForbidden)
})) }))
@ -224,7 +222,7 @@ func TestFetchPage_Bad_StatusCode_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestFetchPage_Bad_InvalidHTML_Good(t *testing.T) { func TestFetchPage_Bad_InvalidHTML(t *testing.T) {
// html.Parse is very forgiving, so serve an empty page. // html.Parse is very forgiving, so serve an empty page.
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -15,12 +13,12 @@ func TestBitcoinTalkCollector_Name_Good(t *testing.T) {
assert.Equal(t, "bitcointalk:12345", b.Name()) assert.Equal(t, "bitcointalk:12345", b.Name())
} }
func TestBitcoinTalkCollector_Name_Good_URL_Good(t *testing.T) { func TestBitcoinTalkCollector_Name_Good_URL(t *testing.T) {
b := &BitcoinTalkCollector{URL: "https://bitcointalk.org/index.php?topic=12345.0"} b := &BitcoinTalkCollector{URL: "https://bitcointalk.org/index.php?topic=12345.0"}
assert.Equal(t, "bitcointalk:url", b.Name()) assert.Equal(t, "bitcointalk:url", b.Name())
} }
func TestBitcoinTalkCollector_Collect_Bad_NoTopicID_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_NoTopicID(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -29,7 +27,7 @@ func TestBitcoinTalkCollector_Collect_Bad_NoTopicID_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestBitcoinTalkCollector_Collect_Good_DryRun_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_DryRun(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -72,7 +70,7 @@ func TestParsePostsFromHTML_Good(t *testing.T) {
assert.Contains(t, posts[1].Content, "Running bitcoin!") assert.Contains(t, posts[1].Content, "Running bitcoin!")
} }
func TestParsePostsFromHTML_Good_Empty_Good(t *testing.T) { func TestParsePostsFromHTML_Good_Empty(t *testing.T) {
posts, err := ParsePostsFromHTML("<html><body></body></html>") posts, err := ParsePostsFromHTML("<html><body></body></html>")
assert.NoError(t, err) assert.NoError(t, err)
assert.Empty(t, posts) assert.Empty(t, posts)
@ -86,7 +84,7 @@ func TestFormatPostMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "Hello, world!") assert.Contains(t, md, "Hello, world!")
} }
func TestFormatPostMarkdown_Good_NoDate_Good(t *testing.T) { func TestFormatPostMarkdown_Good_NoDate(t *testing.T) {
md := FormatPostMarkdown(5, "user", "", "Content here") md := FormatPostMarkdown(5, "user", "", "Content here")
assert.Contains(t, md, "# Post 5 by user") assert.Contains(t, md, "# Post 5 by user")

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package collect provides a data collection subsystem for gathering information // Package collect provides a data collection subsystem for gathering information
// from multiple sources including GitHub, BitcoinTalk, CoinGecko, and academic // from multiple sources including GitHub, BitcoinTalk, CoinGecko, and academic
// paper repositories. It supports rate limiting, incremental state tracking, // paper repositories. It supports rate limiting, incremental state tracking,
@ -8,11 +6,9 @@ package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "path/filepath"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"dappco.re/go/core/io" "dappco.re/go/core/io"
core "dappco.re/go/core/log"
) )
// Collector is the interface all collection sources implement. // Collector is the interface all collection sources implement.
@ -69,7 +65,6 @@ type Result struct {
// NewConfig creates a Config with sensible defaults. // NewConfig creates a Config with sensible defaults.
// It initialises a MockMedium for output if none is provided, // It initialises a MockMedium for output if none is provided,
// sets up a rate limiter, state tracker, and event dispatcher. // sets up a rate limiter, state tracker, and event dispatcher.
// Usage: NewConfig(...)
func NewConfig(outputDir string) *Config { func NewConfig(outputDir string) *Config {
m := io.NewMockMedium() m := io.NewMockMedium()
return &Config{ return &Config{
@ -82,7 +77,6 @@ func NewConfig(outputDir string) *Config {
} }
// NewConfigWithMedium creates a Config using the specified storage medium. // NewConfigWithMedium creates a Config using the specified storage medium.
// Usage: NewConfigWithMedium(...)
func NewConfigWithMedium(m io.Medium, outputDir string) *Config { func NewConfigWithMedium(m io.Medium, outputDir string) *Config {
return &Config{ return &Config{
Output: m, Output: m,
@ -93,21 +87,7 @@ func NewConfigWithMedium(m io.Medium, outputDir string) *Config {
} }
} }
// verboseProgress emits a progress event when verbose mode is enabled.
// Usage: verboseProgress(cfg, "excavator", "loading state")
func verboseProgress(cfg *Config, source, message string) {
if cfg == nil || !cfg.Verbose {
return
}
if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitProgress(source, message, nil)
return
}
core.Warn(fmt.Sprintf("%s: %s", source, message))
}
// MergeResults combines multiple results into a single aggregated result. // MergeResults combines multiple results into a single aggregated result.
// Usage: MergeResults(...)
func MergeResults(source string, results ...*Result) *Result { func MergeResults(source string, results ...*Result) *Result {
merged := &Result{Source: source} merged := &Result{Source: source}
for _, r := range results { for _, r := range results {

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -56,13 +54,13 @@ func TestMergeResults_Good(t *testing.T) {
assert.Len(t, merged.Files, 3) assert.Len(t, merged.Files, 3)
} }
func TestMergeResults_Good_NilResults_Good(t *testing.T) { func TestMergeResults_Good_NilResults(t *testing.T) {
r1 := &Result{Items: 3} r1 := &Result{Items: 3}
merged := MergeResults("test", r1, nil, nil) merged := MergeResults("test", r1, nil, nil)
assert.Equal(t, 3, merged.Items) assert.Equal(t, 3, merged.Items)
} }
func TestMergeResults_Good_Empty_Good(t *testing.T) { func TestMergeResults_Good_Empty(t *testing.T) {
merged := MergeResults("empty") merged := MergeResults("empty")
assert.Equal(t, 0, merged.Items) assert.Equal(t, 0, merged.Items)
assert.Equal(t, 0, merged.Errors) assert.Equal(t, 0, merged.Errors)

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
@ -17,7 +15,7 @@ import (
// --- GitHub collector: context cancellation and orchestration --- // --- GitHub collector: context cancellation and orchestration ---
func TestGitHubCollector_Collect_Good_ContextCancelledInLoop_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_ContextCancelledInLoop(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = false cfg.DryRun = false
@ -33,7 +31,7 @@ func TestGitHubCollector_Collect_Good_ContextCancelledInLoop_Good(t *testing.T)
assert.NotNil(t, result) assert.NotNil(t, result)
} }
func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -49,7 +47,7 @@ func TestGitHubCollector_Collect_Good_IssuesOnlyDryRunProgress_Good(t *testing.T
assert.GreaterOrEqual(t, progressCount, 1) assert.GreaterOrEqual(t, progressCount, 1)
} }
func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -61,7 +59,7 @@ func TestGitHubCollector_Collect_Good_PRsOnlyDryRunSkipsIssues_Good(t *testing.T
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestGitHubCollector_Collect_Good_EmitsStartAndComplete_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_EmitsStartAndComplete(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -78,7 +76,7 @@ func TestGitHubCollector_Collect_Good_EmitsStartAndComplete_Good(t *testing.T) {
assert.Equal(t, 1, completes) assert.Equal(t, 1, completes)
} }
func TestGitHubCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -91,7 +89,7 @@ func TestGitHubCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestFormatIssueMarkdown_Good_NoBodyNoURL_Good(t *testing.T) { func TestFormatIssueMarkdown_Good_NoBodyNoURL(t *testing.T) {
issue := ghIssue{ issue := ghIssue{
Number: 1, Number: 1,
Title: "No Body Issue", Title: "No Body Issue",
@ -108,7 +106,7 @@ func TestFormatIssueMarkdown_Good_NoBodyNoURL_Good(t *testing.T) {
// --- Market collector: fetchJSON edge cases --- // --- Market collector: fetchJSON edge cases ---
func TestFetchJSON_Bad_NonJSONBody_Good(t *testing.T) { func TestFetchJSON_Bad_NonJSONBody(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(`<html>not json</html>`)) _, _ = w.Write([]byte(`<html>not json</html>`))
@ -119,17 +117,17 @@ func TestFetchJSON_Bad_NonJSONBody_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestFetchJSON_Bad_MalformedURL_Good(t *testing.T) { func TestFetchJSON_Bad_MalformedURL(t *testing.T) {
_, err := fetchJSON[coinData](context.Background(), "://bad-url") _, err := fetchJSON[coinData](context.Background(), "://bad-url")
assert.Error(t, err) assert.Error(t, err)
} }
func TestFetchJSON_Bad_ServerUnavailable_Good(t *testing.T) { func TestFetchJSON_Bad_ServerUnavailable(t *testing.T) {
_, err := fetchJSON[coinData](context.Background(), "http://127.0.0.1:1") _, err := fetchJSON[coinData](context.Background(), "http://127.0.0.1:1")
assert.Error(t, err) assert.Error(t, err)
} }
func TestFetchJSON_Bad_Non200StatusCode_Good(t *testing.T) { func TestFetchJSON_Bad_Non200StatusCode(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound) w.WriteHeader(http.StatusNotFound)
})) }))
@ -140,7 +138,7 @@ func TestFetchJSON_Bad_Non200StatusCode_Good(t *testing.T) {
assert.Contains(t, err.Error(), "unexpected status code") assert.Contains(t, err.Error(), "unexpected status code")
} }
func TestMarketCollector_Collect_Bad_MissingCoinID_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_MissingCoinID(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -150,7 +148,7 @@ func TestMarketCollector_Collect_Bad_MissingCoinID_Good(t *testing.T) {
assert.Contains(t, err.Error(), "coin ID is required") assert.Contains(t, err.Error(), "coin ID is required")
} }
func TestMarketCollector_Collect_Good_NoDispatcher_Good(t *testing.T) { func TestMarketCollector_Collect_Good_NoDispatcher(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
data := coinData{ID: "test", Symbol: "tst", Name: "Test", data := coinData{ID: "test", Symbol: "tst", Name: "Test",
@ -175,7 +173,7 @@ func TestMarketCollector_Collect_Good_NoDispatcher_Good(t *testing.T) {
assert.Equal(t, 2, result.Items) assert.Equal(t, 2, result.Items)
} }
func TestMarketCollector_Collect_Bad_CurrentFetchFails_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_CurrentFetchFails(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))
@ -197,7 +195,7 @@ func TestMarketCollector_Collect_Bad_CurrentFetchFails_Good(t *testing.T) {
assert.Equal(t, 1, result.Errors) assert.Equal(t, 1, result.Errors)
} }
func TestMarketCollector_CollectHistorical_Good_DefaultDays_Good(t *testing.T) { func TestMarketCollector_CollectHistorical_Good_DefaultDays(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -229,7 +227,7 @@ func TestMarketCollector_CollectHistorical_Good_DefaultDays_Good(t *testing.T) {
assert.Equal(t, 3, result.Items) assert.Equal(t, 3, result.Items)
} }
func TestMarketCollector_CollectHistorical_Good_WithRateLimiter_Good(t *testing.T) { func TestMarketCollector_CollectHistorical_Good_WithRateLimiter(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -263,7 +261,7 @@ func TestMarketCollector_CollectHistorical_Good_WithRateLimiter_Good(t *testing.
// --- State: error paths --- // --- State: error paths ---
func TestState_Load_Bad_MalformedJSON_Good(t *testing.T) { func TestState_Load_Bad_MalformedJSON(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Files["/state.json"] = `{invalid json` m.Files["/state.json"] = `{invalid json`
@ -274,7 +272,7 @@ func TestState_Load_Bad_MalformedJSON_Good(t *testing.T) {
// --- Process: additional coverage for uncovered branches --- // --- Process: additional coverage for uncovered branches ---
func TestHTMLToMarkdown_Good_PreCodeBlock_Good(t *testing.T) { func TestHTMLToMarkdown_Good_PreCodeBlock(t *testing.T) {
input := `<pre>some code here</pre>` input := `<pre>some code here</pre>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -282,7 +280,7 @@ func TestHTMLToMarkdown_Good_PreCodeBlock_Good(t *testing.T) {
assert.Contains(t, result, "some code here") assert.Contains(t, result, "some code here")
} }
func TestHTMLToMarkdown_Good_StrongAndEmElements_Good(t *testing.T) { func TestHTMLToMarkdown_Good_StrongAndEmElements(t *testing.T) {
input := `<strong>bold</strong> and <em>italic</em>` input := `<strong>bold</strong> and <em>italic</em>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -290,21 +288,21 @@ func TestHTMLToMarkdown_Good_StrongAndEmElements_Good(t *testing.T) {
assert.Contains(t, result, "*italic*") assert.Contains(t, result, "*italic*")
} }
func TestHTMLToMarkdown_Good_InlineCode_Good(t *testing.T) { func TestHTMLToMarkdown_Good_InlineCode(t *testing.T) {
input := `<code>var x = 1</code>` input := `<code>var x = 1</code>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "`var x = 1`") assert.Contains(t, result, "`var x = 1`")
} }
func TestHTMLToMarkdown_Good_AnchorWithHref_Good(t *testing.T) { func TestHTMLToMarkdown_Good_AnchorWithHref(t *testing.T) {
input := `<a href="https://example.com">Click here</a>` input := `<a href="https://example.com">Click here</a>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "[Click here](https://example.com)") assert.Contains(t, result, "[Click here](https://example.com)")
} }
func TestHTMLToMarkdown_Good_ScriptTagRemoved_Good(t *testing.T) { func TestHTMLToMarkdown_Good_ScriptTagRemoved(t *testing.T) {
input := `<html><body><script>alert('xss')</script><p>Safe text</p></body></html>` input := `<html><body><script>alert('xss')</script><p>Safe text</p></body></html>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -312,7 +310,7 @@ func TestHTMLToMarkdown_Good_ScriptTagRemoved_Good(t *testing.T) {
assert.NotContains(t, result, "alert") assert.NotContains(t, result, "alert")
} }
func TestHTMLToMarkdown_Good_H1H2H3Headers_Good(t *testing.T) { func TestHTMLToMarkdown_Good_H1H2H3Headers(t *testing.T) {
input := `<h1>One</h1><h2>Two</h2><h3>Three</h3>` input := `<h1>One</h1><h2>Two</h2><h3>Three</h3>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -321,7 +319,7 @@ func TestHTMLToMarkdown_Good_H1H2H3Headers_Good(t *testing.T) {
assert.Contains(t, result, "### Three") assert.Contains(t, result, "### Three")
} }
func TestHTMLToMarkdown_Good_MultiParagraph_Good(t *testing.T) { func TestHTMLToMarkdown_Good_MultiParagraph(t *testing.T) {
input := `<p>First paragraph</p><p>Second paragraph</p>` input := `<p>First paragraph</p><p>Second paragraph</p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -329,12 +327,12 @@ func TestHTMLToMarkdown_Good_MultiParagraph_Good(t *testing.T) {
assert.Contains(t, result, "Second paragraph") assert.Contains(t, result, "Second paragraph")
} }
func TestJSONToMarkdown_Bad_Malformed_Good(t *testing.T) { func TestJSONToMarkdown_Bad_Malformed(t *testing.T) {
_, err := JSONToMarkdown(`{invalid}`) _, err := JSONToMarkdown(`{invalid}`)
assert.Error(t, err) assert.Error(t, err)
} }
func TestJSONToMarkdown_Good_FlatObject_Good(t *testing.T) { func TestJSONToMarkdown_Good_FlatObject(t *testing.T) {
input := `{"name": "Alice", "age": 30}` input := `{"name": "Alice", "age": 30}`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -342,7 +340,7 @@ func TestJSONToMarkdown_Good_FlatObject_Good(t *testing.T) {
assert.Contains(t, result, "**age:** 30") assert.Contains(t, result, "**age:** 30")
} }
func TestJSONToMarkdown_Good_ScalarList_Good(t *testing.T) { func TestJSONToMarkdown_Good_ScalarList(t *testing.T) {
input := `["hello", "world"]` input := `["hello", "world"]`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -350,14 +348,14 @@ func TestJSONToMarkdown_Good_ScalarList_Good(t *testing.T) {
assert.Contains(t, result, "- world") assert.Contains(t, result, "- world")
} }
func TestJSONToMarkdown_Good_ObjectContainingArray_Good(t *testing.T) { func TestJSONToMarkdown_Good_ObjectContainingArray(t *testing.T) {
input := `{"items": [1, 2, 3]}` input := `{"items": [1, 2, 3]}`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "**items:**") assert.Contains(t, result, "**items:**")
} }
func TestProcessor_Process_Bad_MissingDir_Good(t *testing.T) { func TestProcessor_Process_Bad_MissingDir(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -367,7 +365,7 @@ func TestProcessor_Process_Bad_MissingDir_Good(t *testing.T) {
assert.Contains(t, err.Error(), "directory is required") assert.Contains(t, err.Error(), "directory is required")
} }
func TestProcessor_Process_Good_DryRunEmitsProgress_Good(t *testing.T) { func TestProcessor_Process_Good_DryRunEmitsProgress(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -383,7 +381,7 @@ func TestProcessor_Process_Good_DryRunEmitsProgress_Good(t *testing.T) {
assert.Equal(t, 1, progressCount) assert.Equal(t, 1, progressCount)
} }
func TestProcessor_Process_Good_SkipsUnsupportedExtension_Good(t *testing.T) { func TestProcessor_Process_Good_SkipsUnsupportedExtension(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/data.csv"] = `a,b,c` m.Files["/input/data.csv"] = `a,b,c`
@ -399,7 +397,7 @@ func TestProcessor_Process_Good_SkipsUnsupportedExtension_Good(t *testing.T) {
assert.Equal(t, 1, result.Skipped) assert.Equal(t, 1, result.Skipped)
} }
func TestProcessor_Process_Good_MarkdownPassthroughTrimmed_Good(t *testing.T) { func TestProcessor_Process_Good_MarkdownPassthroughTrimmed(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/readme.md"] = `# Hello World ` m.Files["/input/readme.md"] = `# Hello World `
@ -418,7 +416,7 @@ func TestProcessor_Process_Good_MarkdownPassthroughTrimmed_Good(t *testing.T) {
assert.Equal(t, "# Hello World", content) assert.Equal(t, "# Hello World", content)
} }
func TestProcessor_Process_Good_HTMExtensionHandled_Good(t *testing.T) { func TestProcessor_Process_Good_HTMExtensionHandled(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/page.htm"] = `<h1>HTM File</h1>` m.Files["/input/page.htm"] = `<h1>HTM File</h1>`
@ -433,7 +431,7 @@ func TestProcessor_Process_Good_HTMExtensionHandled_Good(t *testing.T) {
assert.Equal(t, 1, result.Items) assert.Equal(t, 1, result.Items)
} }
func TestProcessor_Process_Good_NilDispatcherHandled_Good(t *testing.T) { func TestProcessor_Process_Good_NilDispatcherHandled(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/test.html"] = `<p>Text</p>` m.Files["/input/test.html"] = `<p>Text</p>`
@ -451,12 +449,12 @@ func TestProcessor_Process_Good_NilDispatcherHandled_Good(t *testing.T) {
// --- BitcoinTalk: additional edge cases --- // --- BitcoinTalk: additional edge cases ---
func TestBitcoinTalkCollector_Name_Good_EmptyTopicAndURL_Good(t *testing.T) { func TestBitcoinTalkCollector_Name_Good_EmptyTopicAndURL(t *testing.T) {
b := &BitcoinTalkCollector{} b := &BitcoinTalkCollector{}
assert.Equal(t, "bitcointalk:", b.Name()) assert.Equal(t, "bitcointalk:", b.Name())
} }
func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(2))) _, _ = w.Write([]byte(sampleBTCTalkPage(2)))
@ -480,7 +478,7 @@ func TestBitcoinTalkCollector_Collect_Good_NilDispatcherHandled_Good(t *testing.
assert.Equal(t, 2, result.Items) assert.Equal(t, 2, result.Items)
} }
func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -496,7 +494,7 @@ func TestBitcoinTalkCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T
assert.True(t, progressEmitted) assert.True(t, progressEmitted)
} }
func TestParsePostsFromHTML_Good_PostWithNoInnerContent_Good(t *testing.T) { func TestParsePostsFromHTML_Good_PostWithNoInnerContent(t *testing.T) {
htmlContent := `<html><body> htmlContent := `<html><body>
<div class="post"> <div class="post">
<div class="poster_info">user1</div> <div class="poster_info">user1</div>
@ -507,7 +505,7 @@ func TestParsePostsFromHTML_Good_PostWithNoInnerContent_Good(t *testing.T) {
assert.Empty(t, posts) assert.Empty(t, posts)
} }
func TestFormatPostMarkdown_Good_WithDateContent_Good(t *testing.T) { func TestFormatPostMarkdown_Good_WithDateContent(t *testing.T) {
md := FormatPostMarkdown(1, "alice", "2025-01-15", "Hello world") md := FormatPostMarkdown(1, "alice", "2025-01-15", "Hello world")
assert.Contains(t, md, "# Post 1 by alice") assert.Contains(t, md, "# Post 1 by alice")
assert.Contains(t, md, "**Date:** 2025-01-15") assert.Contains(t, md, "**Date:** 2025-01-15")
@ -516,7 +514,7 @@ func TestFormatPostMarkdown_Good_WithDateContent_Good(t *testing.T) {
// --- Papers collector: edge cases --- // --- Papers collector: edge cases ---
func TestPapersCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T) { func TestPapersCollector_Collect_Good_DryRunEmitsProgress(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -532,7 +530,7 @@ func TestPapersCollector_Collect_Good_DryRunEmitsProgress_Good(t *testing.T) {
assert.True(t, progressEmitted) assert.True(t, progressEmitted)
} }
func TestPapersCollector_Collect_Good_NilDispatcherIACR_Good(t *testing.T) { func TestPapersCollector_Collect_Good_NilDispatcherIACR(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML)) _, _ = w.Write([]byte(sampleIACRHTML))
@ -556,7 +554,7 @@ func TestPapersCollector_Collect_Good_NilDispatcherIACR_Good(t *testing.T) {
assert.Equal(t, 2, result.Items) assert.Equal(t, 2, result.Items)
} }
func TestArXivEntryToPaper_Good_NoAlternateLink_Good(t *testing.T) { func TestArXivEntryToPaper_Good_NoAlternateLink(t *testing.T) {
entry := arxivEntry{ entry := arxivEntry{
ID: "http://arxiv.org/abs/2501.99999v1", ID: "http://arxiv.org/abs/2501.99999v1",
Title: "No Alternate", Title: "No Alternate",
@ -571,7 +569,7 @@ func TestArXivEntryToPaper_Good_NoAlternateLink_Good(t *testing.T) {
// --- Excavator: additional edge cases --- // --- Excavator: additional edge cases ---
func TestExcavator_Run_Good_ResumeLoadError_Good(t *testing.T) { func TestExcavator_Run_Good_ResumeLoadError(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Files["/output/.collect-state.json"] = `{invalid` m.Files["/output/.collect-state.json"] = `{invalid`
@ -591,7 +589,7 @@ func TestExcavator_Run_Good_ResumeLoadError_Good(t *testing.T) {
// --- RateLimiter: additional edge cases --- // --- RateLimiter: additional edge cases ---
func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay_Good(t *testing.T) { func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay(t *testing.T) {
rl := NewRateLimiter() rl := NewRateLimiter()
rl.SetDelay("fast", 1*time.Millisecond) rl.SetDelay("fast", 1*time.Millisecond)
@ -610,7 +608,7 @@ func TestRateLimiter_Wait_Good_QuickSuccessiveCallsAfterDelay_Good(t *testing.T)
// --- FormatMarketSummary: with empty market data values --- // --- FormatMarketSummary: with empty market data values ---
func TestFormatMarketSummary_Good_ZeroRank_Good(t *testing.T) { func TestFormatMarketSummary_Good_ZeroRank(t *testing.T) {
data := &coinData{ data := &coinData{
Name: "Tiny Token", Name: "Tiny Token",
Symbol: "tiny", Symbol: "tiny",
@ -624,7 +622,7 @@ func TestFormatMarketSummary_Good_ZeroRank_Good(t *testing.T) {
assert.NotContains(t, summary, "Market Cap Rank") assert.NotContains(t, summary, "Market Cap Rank")
} }
func TestFormatMarketSummary_Good_ZeroSupply_Good(t *testing.T) { func TestFormatMarketSummary_Good_ZeroSupply(t *testing.T) {
data := &coinData{ data := &coinData{
Name: "Zero Supply", Name: "Zero Supply",
Symbol: "zs", Symbol: "zs",
@ -638,7 +636,7 @@ func TestFormatMarketSummary_Good_ZeroSupply_Good(t *testing.T) {
assert.NotContains(t, summary, "Total Supply") assert.NotContains(t, summary, "Total Supply")
} }
func TestFormatMarketSummary_Good_NoLastUpdated_Good(t *testing.T) { func TestFormatMarketSummary_Good_NoLastUpdated(t *testing.T) {
data := &coinData{ data := &coinData{
Name: "No Update", Name: "No Update",
Symbol: "nu", Symbol: "nu",

View file

@ -1,17 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
core "dappco.re/go/core" "encoding/json"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
goio "io" goio "io"
"io/fs" "io/fs"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"time" "time"
@ -20,14 +17,6 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func testErr(msg string) error {
return core.E("collect.test", msg, nil)
}
func testErrf(format string, args ...any) error {
return core.E("collect.test", fmt.Sprintf(format, args...), nil)
}
// errorMedium wraps MockMedium and injects errors on specific operations. // errorMedium wraps MockMedium and injects errors on specific operations.
type errorMedium struct { type errorMedium struct {
*io.MockMedium *io.MockMedium
@ -70,9 +59,7 @@ func (e *errorMedium) Stat(path string) (fs.FileInfo, error) { return e.M
func (e *errorMedium) Open(path string) (fs.File, error) { return e.MockMedium.Open(path) } func (e *errorMedium) Open(path string) (fs.File, error) { return e.MockMedium.Open(path) }
func (e *errorMedium) Create(path string) (goio.WriteCloser, error) { return e.MockMedium.Create(path) } func (e *errorMedium) Create(path string) (goio.WriteCloser, error) { return e.MockMedium.Create(path) }
func (e *errorMedium) Append(path string) (goio.WriteCloser, error) { return e.MockMedium.Append(path) } func (e *errorMedium) Append(path string) (goio.WriteCloser, error) { return e.MockMedium.Append(path) }
func (e *errorMedium) ReadStream(path string) (goio.ReadCloser, error) { func (e *errorMedium) ReadStream(path string) (goio.ReadCloser, error) { return e.MockMedium.ReadStream(path) }
return e.MockMedium.ReadStream(path)
}
func (e *errorMedium) WriteStream(path string) (goio.WriteCloser, error) { func (e *errorMedium) WriteStream(path string) (goio.WriteCloser, error) {
return e.MockMedium.WriteStream(path) return e.MockMedium.WriteStream(path)
} }
@ -86,8 +73,8 @@ type errorLimiterWaiter struct{}
// --- Processor: list error --- // --- Processor: list error ---
func TestProcessor_Process_Bad_ListError_Good(t *testing.T) { func TestProcessor_Process_Bad_ListError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), listErr: testErr("list denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), listErr: fmt.Errorf("list denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
p := &Processor{Source: "test", Dir: "/input"} p := &Processor{Source: "test", Dir: "/input"}
@ -98,8 +85,8 @@ func TestProcessor_Process_Bad_ListError_Good(t *testing.T) {
// --- Processor: ensureDir error --- // --- Processor: ensureDir error ---
func TestProcessor_Process_Bad_EnsureDirError_Good(t *testing.T) { func TestProcessor_Process_Bad_EnsureDirError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
// Need to ensure List returns entries // Need to ensure List returns entries
em.MockMedium.Dirs["/input"] = true em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>" em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>"
@ -114,7 +101,7 @@ func TestProcessor_Process_Bad_EnsureDirError_Good(t *testing.T) {
// --- Processor: context cancellation during processing --- // --- Processor: context cancellation during processing ---
func TestProcessor_Process_Bad_ContextCancelledDuringLoop_Good(t *testing.T) { func TestProcessor_Process_Bad_ContextCancelledDuringLoop(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/a.html"] = "<h1>Test</h1>" m.Files["/input/a.html"] = "<h1>Test</h1>"
@ -133,8 +120,8 @@ func TestProcessor_Process_Bad_ContextCancelledDuringLoop_Good(t *testing.T) {
// --- Processor: read error during file processing --- // --- Processor: read error during file processing ---
func TestProcessor_Process_Bad_ReadError_Good(t *testing.T) { func TestProcessor_Process_Bad_ReadError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: testErr("read denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: fmt.Errorf("read denied")}
em.MockMedium.Dirs["/input"] = true em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>" em.MockMedium.Files["/input/test.html"] = "<h1>Test</h1>"
@ -148,7 +135,7 @@ func TestProcessor_Process_Bad_ReadError_Good(t *testing.T) {
// --- Processor: JSON conversion error --- // --- Processor: JSON conversion error ---
func TestProcessor_Process_Bad_InvalidJSONFile_Good(t *testing.T) { func TestProcessor_Process_Bad_InvalidJSONFile(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/bad.json"] = "not valid json {" m.Files["/input/bad.json"] = "not valid json {"
@ -166,8 +153,8 @@ func TestProcessor_Process_Bad_InvalidJSONFile_Good(t *testing.T) {
// --- Processor: write error during output --- // --- Processor: write error during output ---
func TestProcessor_Process_Bad_WriteError_Good(t *testing.T) { func TestProcessor_Process_Bad_WriteError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
em.MockMedium.Dirs["/input"] = true em.MockMedium.Dirs["/input"] = true
em.MockMedium.Files["/input/page.html"] = "<h1>Title</h1>" em.MockMedium.Files["/input/page.html"] = "<h1>Title</h1>"
@ -181,7 +168,7 @@ func TestProcessor_Process_Bad_WriteError_Good(t *testing.T) {
// --- Processor: successful processing with events --- // --- Processor: successful processing with events ---
func TestProcessor_Process_Good_EmitsItemAndComplete_Good(t *testing.T) { func TestProcessor_Process_Good_EmitsItemAndComplete(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/page.html"] = "<h1>Title</h1><p>Body</p>" m.Files["/input/page.html"] = "<h1>Title</h1><p>Body</p>"
@ -201,7 +188,7 @@ func TestProcessor_Process_Good_EmitsItemAndComplete_Good(t *testing.T) {
// --- Papers: with rate limiter that fails --- // --- Papers: with rate limiter that fails ---
func TestPapersCollector_CollectIACR_Bad_LimiterError_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_LimiterError(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -215,7 +202,7 @@ func TestPapersCollector_CollectIACR_Bad_LimiterError_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_CollectArXiv_Bad_LimiterError_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_LimiterError(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -231,7 +218,7 @@ func TestPapersCollector_CollectArXiv_Bad_LimiterError_Good(t *testing.T) {
// --- Papers: IACR with bad HTML response --- // --- Papers: IACR with bad HTML response ---
func TestPapersCollector_CollectIACR_Bad_InvalidHTML_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_InvalidHTML(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
// Serve valid-ish HTML but with no papers - the parse succeeds but returns empty. // Serve valid-ish HTML but with no papers - the parse succeeds but returns empty.
@ -256,7 +243,7 @@ func TestPapersCollector_CollectIACR_Bad_InvalidHTML_Good(t *testing.T) {
// --- Papers: IACR write error --- // --- Papers: IACR write error ---
func TestPapersCollector_CollectIACR_Bad_WriteError_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_WriteError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML)) _, _ = w.Write([]byte(sampleIACRHTML))
@ -268,7 +255,7 @@ func TestPapersCollector_CollectIACR_Bad_WriteError_Good(t *testing.T) {
httpClient = &http.Client{Transport: transport} httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }() defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -280,7 +267,7 @@ func TestPapersCollector_CollectIACR_Bad_WriteError_Good(t *testing.T) {
// --- Papers: IACR EnsureDir error --- // --- Papers: IACR EnsureDir error ---
func TestPapersCollector_CollectIACR_Bad_EnsureDirError_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_EnsureDirError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML)) _, _ = w.Write([]byte(sampleIACRHTML))
@ -292,7 +279,7 @@ func TestPapersCollector_CollectIACR_Bad_EnsureDirError_Good(t *testing.T) {
httpClient = &http.Client{Transport: transport} httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }() defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -304,7 +291,7 @@ func TestPapersCollector_CollectIACR_Bad_EnsureDirError_Good(t *testing.T) {
// --- Papers: arXiv write error --- // --- Papers: arXiv write error ---
func TestPapersCollector_CollectArXiv_Bad_WriteError_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_WriteError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml") w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML)) _, _ = w.Write([]byte(sampleArXivXML))
@ -316,7 +303,7 @@ func TestPapersCollector_CollectArXiv_Bad_WriteError_Good(t *testing.T) {
httpClient = &http.Client{Transport: transport} httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }() defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -328,7 +315,7 @@ func TestPapersCollector_CollectArXiv_Bad_WriteError_Good(t *testing.T) {
// --- Papers: arXiv EnsureDir error --- // --- Papers: arXiv EnsureDir error ---
func TestPapersCollector_CollectArXiv_Bad_EnsureDirError_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_EnsureDirError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml") w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML)) _, _ = w.Write([]byte(sampleArXivXML))
@ -340,7 +327,7 @@ func TestPapersCollector_CollectArXiv_Bad_EnsureDirError_Good(t *testing.T) {
httpClient = &http.Client{Transport: transport} httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }() defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -352,7 +339,7 @@ func TestPapersCollector_CollectArXiv_Bad_EnsureDirError_Good(t *testing.T) {
// --- Papers: collectAll with dispatcher events --- // --- Papers: collectAll with dispatcher events ---
func TestPapersCollector_CollectAll_Good_WithDispatcher_Good(t *testing.T) { func TestPapersCollector_CollectAll_Good_WithDispatcher(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -387,7 +374,7 @@ func TestPapersCollector_CollectAll_Good_WithDispatcher_Good(t *testing.T) {
// --- Papers: IACR with events on item emit --- // --- Papers: IACR with events on item emit ---
func TestPapersCollector_CollectIACR_Good_EmitsItemEvents_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Good_EmitsItemEvents(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleIACRHTML)) _, _ = w.Write([]byte(sampleIACRHTML))
@ -415,7 +402,7 @@ func TestPapersCollector_CollectIACR_Good_EmitsItemEvents_Good(t *testing.T) {
// --- Papers: arXiv with events on item emit --- // --- Papers: arXiv with events on item emit ---
func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml") w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(sampleArXivXML)) _, _ = w.Write([]byte(sampleArXivXML))
@ -443,7 +430,7 @@ func TestPapersCollector_CollectArXiv_Good_EmitsItemEvents_Good(t *testing.T) {
// --- Market: collectCurrent write error (summary path) --- // --- Market: collectCurrent write error (summary path) ---
func TestMarketCollector_Collect_Bad_WriteError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_WriteError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") { if strings.Contains(r.URL.Path, "/market_chart") {
@ -466,7 +453,7 @@ func TestMarketCollector_Collect_Bad_WriteError_Good(t *testing.T) {
coinGeckoBaseURL = server.URL coinGeckoBaseURL = server.URL
defer func() { coinGeckoBaseURL = oldURL }() defer func() { coinGeckoBaseURL = oldURL }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -479,7 +466,7 @@ func TestMarketCollector_Collect_Bad_WriteError_Good(t *testing.T) {
// --- Market: EnsureDir error --- // --- Market: EnsureDir error ---
func TestMarketCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_EnsureDirError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"}) _ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"})
@ -490,7 +477,7 @@ func TestMarketCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) {
coinGeckoBaseURL = server.URL coinGeckoBaseURL = server.URL
defer func() { coinGeckoBaseURL = oldURL }() defer func() { coinGeckoBaseURL = oldURL }()
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -502,7 +489,7 @@ func TestMarketCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) {
// --- Market: collectCurrent with limiter wait error --- // --- Market: collectCurrent with limiter wait error ---
func TestMarketCollector_Collect_Bad_LimiterError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_LimiterError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"}) _ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin"})
@ -529,7 +516,7 @@ func TestMarketCollector_Collect_Bad_LimiterError_Good(t *testing.T) {
// --- Market: collectHistorical with custom FromDate --- // --- Market: collectHistorical with custom FromDate ---
func TestMarketCollector_Collect_Good_HistoricalCustomDate_Good(t *testing.T) { func TestMarketCollector_Collect_Good_HistoricalCustomDate(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") { if strings.Contains(r.URL.Path, "/market_chart") {
@ -564,8 +551,8 @@ func TestMarketCollector_Collect_Good_HistoricalCustomDate_Good(t *testing.T) {
// --- BitcoinTalk: EnsureDir error --- // --- BitcoinTalk: EnsureDir error ---
func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: testErr("mkdir denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), ensureDirErr: fmt.Errorf("mkdir denied")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -577,7 +564,7 @@ func TestBitcoinTalkCollector_Collect_Bad_EnsureDirError_Good(t *testing.T) {
// --- BitcoinTalk: limiter error --- // --- BitcoinTalk: limiter error ---
func TestBitcoinTalkCollector_Collect_Bad_LimiterError_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_LimiterError(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -593,7 +580,7 @@ func TestBitcoinTalkCollector_Collect_Bad_LimiterError_Good(t *testing.T) {
// --- BitcoinTalk: write error during post saving --- // --- BitcoinTalk: write error during post saving ---
func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte(sampleBTCTalkPage(3))) _, _ = w.Write([]byte(sampleBTCTalkPage(3)))
@ -605,7 +592,7 @@ func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts_Good(t *testing.T) {
httpClient = &http.Client{Transport: transport} httpClient = &http.Client{Transport: transport}
defer func() { httpClient = old }() defer func() { httpClient = old }()
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()} cfg := &Config{Output: em, OutputDir: "/output", Dispatcher: NewDispatcher()}
cfg.Limiter = nil cfg.Limiter = nil
@ -618,7 +605,7 @@ func TestBitcoinTalkCollector_Collect_Bad_WriteErrorOnPosts_Good(t *testing.T) {
// --- BitcoinTalk: fetchPage with bad HTTP status --- // --- BitcoinTalk: fetchPage with bad HTTP status ---
func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus_Good(t *testing.T) { func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden) w.WriteHeader(http.StatusForbidden)
})) }))
@ -632,7 +619,7 @@ func TestBitcoinTalkCollector_FetchPage_Bad_NonOKStatus_Good(t *testing.T) {
// --- BitcoinTalk: fetchPage with request error --- // --- BitcoinTalk: fetchPage with request error ---
func TestBitcoinTalkCollector_FetchPage_Bad_RequestError_Good(t *testing.T) { func TestBitcoinTalkCollector_FetchPage_Bad_RequestError(t *testing.T) {
old := httpClient old := httpClient
httpClient = &http.Client{Transport: &rewriteTransport{target: "http://127.0.0.1:1"}} // Connection refused httpClient = &http.Client{Transport: &rewriteTransport{target: "http://127.0.0.1:1"}} // Connection refused
defer func() { httpClient = old }() defer func() { httpClient = old }()
@ -645,7 +632,7 @@ func TestBitcoinTalkCollector_FetchPage_Bad_RequestError_Good(t *testing.T) {
// --- BitcoinTalk: fetchPage with valid but empty page --- // --- BitcoinTalk: fetchPage with valid but empty page ---
func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage_Good(t *testing.T) { func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
_, _ = w.Write([]byte("<html><body></body></html>")) _, _ = w.Write([]byte("<html><body></body></html>"))
@ -664,7 +651,7 @@ func TestBitcoinTalkCollector_FetchPage_Good_EmptyPage_Good(t *testing.T) {
// --- BitcoinTalk: Collect with fetch error + dispatcher --- // --- BitcoinTalk: Collect with fetch error + dispatcher ---
func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))
@ -691,7 +678,7 @@ func TestBitcoinTalkCollector_Collect_Bad_FetchErrorWithDispatcher_Good(t *testi
// --- State: Save with a populated state --- // --- State: Save with a populated state ---
func TestState_Save_Good_RoundTrip_Good(t *testing.T) { func TestState_Save_Good_RoundTrip(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/data/state.json") s := NewState(m, "/data/state.json")
@ -717,7 +704,7 @@ func TestState_Save_Good_RoundTrip_Good(t *testing.T) {
// --- GitHub: Collect with Repo set triggers collectIssues/collectPRs (which fail via gh) --- // --- GitHub: Collect with Repo set triggers collectIssues/collectPRs (which fail via gh) ---
func TestGitHubCollector_Collect_Bad_GhNotAuthenticated_Good(t *testing.T) { func TestGitHubCollector_Collect_Bad_GhNotAuthenticated(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -737,7 +724,7 @@ func TestGitHubCollector_Collect_Bad_GhNotAuthenticated_Good(t *testing.T) {
// --- GitHub: Collect IssuesOnly triggers only issues, not PRs --- // --- GitHub: Collect IssuesOnly triggers only issues, not PRs ---
func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails_Good(t *testing.T) { func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -750,7 +737,7 @@ func TestGitHubCollector_Collect_Bad_IssuesOnlyGhFails_Good(t *testing.T) {
// --- GitHub: Collect PRsOnly triggers only PRs, not issues --- // --- GitHub: Collect PRsOnly triggers only PRs, not issues ---
func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails_Good(t *testing.T) { func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -763,7 +750,7 @@ func TestGitHubCollector_Collect_Bad_PRsOnlyGhFails_Good(t *testing.T) {
// --- extractText: text before a br/p/div element adds newline --- // --- extractText: text before a br/p/div element adds newline ---
func TestExtractText_Good_TextBeforeBR_Good(t *testing.T) { func TestExtractText_Good_TextBeforeBR(t *testing.T) {
htmlStr := `<div class="inner">Hello<br>World<p>End</p></div>` htmlStr := `<div class="inner">Hello<br>World<p>End</p></div>`
posts, err := ParsePostsFromHTML(fmt.Sprintf(`<html><body><div class="post"><div class="inner">%s</div></div></body></html>`, posts, err := ParsePostsFromHTML(fmt.Sprintf(`<html><body><div class="post"><div class="inner">%s</div></div></body></html>`,
"First text<br>Second text<div>Third text</div>")) "First text<br>Second text<div>Third text</div>"))
@ -777,7 +764,7 @@ func TestExtractText_Good_TextBeforeBR_Good(t *testing.T) {
// --- ParsePostsFromHTML: posts with full structure --- // --- ParsePostsFromHTML: posts with full structure ---
func TestParsePostsFromHTML_Good_FullStructure_Good(t *testing.T) { func TestParsePostsFromHTML_Good_FullStructure(t *testing.T) {
htmlContent := `<html><body> htmlContent := `<html><body>
<div class="post"> <div class="post">
<div class="poster_info">TestAuthor</div> <div class="poster_info">TestAuthor</div>
@ -796,7 +783,7 @@ func TestParsePostsFromHTML_Good_FullStructure_Good(t *testing.T) {
// --- getChildrenText: nested element node path --- // --- getChildrenText: nested element node path ---
func TestHTMLToMarkdown_Good_NestedElements_Good(t *testing.T) { func TestHTMLToMarkdown_Good_NestedElements(t *testing.T) {
// <a> with nested <span> triggers getChildrenText with non-text child nodes // <a> with nested <span> triggers getChildrenText with non-text child nodes
input := `<p><a href="https://example.com"><span>Nested</span> Link</a></p>` input := `<p><a href="https://example.com"><span>Nested</span> Link</a></p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
@ -806,7 +793,7 @@ func TestHTMLToMarkdown_Good_NestedElements_Good(t *testing.T) {
// --- HTML: ordered list --- // --- HTML: ordered list ---
func TestHTMLToMarkdown_Good_OL_Good(t *testing.T) { func TestHTMLToMarkdown_Good_OL(t *testing.T) {
input := `<ol><li>First</li><li>Second</li></ol>` input := `<ol><li>First</li><li>Second</li></ol>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -816,7 +803,7 @@ func TestHTMLToMarkdown_Good_OL_Good(t *testing.T) {
// --- HTML: blockquote --- // --- HTML: blockquote ---
func TestHTMLToMarkdown_Good_BlockquoteElement_Good(t *testing.T) { func TestHTMLToMarkdown_Good_BlockquoteElement(t *testing.T) {
input := `<blockquote>Quoted text</blockquote>` input := `<blockquote>Quoted text</blockquote>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -825,7 +812,7 @@ func TestHTMLToMarkdown_Good_BlockquoteElement_Good(t *testing.T) {
// --- HTML: hr --- // --- HTML: hr ---
func TestHTMLToMarkdown_Good_HR_Good(t *testing.T) { func TestHTMLToMarkdown_Good_HR(t *testing.T) {
input := `<p>Before</p><hr><p>After</p>` input := `<p>Before</p><hr><p>After</p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -834,7 +821,7 @@ func TestHTMLToMarkdown_Good_HR_Good(t *testing.T) {
// --- HTML: h4, h5, h6 --- // --- HTML: h4, h5, h6 ---
func TestHTMLToMarkdown_Good_AllHeadingLevels_Good(t *testing.T) { func TestHTMLToMarkdown_Good_AllHeadingLevels(t *testing.T) {
input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>` input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -845,7 +832,7 @@ func TestHTMLToMarkdown_Good_AllHeadingLevels_Good(t *testing.T) {
// --- HTML: link without href --- // --- HTML: link without href ---
func TestHTMLToMarkdown_Good_LinkNoHref_Good(t *testing.T) { func TestHTMLToMarkdown_Good_LinkNoHref(t *testing.T) {
input := `<a>bare link text</a>` input := `<a>bare link text</a>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -855,7 +842,7 @@ func TestHTMLToMarkdown_Good_LinkNoHref_Good(t *testing.T) {
// --- HTML: unordered list --- // --- HTML: unordered list ---
func TestHTMLToMarkdown_Good_UL_Good(t *testing.T) { func TestHTMLToMarkdown_Good_UL(t *testing.T) {
input := `<ul><li>Item A</li><li>Item B</li></ul>` input := `<ul><li>Item A</li><li>Item B</li></ul>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -865,7 +852,7 @@ func TestHTMLToMarkdown_Good_UL_Good(t *testing.T) {
// --- HTML: br tag --- // --- HTML: br tag ---
func TestHTMLToMarkdown_Good_BRTag_Good(t *testing.T) { func TestHTMLToMarkdown_Good_BRTag(t *testing.T) {
input := `<p>Line one<br>Line two</p>` input := `<p>Line one<br>Line two</p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -875,7 +862,7 @@ func TestHTMLToMarkdown_Good_BRTag_Good(t *testing.T) {
// --- HTML: style tag stripped --- // --- HTML: style tag stripped ---
func TestHTMLToMarkdown_Good_StyleStripped_Good(t *testing.T) { func TestHTMLToMarkdown_Good_StyleStripped(t *testing.T) {
input := `<html><head><style>body{color:red}</style></head><body><p>Clean</p></body></html>` input := `<html><head><style>body{color:red}</style></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -885,7 +872,7 @@ func TestHTMLToMarkdown_Good_StyleStripped_Good(t *testing.T) {
// --- HTML: i and b tags --- // --- HTML: i and b tags ---
func TestHTMLToMarkdown_Good_AlternateBoldItalic_Good(t *testing.T) { func TestHTMLToMarkdown_Good_AlternateBoldItalic(t *testing.T) {
input := `<p><b>bold</b> and <i>italic</i></p>` input := `<p><b>bold</b> and <i>italic</i></p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -895,7 +882,7 @@ func TestHTMLToMarkdown_Good_AlternateBoldItalic_Good(t *testing.T) {
// --- Market: collectCurrent with limiter that actually blocks --- // --- Market: collectCurrent with limiter that actually blocks ---
func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin", Symbol: "btc", Name: "Bitcoin", _ = json.NewEncoder(w).Encode(coinData{ID: "bitcoin", Symbol: "btc", Name: "Bitcoin",
@ -927,7 +914,7 @@ func TestMarketCollector_Collect_Bad_LimiterBlocksThenCancelled_Good(t *testing.
// --- Papers: IACR with limiter that blocks --- // --- Papers: IACR with limiter that blocks ---
func TestPapersCollector_CollectIACR_Bad_LimiterBlocks_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_LimiterBlocks(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -944,7 +931,7 @@ func TestPapersCollector_CollectIACR_Bad_LimiterBlocks_Good(t *testing.T) {
// --- Papers: arXiv with limiter that blocks --- // --- Papers: arXiv with limiter that blocks ---
func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -961,7 +948,7 @@ func TestPapersCollector_CollectArXiv_Bad_LimiterBlocks_Good(t *testing.T) {
// --- BitcoinTalk: limiter that blocks --- // --- BitcoinTalk: limiter that blocks ---
func TestBitcoinTalkCollector_Collect_Bad_LimiterBlocks_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Bad_LimiterBlocks(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = NewRateLimiter() cfg.Limiter = NewRateLimiter()
@ -988,7 +975,7 @@ type writeCountMedium struct {
func (w *writeCountMedium) Write(path, content string) error { func (w *writeCountMedium) Write(path, content string) error {
w.writeCount++ w.writeCount++
if w.writeCount > w.failAfterN { if w.writeCount > w.failAfterN {
return testErrf("write %d: disk full", w.writeCount) return fmt.Errorf("write %d: disk full", w.writeCount)
} }
return w.MockMedium.Write(path, content) return w.MockMedium.Write(path, content)
} }
@ -997,31 +984,21 @@ func (w *writeCountMedium) Read(path string) (string, error) { return w.M
func (w *writeCountMedium) List(path string) ([]fs.DirEntry, error) { return w.MockMedium.List(path) } func (w *writeCountMedium) List(path string) ([]fs.DirEntry, error) { return w.MockMedium.List(path) }
func (w *writeCountMedium) IsFile(path string) bool { return w.MockMedium.IsFile(path) } func (w *writeCountMedium) IsFile(path string) bool { return w.MockMedium.IsFile(path) }
func (w *writeCountMedium) FileGet(path string) (string, error) { return w.MockMedium.FileGet(path) } func (w *writeCountMedium) FileGet(path string) (string, error) { return w.MockMedium.FileGet(path) }
func (w *writeCountMedium) FileSet(path, content string) error { func (w *writeCountMedium) FileSet(path, content string) error { return w.MockMedium.FileSet(path, content) }
return w.MockMedium.FileSet(path, content)
}
func (w *writeCountMedium) Delete(path string) error { return w.MockMedium.Delete(path) } func (w *writeCountMedium) Delete(path string) error { return w.MockMedium.Delete(path) }
func (w *writeCountMedium) DeleteAll(path string) error { return w.MockMedium.DeleteAll(path) } func (w *writeCountMedium) DeleteAll(path string) error { return w.MockMedium.DeleteAll(path) }
func (w *writeCountMedium) Rename(old, new string) error { return w.MockMedium.Rename(old, new) } func (w *writeCountMedium) Rename(old, new string) error { return w.MockMedium.Rename(old, new) }
func (w *writeCountMedium) Stat(path string) (fs.FileInfo, error) { return w.MockMedium.Stat(path) } func (w *writeCountMedium) Stat(path string) (fs.FileInfo, error) { return w.MockMedium.Stat(path) }
func (w *writeCountMedium) Open(path string) (fs.File, error) { return w.MockMedium.Open(path) } func (w *writeCountMedium) Open(path string) (fs.File, error) { return w.MockMedium.Open(path) }
func (w *writeCountMedium) Create(path string) (goio.WriteCloser, error) { func (w *writeCountMedium) Create(path string) (goio.WriteCloser, error) { return w.MockMedium.Create(path) }
return w.MockMedium.Create(path) func (w *writeCountMedium) Append(path string) (goio.WriteCloser, error) { return w.MockMedium.Append(path) }
} func (w *writeCountMedium) ReadStream(path string) (goio.ReadCloser, error) { return w.MockMedium.ReadStream(path) }
func (w *writeCountMedium) Append(path string) (goio.WriteCloser, error) { func (w *writeCountMedium) WriteStream(path string) (goio.WriteCloser, error) { return w.MockMedium.WriteStream(path) }
return w.MockMedium.Append(path)
}
func (w *writeCountMedium) ReadStream(path string) (goio.ReadCloser, error) {
return w.MockMedium.ReadStream(path)
}
func (w *writeCountMedium) WriteStream(path string) (goio.WriteCloser, error) {
return w.MockMedium.WriteStream(path)
}
func (w *writeCountMedium) Exists(path string) bool { return w.MockMedium.Exists(path) } func (w *writeCountMedium) Exists(path string) bool { return w.MockMedium.Exists(path) }
func (w *writeCountMedium) IsDir(path string) bool { return w.MockMedium.IsDir(path) } func (w *writeCountMedium) IsDir(path string) bool { return w.MockMedium.IsDir(path) }
// Test that the summary.md write error in collectCurrent is handled. // Test that the summary.md write error in collectCurrent is handled.
func TestMarketCollector_Collect_Bad_SummaryWriteError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_SummaryWriteError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
if strings.Contains(r.URL.Path, "/market_chart") { if strings.Contains(r.URL.Path, "/market_chart") {
@ -1058,7 +1035,7 @@ func TestMarketCollector_Collect_Bad_SummaryWriteError_Good(t *testing.T) {
// --- Market: collectHistorical write error --- // --- Market: collectHistorical write error ---
func TestMarketCollector_Collect_Bad_HistoricalWriteError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_HistoricalWriteError(t *testing.T) {
callCount := 0 callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -1097,8 +1074,8 @@ func TestMarketCollector_Collect_Bad_HistoricalWriteError_Good(t *testing.T) {
// --- State: Save write error --- // --- State: Save write error ---
func TestState_Save_Bad_WriteError_Good(t *testing.T) { func TestState_Save_Bad_WriteError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("disk full")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("disk full")}
s := NewState(em, "/state.json") s := NewState(em, "/state.json")
s.Set("test", &StateEntry{Source: "test", Items: 1}) s.Set("test", &StateEntry{Source: "test", Items: 1})
@ -1109,7 +1086,7 @@ func TestState_Save_Bad_WriteError_Good(t *testing.T) {
// --- Excavator: collector with state error --- // --- Excavator: collector with state error ---
func TestExcavator_Run_Bad_CollectorStateError_Good(t *testing.T) { func TestExcavator_Run_Bad_CollectorStateError(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.State = NewState(m, "/state.json") cfg.State = NewState(m, "/state.json")
@ -1131,7 +1108,7 @@ func TestExcavator_Run_Bad_CollectorStateError_Good(t *testing.T) {
// --- BitcoinTalk: page returns zero posts (empty content) --- // --- BitcoinTalk: page returns zero posts (empty content) ---
func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage_Good(t *testing.T) { func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
// Valid HTML with no post divs at all // Valid HTML with no post divs at all
@ -1156,8 +1133,8 @@ func TestBitcoinTalkCollector_Collect_Good_ZeroPostsPage_Good(t *testing.T) {
// --- Excavator: state save error after collection --- // --- Excavator: state save error after collection ---
func TestExcavator_Run_Bad_StateSaveError_Good(t *testing.T) { func TestExcavator_Run_Bad_StateSaveError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: testErr("state write failed")} em := &errorMedium{MockMedium: io.NewMockMedium(), writeErr: fmt.Errorf("state write failed")}
cfg := &Config{ cfg := &Config{
Output: io.NewMockMedium(), // Use regular medium for output Output: io.NewMockMedium(), // Use regular medium for output
OutputDir: "/output", OutputDir: "/output",
@ -1180,8 +1157,8 @@ func TestExcavator_Run_Bad_StateSaveError_Good(t *testing.T) {
// --- State: Load with read error --- // --- State: Load with read error ---
func TestState_Load_Bad_ReadError_Good(t *testing.T) { func TestState_Load_Bad_ReadError(t *testing.T) {
em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: testErr("read denied")} em := &errorMedium{MockMedium: io.NewMockMedium(), readErr: fmt.Errorf("read denied")}
em.MockMedium.Files["/state.json"] = "{}" // File exists but read will fail em.MockMedium.Files["/state.json"] = "{}" // File exists but read will fail
s := NewState(em, "/state.json") s := NewState(em, "/state.json")
@ -1192,7 +1169,7 @@ func TestState_Load_Bad_ReadError_Good(t *testing.T) {
// --- Papers: PaperSourceAll emits complete --- // --- Papers: PaperSourceAll emits complete ---
func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR_Good(t *testing.T) { func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -1229,7 +1206,7 @@ func TestPapersCollector_CollectAll_Good_ArxivFailsWithIACR_Good(t *testing.T) {
// --- Papers: IACR with cancelled context (request creation fails) --- // --- Papers: IACR with cancelled context (request creation fails) ---
func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails(t *testing.T) {
// Don't set up any server - the request should fail because context is cancelled. // Don't set up any server - the request should fail because context is cancelled.
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -1245,7 +1222,7 @@ func TestPapersCollector_CollectIACR_Bad_CancelledContextRequestFails_Good(t *te
// --- Papers: arXiv with cancelled context --- // --- Papers: arXiv with cancelled context ---
func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -1260,7 +1237,7 @@ func TestPapersCollector_CollectArXiv_Bad_CancelledContextRequestFails_Good(t *t
// --- Market: collectHistorical limiter blocks --- // --- Market: collectHistorical limiter blocks ---
func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(coinData{ _ = json.NewEncoder(w).Encode(coinData{
@ -1299,7 +1276,7 @@ func TestMarketCollector_Collect_Bad_HistoricalLimiterBlocks_Good(t *testing.T)
// --- BitcoinTalk: fetchPage with invalid URL --- // --- BitcoinTalk: fetchPage with invalid URL ---
func TestBitcoinTalkCollector_FetchPage_Bad_InvalidURL_Good(t *testing.T) { func TestBitcoinTalkCollector_FetchPage_Bad_InvalidURL(t *testing.T) {
b := &BitcoinTalkCollector{TopicID: "12345"} b := &BitcoinTalkCollector{TopicID: "12345"}
// Use a URL with control character that will fail NewRequestWithContext // Use a URL with control character that will fail NewRequestWithContext
_, err := b.fetchPage(context.Background(), "http://\x7f/invalid") _, err := b.fetchPage(context.Background(), "http://\x7f/invalid")

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -10,23 +8,18 @@ import (
// Event types used by the collection subsystem. // Event types used by the collection subsystem.
const ( const (
// EventStart is emitted when a collector begins its run. // EventStart is emitted when a collector begins its run.
//
EventStart = "start" EventStart = "start"
// EventProgress is emitted to report incremental progress. // EventProgress is emitted to report incremental progress.
//
EventProgress = "progress" EventProgress = "progress"
// EventItem is emitted when a single item is collected. // EventItem is emitted when a single item is collected.
//
EventItem = "item" EventItem = "item"
// EventError is emitted when an error occurs during collection. // EventError is emitted when an error occurs during collection.
//
EventError = "error" EventError = "error"
// EventComplete is emitted when a collector finishes its run. // EventComplete is emitted when a collector finishes its run.
//
EventComplete = "complete" EventComplete = "complete"
) )
@ -59,7 +52,6 @@ type Dispatcher struct {
} }
// NewDispatcher creates a new event dispatcher. // NewDispatcher creates a new event dispatcher.
// Usage: NewDispatcher(...)
func NewDispatcher() *Dispatcher { func NewDispatcher() *Dispatcher {
return &Dispatcher{ return &Dispatcher{
handlers: make(map[string][]EventHandler), handlers: make(map[string][]EventHandler),
@ -68,7 +60,6 @@ func NewDispatcher() *Dispatcher {
// On registers a handler for an event type. Multiple handlers can be // On registers a handler for an event type. Multiple handlers can be
// registered for the same event type and will be called in order. // registered for the same event type and will be called in order.
// Usage: On(...)
func (d *Dispatcher) On(eventType string, handler EventHandler) { func (d *Dispatcher) On(eventType string, handler EventHandler) {
d.mu.Lock() d.mu.Lock()
defer d.mu.Unlock() defer d.mu.Unlock()
@ -78,7 +69,6 @@ func (d *Dispatcher) On(eventType string, handler EventHandler) {
// Emit dispatches an event to all registered handlers for that event type. // Emit dispatches an event to all registered handlers for that event type.
// If no handlers are registered for the event type, the event is silently dropped. // If no handlers are registered for the event type, the event is silently dropped.
// The event's Time field is set to now if it is zero. // The event's Time field is set to now if it is zero.
// Usage: Emit(...)
func (d *Dispatcher) Emit(event Event) { func (d *Dispatcher) Emit(event Event) {
if event.Time.IsZero() { if event.Time.IsZero() {
event.Time = time.Now() event.Time = time.Now()
@ -94,7 +84,6 @@ func (d *Dispatcher) Emit(event Event) {
} }
// EmitStart emits a start event for the given source. // EmitStart emits a start event for the given source.
// Usage: EmitStart(...)
func (d *Dispatcher) EmitStart(source, message string) { func (d *Dispatcher) EmitStart(source, message string) {
d.Emit(Event{ d.Emit(Event{
Type: EventStart, Type: EventStart,
@ -104,7 +93,6 @@ func (d *Dispatcher) EmitStart(source, message string) {
} }
// EmitProgress emits a progress event. // EmitProgress emits a progress event.
// Usage: EmitProgress(...)
func (d *Dispatcher) EmitProgress(source, message string, data any) { func (d *Dispatcher) EmitProgress(source, message string, data any) {
d.Emit(Event{ d.Emit(Event{
Type: EventProgress, Type: EventProgress,
@ -115,7 +103,6 @@ func (d *Dispatcher) EmitProgress(source, message string, data any) {
} }
// EmitItem emits an item event. // EmitItem emits an item event.
// Usage: EmitItem(...)
func (d *Dispatcher) EmitItem(source, message string, data any) { func (d *Dispatcher) EmitItem(source, message string, data any) {
d.Emit(Event{ d.Emit(Event{
Type: EventItem, Type: EventItem,
@ -126,7 +113,6 @@ func (d *Dispatcher) EmitItem(source, message string, data any) {
} }
// EmitError emits an error event. // EmitError emits an error event.
// Usage: EmitError(...)
func (d *Dispatcher) EmitError(source, message string, data any) { func (d *Dispatcher) EmitError(source, message string, data any) {
d.Emit(Event{ d.Emit(Event{
Type: EventError, Type: EventError,
@ -137,7 +123,6 @@ func (d *Dispatcher) EmitError(source, message string, data any) {
} }
// EmitComplete emits a complete event. // EmitComplete emits a complete event.
// Usage: EmitComplete(...)
func (d *Dispatcher) EmitComplete(source, message string, data any) { func (d *Dispatcher) EmitComplete(source, message string, data any) {
d.Emit(Event{ d.Emit(Event{
Type: EventComplete, Type: EventComplete,

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -43,7 +41,7 @@ func TestDispatcher_On_Good(t *testing.T) {
assert.Equal(t, 3, count, "All three handlers should be called") assert.Equal(t, 3, count, "All three handlers should be called")
} }
func TestDispatcher_Emit_Good_NoHandlers_Good(t *testing.T) { func TestDispatcher_Emit_Good_NoHandlers(t *testing.T) {
d := NewDispatcher() d := NewDispatcher()
// Should not panic when emitting an event with no handlers // Should not panic when emitting an event with no handlers
@ -56,7 +54,7 @@ func TestDispatcher_Emit_Good_NoHandlers_Good(t *testing.T) {
}) })
} }
func TestDispatcher_Emit_Good_MultipleEventTypes_Good(t *testing.T) { func TestDispatcher_Emit_Good_MultipleEventTypes(t *testing.T) {
d := NewDispatcher() d := NewDispatcher()
var starts, errors int var starts, errors int
@ -71,7 +69,7 @@ func TestDispatcher_Emit_Good_MultipleEventTypes_Good(t *testing.T) {
assert.Equal(t, 1, errors) assert.Equal(t, 1, errors)
} }
func TestDispatcher_Emit_Good_SetsTime_Good(t *testing.T) { func TestDispatcher_Emit_Good_SetsTime(t *testing.T) {
d := NewDispatcher() d := NewDispatcher()
var received Event var received Event
@ -87,7 +85,7 @@ func TestDispatcher_Emit_Good_SetsTime_Good(t *testing.T) {
assert.True(t, received.Time.Before(after) || received.Time.Equal(after)) assert.True(t, received.Time.Before(after) || received.Time.Equal(after))
} }
func TestDispatcher_Emit_Good_PreservesExistingTime_Good(t *testing.T) { func TestDispatcher_Emit_Good_PreservesExistingTime(t *testing.T) {
d := NewDispatcher() d := NewDispatcher()
customTime := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC) customTime := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
"time" "time"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
@ -25,14 +23,12 @@ type Excavator struct {
} }
// Name returns the orchestrator name. // Name returns the orchestrator name.
// Usage: Name(...)
func (e *Excavator) Name() string { func (e *Excavator) Name() string {
return "excavator" return "excavator"
} }
// Run executes all collectors sequentially, respecting rate limits and // Run executes all collectors sequentially, respecting rate limits and
// using state for resume support. Results are aggregated from all collectors. // using state for resume support. Results are aggregated from all collectors.
// Usage: Run(...)
func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) { func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: e.Name()} result := &Result{Source: e.Name()}
@ -43,11 +39,9 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
if cfg.Dispatcher != nil { if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitStart(e.Name(), fmt.Sprintf("Starting excavation with %d collectors", len(e.Collectors))) cfg.Dispatcher.EmitStart(e.Name(), fmt.Sprintf("Starting excavation with %d collectors", len(e.Collectors)))
} }
verboseProgress(cfg, e.Name(), fmt.Sprintf("queueing %d collectors", len(e.Collectors)))
// Load state if resuming // Load state if resuming
if e.Resume && cfg.State != nil { if e.Resume && cfg.State != nil {
verboseProgress(cfg, e.Name(), "loading resume state")
if err := cfg.State.Load(); err != nil { if err := cfg.State.Load(); err != nil {
return result, core.E("collect.Excavator.Run", "failed to load state", err) return result, core.E("collect.Excavator.Run", "failed to load state", err)
} }
@ -59,7 +53,6 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
if cfg.Dispatcher != nil { if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitProgress(e.Name(), fmt.Sprintf("[scan] Would run collector: %s", c.Name()), nil) cfg.Dispatcher.EmitProgress(e.Name(), fmt.Sprintf("[scan] Would run collector: %s", c.Name()), nil)
} }
verboseProgress(cfg, e.Name(), fmt.Sprintf("scan-only collector: %s", c.Name()))
} }
return result, nil return result, nil
} }
@ -73,7 +66,6 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
cfg.Dispatcher.EmitProgress(e.Name(), cfg.Dispatcher.EmitProgress(e.Name(),
fmt.Sprintf("Running collector %d/%d: %s", i+1, len(e.Collectors), c.Name()), nil) fmt.Sprintf("Running collector %d/%d: %s", i+1, len(e.Collectors), c.Name()), nil)
} }
verboseProgress(cfg, e.Name(), fmt.Sprintf("dispatching collector %d/%d: %s", i+1, len(e.Collectors), c.Name()))
// Check if we should skip (already completed in a previous run) // Check if we should skip (already completed in a previous run)
if e.Resume && cfg.State != nil { if e.Resume && cfg.State != nil {
@ -84,7 +76,6 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
fmt.Sprintf("Skipping %s (already collected %d items on %s)", fmt.Sprintf("Skipping %s (already collected %d items on %s)",
c.Name(), entry.Items, entry.LastRun.Format(time.RFC3339)), nil) c.Name(), entry.Items, entry.LastRun.Format(time.RFC3339)), nil)
} }
verboseProgress(cfg, e.Name(), fmt.Sprintf("resume skip: %s", c.Name()))
result.Skipped++ result.Skipped++
continue continue
} }
@ -120,7 +111,6 @@ func (e *Excavator) Run(ctx context.Context, cfg *Config) (*Result, error) {
// Save state // Save state
if cfg.State != nil { if cfg.State != nil {
verboseProgress(cfg, e.Name(), "saving resume state")
if err := cfg.State.Save(); err != nil { if err := cfg.State.Save(); err != nil {
if cfg.Dispatcher != nil { if cfg.Dispatcher != nil {
cfg.Dispatcher.EmitError(e.Name(), fmt.Sprintf("Failed to save state: %v", err), nil) cfg.Dispatcher.EmitError(e.Name(), fmt.Sprintf("Failed to save state: %v", err), nil)

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -12,7 +10,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestExcavator_Run_Good_ResumeSkipsCompleted_Good(t *testing.T) { func TestExcavator_Run_Good_ResumeSkipsCompleted(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -41,7 +39,7 @@ func TestExcavator_Run_Good_ResumeSkipsCompleted_Good(t *testing.T) {
assert.Equal(t, 1, result.Skipped) assert.Equal(t, 1, result.Skipped)
} }
func TestExcavator_Run_Good_ResumeRunsIncomplete_Good(t *testing.T) { func TestExcavator_Run_Good_ResumeRunsIncomplete(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -67,7 +65,7 @@ func TestExcavator_Run_Good_ResumeRunsIncomplete_Good(t *testing.T) {
assert.Equal(t, 5, result.Items) assert.Equal(t, 5, result.Items)
} }
func TestExcavator_Run_Good_NilState_Good(t *testing.T) { func TestExcavator_Run_Good_NilState(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.State = nil cfg.State = nil
@ -85,7 +83,7 @@ func TestExcavator_Run_Good_NilState_Good(t *testing.T) {
assert.Equal(t, 3, result.Items) assert.Equal(t, 3, result.Items)
} }
func TestExcavator_Run_Good_NilDispatcher_Good(t *testing.T) { func TestExcavator_Run_Good_NilDispatcher(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Dispatcher = nil cfg.Dispatcher = nil
@ -103,7 +101,7 @@ func TestExcavator_Run_Good_NilDispatcher_Good(t *testing.T) {
assert.Equal(t, 2, result.Items) assert.Equal(t, 2, result.Items)
} }
func TestExcavator_Run_Good_ProgressEvents_Good(t *testing.T) { func TestExcavator_Run_Good_ProgressEvents(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil

View file

@ -1,11 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
core "dappco.re/go/core" "fmt"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
"testing" "testing"
"dappco.re/go/core/io" "dappco.re/go/core/io"
@ -66,7 +63,7 @@ func TestExcavator_Run_Good(t *testing.T) {
assert.Len(t, result.Files, 8) assert.Len(t, result.Files, 8)
} }
func TestExcavator_Run_Good_Empty_Good(t *testing.T) { func TestExcavator_Run_Good_Empty(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -77,7 +74,7 @@ func TestExcavator_Run_Good_Empty_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestExcavator_Run_Good_DryRun_Good(t *testing.T) { func TestExcavator_Run_Good_DryRun(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -98,7 +95,7 @@ func TestExcavator_Run_Good_DryRun_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestExcavator_Run_Good_ScanOnly_Good(t *testing.T) { func TestExcavator_Run_Good_ScanOnly(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -123,13 +120,13 @@ func TestExcavator_Run_Good_ScanOnly_Good(t *testing.T) {
assert.Contains(t, progressMessages[0], "source-a") assert.Contains(t, progressMessages[0], "source-a")
} }
func TestExcavator_Run_Good_WithErrors_Good(t *testing.T) { func TestExcavator_Run_Good_WithErrors(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
c1 := &mockCollector{name: "good", items: 5} c1 := &mockCollector{name: "good", items: 5}
c2 := &mockCollector{name: "bad", err: core.E("collect.mockCollector.Collect", "network error", nil)} c2 := &mockCollector{name: "bad", err: fmt.Errorf("network error")}
c3 := &mockCollector{name: "also-good", items: 3} c3 := &mockCollector{name: "also-good", items: 3}
e := &Excavator{ e := &Excavator{
@ -146,7 +143,7 @@ func TestExcavator_Run_Good_WithErrors_Good(t *testing.T) {
assert.True(t, c3.called) assert.True(t, c3.called)
} }
func TestExcavator_Run_Good_CancelledContext_Good(t *testing.T) { func TestExcavator_Run_Good_CancelledContext(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -163,7 +160,7 @@ func TestExcavator_Run_Good_CancelledContext_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestExcavator_Run_Good_SavesState_Good(t *testing.T) { func TestExcavator_Run_Good_SavesState(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -184,7 +181,7 @@ func TestExcavator_Run_Good_SavesState_Good(t *testing.T) {
assert.Equal(t, "source-a", entry.Source) assert.Equal(t, "source-a", entry.Source)
} }
func TestExcavator_Run_Good_Events_Good(t *testing.T) { func TestExcavator_Run_Good_Events(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil cfg.Limiter = nil
@ -203,24 +200,3 @@ func TestExcavator_Run_Good_Events_Good(t *testing.T) {
assert.Equal(t, 1, startCount) assert.Equal(t, 1, startCount)
assert.Equal(t, 1, completeCount) assert.Equal(t, 1, completeCount)
} }
func TestExcavator_Run_Good_VerboseProgress_Good(t *testing.T) {
m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output")
cfg.Limiter = nil
cfg.Verbose = true
var progressCount int
cfg.Dispatcher.On(EventProgress, func(e Event) {
progressCount++
})
c1 := &mockCollector{name: "source-a", items: 1}
e := &Excavator{
Collectors: []Collector{c1},
}
_, err := e.Run(context.Background(), cfg)
assert.NoError(t, err)
assert.GreaterOrEqual(t, progressCount, 2)
}

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "encoding/json"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
json "dappco.re/go/core/scm/internal/ax/jsonx" "os/exec"
strings "dappco.re/go/core/scm/internal/ax/stringsx" "path/filepath"
exec "golang.org/x/sys/execabs" "strings"
"time" "time"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
@ -55,7 +53,6 @@ type GitHubCollector struct {
} }
// Name returns the collector name. // Name returns the collector name.
// Usage: Name(...)
func (g *GitHubCollector) Name() string { func (g *GitHubCollector) Name() string {
if g.Repo != "" { if g.Repo != "" {
return fmt.Sprintf("github:%s/%s", g.Org, g.Repo) return fmt.Sprintf("github:%s/%s", g.Org, g.Repo)
@ -64,7 +61,6 @@ func (g *GitHubCollector) Name() string {
} }
// Collect gathers issues and/or PRs from GitHub repositories. // Collect gathers issues and/or PRs from GitHub repositories.
// Usage: Collect(...)
func (g *GitHubCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) { func (g *GitHubCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: g.Name()} result := &Result{Source: g.Name()}

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -16,12 +14,12 @@ func TestGitHubCollector_Name_Good(t *testing.T) {
assert.Equal(t, "github:host-uk/core", g.Name()) assert.Equal(t, "github:host-uk/core", g.Name())
} }
func TestGitHubCollector_Name_Good_OrgOnly_Good(t *testing.T) { func TestGitHubCollector_Name_Good_OrgOnly(t *testing.T) {
g := &GitHubCollector{Org: "host-uk"} g := &GitHubCollector{Org: "host-uk"}
assert.Equal(t, "github:host-uk", g.Name()) assert.Equal(t, "github:host-uk", g.Name())
} }
func TestGitHubCollector_Collect_Good_DryRun_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_DryRun(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -40,7 +38,7 @@ func TestGitHubCollector_Collect_Good_DryRun_Good(t *testing.T) {
assert.True(t, progressEmitted, "Should emit progress event in dry-run mode") assert.True(t, progressEmitted, "Should emit progress event in dry-run mode")
} }
func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -52,7 +50,7 @@ func TestGitHubCollector_Collect_Good_DryRun_IssuesOnly_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestGitHubCollector_Collect_Good_DryRun_PRsOnly_Good(t *testing.T) { func TestGitHubCollector_Collect_Good_DryRun_PRsOnly(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -90,7 +88,7 @@ func TestFormatIssueMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "**URL:** https://github.com/test/repo/issues/42") assert.Contains(t, md, "**URL:** https://github.com/test/repo/issues/42")
} }
func TestFormatIssueMarkdown_Good_NoLabels_Good(t *testing.T) { func TestFormatIssueMarkdown_Good_NoLabels(t *testing.T) {
issue := ghIssue{ issue := ghIssue{
Number: 1, Number: 1,
Title: "Simple", Title: "Simple",

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "encoding/json"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http" "net/http"
"path/filepath"
"strings"
"time" "time"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
@ -31,7 +29,6 @@ type MarketCollector struct {
} }
// Name returns the collector name. // Name returns the collector name.
// Usage: Name(...)
func (m *MarketCollector) Name() string { func (m *MarketCollector) Name() string {
return fmt.Sprintf("market:%s", m.CoinID) return fmt.Sprintf("market:%s", m.CoinID)
} }
@ -66,7 +63,6 @@ type historicalData struct {
} }
// Collect gathers market data from CoinGecko. // Collect gathers market data from CoinGecko.
// Usage: Collect(...)
func (m *MarketCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) { func (m *MarketCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: m.Name()} result := &Result{Source: m.Name()}
@ -276,7 +272,6 @@ func formatMarketSummary(data *coinData) string {
} }
// FormatMarketSummary is exported for testing. // FormatMarketSummary is exported for testing.
// Usage: FormatMarketSummary(...)
func FormatMarketSummary(data *coinData) string { func FormatMarketSummary(data *coinData) string {
return formatMarketSummary(data) return formatMarketSummary(data)
} }

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
@ -14,7 +12,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestMarketCollector_Collect_Good_HistoricalWithFromDate_Good(t *testing.T) { func TestMarketCollector_Collect_Good_HistoricalWithFromDate(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -58,7 +56,7 @@ func TestMarketCollector_Collect_Good_HistoricalWithFromDate_Good(t *testing.T)
assert.Equal(t, 3, result.Items) assert.Equal(t, 3, result.Items)
} }
func TestMarketCollector_Collect_Good_HistoricalInvalidDate_Good(t *testing.T) { func TestMarketCollector_Collect_Good_HistoricalInvalidDate(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -100,7 +98,7 @@ func TestMarketCollector_Collect_Good_HistoricalInvalidDate_Good(t *testing.T) {
assert.Equal(t, 3, result.Items) assert.Equal(t, 3, result.Items)
} }
func TestMarketCollector_Collect_Bad_HistoricalServerError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_HistoricalServerError(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -139,7 +137,7 @@ func TestMarketCollector_Collect_Bad_HistoricalServerError_Good(t *testing.T) {
assert.Equal(t, 1, result.Errors) // historical failed assert.Equal(t, 1, result.Errors) // historical failed
} }
func TestMarketCollector_Collect_Good_EmitsEvents_Good(t *testing.T) { func TestMarketCollector_Collect_Good_EmitsEvents(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
data := coinData{ data := coinData{
@ -174,7 +172,7 @@ func TestMarketCollector_Collect_Good_EmitsEvents_Good(t *testing.T) {
assert.Equal(t, 1, completes) assert.Equal(t, 1, completes)
} }
func TestMarketCollector_Collect_Good_CancelledContext_Good(t *testing.T) { func TestMarketCollector_Collect_Good_CancelledContext(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
})) }))
@ -199,7 +197,7 @@ func TestMarketCollector_Collect_Good_CancelledContext_Good(t *testing.T) {
assert.Equal(t, 1, result.Errors) assert.Equal(t, 1, result.Errors)
} }
func TestFormatMarketSummary_Good_AllFields_Good(t *testing.T) { func TestFormatMarketSummary_Good_AllFields(t *testing.T) {
data := &coinData{ data := &coinData{
Name: "Lethean", Name: "Lethean",
Symbol: "lthn", Symbol: "lthn",
@ -231,7 +229,7 @@ func TestFormatMarketSummary_Good_AllFields_Good(t *testing.T) {
assert.Contains(t, summary, "Last updated") assert.Contains(t, summary, "Last updated")
} }
func TestFormatMarketSummary_Good_Minimal_Good(t *testing.T) { func TestFormatMarketSummary_Good_Minimal(t *testing.T) {
data := &coinData{ data := &coinData{
Name: "Unknown", Name: "Unknown",
Symbol: "ukn", Symbol: "ukn",

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
@ -18,7 +16,7 @@ func TestMarketCollector_Name_Good(t *testing.T) {
assert.Equal(t, "market:bitcoin", m.Name()) assert.Equal(t, "market:bitcoin", m.Name())
} }
func TestMarketCollector_Collect_Bad_NoCoinID_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_NoCoinID(t *testing.T) {
mock := io.NewMockMedium() mock := io.NewMockMedium()
cfg := NewConfigWithMedium(mock, "/output") cfg := NewConfigWithMedium(mock, "/output")
@ -27,7 +25,7 @@ func TestMarketCollector_Collect_Bad_NoCoinID_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestMarketCollector_Collect_Good_DryRun_Good(t *testing.T) { func TestMarketCollector_Collect_Good_DryRun(t *testing.T) {
mock := io.NewMockMedium() mock := io.NewMockMedium()
cfg := NewConfigWithMedium(mock, "/output") cfg := NewConfigWithMedium(mock, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -39,7 +37,7 @@ func TestMarketCollector_Collect_Good_DryRun_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestMarketCollector_Collect_Good_CurrentData_Good(t *testing.T) { func TestMarketCollector_Collect_Good_CurrentData(t *testing.T) {
// Set up a mock CoinGecko server // Set up a mock CoinGecko server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
data := coinData{ data := coinData{
@ -94,7 +92,7 @@ func TestMarketCollector_Collect_Good_CurrentData_Good(t *testing.T) {
assert.Contains(t, summary, "42000.50") assert.Contains(t, summary, "42000.50")
} }
func TestMarketCollector_Collect_Good_Historical_Good(t *testing.T) { func TestMarketCollector_Collect_Good_Historical(t *testing.T) {
callCount := 0 callCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -166,7 +164,7 @@ func TestFormatMarketSummary_Good(t *testing.T) {
assert.Contains(t, summary, "Total Supply") assert.Contains(t, summary, "Total Supply")
} }
func TestMarketCollector_Collect_Bad_ServerError_Good(t *testing.T) { func TestMarketCollector_Collect_Bad_ServerError(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))

View file

@ -1,16 +1,14 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx"
fmt "dappco.re/go/core/scm/internal/ax/fmtx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"encoding/xml" "encoding/xml"
"fmt"
"iter" "iter"
"net/http" "net/http"
"net/url" "net/url"
"path/filepath"
"strings"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
"golang.org/x/net/html" "golang.org/x/net/html"
@ -18,11 +16,8 @@ import (
// Paper source identifiers. // Paper source identifiers.
const ( const (
//
PaperSourceIACR = "iacr" PaperSourceIACR = "iacr"
//
PaperSourceArXiv = "arxiv" PaperSourceArXiv = "arxiv"
//
PaperSourceAll = "all" PaperSourceAll = "all"
) )
@ -39,7 +34,6 @@ type PapersCollector struct {
} }
// Name returns the collector name. // Name returns the collector name.
// Usage: Name(...)
func (p *PapersCollector) Name() string { func (p *PapersCollector) Name() string {
return fmt.Sprintf("papers:%s", p.Source) return fmt.Sprintf("papers:%s", p.Source)
} }
@ -56,7 +50,6 @@ type paper struct {
} }
// Collect gathers papers from the configured sources. // Collect gathers papers from the configured sources.
// Usage: Collect(...)
func (p *PapersCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) { func (p *PapersCollector) Collect(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: p.Name()} result := &Result{Source: p.Name()}
@ -410,7 +403,6 @@ func formatPaperMarkdown(ppr paper) string {
} }
// FormatPaperMarkdown is exported for testing. // FormatPaperMarkdown is exported for testing.
// Usage: FormatPaperMarkdown(...)
func FormatPaperMarkdown(title string, authors []string, date, paperURL, source, abstract string) string { func FormatPaperMarkdown(title string, authors []string, date, paperURL, source, abstract string) string {
return formatPaperMarkdown(paper{ return formatPaperMarkdown(paper{
Title: title, Title: title,

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"dappco.re/go/core/io" "dappco.re/go/core/io"
@ -111,7 +109,7 @@ func TestPapersCollector_CollectArXiv_Good(t *testing.T) {
assert.Contains(t, content, "Alice") assert.Contains(t, content, "Alice")
} }
func TestPapersCollector_CollectArXiv_Good_WithCategory_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Good_WithCategory(t *testing.T) {
var capturedQuery string var capturedQuery string
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
capturedQuery = r.URL.RawQuery capturedQuery = r.URL.RawQuery
@ -167,7 +165,7 @@ func TestPapersCollector_CollectAll_Good(t *testing.T) {
assert.Equal(t, 4, result.Items) // 2 IACR + 2 arXiv assert.Equal(t, 4, result.Items) // 2 IACR + 2 arXiv
} }
func TestPapersCollector_CollectIACR_Bad_ServerError_Good(t *testing.T) { func TestPapersCollector_CollectIACR_Bad_ServerError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))
@ -187,7 +185,7 @@ func TestPapersCollector_CollectIACR_Bad_ServerError_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_CollectArXiv_Bad_ServerError_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_ServerError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable) w.WriteHeader(http.StatusServiceUnavailable)
})) }))
@ -207,7 +205,7 @@ func TestPapersCollector_CollectArXiv_Bad_ServerError_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_CollectArXiv_Bad_InvalidXML_Good(t *testing.T) { func TestPapersCollector_CollectArXiv_Bad_InvalidXML(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/xml") w.Header().Set("Content-Type", "application/xml")
_, _ = w.Write([]byte(`not xml at all`)) _, _ = w.Write([]byte(`not xml at all`))
@ -228,7 +226,7 @@ func TestPapersCollector_CollectArXiv_Bad_InvalidXML_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_CollectAll_Bad_BothFail_Good(t *testing.T) { func TestPapersCollector_CollectAll_Bad_BothFail(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError) w.WriteHeader(http.StatusInternalServerError)
})) }))
@ -248,7 +246,7 @@ func TestPapersCollector_CollectAll_Bad_BothFail_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_CollectAll_Good_OneFails_Good(t *testing.T) { func TestPapersCollector_CollectAll_Good_OneFails(t *testing.T) {
callCount := 0 callCount := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
callCount++ callCount++
@ -297,7 +295,7 @@ func TestExtractIACRPapers_Good(t *testing.T) {
assert.Equal(t, "Lattice Cryptography", papers[1].Title) assert.Equal(t, "Lattice Cryptography", papers[1].Title)
} }
func TestExtractIACRPapers_Good_Empty_Good(t *testing.T) { func TestExtractIACRPapers_Good_Empty(t *testing.T) {
doc, err := html.Parse(strings.NewReader(`<html><body></body></html>`)) doc, err := html.Parse(strings.NewReader(`<html><body></body></html>`))
require.NoError(t, err) require.NoError(t, err)
@ -305,7 +303,7 @@ func TestExtractIACRPapers_Good_Empty_Good(t *testing.T) {
assert.Empty(t, papers) assert.Empty(t, papers)
} }
func TestExtractIACRPapers_Good_NoTitle_Good(t *testing.T) { func TestExtractIACRPapers_Good_NoTitle(t *testing.T) {
doc, err := html.Parse(strings.NewReader(`<html><body><div class="paperentry"></div></body></html>`)) doc, err := html.Parse(strings.NewReader(`<html><body><div class="paperentry"></div></body></html>`))
require.NoError(t, err) require.NoError(t, err)

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -15,17 +13,17 @@ func TestPapersCollector_Name_Good(t *testing.T) {
assert.Equal(t, "papers:iacr", p.Name()) assert.Equal(t, "papers:iacr", p.Name())
} }
func TestPapersCollector_Name_Good_ArXiv_Good(t *testing.T) { func TestPapersCollector_Name_Good_ArXiv(t *testing.T) {
p := &PapersCollector{Source: PaperSourceArXiv} p := &PapersCollector{Source: PaperSourceArXiv}
assert.Equal(t, "papers:arxiv", p.Name()) assert.Equal(t, "papers:arxiv", p.Name())
} }
func TestPapersCollector_Name_Good_All_Good(t *testing.T) { func TestPapersCollector_Name_Good_All(t *testing.T) {
p := &PapersCollector{Source: PaperSourceAll} p := &PapersCollector{Source: PaperSourceAll}
assert.Equal(t, "papers:all", p.Name()) assert.Equal(t, "papers:all", p.Name())
} }
func TestPapersCollector_Collect_Bad_NoQuery_Good(t *testing.T) { func TestPapersCollector_Collect_Bad_NoQuery(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -34,7 +32,7 @@ func TestPapersCollector_Collect_Bad_NoQuery_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_Collect_Bad_UnknownSource_Good(t *testing.T) { func TestPapersCollector_Collect_Bad_UnknownSource(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -43,7 +41,7 @@ func TestPapersCollector_Collect_Bad_UnknownSource_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestPapersCollector_Collect_Good_DryRun_Good(t *testing.T) { func TestPapersCollector_Collect_Good_DryRun(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -74,7 +72,7 @@ func TestFormatPaperMarkdown_Good(t *testing.T) {
assert.Contains(t, md, "zero-knowledge proofs") assert.Contains(t, md, "zero-knowledge proofs")
} }
func TestFormatPaperMarkdown_Good_Minimal_Good(t *testing.T) { func TestFormatPaperMarkdown_Good_Minimal(t *testing.T) {
md := FormatPaperMarkdown("Title Only", nil, "", "", "", "") md := FormatPaperMarkdown("Title Only", nil, "", "", "", "")
assert.Contains(t, md, "# Title Only") assert.Contains(t, md, "# Title Only")

View file

@ -1,15 +1,13 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
filepath "dappco.re/go/core/scm/internal/ax/filepathx" "encoding/json"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
json "dappco.re/go/core/scm/internal/ax/jsonx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"maps" "maps"
"path/filepath"
"slices" "slices"
"strings"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
"golang.org/x/net/html" "golang.org/x/net/html"
@ -25,14 +23,12 @@ type Processor struct {
} }
// Name returns the processor name. // Name returns the processor name.
// Usage: Name(...)
func (p *Processor) Name() string { func (p *Processor) Name() string {
return fmt.Sprintf("process:%s", p.Source) return fmt.Sprintf("process:%s", p.Source)
} }
// Process reads files from the source directory, converts HTML or JSON // Process reads files from the source directory, converts HTML or JSON
// to clean markdown, and writes the results to the output directory. // to clean markdown, and writes the results to the output directory.
// Usage: Process(...)
func (p *Processor) Process(ctx context.Context, cfg *Config) (*Result, error) { func (p *Processor) Process(ctx context.Context, cfg *Config) (*Result, error) {
result := &Result{Source: p.Name()} result := &Result{Source: p.Name()}
@ -335,13 +331,11 @@ func jsonValueToMarkdown(b *strings.Builder, data any, depth int) {
} }
// HTMLToMarkdown is exported for testing. // HTMLToMarkdown is exported for testing.
// Usage: HTMLToMarkdown(...)
func HTMLToMarkdown(content string) (string, error) { func HTMLToMarkdown(content string) (string, error) {
return htmlToMarkdown(content) return htmlToMarkdown(content)
} }
// JSONToMarkdown is exported for testing. // JSONToMarkdown is exported for testing.
// Usage: JSONToMarkdown(...)
func JSONToMarkdown(content string) (string, error) { func JSONToMarkdown(content string) (string, error) {
return jsonToMarkdown(content) return jsonToMarkdown(content)
} }

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -11,7 +9,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestHTMLToMarkdown_Good_OrderedList_Good(t *testing.T) { func TestHTMLToMarkdown_Good_OrderedList(t *testing.T) {
input := `<ol><li>First</li><li>Second</li><li>Third</li></ol>` input := `<ol><li>First</li><li>Second</li><li>Third</li></ol>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -20,7 +18,7 @@ func TestHTMLToMarkdown_Good_OrderedList_Good(t *testing.T) {
assert.Contains(t, result, "3. Third") assert.Contains(t, result, "3. Third")
} }
func TestHTMLToMarkdown_Good_UnorderedList_Good(t *testing.T) { func TestHTMLToMarkdown_Good_UnorderedList(t *testing.T) {
input := `<ul><li>Alpha</li><li>Beta</li></ul>` input := `<ul><li>Alpha</li><li>Beta</li></ul>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -28,21 +26,21 @@ func TestHTMLToMarkdown_Good_UnorderedList_Good(t *testing.T) {
assert.Contains(t, result, "- Beta") assert.Contains(t, result, "- Beta")
} }
func TestHTMLToMarkdown_Good_Blockquote_Good(t *testing.T) { func TestHTMLToMarkdown_Good_Blockquote(t *testing.T) {
input := `<blockquote>A wise quote</blockquote>` input := `<blockquote>A wise quote</blockquote>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "> A wise quote") assert.Contains(t, result, "> A wise quote")
} }
func TestHTMLToMarkdown_Good_HorizontalRule_Good(t *testing.T) { func TestHTMLToMarkdown_Good_HorizontalRule(t *testing.T) {
input := `<p>Before</p><hr/><p>After</p>` input := `<p>Before</p><hr/><p>After</p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "---") assert.Contains(t, result, "---")
} }
func TestHTMLToMarkdown_Good_LinkWithoutHref_Good(t *testing.T) { func TestHTMLToMarkdown_Good_LinkWithoutHref(t *testing.T) {
input := `<a>bare link text</a>` input := `<a>bare link text</a>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -50,7 +48,7 @@ func TestHTMLToMarkdown_Good_LinkWithoutHref_Good(t *testing.T) {
assert.NotContains(t, result, "[") assert.NotContains(t, result, "[")
} }
func TestHTMLToMarkdown_Good_H4H5H6_Good(t *testing.T) { func TestHTMLToMarkdown_Good_H4H5H6(t *testing.T) {
input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>` input := `<h4>H4</h4><h5>H5</h5><h6>H6</h6>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -59,7 +57,7 @@ func TestHTMLToMarkdown_Good_H4H5H6_Good(t *testing.T) {
assert.Contains(t, result, "###### H6") assert.Contains(t, result, "###### H6")
} }
func TestHTMLToMarkdown_Good_StripsStyle_Good(t *testing.T) { func TestHTMLToMarkdown_Good_StripsStyle(t *testing.T) {
input := `<html><head><style>.foo{color:red}</style></head><body><p>Clean</p></body></html>` input := `<html><head><style>.foo{color:red}</style></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -67,7 +65,7 @@ func TestHTMLToMarkdown_Good_StripsStyle_Good(t *testing.T) {
assert.NotContains(t, result, "color") assert.NotContains(t, result, "color")
} }
func TestHTMLToMarkdown_Good_LineBreak_Good(t *testing.T) { func TestHTMLToMarkdown_Good_LineBreak(t *testing.T) {
input := `<p>Line one<br/>Line two</p>` input := `<p>Line one<br/>Line two</p>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -75,7 +73,7 @@ func TestHTMLToMarkdown_Good_LineBreak_Good(t *testing.T) {
assert.Contains(t, result, "Line two") assert.Contains(t, result, "Line two")
} }
func TestHTMLToMarkdown_Good_NestedBoldItalic_Good(t *testing.T) { func TestHTMLToMarkdown_Good_NestedBoldItalic(t *testing.T) {
input := `<b>bold text</b> and <i>italic text</i>` input := `<b>bold text</b> and <i>italic text</i>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -83,7 +81,7 @@ func TestHTMLToMarkdown_Good_NestedBoldItalic_Good(t *testing.T) {
assert.Contains(t, result, "*italic text*") assert.Contains(t, result, "*italic text*")
} }
func TestJSONToMarkdown_Good_NestedObject_Good(t *testing.T) { func TestJSONToMarkdown_Good_NestedObject(t *testing.T) {
input := `{"outer": {"inner_key": "inner_value"}}` input := `{"outer": {"inner_key": "inner_value"}}`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -91,7 +89,7 @@ func TestJSONToMarkdown_Good_NestedObject_Good(t *testing.T) {
assert.Contains(t, result, "**inner_key:** inner_value") assert.Contains(t, result, "**inner_key:** inner_value")
} }
func TestJSONToMarkdown_Good_NestedArray_Good(t *testing.T) { func TestJSONToMarkdown_Good_NestedArray(t *testing.T) {
input := `[["a", "b"], ["c"]]` input := `[["a", "b"], ["c"]]`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -100,14 +98,14 @@ func TestJSONToMarkdown_Good_NestedArray_Good(t *testing.T) {
assert.Contains(t, result, "b") assert.Contains(t, result, "b")
} }
func TestJSONToMarkdown_Good_ScalarValue_Good(t *testing.T) { func TestJSONToMarkdown_Good_ScalarValue(t *testing.T) {
input := `42` input := `42`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
assert.Contains(t, result, "42") assert.Contains(t, result, "42")
} }
func TestJSONToMarkdown_Good_ArrayOfObjects_Good(t *testing.T) { func TestJSONToMarkdown_Good_ArrayOfObjects(t *testing.T) {
input := `[{"name": "Alice"}, {"name": "Bob"}]` input := `[{"name": "Alice"}, {"name": "Bob"}]`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
require.NoError(t, err) require.NoError(t, err)
@ -117,7 +115,7 @@ func TestJSONToMarkdown_Good_ArrayOfObjects_Good(t *testing.T) {
assert.Contains(t, result, "Bob") assert.Contains(t, result, "Bob")
} }
func TestProcessor_Process_Good_CancelledContext_Good(t *testing.T) { func TestProcessor_Process_Good_CancelledContext(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/file.html"] = `<h1>Test</h1>` m.Files["/input/file.html"] = `<h1>Test</h1>`
@ -133,7 +131,7 @@ func TestProcessor_Process_Good_CancelledContext_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestProcessor_Process_Good_EmitsEvents_Good(t *testing.T) { func TestProcessor_Process_Good_EmitsEvents(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/a.html"] = `<h1>Title</h1>` m.Files["/input/a.html"] = `<h1>Title</h1>`
@ -157,7 +155,7 @@ func TestProcessor_Process_Good_EmitsEvents_Good(t *testing.T) {
assert.Equal(t, 1, completes) assert.Equal(t, 1, completes)
} }
func TestProcessor_Process_Good_BadHTML_Good(t *testing.T) { func TestProcessor_Process_Good_BadHTML(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
// html.Parse is very tolerant, so even bad HTML will parse. But we test // html.Parse is very tolerant, so even bad HTML will parse. But we test
@ -174,7 +172,7 @@ func TestProcessor_Process_Good_BadHTML_Good(t *testing.T) {
assert.Equal(t, 1, result.Items) assert.Equal(t, 1, result.Items)
} }
func TestProcessor_Process_Good_BadJSON_Good(t *testing.T) { func TestProcessor_Process_Good_BadJSON(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/bad.json"] = `not valid json` m.Files["/input/bad.json"] = `not valid json`

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -15,7 +13,7 @@ func TestProcessor_Name_Good(t *testing.T) {
assert.Equal(t, "process:github", p.Name()) assert.Equal(t, "process:github", p.Name())
} }
func TestProcessor_Process_Bad_NoDir_Good(t *testing.T) { func TestProcessor_Process_Bad_NoDir(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
@ -24,7 +22,7 @@ func TestProcessor_Process_Bad_NoDir_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestProcessor_Process_Good_DryRun_Good(t *testing.T) { func TestProcessor_Process_Good_DryRun(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
cfg := NewConfigWithMedium(m, "/output") cfg := NewConfigWithMedium(m, "/output")
cfg.DryRun = true cfg.DryRun = true
@ -36,7 +34,7 @@ func TestProcessor_Process_Good_DryRun_Good(t *testing.T) {
assert.Equal(t, 0, result.Items) assert.Equal(t, 0, result.Items)
} }
func TestProcessor_Process_Good_HTMLFiles_Good(t *testing.T) { func TestProcessor_Process_Good_HTMLFiles(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/page.html"] = `<html><body><h1>Hello</h1><p>World</p></body></html>` m.Files["/input/page.html"] = `<html><body><h1>Hello</h1><p>World</p></body></html>`
@ -57,7 +55,7 @@ func TestProcessor_Process_Good_HTMLFiles_Good(t *testing.T) {
assert.Contains(t, content, "World") assert.Contains(t, content, "World")
} }
func TestProcessor_Process_Good_JSONFiles_Good(t *testing.T) { func TestProcessor_Process_Good_JSONFiles(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/data.json"] = `{"name": "Bitcoin", "price": 42000}` m.Files["/input/data.json"] = `{"name": "Bitcoin", "price": 42000}`
@ -77,7 +75,7 @@ func TestProcessor_Process_Good_JSONFiles_Good(t *testing.T) {
assert.Contains(t, content, "Bitcoin") assert.Contains(t, content, "Bitcoin")
} }
func TestProcessor_Process_Good_MarkdownPassthrough_Good(t *testing.T) { func TestProcessor_Process_Good_MarkdownPassthrough(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/readme.md"] = "# Already Markdown\n\nThis is already formatted." m.Files["/input/readme.md"] = "# Already Markdown\n\nThis is already formatted."
@ -96,7 +94,7 @@ func TestProcessor_Process_Good_MarkdownPassthrough_Good(t *testing.T) {
assert.Contains(t, content, "# Already Markdown") assert.Contains(t, content, "# Already Markdown")
} }
func TestProcessor_Process_Good_SkipUnknownTypes_Good(t *testing.T) { func TestProcessor_Process_Good_SkipUnknownTypes(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Dirs["/input"] = true m.Dirs["/input"] = true
m.Files["/input/image.png"] = "binary data" m.Files["/input/image.png"] = "binary data"
@ -172,7 +170,7 @@ func TestHTMLToMarkdown_Good(t *testing.T) {
} }
} }
func TestHTMLToMarkdown_Good_StripsScripts_Good(t *testing.T) { func TestHTMLToMarkdown_Good_StripsScripts(t *testing.T) {
input := `<html><head><script>alert('xss')</script></head><body><p>Clean</p></body></html>` input := `<html><head><script>alert('xss')</script></head><body><p>Clean</p></body></html>`
result, err := HTMLToMarkdown(input) result, err := HTMLToMarkdown(input)
assert.NoError(t, err) assert.NoError(t, err)
@ -190,14 +188,14 @@ func TestJSONToMarkdown_Good(t *testing.T) {
assert.Contains(t, result, "42") assert.Contains(t, result, "42")
} }
func TestJSONToMarkdown_Good_Array_Good(t *testing.T) { func TestJSONToMarkdown_Good_Array(t *testing.T) {
input := `[{"id": 1}, {"id": 2}]` input := `[{"id": 1}, {"id": 2}]`
result, err := JSONToMarkdown(input) result, err := JSONToMarkdown(input)
assert.NoError(t, err) assert.NoError(t, err)
assert.Contains(t, result, "# Data") assert.Contains(t, result, "# Data")
} }
func TestJSONToMarkdown_Bad_InvalidJSON_Good(t *testing.T) { func TestJSONToMarkdown_Bad_InvalidJSON(t *testing.T) {
_, err := JSONToMarkdown("not json") _, err := JSONToMarkdown("not json")
assert.Error(t, err) assert.Error(t, err)
} }

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
"context" "context"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "fmt"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"maps" "maps"
"os/exec"
"strconv" "strconv"
"strings"
"sync" "sync"
"time" "time"
@ -32,7 +30,6 @@ var defaultDelays = map[string]time.Duration{
} }
// NewRateLimiter creates a limiter with default delays. // NewRateLimiter creates a limiter with default delays.
// Usage: NewRateLimiter(...)
func NewRateLimiter() *RateLimiter { func NewRateLimiter() *RateLimiter {
delays := make(map[string]time.Duration, len(defaultDelays)) delays := make(map[string]time.Duration, len(defaultDelays))
maps.Copy(delays, defaultDelays) maps.Copy(delays, defaultDelays)
@ -44,7 +41,6 @@ func NewRateLimiter() *RateLimiter {
// Wait blocks until the rate limit allows the next request for the given source. // Wait blocks until the rate limit allows the next request for the given source.
// It respects context cancellation. // It respects context cancellation.
// Usage: Wait(...)
func (r *RateLimiter) Wait(ctx context.Context, source string) error { func (r *RateLimiter) Wait(ctx context.Context, source string) error {
r.mu.Lock() r.mu.Lock()
delay, ok := r.delays[source] delay, ok := r.delays[source]
@ -79,7 +75,6 @@ func (r *RateLimiter) Wait(ctx context.Context, source string) error {
} }
// SetDelay sets the delay for a source. // SetDelay sets the delay for a source.
// Usage: SetDelay(...)
func (r *RateLimiter) SetDelay(source string, d time.Duration) { func (r *RateLimiter) SetDelay(source string, d time.Duration) {
r.mu.Lock() r.mu.Lock()
defer r.mu.Unlock() defer r.mu.Unlock()
@ -87,7 +82,6 @@ func (r *RateLimiter) SetDelay(source string, d time.Duration) {
} }
// GetDelay returns the delay configured for a source. // GetDelay returns the delay configured for a source.
// Usage: GetDelay(...)
func (r *RateLimiter) GetDelay(source string) time.Duration { func (r *RateLimiter) GetDelay(source string) time.Duration {
r.mu.Lock() r.mu.Lock()
defer r.mu.Unlock() defer r.mu.Unlock()
@ -101,7 +95,6 @@ func (r *RateLimiter) GetDelay(source string) time.Duration {
// Returns used and limit counts. Auto-pauses at 75% usage by increasing // Returns used and limit counts. Auto-pauses at 75% usage by increasing
// the GitHub rate limit delay. // the GitHub rate limit delay.
// Deprecated: Use CheckGitHubRateLimitCtx for context-aware cancellation. // Deprecated: Use CheckGitHubRateLimitCtx for context-aware cancellation.
// Usage: CheckGitHubRateLimit(...)
func (r *RateLimiter) CheckGitHubRateLimit() (used, limit int, err error) { func (r *RateLimiter) CheckGitHubRateLimit() (used, limit int, err error) {
return r.CheckGitHubRateLimitCtx(context.Background()) return r.CheckGitHubRateLimitCtx(context.Background())
} }
@ -109,7 +102,6 @@ func (r *RateLimiter) CheckGitHubRateLimit() (used, limit int, err error) {
// CheckGitHubRateLimitCtx checks GitHub API rate limit status via gh api with context support. // CheckGitHubRateLimitCtx checks GitHub API rate limit status via gh api with context support.
// Returns used and limit counts. Auto-pauses at 75% usage by increasing // Returns used and limit counts. Auto-pauses at 75% usage by increasing
// the GitHub rate limit delay. // the GitHub rate limit delay.
// Usage: CheckGitHubRateLimitCtx(...)
func (r *RateLimiter) CheckGitHubRateLimitCtx(ctx context.Context) (used, limit int, err error) { func (r *RateLimiter) CheckGitHubRateLimitCtx(ctx context.Context) (used, limit int, err error) {
cmd := exec.CommandContext(ctx, "gh", "api", "rate_limit", "--jq", ".rate | \"\\(.used) \\(.limit)\"") cmd := exec.CommandContext(ctx, "gh", "api", "rate_limit", "--jq", ".rate | \"\\(.used) \\(.limit)\"")
out, err := cmd.Output() out, err := cmd.Output()

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -29,7 +27,7 @@ func TestRateLimiter_Wait_Good(t *testing.T) {
assert.GreaterOrEqual(t, time.Since(start), 40*time.Millisecond) // allow small timing variance assert.GreaterOrEqual(t, time.Since(start), 40*time.Millisecond) // allow small timing variance
} }
func TestRateLimiter_Wait_Bad_ContextCancelled_Good(t *testing.T) { func TestRateLimiter_Wait_Bad_ContextCancelled(t *testing.T) {
rl := NewRateLimiter() rl := NewRateLimiter()
rl.SetDelay("test", 5*time.Second) rl.SetDelay("test", 5*time.Second)
@ -53,7 +51,7 @@ func TestRateLimiter_SetDelay_Good(t *testing.T) {
assert.Equal(t, 3*time.Second, rl.GetDelay("custom")) assert.Equal(t, 3*time.Second, rl.GetDelay("custom"))
} }
func TestRateLimiter_GetDelay_Good_Defaults_Good(t *testing.T) { func TestRateLimiter_GetDelay_Good_Defaults(t *testing.T) {
rl := NewRateLimiter() rl := NewRateLimiter()
assert.Equal(t, 500*time.Millisecond, rl.GetDelay("github")) assert.Equal(t, 500*time.Millisecond, rl.GetDelay("github"))
@ -62,13 +60,13 @@ func TestRateLimiter_GetDelay_Good_Defaults_Good(t *testing.T) {
assert.Equal(t, 1*time.Second, rl.GetDelay("iacr")) assert.Equal(t, 1*time.Second, rl.GetDelay("iacr"))
} }
func TestRateLimiter_GetDelay_Good_UnknownSource_Good(t *testing.T) { func TestRateLimiter_GetDelay_Good_UnknownSource(t *testing.T) {
rl := NewRateLimiter() rl := NewRateLimiter()
// Unknown sources should get the default 500ms delay // Unknown sources should get the default 500ms delay
assert.Equal(t, 500*time.Millisecond, rl.GetDelay("unknown")) assert.Equal(t, 500*time.Millisecond, rl.GetDelay("unknown"))
} }
func TestRateLimiter_Wait_Good_UnknownSource_Good(t *testing.T) { func TestRateLimiter_Wait_Good_UnknownSource(t *testing.T) {
rl := NewRateLimiter() rl := NewRateLimiter()
ctx := context.Background() ctx := context.Background()

View file

@ -1,14 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
"sync" "sync"
"time" "time"
"dappco.re/go/core/io"
core "dappco.re/go/core/log" core "dappco.re/go/core/log"
"dappco.re/go/core/io"
) )
// State tracks collection progress for incremental runs. // State tracks collection progress for incremental runs.
@ -41,7 +39,6 @@ type StateEntry struct {
// NewState creates a state tracker that persists to the given path // NewState creates a state tracker that persists to the given path
// using the provided storage medium. // using the provided storage medium.
// Usage: NewState(...)
func NewState(m io.Medium, path string) *State { func NewState(m io.Medium, path string) *State {
return &State{ return &State{
medium: m, medium: m,
@ -52,7 +49,6 @@ func NewState(m io.Medium, path string) *State {
// Load reads state from disk. If the file does not exist, the state // Load reads state from disk. If the file does not exist, the state
// is initialised as empty without error. // is initialised as empty without error.
// Usage: Load(...)
func (s *State) Load() error { func (s *State) Load() error {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
@ -79,7 +75,6 @@ func (s *State) Load() error {
} }
// Save writes state to disk. // Save writes state to disk.
// Usage: Save(...)
func (s *State) Save() error { func (s *State) Save() error {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
@ -98,7 +93,6 @@ func (s *State) Save() error {
// Get returns a copy of the state for a source. The second return value // Get returns a copy of the state for a source. The second return value
// indicates whether the entry was found. // indicates whether the entry was found.
// Usage: Get(...)
func (s *State) Get(source string) (*StateEntry, bool) { func (s *State) Get(source string) (*StateEntry, bool) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()
@ -112,7 +106,6 @@ func (s *State) Get(source string) (*StateEntry, bool) {
} }
// Set updates state for a source. // Set updates state for a source.
// Usage: Set(...)
func (s *State) Set(source string, entry *StateEntry) { func (s *State) Set(source string, entry *StateEntry) {
s.mu.Lock() s.mu.Lock()
defer s.mu.Unlock() defer s.mu.Unlock()

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -10,7 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestState_Get_Good_ReturnsCopy_Good(t *testing.T) { func TestState_Get_Good_ReturnsCopy(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/state.json") s := NewState(m, "/state.json")
@ -26,7 +24,7 @@ func TestState_Get_Good_ReturnsCopy_Good(t *testing.T) {
assert.Equal(t, 5, again.Items, "internal state should not be mutated") assert.Equal(t, 5, again.Items, "internal state should not be mutated")
} }
func TestState_Save_Good_WritesJSON_Good(t *testing.T) { func TestState_Save_Good_WritesJSON(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/data/state.json") s := NewState(m, "/data/state.json")
@ -42,7 +40,7 @@ func TestState_Save_Good_WritesJSON_Good(t *testing.T) {
assert.Contains(t, content, `"abc"`) assert.Contains(t, content, `"abc"`)
} }
func TestState_Load_Good_NullJSON_Good(t *testing.T) { func TestState_Load_Good_NullJSON(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Files["/state.json"] = "null" m.Files["/state.json"] = "null"
@ -55,7 +53,7 @@ func TestState_Load_Good_NullJSON_Good(t *testing.T) {
assert.False(t, ok) assert.False(t, ok)
} }
func TestState_SaveLoad_Good_WithCursor_Good(t *testing.T) { func TestState_SaveLoad_Good_WithCursor(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/state.json") s := NewState(m, "/state.json")

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package collect package collect
import ( import (
@ -75,7 +73,7 @@ func TestState_SaveLoad_Good(t *testing.T) {
assert.True(t, now.Equal(got.LastRun)) assert.True(t, now.Equal(got.LastRun))
} }
func TestState_Load_Good_NoFile_Good(t *testing.T) { func TestState_Load_Good_NoFile(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/nonexistent.json") s := NewState(m, "/nonexistent.json")
@ -88,7 +86,7 @@ func TestState_Load_Good_NoFile_Good(t *testing.T) {
assert.False(t, ok) assert.False(t, ok)
} }
func TestState_Load_Bad_InvalidJSON_Good(t *testing.T) { func TestState_Load_Bad_InvalidJSON(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
m.Files["/state.json"] = "not valid json" m.Files["/state.json"] = "not valid json"
@ -97,7 +95,7 @@ func TestState_Load_Bad_InvalidJSON_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestState_SaveLoad_Good_MultipleEntries_Good(t *testing.T) { func TestState_SaveLoad_Good_MultipleEntries(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/state.json") s := NewState(m, "/state.json")
@ -125,7 +123,7 @@ func TestState_SaveLoad_Good_MultipleEntries_Good(t *testing.T) {
assert.Equal(t, 30, c.Items) assert.Equal(t, 30, c.Items)
} }
func TestState_Set_Good_Overwrite_Good(t *testing.T) { func TestState_Set_Good_Overwrite(t *testing.T) {
m := io.NewMockMedium() m := io.NewMockMedium()
s := NewState(m, "/state.json") s := NewState(m, "/state.json")

View file

@ -88,9 +88,9 @@ The `gitea/` package mirrors this using `GITEA_URL`/`GITEA_TOKEN` and `gitea.*`
|------|-----------| |------|-----------|
| `client.go` | `New`, `NewFromConfig`, `GetCurrentUser`, `ForkRepo`, `CreatePullRequest` | | `client.go` | `New`, `NewFromConfig`, `GetCurrentUser`, `ForkRepo`, `CreatePullRequest` |
| `repos.go` | `ListOrgRepos`, `ListOrgReposIter`, `ListUserRepos`, `ListUserReposIter`, `GetRepo`, `CreateOrgRepo`, `DeleteRepo`, `MigrateRepo` | | `repos.go` | `ListOrgRepos`, `ListOrgReposIter`, `ListUserRepos`, `ListUserReposIter`, `GetRepo`, `CreateOrgRepo`, `DeleteRepo`, `MigrateRepo` |
| `issues.go` | `ListIssues`, `ListIssuesIter`, `GetIssue`, `CreateIssue`, `EditIssue`, `AssignIssue`, `ListPullRequests`, `ListPullRequestsIter`, `GetPullRequest`, `CreateIssueComment`, `GetIssueLabels`, `ListIssueComments`, `ListIssueCommentsIter`, `CloseIssue` | | `issues.go` | `ListIssues`, `GetIssue`, `CreateIssue`, `EditIssue`, `AssignIssue`, `ListPullRequests`, `ListPullRequestsIter`, `GetPullRequest`, `CreateIssueComment`, `ListIssueComments`, `CloseIssue` |
| `labels.go` | `ListOrgLabels`, `ListOrgLabelsIter`, `ListRepoLabels`, `ListRepoLabelsIter`, `CreateRepoLabel`, `GetLabelByName`, `EnsureLabel`, `AddIssueLabels`, `RemoveIssueLabel` | | `labels.go` | `ListOrgLabels`, `ListRepoLabels`, `CreateRepoLabel`, `GetLabelByName`, `EnsureLabel`, `AddIssueLabels`, `RemoveIssueLabel` |
| `prs.go` | `MergePullRequest`, `SetPRDraft`, `ListPRReviews`, `GetCombinedStatus`, `DismissReview`, `UndismissReview` | | `prs.go` | `MergePullRequest`, `SetPRDraft`, `ListPRReviews`, `GetCombinedStatus`, `DismissReview` |
| `webhooks.go` | `CreateRepoWebhook`, `ListRepoWebhooks` | | `webhooks.go` | `CreateRepoWebhook`, `ListRepoWebhooks` |
| `orgs.go` | `ListMyOrgs`, `GetOrg`, `CreateOrg` | | `orgs.go` | `ListMyOrgs`, `GetOrg`, `CreateOrg` |
| `meta.go` | `GetPRMeta`, `GetCommentBodies`, `GetIssueBody` | | `meta.go` | `GetPRMeta`, `GetCommentBodies`, `GetIssueBody` |
@ -119,7 +119,7 @@ The two packages are structurally parallel but intentionally not unified behind
- PR merge, draft status, reviews, combined status, review dismissal - PR merge, draft status, reviews, combined status, review dismissal
- Repository migration (full import with issues/labels/PRs) - Repository migration (full import with issues/labels/PRs)
The Gitea client has a `GetCurrentUser` helper and a `CreateMirror` method for setting up pull mirrors from GitHub -- a capability specific to the public mirror workflow. The Gitea client has a `CreateMirror` method for setting up pull mirrors from GitHub -- a capability specific to the public mirror workflow.
**SDK limitation:** The Forgejo SDK v2 does not accept `context.Context` on API methods. All SDK calls are synchronous. Context propagation through the wrapper layer is nominal -- contexts are accepted at the boundary but cannot be forwarded. **SDK limitation:** The Forgejo SDK v2 does not accept `context.Context` on API methods. All SDK calls are synchronous. Context propagation through the wrapper layer is nominal -- contexts are accepted at the boundary but cannot be forwarded.
@ -350,7 +350,7 @@ agentci:
3. If the repository name is `core` or contains `security`, dual (Axiom 1: critical repos always verified). 3. If the repository name is `core` or contains `security`, dual (Axiom 1: critical repos always verified).
4. Otherwise, standard. 4. Otherwise, standard.
In dual-run mode, `DispatchHandler` populates `DispatchTicket.VerifyModel` and `DispatchTicket.DualRun=true`. The `Weave` method compares primary and verifier outputs for convergence using a deterministic token-overlap score against `validation_threshold`; richer semantic diffing remains a future phase. In dual-run mode, `DispatchHandler` populates `DispatchTicket.VerifyModel` and `DispatchTicket.DualRun=true`. The `Weave` method compares primary and verifier outputs for convergence (currently byte-equal; semantic diff reserved for a future phase).
### Dispatch Ticket Transfer ### Dispatch Ticket Transfer

View file

@ -0,0 +1,217 @@
# Convention Drift Check — 2026-03-23
`CODEX.md` is not present under `/workspace`, so this pass uses [CLAUDE.md](/workspace/CLAUDE.md) and [docs/development.md](/workspace/docs/development.md) as the convention baseline.
Scope used for this pass:
- Import ordering was checked on non-test Go files for `stdlib -> core/internal -> third-party`, including blank-line separation between groups.
- UK English findings are limited to repo-authored prose/comments/schema terms, not external API field names, CSS properties, or shield/vendor text beyond repo-owned alt text.
- Missing-test findings come from `go test -coverprofile=/tmp/convention_drift_cover.out ./...` plus `go tool cover -func`.
- SPDX findings are limited to non-test `.go` and `.ts` source files.
## Import Grouping Drift (36)
### Core/internal imports placed after third-party imports (24)
- `cmd/forge/cmd_issues.go:9`: internal/core import follows a third-party SDK import.
- `cmd/forge/cmd_labels.go:8`: internal/core import follows a third-party SDK import.
- `cmd/forge/cmd_migrate.go:8`: internal/core import follows a third-party SDK import.
- `cmd/forge/cmd_prs.go:9`: internal/core import follows a third-party SDK import.
- `cmd/forge/cmd_repos.go:8`: internal/core import follows a third-party SDK import.
- `cmd/forge/cmd_sync.go:12`: internal/core import follows a third-party SDK import.
- `cmd/gitea/cmd_issues.go:9`: internal/core import follows a third-party SDK import.
- `cmd/gitea/cmd_prs.go:9`: internal/core import follows a third-party SDK import.
- `cmd/gitea/cmd_sync.go:12`: internal/core import follows a third-party SDK import.
- `forge/client.go:14`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/issues.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/labels.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/meta.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/orgs.go:6`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/prs.go:11`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/repos.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `forge/webhooks.go:6`: `dappco.re/go/core/log` follows a third-party SDK import.
- `gitea/client.go:14`: `dappco.re/go/core/log` follows a third-party SDK import.
- `gitea/issues.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `gitea/meta.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `gitea/repos.go:8`: `dappco.re/go/core/log` follows a third-party SDK import.
- `jobrunner/forgejo/signals.go:9`: internal jobrunner import follows a third-party SDK import.
- `jobrunner/handlers/resolve_threads.go:10`: `dappco.re/go/core/log` follows a third-party SDK import.
- `jobrunner/handlers/tick_parent.go:11`: `dappco.re/go/core/log` follows a third-party SDK import.
### Missing blank line between import groups (12)
- `collect/bitcointalk.go:13`: no blank line between internal/core and third-party imports.
- `collect/papers.go:14`: no blank line between internal/core and third-party imports.
- `collect/process.go:13`: no blank line between internal/core and third-party imports.
- `manifest/loader.go:9`: no blank line before the third-party YAML import.
- `manifest/manifest.go:5`: no blank line before the third-party YAML import.
- `manifest/sign.go:8`: no blank line before the third-party YAML import.
- `marketplace/discovery.go:11`: no blank line before the third-party YAML import.
- `pkg/api/provider.go:21`: no blank line before the third-party Gin import.
- `repos/gitstate.go:9`: no blank line before the third-party YAML import.
- `repos/kbconfig.go:9`: no blank line before the third-party YAML import.
- `repos/registry.go:13`: no blank line before the third-party YAML import.
- `repos/workconfig.go:9`: no blank line before the third-party YAML import.
## UK English Drift (7)
- `README.md:2`: badge alt text and shield label use `License` rather than `Licence`.
- `CONTRIBUTING.md:34`: section heading uses `License` rather than `Licence`.
- `jobrunner/journal.go:76`: comment uses `normalize` rather than `normalise`.
- `marketplace/marketplace.go:19`: comment uses `catalog` rather than `catalogue`.
- `repos/registry.go:29`: schema field and YAML tag use `License`/`license` rather than `Licence`/`licence`.
- `repos/registry_test.go:48`: fixture uses `license:` rather than `licence:`.
- `docs/architecture.md:508`: `repos.yaml` example uses `license:` rather than `licence:`.
## Missing Tests (50)
### Files or packages with 0% statement coverage, or no tests at all (36)
- `agentci/clotho.go:25`: whole file is at 0% statement coverage.
- `cmd/collect/cmd.go:12`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_bitcointalk.go:16`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_dispatch.go:13`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_excavate.go:19`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_github.go:20`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_market.go:18`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_papers.go:19`: whole file is at 0% statement coverage.
- `cmd/collect/cmd_process.go:12`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_auth.go:17`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_config.go:18`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_forge.go:19`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_issues.go:21`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_labels.go:20`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_migrate.go:21`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_orgs.go:11`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_prs.go:19`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_repos.go:19`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_status.go:11`: whole file is at 0% statement coverage.
- `cmd/forge/cmd_sync.go:25`: whole file is at 0% statement coverage.
- `cmd/forge/helpers.go:11`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_config.go:18`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_gitea.go:16`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_issues.go:21`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_mirror.go:19`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_prs.go:19`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_repos.go:17`: whole file is at 0% statement coverage.
- `cmd/gitea/cmd_sync.go:25`: whole file is at 0% statement coverage.
- `cmd/scm/cmd_compile.go:14`: whole file is at 0% statement coverage.
- `cmd/scm/cmd_export.go:11`: whole file is at 0% statement coverage.
- `cmd/scm/cmd_index.go:11`: whole file is at 0% statement coverage.
- `cmd/scm/cmd_scm.go:14`: whole file is at 0% statement coverage.
- `git/git.go:31`: whole file is at 0% statement coverage.
- `git/service.go:57`: whole file is at 0% statement coverage.
- `jobrunner/handlers/completion.go:23`: whole file is at 0% statement coverage.
- `locales/embed.go:1`: package has no test files.
### 0%-covered functions inside otherwise tested files (14)
- `collect/github.go:123`: `listOrgRepos` has 0% coverage.
- `collect/ratelimit.go:98`: `CheckGitHubRateLimit` has 0% coverage.
- `collect/ratelimit.go:105`: `CheckGitHubRateLimitCtx` has 0% coverage.
- `forge/config.go:73`: `SaveConfig` has 0% coverage.
- `forge/issues.go:129`: `ListPullRequestsIter` has 0% coverage.
- `forge/repos.go:36`: `ListOrgReposIter` has 0% coverage.
- `forge/repos.go:85`: `ListUserReposIter` has 0% coverage.
- `gitea/issues.go:104`: `ListPullRequestsIter` has 0% coverage.
- `gitea/repos.go:36`: `ListOrgReposIter` has 0% coverage.
- `gitea/repos.go:85`: `ListUserReposIter` has 0% coverage.
- `jobrunner/handlers/dispatch.go:272`: `runRemote` has 0% coverage.
- `pkg/api/provider.go:442`: `emitEvent` has 0% coverage.
- `plugin/installer.go:152`: `cloneRepo` has 0% coverage.
- `repos/registry.go:105`: `FindRegistry` has 0% coverage.
## Missing SPDX Headers In Non-Test Go/TS Sources (92)
- `agentci/clotho.go:1`
- `agentci/config.go:1`
- `agentci/security.go:1`
- `cmd/collect/cmd.go:1`
- `cmd/collect/cmd_bitcointalk.go:1`
- `cmd/collect/cmd_dispatch.go:1`
- `cmd/collect/cmd_excavate.go:1`
- `cmd/collect/cmd_github.go:1`
- `cmd/collect/cmd_market.go:1`
- `cmd/collect/cmd_papers.go:1`
- `cmd/collect/cmd_process.go:1`
- `cmd/forge/cmd_auth.go:1`
- `cmd/forge/cmd_config.go:1`
- `cmd/forge/cmd_forge.go:1`
- `cmd/forge/cmd_issues.go:1`
- `cmd/forge/cmd_labels.go:1`
- `cmd/forge/cmd_migrate.go:1`
- `cmd/forge/cmd_orgs.go:1`
- `cmd/forge/cmd_prs.go:1`
- `cmd/forge/cmd_repos.go:1`
- `cmd/forge/cmd_status.go:1`
- `cmd/forge/cmd_sync.go:1`
- `cmd/forge/helpers.go:1`
- `cmd/gitea/cmd_config.go:1`
- `cmd/gitea/cmd_gitea.go:1`
- `cmd/gitea/cmd_issues.go:1`
- `cmd/gitea/cmd_mirror.go:1`
- `cmd/gitea/cmd_prs.go:1`
- `cmd/gitea/cmd_repos.go:1`
- `cmd/gitea/cmd_sync.go:1`
- `cmd/scm/cmd_compile.go:1`
- `cmd/scm/cmd_export.go:1`
- `cmd/scm/cmd_index.go:1`
- `cmd/scm/cmd_scm.go:1`
- `collect/bitcointalk.go:1`
- `collect/collect.go:1`
- `collect/events.go:1`
- `collect/excavate.go:1`
- `collect/github.go:1`
- `collect/market.go:1`
- `collect/papers.go:1`
- `collect/process.go:1`
- `collect/ratelimit.go:1`
- `collect/state.go:1`
- `forge/client.go:1`
- `forge/config.go:1`
- `forge/issues.go:1`
- `forge/labels.go:1`
- `forge/meta.go:1`
- `forge/orgs.go:1`
- `forge/prs.go:1`
- `forge/repos.go:1`
- `forge/webhooks.go:1`
- `git/git.go:1`
- `git/service.go:1`
- `gitea/client.go:1`
- `gitea/config.go:1`
- `gitea/issues.go:1`
- `gitea/meta.go:1`
- `gitea/repos.go:1`
- `jobrunner/forgejo/signals.go:1`
- `jobrunner/forgejo/source.go:1`
- `jobrunner/handlers/completion.go:1`
- `jobrunner/handlers/dispatch.go:1`
- `jobrunner/handlers/enable_auto_merge.go:1`
- `jobrunner/handlers/publish_draft.go:1`
- `jobrunner/handlers/resolve_threads.go:1`
- `jobrunner/handlers/send_fix_command.go:1`
- `jobrunner/handlers/tick_parent.go:1`
- `jobrunner/journal.go:1`
- `jobrunner/poller.go:1`
- `jobrunner/types.go:1`
- `locales/embed.go:1`
- `manifest/compile.go:1`
- `manifest/loader.go:1`
- `manifest/manifest.go:1`
- `manifest/sign.go:1`
- `marketplace/builder.go:1`
- `marketplace/discovery.go:1`
- `marketplace/installer.go:1`
- `marketplace/marketplace.go:1`
- `plugin/config.go:1`
- `plugin/installer.go:1`
- `plugin/loader.go:1`
- `plugin/manifest.go:1`
- `plugin/plugin.go:1`
- `plugin/registry.go:1`
- `repos/gitstate.go:1`
- `repos/kbconfig.go:1`
- `repos/registry.go:1`
- `repos/workconfig.go:1`
- `ui/vite.config.ts:1`

View file

@ -122,17 +122,17 @@ Full signal-to-result flow tested for all five handlers via a mock Forgejo serve
The Forgejo SDK v2 and Gitea SDK do not accept `context.Context`. All Forgejo/Gitea API calls are blocking with no cancellation path. When the SDK is updated to support context (v3 or later), a follow-up task should thread `ctx` through all forge/ and gitea/ wrapper signatures. The Forgejo SDK v2 and Gitea SDK do not accept `context.Context`. All Forgejo/Gitea API calls are blocking with no cancellation path. When the SDK is updated to support context (v3 or later), a follow-up task should thread `ctx` through all forge/ and gitea/ wrapper signatures.
**Clotho Weave — thresholded token overlap** **Clotho Weave — byte-equal only**
`Spinner.Weave(ctx, primary, signed)` now uses the configured `validation_threshold` to decide convergence from a deterministic token-overlap score. This is still a lightweight approximation rather than full semantic diffing, but it now honours the config knob already exposed by `ClothoConfig`. `Spinner.Weave(ctx, primary, signed)` currently returns `string(primaryOutput) == string(signedOutput)`. This is a placeholder. Meaningful dual-run verification requires semantic diff logic (e.g., normalised AST comparison, embedding cosine similarity, or LLM-assisted diffing). The interface signature is stable; the implementation is not production-ready for divergent outputs.
**collect/ HTTP collectors — no retry** **collect/ HTTP collectors — no retry**
None of the HTTP-dependent collectors (`bitcointalk.go`, `github.go`, `market.go`, `papers.go`) implement retry on transient failures. A single HTTP error causes the collector to return an error and increment the `Errors` count in the result. The `Excavator` continues to the next collector. For long-running collection runs, transient network errors cause silent data gaps. None of the HTTP-dependent collectors (`bitcointalk.go`, `github.go`, `market.go`, `papers.go`) implement retry on transient failures. A single HTTP error causes the collector to return an error and increment the `Errors` count in the result. The `Excavator` continues to the next collector. For long-running collection runs, transient network errors cause silent data gaps.
**Journal replay** **Journal replay — no public API**
The journal now exposes `Journal.Query(...)` for replay and filtering over the JSONL archive. It supports repo, action, and time-range filters while preserving the date-partitioned storage layout used by `Append(...)`. The journal can be replayed by scanning the JSONL files directly, but there is no exported `Query` or `Filter` function. Replay filtering patterns exist only in tests. A future phase should export a query interface.
**git.Service framework integration** **git.Service framework integration**

View file

@ -1,141 +0,0 @@
# Verification Pass 2026-03-27
- Repository note: `CODEX.md` was not present under `/workspace`; conventions were taken from `CLAUDE.md`.
- Commands run: `go build ./...`, `go vet ./...`, `go test ./...`
- Command status: all passed
## Banned imports
ZERO FINDINGS for banned imports: `os`, `os/exec`, `encoding/json`, `fmt`, `errors`, `strings`, `path/filepath`.
## Test names not matching `TestFile_Function_{Good,Bad,Ugly}`
597 findings across 67 files:
```text
agentci/config_test.go:23,46,67,85,102,120,127,140,162,177,185,205,223,237,257,270,277,295,302
agentci/security_test.go:12,34,61,79,92,109
cmd/forge/cmd_sync_test.go:11,21,29,39,47
cmd/gitea/cmd_sync_test.go:11,21,29,39,47
collect/bitcointalk_http_test.go:36,76,104,128,152,183,192,210,225
collect/bitcointalk_test.go:16,21,30,42,73,79,87
collect/collect_test.go:10,23,35,57,63
collect/coverage_boost_test.go:18,34,50,62,79,92,109,120,125,130,141,151,176,198,230,264,275,283,291,298,305,313,322,330,335,343,351,358,368,384,400,419,434,452,457,481,497,508,517,533,557,572,592,611,625,639
collect/coverage_phase2_test.go:87,99,115,134,149,167,182,202,216,232,257,281,305,329,353,388,416,444,480,503,530,565,578,594,619,633,646,665,692,718,738,751,764,778,797,807,817,826,835,846,856,866,876,886,896,928,945,962,1022,1059,1098,1110,1132,1157,1181,1193,1230,1246,1261,1300
collect/events_test.go:44,57,72,88,129
collect/excavate_extra_test.go:13,42,68,86,104
collect/excavate_test.go:67,78,99,124,147,164,185
collect/github_test.go:17,22,41,53,65,91
collect/market_extra_test.go:15,59,101,140,175,200,232
collect/market_test.go:19,28,40,95,145,167
collect/papers_http_test.go:112,168,188,208,229,249,281,298,306
collect/papers_test.go:16,21,26,35,44,56,75,83
collect/process_extra_test.go:12,21,29,36,43,51,60,68,76,84,92,101,108,118,134,158,175
collect/process_test.go:16,25,37,58,78,97,114,173,182,191,198
collect/ratelimit_test.go:30,54,63,69,78
collect/state_extra_test.go:11,27,43,56
collect/state_test.go:76,89,98,126,138
forge/client_test.go:16,28,64,92,100,124,135,159,185,211,225,263,326,356,387,409,435,441
forge/config_test.go:19,28,39,50,61,71,77,85,100
forge/issues_test.go:22,44,55,73,94,116,135,154,176,194,211,230,247
forge/labels_test.go:27,45,63,72,81,91,111,127,144
forge/meta_test.go:26,48,66
forge/orgs_test.go:22,40,62
forge/prs_test.go:22,30,38,61,79,96,105,133,142
forge/repos_test.go:22,42,60,81,100,122
forge/webhooks_test.go:26,46
gitea/client_test.go:10,21
gitea/config_test.go:18,27,38,49,59,69,75,83,95
gitea/coverage_boost_test.go:15,26,35,44,90,124,176,220,239,264,294,306
gitea/issues_test.go:22,44,55,73,94,115,137,155,166
gitea/meta_test.go:26,48,66,77,103,115
gitea/repos_test.go:22,42,60,69,79,89,106,127
jobrunner/forgejo/source_test.go:38,109,155,162,166
jobrunner/handlers/dispatch_test.go:38,50,63,76,88,100,120,145,210,232,255,276,302,339
jobrunner/handlers/enable_auto_merge_test.go:29,42,84
jobrunner/handlers/publish_draft_test.go:26,36
jobrunner/handlers/resolve_threads_test.go:26
jobrunner/handlers/send_fix_command_test.go:16,25,37,49
jobrunner/handlers/tick_parent_test.go:26
jobrunner/journal_test.go:116,198,233,249
jobrunner/poller_test.go:121,152,193
jobrunner/types_test.go:28
manifest/compile_test.go:14,38,61,67,74,81,106,123,128,152,163,169
manifest/loader_test.go:12,28,34,51
manifest/manifest_test.go:10,49,67,127,135,146,157,205,210,215,220
manifest/sign_test.go:11,32,44
marketplace/builder_test.go:39,56,71,87,102,121,138,147,154,166,178,189,198,221
marketplace/discovery_test.go:25,59,84,105,122,130,136,147,195,234
marketplace/installer_test.go:78,104,125,142,166,189,212,223,243,270,281,309
marketplace/marketplace_test.go:10,26,38,50,61
pkg/api/provider_handlers_test.go:20,47,66,79,90,105,118,131,142,155,169,183
pkg/api/provider_security_test.go:12,18,23
pkg/api/provider_test.go:92,116,167,181,200,230
plugin/installer_test.go:14,26,36,47,60,81,95,105,120,132,140,148,156,162,168,174,180,186,192,198,204
plugin/loader_test.go:46,71,92,123,132
plugin/manifest_test.go:10,33,50,58,78,89,100
plugin/plugin_test.go:10,25,37
plugin/registry_test.go:28,69,78,129,138,149,160,173
repos/gitstate_test.go:14,50,61,110,132,158,178,189,199,204,210
repos/kbconfig_test.go:13,48,57,76,93
repos/registry_test.go:13,40,72,93,120,127,181,201,209,243,266,286,309,334,350,363,373,382,390,398,403,411,421,427,474,481
repos/workconfig_test.go:14,51,60,79,97,102
```
## Exported functions missing usage-example comments
199 findings across 51 files:
```text
agentci/clotho.go:39,61,71,90
collect/bitcointalk.go:37,46
collect/events.go:72,81,96,105,115,125,135
collect/excavate.go:27,33
collect/github.go:57,65
collect/market.go:33,67
collect/papers.go:41,57
collect/process.go:27,33
collect/ratelimit.go:46,80,87,100,107
collect/state.go:55,81,99,112
forge/client.go:37,40,43,46,55,69
forge/issues.go:21,56,66,76,86,97,130,164,174,185,209
forge/labels.go:15,31,55,65,81,94,105
forge/meta.go:43,100,139
forge/orgs.go:10,34,44
forge/prs.go:18,43,84,108,117
forge/repos.go:12,36,61,85,110,120,130,141
forge/webhooks.go:10,20
git/git.go:32,37,42,283,293
git/service.go:75,113,116,121,132,145,156
gitea/client.go:36,39
gitea/issues.go:20,52,62,72,105,139
gitea/meta.go:43,101,141
gitea/repos.go:12,36,61,85,110,122,145,155
internal/ax/stdio/stdio.go:12,24
jobrunner/forgejo/source.go:36,42,65
jobrunner/handlers/completion.go:33,38,43
jobrunner/handlers/dispatch.go:76,82,91
jobrunner/handlers/enable_auto_merge.go:25,31,40
jobrunner/handlers/publish_draft.go:25,30,37
jobrunner/handlers/resolve_threads.go:30,35,40
jobrunner/handlers/send_fix_command.go:26,32,46
jobrunner/handlers/tick_parent.go:30,35,41
jobrunner/journal.go:98
jobrunner/poller.go:51,58,65,72,79,88,110
jobrunner/types.go:37,42
manifest/manifest.go:46,79,95
marketplace/builder.go:35
marketplace/discovery.go:132,140,145,151,160
marketplace/installer.go:53,115,133,179
marketplace/marketplace.go:39,53,64
pkg/api/provider.go:61,64,67,75,86,108
plugin/installer.go:35,99,133
plugin/loader.go:28,53
plugin/manifest.go:41
plugin/plugin.go:44,47,50,53,56
plugin/registry.go:35,48,54,63,78,105
repos/gitstate.go:101,106,111,120,131,144,162
repos/kbconfig.go:116,121
repos/registry.go:255,265,271,283,325,330
repos/workconfig.go:104
```

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package forge provides a thin wrapper around the Forgejo Go SDK // Package forge provides a thin wrapper around the Forgejo Go SDK
// for managing repositories, issues, and pull requests on a Forgejo instance. // for managing repositories, issues, and pull requests on a Forgejo instance.
// //
@ -24,7 +22,6 @@ type Client struct {
} }
// New creates a new Forgejo API client for the given URL and token. // New creates a new Forgejo API client for the given URL and token.
// Usage: New(...)
func New(url, token string) (*Client, error) { func New(url, token string) (*Client, error) {
api, err := forgejo.NewClient(url, forgejo.SetToken(token)) api, err := forgejo.NewClient(url, forgejo.SetToken(token))
if err != nil { if err != nil {
@ -35,19 +32,15 @@ func New(url, token string) (*Client, error) {
} }
// API exposes the underlying SDK client for direct access. // API exposes the underlying SDK client for direct access.
// Usage: API(...)
func (c *Client) API() *forgejo.Client { return c.api } func (c *Client) API() *forgejo.Client { return c.api }
// URL returns the Forgejo instance URL. // URL returns the Forgejo instance URL.
// Usage: URL(...)
func (c *Client) URL() string { return c.url } func (c *Client) URL() string { return c.url }
// Token returns the Forgejo API token. // Token returns the Forgejo API token.
// Usage: Token(...)
func (c *Client) Token() string { return c.token } func (c *Client) Token() string { return c.token }
// GetCurrentUser returns the authenticated user's information. // GetCurrentUser returns the authenticated user's information.
// Usage: GetCurrentUser(...)
func (c *Client) GetCurrentUser() (*forgejo.User, error) { func (c *Client) GetCurrentUser() (*forgejo.User, error) {
user, _, err := c.api.GetMyUserInfo() user, _, err := c.api.GetMyUserInfo()
if err != nil { if err != nil {
@ -57,7 +50,6 @@ func (c *Client) GetCurrentUser() (*forgejo.User, error) {
} }
// ForkRepo forks a repository. If org is non-empty, forks into that organisation. // ForkRepo forks a repository. If org is non-empty, forks into that organisation.
// Usage: ForkRepo(...)
func (c *Client) ForkRepo(owner, repo string, org string) (*forgejo.Repository, error) { func (c *Client) ForkRepo(owner, repo string, org string) (*forgejo.Repository, error) {
opts := forgejo.CreateForkOption{} opts := forgejo.CreateForkOption{}
if org != "" { if org != "" {
@ -72,7 +64,6 @@ func (c *Client) ForkRepo(owner, repo string, org string) (*forgejo.Repository,
} }
// CreatePullRequest creates a pull request on the given repository. // CreatePullRequest creates a pull request on the given repository.
// Usage: CreatePullRequest(...)
func (c *Client) CreatePullRequest(owner, repo string, opts forgejo.CreatePullRequestOption) (*forgejo.PullRequest, error) { func (c *Client) CreatePullRequest(owner, repo string, opts forgejo.CreatePullRequestOption) (*forgejo.PullRequest, error) {
pr, _, err := c.api.CreatePullRequest(owner, repo, opts) pr, _, err := c.api.CreatePullRequest(owner, repo, opts)
if err != nil { if err != nil {

View file

@ -1,10 +1,8 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx" "fmt"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"testing" "testing"
@ -27,7 +25,7 @@ func TestNew_Good(t *testing.T) {
assert.Equal(t, "test-token-123", client.Token()) assert.Equal(t, "test-token-123", client.Token())
} }
func TestNew_Bad_InvalidURL_Good(t *testing.T) { func TestNew_Bad_InvalidURL(t *testing.T) {
// The Forgejo SDK may reject certain URL formats. // The Forgejo SDK may reject certain URL formats.
_, err := New("://invalid-url", "token") _, err := New("://invalid-url", "token")
assert.Error(t, err) assert.Error(t, err)
@ -63,7 +61,7 @@ func TestClient_GetCurrentUser_Good(t *testing.T) {
assert.Equal(t, "test-user", user.UserName) assert.Equal(t, "test-user", user.UserName)
} }
func TestClient_GetCurrentUser_Bad_ServerError_Good(t *testing.T) { func TestClient_GetCurrentUser_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -91,7 +89,7 @@ func TestClient_SetPRDraft_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_SetPRDraft_Good_Undraft_Good(t *testing.T) { func TestClient_SetPRDraft_Good_Undraft(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -99,7 +97,7 @@ func TestClient_SetPRDraft_Good_Undraft_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_SetPRDraft_Bad_ServerError_Good(t *testing.T) { func TestClient_SetPRDraft_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -123,7 +121,7 @@ func TestClient_SetPRDraft_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "unexpected status 403") assert.Contains(t, err.Error(), "unexpected status 403")
} }
func TestClient_SetPRDraft_Bad_ConnectionRefused_Good(t *testing.T) { func TestClient_SetPRDraft_Bad_ConnectionRefused(t *testing.T) {
// Use a closed server to simulate connection errors. // Use a closed server to simulate connection errors.
srv := newMockForgejoServer(t) srv := newMockForgejoServer(t)
client, err := New(srv.URL, "token") client, err := New(srv.URL, "token")
@ -134,7 +132,7 @@ func TestClient_SetPRDraft_Bad_ConnectionRefused_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
} }
func TestClient_SetPRDraft_Good_URLConstruction_Good(t *testing.T) { func TestClient_SetPRDraft_URLConstruction(t *testing.T) {
// Verify the URL is constructed correctly by checking the request path. // Verify the URL is constructed correctly by checking the request path.
var capturedPath string var capturedPath string
mux := http.NewServeMux() mux := http.NewServeMux()
@ -158,7 +156,7 @@ func TestClient_SetPRDraft_Good_URLConstruction_Good(t *testing.T) {
assert.Equal(t, "/api/v1/repos/my-org/my-repo/pulls/42", capturedPath) assert.Equal(t, "/api/v1/repos/my-org/my-repo/pulls/42", capturedPath)
} }
func TestClient_SetPRDraft_Good_AuthHeader_Good(t *testing.T) { func TestClient_SetPRDraft_AuthHeader(t *testing.T) {
// Verify the authorisation header is set correctly. // Verify the authorisation header is set correctly.
var capturedAuth string var capturedAuth string
mux := http.NewServeMux() mux := http.NewServeMux()
@ -184,7 +182,7 @@ func TestClient_SetPRDraft_Good_AuthHeader_Good(t *testing.T) {
// --- PRMeta and Comment struct tests --- // --- PRMeta and Comment struct tests ---
func TestPRMeta_Good_Fields_Good(t *testing.T) { func TestPRMeta_Fields(t *testing.T) {
meta := &PRMeta{ meta := &PRMeta{
Number: 42, Number: 42,
Title: "Test PR", Title: "Test PR",
@ -210,7 +208,7 @@ func TestPRMeta_Good_Fields_Good(t *testing.T) {
assert.Equal(t, 5, meta.CommentCount) assert.Equal(t, 5, meta.CommentCount)
} }
func TestComment_Good_Fields_Good(t *testing.T) { func TestComment_Fields(t *testing.T) {
comment := Comment{ comment := Comment{
ID: 123, ID: 123,
Author: "reviewer", Author: "reviewer",
@ -224,7 +222,7 @@ func TestComment_Good_Fields_Good(t *testing.T) {
// --- MergePullRequest merge style mapping --- // --- MergePullRequest merge style mapping ---
func TestMergePullRequest_Good_StyleMapping_Good(t *testing.T) { func TestMergePullRequest_StyleMapping(t *testing.T) {
// We can't easily test the SDK call, but we can verify the method // We can't easily test the SDK call, but we can verify the method
// errors when the server returns failure. This exercises the style mapping code. // errors when the server returns failure. This exercises the style mapping code.
tests := []struct { tests := []struct {
@ -262,7 +260,7 @@ func TestMergePullRequest_Good_StyleMapping_Good(t *testing.T) {
// --- ListIssuesOpts defaulting --- // --- ListIssuesOpts defaulting ---
func TestListIssuesOpts_Good_Defaults_Good(t *testing.T) { func TestListIssuesOpts_Defaults(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
opts ListIssuesOpts opts ListIssuesOpts
@ -325,7 +323,7 @@ func TestListIssuesOpts_Good_Defaults_Good(t *testing.T) {
// --- ForkRepo error handling --- // --- ForkRepo error handling ---
func TestClient_ForkRepo_Good_WithOrg_Good(t *testing.T) { func TestClient_ForkRepo_Good_WithOrg(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -355,7 +353,7 @@ func TestClient_ForkRepo_Good_WithOrg_Good(t *testing.T) {
assert.Equal(t, "target-org", capturedBody["organization"]) assert.Equal(t, "target-org", capturedBody["organization"])
} }
func TestClient_ForkRepo_Good_WithoutOrg_Good(t *testing.T) { func TestClient_ForkRepo_Good_WithoutOrg(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -386,7 +384,7 @@ func TestClient_ForkRepo_Good_WithoutOrg_Good(t *testing.T) {
// The SDK may or may not include it in the JSON; just verify the fork succeeded. // The SDK may or may not include it in the JSON; just verify the fork succeeded.
} }
func TestClient_ForkRepo_Bad_ServerError_Good(t *testing.T) { func TestClient_ForkRepo_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -408,7 +406,7 @@ func TestClient_ForkRepo_Bad_ServerError_Good(t *testing.T) {
// --- CreatePullRequest error handling --- // --- CreatePullRequest error handling ---
func TestClient_CreatePullRequest_Bad_ServerError_Good(t *testing.T) { func TestClient_CreatePullRequest_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux() mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
@ -434,13 +432,13 @@ func TestClient_CreatePullRequest_Bad_ServerError_Good(t *testing.T) {
// --- commentPageSize constant test --- // --- commentPageSize constant test ---
func TestCommentPageSize_Good(t *testing.T) { func TestCommentPageSize(t *testing.T) {
assert.Equal(t, 50, commentPageSize, "comment page size should be 50") assert.Equal(t, 50, commentPageSize, "comment page size should be 50")
} }
// --- ListPullRequests state mapping --- // --- ListPullRequests state mapping ---
func TestListPullRequests_Good_StateMapping_Good(t *testing.T) { func TestListPullRequests_StateMapping(t *testing.T) {
// Verify state mapping via error path (server returns error). // Verify state mapping via error path (server returns error).
tests := []struct { tests := []struct {
name string name string

View file

@ -1,24 +1,19 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
os "dappco.re/go/core/scm/internal/ax/osx" "os"
"dappco.re/go/core/log"
"forge.lthn.ai/core/config" "forge.lthn.ai/core/config"
"dappco.re/go/core/log"
) )
const ( const (
// ConfigKeyURL is the config key for the Forgejo instance URL. // ConfigKeyURL is the config key for the Forgejo instance URL.
//
ConfigKeyURL = "forge.url" ConfigKeyURL = "forge.url"
// ConfigKeyToken is the config key for the Forgejo API token. // ConfigKeyToken is the config key for the Forgejo API token.
//
ConfigKeyToken = "forge.token" ConfigKeyToken = "forge.token"
// DefaultURL is the default Forgejo instance URL. // DefaultURL is the default Forgejo instance URL.
//
DefaultURL = "http://localhost:4000" DefaultURL = "http://localhost:4000"
) )
@ -27,8 +22,6 @@ const (
// 1. ~/.core/config.yaml keys: forge.token, forge.url // 1. ~/.core/config.yaml keys: forge.token, forge.url
// 2. FORGE_TOKEN + FORGE_URL environment variables (override config file) // 2. FORGE_TOKEN + FORGE_URL environment variables (override config file)
// 3. Provided flag overrides (highest priority; pass empty to skip) // 3. Provided flag overrides (highest priority; pass empty to skip)
//
// Usage: NewFromConfig(...)
func NewFromConfig(flagURL, flagToken string) (*Client, error) { func NewFromConfig(flagURL, flagToken string) (*Client, error) {
url, token, err := ResolveConfig(flagURL, flagToken) url, token, err := ResolveConfig(flagURL, flagToken)
if err != nil { if err != nil {
@ -44,7 +37,6 @@ func NewFromConfig(flagURL, flagToken string) (*Client, error) {
// ResolveConfig resolves the Forgejo URL and token from all config sources. // ResolveConfig resolves the Forgejo URL and token from all config sources.
// Flag values take highest priority, then env vars, then config file. // Flag values take highest priority, then env vars, then config file.
// Usage: ResolveConfig(...)
func ResolveConfig(flagURL, flagToken string) (url, token string, err error) { func ResolveConfig(flagURL, flagToken string) (url, token string, err error) {
// Start with config file values // Start with config file values
cfg, cfgErr := config.New() cfg, cfgErr := config.New()
@ -78,7 +70,6 @@ func ResolveConfig(flagURL, flagToken string) (url, token string, err error) {
} }
// SaveConfig persists the Forgejo URL and/or token to the config file. // SaveConfig persists the Forgejo URL and/or token to the config file.
// Usage: SaveConfig(...)
func SaveConfig(url, token string) error { func SaveConfig(url, token string) error {
cfg, err := config.New() cfg, err := config.New()
if err != nil { if err != nil {

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
@ -18,7 +16,7 @@ func isolateConfigEnv(t *testing.T) {
t.Setenv("HOME", t.TempDir()) t.Setenv("HOME", t.TempDir())
} }
func TestResolveConfig_Good_Defaults_Good(t *testing.T) { func TestResolveConfig_Good_Defaults(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
url, token, err := ResolveConfig("", "") url, token, err := ResolveConfig("", "")
@ -27,7 +25,7 @@ func TestResolveConfig_Good_Defaults_Good(t *testing.T) {
assert.Empty(t, token, "token should be empty when nothing configured") assert.Empty(t, token, "token should be empty when nothing configured")
} }
func TestResolveConfig_Good_FlagsOverrideAll_Good(t *testing.T) { func TestResolveConfig_Good_FlagsOverrideAll(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
t.Setenv("FORGE_URL", "https://env-url.example.com") t.Setenv("FORGE_URL", "https://env-url.example.com")
t.Setenv("FORGE_TOKEN", "env-token-abc") t.Setenv("FORGE_TOKEN", "env-token-abc")
@ -38,7 +36,7 @@ func TestResolveConfig_Good_FlagsOverrideAll_Good(t *testing.T) {
assert.Equal(t, "flag-token-xyz", token, "flag token should override env") assert.Equal(t, "flag-token-xyz", token, "flag token should override env")
} }
func TestResolveConfig_Good_EnvVarsOverrideConfig_Good(t *testing.T) { func TestResolveConfig_Good_EnvVarsOverrideConfig(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
t.Setenv("FORGE_URL", "https://env-url.example.com") t.Setenv("FORGE_URL", "https://env-url.example.com")
t.Setenv("FORGE_TOKEN", "env-token-123") t.Setenv("FORGE_TOKEN", "env-token-123")
@ -49,7 +47,7 @@ func TestResolveConfig_Good_EnvVarsOverrideConfig_Good(t *testing.T) {
assert.Equal(t, "env-token-123", token) assert.Equal(t, "env-token-123", token)
} }
func TestResolveConfig_Good_PartialOverrides_Good(t *testing.T) { func TestResolveConfig_Good_PartialOverrides(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
// Set only env URL, flag token. // Set only env URL, flag token.
t.Setenv("FORGE_URL", "https://env-only.example.com") t.Setenv("FORGE_URL", "https://env-only.example.com")
@ -60,7 +58,7 @@ func TestResolveConfig_Good_PartialOverrides_Good(t *testing.T) {
assert.Equal(t, "flag-only-token", token, "flag token should be used") assert.Equal(t, "flag-only-token", token, "flag token should be used")
} }
func TestResolveConfig_Good_URLDefaultsWhenEmpty_Good(t *testing.T) { func TestResolveConfig_Good_URLDefaultsWhenEmpty(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
t.Setenv("FORGE_TOKEN", "some-token") t.Setenv("FORGE_TOKEN", "some-token")
@ -70,13 +68,13 @@ func TestResolveConfig_Good_URLDefaultsWhenEmpty_Good(t *testing.T) {
assert.Equal(t, "some-token", token) assert.Equal(t, "some-token", token)
} }
func TestConstants_Good(t *testing.T) { func TestConstants(t *testing.T) {
assert.Equal(t, "forge.url", ConfigKeyURL) assert.Equal(t, "forge.url", ConfigKeyURL)
assert.Equal(t, "forge.token", ConfigKeyToken) assert.Equal(t, "forge.token", ConfigKeyToken)
assert.Equal(t, "http://localhost:4000", DefaultURL) assert.Equal(t, "http://localhost:4000", DefaultURL)
} }
func TestNewFromConfig_Bad_NoToken_Good(t *testing.T) { func TestNewFromConfig_Bad_NoToken(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
_, err := NewFromConfig("", "") _, err := NewFromConfig("", "")
@ -84,7 +82,7 @@ func TestNewFromConfig_Bad_NoToken_Good(t *testing.T) {
assert.Contains(t, err.Error(), "no API token configured") assert.Contains(t, err.Error(), "no API token configured")
} }
func TestNewFromConfig_Good_WithFlagToken_Good(t *testing.T) { func TestNewFromConfig_Good_WithFlagToken(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
// The Forgejo SDK NewClient validates the token by calling /api/v1/version, // The Forgejo SDK NewClient validates the token by calling /api/v1/version,
@ -99,7 +97,7 @@ func TestNewFromConfig_Good_WithFlagToken_Good(t *testing.T) {
assert.Equal(t, "test-token", client.Token()) assert.Equal(t, "test-token", client.Token())
} }
func TestNewFromConfig_Good_EnvToken_Good(t *testing.T) { func TestNewFromConfig_Good_EnvToken(t *testing.T) {
isolateConfigEnv(t) isolateConfigEnv(t)
srv := newMockForgejoServer(t) srv := newMockForgejoServer(t)

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
@ -19,7 +17,6 @@ type ListIssuesOpts struct {
} }
// ListIssues returns issues for the given repository. // ListIssues returns issues for the given repository.
// Usage: ListIssues(...)
func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo.Issue, error) { func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo.Issue, error) {
state := forgejo.StateOpen state := forgejo.StateOpen
switch opts.State { switch opts.State {
@ -39,9 +36,6 @@ func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo
page = 1 page = 1
} }
var all []*forgejo.Issue
for {
listOpt := forgejo.ListIssueOption{ listOpt := forgejo.ListIssueOption{
ListOptions: forgejo.ListOptions{Page: page, PageSize: limit}, ListOptions: forgejo.ListOptions{Page: page, PageSize: limit},
State: state, State: state,
@ -49,75 +43,15 @@ func (c *Client) ListIssues(owner, repo string, opts ListIssuesOpts) ([]*forgejo
Labels: opts.Labels, Labels: opts.Labels,
} }
issues, resp, err := c.api.ListRepoIssues(owner, repo, listOpt) issues, _, err := c.api.ListRepoIssues(owner, repo, listOpt)
if err != nil { if err != nil {
return nil, log.E("forge.ListIssues", "failed to list issues", err) return nil, log.E("forge.ListIssues", "failed to list issues", err)
} }
all = append(all, issues...) return issues, nil
if len(issues) < limit || len(issues) == 0 {
break
}
if resp != nil && resp.LastPage > 0 && page >= resp.LastPage {
break
}
page++
}
return all, nil
}
// ListIssuesIter returns an iterator over issues for the given repository.
// Usage: ListIssuesIter(...)
func (c *Client) ListIssuesIter(owner, repo string, opts ListIssuesOpts) iter.Seq2[*forgejo.Issue, error] {
state := forgejo.StateOpen
switch opts.State {
case "closed":
state = forgejo.StateClosed
case "all":
state = forgejo.StateAll
}
limit := opts.Limit
if limit == 0 {
limit = 50
}
page := opts.Page
if page == 0 {
page = 1
}
return func(yield func(*forgejo.Issue, error) bool) {
for {
issues, resp, err := c.api.ListRepoIssues(owner, repo, forgejo.ListIssueOption{
ListOptions: forgejo.ListOptions{Page: page, PageSize: limit},
State: state,
Type: forgejo.IssueTypeIssue,
Labels: opts.Labels,
})
if err != nil {
yield(nil, log.E("forge.ListIssues", "failed to list issues", err))
return
}
for _, issue := range issues {
if !yield(issue, nil) {
return
}
}
if len(issues) < limit || len(issues) == 0 {
break
}
if resp != nil && resp.LastPage > 0 && page >= resp.LastPage {
break
}
page++
}
}
} }
// GetIssue returns a single issue by number. // GetIssue returns a single issue by number.
// Usage: GetIssue(...)
func (c *Client) GetIssue(owner, repo string, number int64) (*forgejo.Issue, error) { func (c *Client) GetIssue(owner, repo string, number int64) (*forgejo.Issue, error) {
issue, _, err := c.api.GetIssue(owner, repo, number) issue, _, err := c.api.GetIssue(owner, repo, number)
if err != nil { if err != nil {
@ -128,7 +62,6 @@ func (c *Client) GetIssue(owner, repo string, number int64) (*forgejo.Issue, err
} }
// CreateIssue creates a new issue in the given repository. // CreateIssue creates a new issue in the given repository.
// Usage: CreateIssue(...)
func (c *Client) CreateIssue(owner, repo string, opts forgejo.CreateIssueOption) (*forgejo.Issue, error) { func (c *Client) CreateIssue(owner, repo string, opts forgejo.CreateIssueOption) (*forgejo.Issue, error) {
issue, _, err := c.api.CreateIssue(owner, repo, opts) issue, _, err := c.api.CreateIssue(owner, repo, opts)
if err != nil { if err != nil {
@ -139,7 +72,6 @@ func (c *Client) CreateIssue(owner, repo string, opts forgejo.CreateIssueOption)
} }
// EditIssue edits an existing issue. // EditIssue edits an existing issue.
// Usage: EditIssue(...)
func (c *Client) EditIssue(owner, repo string, number int64, opts forgejo.EditIssueOption) (*forgejo.Issue, error) { func (c *Client) EditIssue(owner, repo string, number int64, opts forgejo.EditIssueOption) (*forgejo.Issue, error) {
issue, _, err := c.api.EditIssue(owner, repo, number, opts) issue, _, err := c.api.EditIssue(owner, repo, number, opts)
if err != nil { if err != nil {
@ -150,7 +82,6 @@ func (c *Client) EditIssue(owner, repo string, number int64, opts forgejo.EditIs
} }
// AssignIssue assigns an issue to the specified users. // AssignIssue assigns an issue to the specified users.
// Usage: AssignIssue(...)
func (c *Client) AssignIssue(owner, repo string, number int64, assignees []string) error { func (c *Client) AssignIssue(owner, repo string, number int64, assignees []string) error {
_, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{ _, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{
Assignees: assignees, Assignees: assignees,
@ -162,7 +93,6 @@ func (c *Client) AssignIssue(owner, repo string, number int64, assignees []strin
} }
// ListPullRequests returns pull requests for the given repository. // ListPullRequests returns pull requests for the given repository.
// Usage: ListPullRequests(...)
func (c *Client) ListPullRequests(owner, repo string, state string) ([]*forgejo.PullRequest, error) { func (c *Client) ListPullRequests(owner, repo string, state string) ([]*forgejo.PullRequest, error) {
st := forgejo.StateOpen st := forgejo.StateOpen
switch state { switch state {
@ -196,7 +126,6 @@ func (c *Client) ListPullRequests(owner, repo string, state string) ([]*forgejo.
} }
// ListPullRequestsIter returns an iterator over pull requests for the given repository. // ListPullRequestsIter returns an iterator over pull requests for the given repository.
// Usage: ListPullRequestsIter(...)
func (c *Client) ListPullRequestsIter(owner, repo string, state string) iter.Seq2[*forgejo.PullRequest, error] { func (c *Client) ListPullRequestsIter(owner, repo string, state string) iter.Seq2[*forgejo.PullRequest, error] {
st := forgejo.StateOpen st := forgejo.StateOpen
switch state { switch state {
@ -231,7 +160,6 @@ func (c *Client) ListPullRequestsIter(owner, repo string, state string) iter.Seq
} }
// GetPullRequest returns a single pull request by number. // GetPullRequest returns a single pull request by number.
// Usage: GetPullRequest(...)
func (c *Client) GetPullRequest(owner, repo string, number int64) (*forgejo.PullRequest, error) { func (c *Client) GetPullRequest(owner, repo string, number int64) (*forgejo.PullRequest, error) {
pr, _, err := c.api.GetPullRequest(owner, repo, number) pr, _, err := c.api.GetPullRequest(owner, repo, number)
if err != nil { if err != nil {
@ -242,7 +170,6 @@ func (c *Client) GetPullRequest(owner, repo string, number int64) (*forgejo.Pull
} }
// CreateIssueComment posts a comment on an issue or pull request. // CreateIssueComment posts a comment on an issue or pull request.
// Usage: CreateIssueComment(...)
func (c *Client) CreateIssueComment(owner, repo string, issue int64, body string) error { func (c *Client) CreateIssueComment(owner, repo string, issue int64, body string) error {
_, _, err := c.api.CreateIssueComment(owner, repo, issue, forgejo.CreateIssueCommentOption{ _, _, err := c.api.CreateIssueComment(owner, repo, issue, forgejo.CreateIssueCommentOption{
Body: body, Body: body,
@ -253,19 +180,7 @@ func (c *Client) CreateIssueComment(owner, repo string, issue int64, body string
return nil return nil
} }
// GetIssueLabels returns the labels currently attached to an issue.
// Usage: GetIssueLabels(...)
func (c *Client) GetIssueLabels(owner, repo string, number int64) ([]*forgejo.Label, error) {
labels, _, err := c.api.GetIssueLabels(owner, repo, number, forgejo.ListLabelsOptions{})
if err != nil {
return nil, log.E("forge.GetIssueLabels", "failed to get issue labels", err)
}
return labels, nil
}
// ListIssueComments returns comments for an issue. // ListIssueComments returns comments for an issue.
// Usage: ListIssueComments(...)
func (c *Client) ListIssueComments(owner, repo string, number int64) ([]*forgejo.Comment, error) { func (c *Client) ListIssueComments(owner, repo string, number int64) ([]*forgejo.Comment, error) {
var all []*forgejo.Comment var all []*forgejo.Comment
page := 1 page := 1
@ -289,34 +204,7 @@ func (c *Client) ListIssueComments(owner, repo string, number int64) ([]*forgejo
return all, nil return all, nil
} }
// ListIssueCommentsIter returns an iterator over comments for an issue.
// Usage: ListIssueCommentsIter(...)
func (c *Client) ListIssueCommentsIter(owner, repo string, number int64) iter.Seq2[*forgejo.Comment, error] {
return func(yield func(*forgejo.Comment, error) bool) {
page := 1
for {
comments, resp, err := c.api.ListIssueComments(owner, repo, number, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
if err != nil {
yield(nil, log.E("forge.ListIssueComments", "failed to list comments", err))
return
}
for _, comment := range comments {
if !yield(comment, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// CloseIssue closes an issue by setting its state to closed. // CloseIssue closes an issue by setting its state to closed.
// Usage: CloseIssue(...)
func (c *Client) CloseIssue(owner, repo string, number int64) error { func (c *Client) CloseIssue(owner, repo string, number int64) error {
closed := forgejo.StateClosed closed := forgejo.StateClosed
_, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{ _, _, err := c.api.EditIssue(owner, repo, number, forgejo.EditIssueOption{

View file

@ -1,11 +1,6 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"net/http"
"net/http/httptest"
"strconv"
"testing" "testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -14,71 +9,6 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func newPaginatedIssuesClient(t *testing.T) (*Client, *httptest.Server) {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/issues", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "number": 2, "title": "Issue 2", "state": "open", "body": "Second issue"},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
jsonResponse(w, []map[string]any{
{"id": 1, "number": 1, "title": "Issue 1", "state": "open", "body": "First issue"},
})
}
})
srv := httptest.NewServer(mux)
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
return client, srv
}
func newPaginatedCommentsClient(t *testing.T) (*Client, *httptest.Server) {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/issues/1/comments", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 150, "body": "comment 51", "user": map[string]any{"login": "user51"}, "created_at": "2026-01-02T00:00:00Z", "updated_at": "2026-01-02T00:00:00Z"},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", `</api/v1/repos/test-org/org-repo/issues/1/comments?page=2>; rel="next", </api/v1/repos/test-org/org-repo/issues/1/comments?page=2>; rel="last"`)
comments := make([]map[string]any, 0, 50)
for i := 1; i <= 50; i++ {
comments = append(comments, map[string]any{
"id": 99 + i,
"body": "comment " + strconv.Itoa(i),
"user": map[string]any{"login": "user" + strconv.Itoa(i)},
"created_at": "2026-01-01T00:00:00Z",
"updated_at": "2026-01-01T00:00:00Z",
})
}
jsonResponse(w, comments)
}
})
srv := httptest.NewServer(mux)
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
return client, srv
}
func TestClient_ListIssues_Good(t *testing.T) { func TestClient_ListIssues_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -89,32 +19,7 @@ func TestClient_ListIssues_Good(t *testing.T) {
assert.Equal(t, "Issue 1", issues[0].Title) assert.Equal(t, "Issue 1", issues[0].Title)
} }
func TestClient_ListIssues_Good_Paginates_Good(t *testing.T) { func TestClient_ListIssues_Good_StateMapping(t *testing.T) {
client, srv := newPaginatedIssuesClient(t)
defer srv.Close()
issues, err := client.ListIssues("test-org", "org-repo", ListIssuesOpts{Limit: 1})
require.NoError(t, err)
require.Len(t, issues, 2)
assert.Equal(t, "Issue 1", issues[0].Title)
assert.Equal(t, "Issue 2", issues[1].Title)
}
func TestClient_ListIssuesIter_Good_Paginates_Good(t *testing.T) {
client, srv := newPaginatedIssuesClient(t)
defer srv.Close()
var titles []string
for issue, err := range client.ListIssuesIter("test-org", "org-repo", ListIssuesOpts{Limit: 1}) {
require.NoError(t, err)
titles = append(titles, issue.Title)
}
require.Len(t, titles, 2)
assert.Equal(t, []string{"Issue 1", "Issue 2"}, titles)
}
func TestClient_ListIssues_Good_StateMapping_Good(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
state string state string
@ -136,7 +41,7 @@ func TestClient_ListIssues_Good_StateMapping_Good(t *testing.T) {
} }
} }
func TestClient_ListIssues_Good_CustomPageAndLimit_Good(t *testing.T) { func TestClient_ListIssues_Good_CustomPageAndLimit(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -147,7 +52,7 @@ func TestClient_ListIssues_Good_CustomPageAndLimit_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_ListIssues_Bad_ServerError_Good(t *testing.T) { func TestClient_ListIssues_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -165,7 +70,7 @@ func TestClient_GetIssue_Good(t *testing.T) {
assert.Equal(t, "Issue 1", issue.Title) assert.Equal(t, "Issue 1", issue.Title)
} }
func TestClient_GetIssue_Bad_ServerError_Good(t *testing.T) { func TestClient_GetIssue_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -186,7 +91,7 @@ func TestClient_CreateIssue_Good(t *testing.T) {
assert.NotNil(t, issue) assert.NotNil(t, issue)
} }
func TestClient_CreateIssue_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateIssue_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -208,7 +113,7 @@ func TestClient_EditIssue_Good(t *testing.T) {
assert.NotNil(t, issue) assert.NotNil(t, issue)
} }
func TestClient_EditIssue_Bad_ServerError_Good(t *testing.T) { func TestClient_EditIssue_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -227,7 +132,7 @@ func TestClient_AssignIssue_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_AssignIssue_Bad_ServerError_Good(t *testing.T) { func TestClient_AssignIssue_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -246,7 +151,7 @@ func TestClient_ListPullRequests_Good(t *testing.T) {
assert.Equal(t, "PR 1", prs[0].Title) assert.Equal(t, "PR 1", prs[0].Title)
} }
func TestClient_ListPullRequests_Good_StateMapping_Good(t *testing.T) { func TestClient_ListPullRequests_Good_StateMapping(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
state string state string
@ -268,7 +173,7 @@ func TestClient_ListPullRequests_Good_StateMapping_Good(t *testing.T) {
} }
} }
func TestClient_ListPullRequests_Bad_ServerError_Good(t *testing.T) { func TestClient_ListPullRequests_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -286,7 +191,7 @@ func TestClient_GetPullRequest_Good(t *testing.T) {
assert.Equal(t, "PR 1", pr.Title) assert.Equal(t, "PR 1", pr.Title)
} }
func TestClient_GetPullRequest_Bad_ServerError_Good(t *testing.T) { func TestClient_GetPullRequest_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -303,7 +208,7 @@ func TestClient_CreateIssueComment_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_CreateIssueComment_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateIssueComment_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -312,25 +217,6 @@ func TestClient_CreateIssueComment_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to create comment") assert.Contains(t, err.Error(), "failed to create comment")
} }
func TestClient_GetIssueLabels_Good(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
labels, err := client.GetIssueLabels("test-org", "org-repo", 1)
require.NoError(t, err)
require.Len(t, labels, 1)
assert.Equal(t, "bug", labels[0].Name)
}
func TestClient_GetIssueLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
_, err := client.GetIssueLabels("test-org", "org-repo", 1)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to get issue labels")
}
func TestClient_ListIssueComments_Good(t *testing.T) { func TestClient_ListIssueComments_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -341,7 +227,7 @@ func TestClient_ListIssueComments_Good(t *testing.T) {
assert.Equal(t, "comment 1", comments[0].Body) assert.Equal(t, "comment 1", comments[0].Body)
} }
func TestClient_ListIssueComments_Bad_ServerError_Good(t *testing.T) { func TestClient_ListIssueComments_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -350,22 +236,6 @@ func TestClient_ListIssueComments_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list comments") assert.Contains(t, err.Error(), "failed to list comments")
} }
func TestClient_ListIssueCommentsIter_Good_Paginates_Good(t *testing.T) {
client, srv := newPaginatedCommentsClient(t)
defer srv.Close()
var bodies []string
for comment, err := range client.ListIssueCommentsIter("test-org", "org-repo", 1) {
require.NoError(t, err)
bodies = append(bodies, comment.Body)
}
require.Len(t, bodies, 51)
assert.Equal(t, "comment 1", bodies[0])
assert.Equal(t, "comment 50", bodies[49])
assert.Equal(t, "comment 51", bodies[50])
}
func TestClient_CloseIssue_Good(t *testing.T) { func TestClient_CloseIssue_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -374,7 +244,7 @@ func TestClient_CloseIssue_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_CloseIssue_Bad_ServerError_Good(t *testing.T) { func TestClient_CloseIssue_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()

View file

@ -1,23 +1,19 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"iter" "strings"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log" "dappco.re/go/core/log"
) )
// ListOrgLabels returns all unique labels across repos in the given organisation. // ListOrgLabels returns all labels for repos in the given organisation.
// Note: The Forgejo SDK does not have a dedicated org-level labels endpoint. // Note: The Forgejo SDK does not have a dedicated org-level labels endpoint.
// We aggregate labels from each repo and deduplicate them by name, preserving // This lists labels from the first repo found, which works when orgs use shared label sets.
// the first seen label metadata. // For org-wide label management, use ListRepoLabels with a specific repo.
// Usage: ListOrgLabels(...)
func (c *Client) ListOrgLabels(org string) ([]*forgejo.Label, error) { func (c *Client) ListOrgLabels(org string) ([]*forgejo.Label, error) {
// Forgejo doesn't expose org-level labels via SDK — list repos and aggregate unique labels.
repos, err := c.ListOrgRepos(org) repos, err := c.ListOrgRepos(org)
if err != nil { if err != nil {
return nil, err return nil, err
@ -27,63 +23,11 @@ func (c *Client) ListOrgLabels(org string) ([]*forgejo.Label, error) {
return nil, nil return nil, nil
} }
seen := make(map[string]struct{}, len(repos)) // Use the first repo's labels as representative of the org's label set.
var all []*forgejo.Label return c.ListRepoLabels(repos[0].Owner.UserName, repos[0].Name)
for _, repo := range repos {
labels, err := c.ListRepoLabels(repo.Owner.UserName, repo.Name)
if err != nil {
return nil, err
}
for _, label := range labels {
key := strings.ToLower(label.Name)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
all = append(all, label)
}
}
return all, nil
}
// ListOrgLabelsIter returns an iterator over unique labels across repos in the given organisation.
// Note: The Forgejo SDK does not have a dedicated org-level labels endpoint.
// Labels are yielded in first-seen order across repositories and deduplicated by name.
// Usage: ListOrgLabelsIter(...)
func (c *Client) ListOrgLabelsIter(org string) iter.Seq2[*forgejo.Label, error] {
return func(yield func(*forgejo.Label, error) bool) {
seen := make(map[string]struct{})
for repo, err := range c.ListOrgReposIter(org) {
if err != nil {
yield(nil, log.E("forge.ListOrgLabels", "failed to list org repos", err))
return
}
for label, err := range c.ListRepoLabelsIter(repo.Owner.UserName, repo.Name) {
if err != nil {
yield(nil, log.E("forge.ListOrgLabels", "failed to list repo labels", err))
return
}
key := strings.ToLower(label.Name)
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
if !yield(label, nil) {
return
}
}
}
}
} }
// ListRepoLabels returns all labels for a repository. // ListRepoLabels returns all labels for a repository.
// Usage: ListRepoLabels(...)
func (c *Client) ListRepoLabels(owner, repo string) ([]*forgejo.Label, error) { func (c *Client) ListRepoLabels(owner, repo string) ([]*forgejo.Label, error) {
var all []*forgejo.Label var all []*forgejo.Label
page := 1 page := 1
@ -107,37 +51,7 @@ func (c *Client) ListRepoLabels(owner, repo string) ([]*forgejo.Label, error) {
return all, nil return all, nil
} }
// ListRepoLabelsIter returns an iterator over labels for a repository.
// Usage: ListRepoLabelsIter(...)
func (c *Client) ListRepoLabelsIter(owner, repo string) iter.Seq2[*forgejo.Label, error] {
return func(yield func(*forgejo.Label, error) bool) {
page := 1
for {
labels, resp, err := c.api.ListRepoLabels(owner, repo, forgejo.ListLabelsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListRepoLabels", "failed to list repo labels", err))
return
}
for _, label := range labels {
if !yield(label, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// CreateRepoLabel creates a label on a repository. // CreateRepoLabel creates a label on a repository.
// Usage: CreateRepoLabel(...)
func (c *Client) CreateRepoLabel(owner, repo string, opts forgejo.CreateLabelOption) (*forgejo.Label, error) { func (c *Client) CreateRepoLabel(owner, repo string, opts forgejo.CreateLabelOption) (*forgejo.Label, error) {
label, _, err := c.api.CreateLabel(owner, repo, opts) label, _, err := c.api.CreateLabel(owner, repo, opts)
if err != nil { if err != nil {
@ -148,7 +62,6 @@ func (c *Client) CreateRepoLabel(owner, repo string, opts forgejo.CreateLabelOpt
} }
// GetLabelByName retrieves a specific label by name from a repository. // GetLabelByName retrieves a specific label by name from a repository.
// Usage: GetLabelByName(...)
func (c *Client) GetLabelByName(owner, repo, name string) (*forgejo.Label, error) { func (c *Client) GetLabelByName(owner, repo, name string) (*forgejo.Label, error) {
labels, err := c.ListRepoLabels(owner, repo) labels, err := c.ListRepoLabels(owner, repo)
if err != nil { if err != nil {
@ -165,7 +78,6 @@ func (c *Client) GetLabelByName(owner, repo, name string) (*forgejo.Label, error
} }
// EnsureLabel checks if a label exists, and creates it if it doesn't. // EnsureLabel checks if a label exists, and creates it if it doesn't.
// Usage: EnsureLabel(...)
func (c *Client) EnsureLabel(owner, repo, name, color string) (*forgejo.Label, error) { func (c *Client) EnsureLabel(owner, repo, name, color string) (*forgejo.Label, error) {
label, err := c.GetLabelByName(owner, repo, name) label, err := c.GetLabelByName(owner, repo, name)
if err == nil { if err == nil {
@ -179,7 +91,6 @@ func (c *Client) EnsureLabel(owner, repo, name, color string) (*forgejo.Label, e
} }
// AddIssueLabels adds labels to an issue. // AddIssueLabels adds labels to an issue.
// Usage: AddIssueLabels(...)
func (c *Client) AddIssueLabels(owner, repo string, number int64, labelIDs []int64) error { func (c *Client) AddIssueLabels(owner, repo string, number int64, labelIDs []int64) error {
_, _, err := c.api.AddIssueLabels(owner, repo, number, forgejo.IssueLabelsOption{ _, _, err := c.api.AddIssueLabels(owner, repo, number, forgejo.IssueLabelsOption{
Labels: labelIDs, Labels: labelIDs,
@ -191,7 +102,6 @@ func (c *Client) AddIssueLabels(owner, repo string, number int64, labelIDs []int
} }
// RemoveIssueLabel removes a label from an issue. // RemoveIssueLabel removes a label from an issue.
// Usage: RemoveIssueLabel(...)
func (c *Client) RemoveIssueLabel(owner, repo string, number int64, labelID int64) error { func (c *Client) RemoveIssueLabel(owner, repo string, number int64, labelID int64) error {
_, err := c.api.DeleteIssueLabel(owner, repo, number, labelID) _, err := c.api.DeleteIssueLabel(owner, repo, number, labelID)
if err != nil { if err != nil {

View file

@ -1,10 +1,6 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"net/http"
"net/http/httptest"
"testing" "testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -28,7 +24,7 @@ func TestClient_ListRepoLabels_Good(t *testing.T) {
assert.Equal(t, "feature", labels[1].Name) assert.Equal(t, "feature", labels[1].Name)
} }
func TestClient_ListRepoLabels_Bad_ServerError_Good(t *testing.T) { func TestClient_ListRepoLabels_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -37,53 +33,6 @@ func TestClient_ListRepoLabels_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list repo labels") assert.Contains(t, err.Error(), "failed to list repo labels")
} }
func TestClient_ListRepoLabelsIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/labels", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 3, "name": "documentation", "color": "#00aa00"},
})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/labels?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/labels?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "name": "bug", "color": "#ff0000"},
{"id": 2, "name": "feature", "color": "#0000ff"},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var names []string
for label, err := range client.ListRepoLabelsIter("test-org", "org-repo") {
require.NoError(t, err)
names = append(names, label.Name)
}
require.Len(t, names, 3)
assert.Equal(t, []string{"bug", "feature", "documentation"}, names)
}
func TestClient_ListRepoLabelsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListRepoLabelsIter("test-org", "org-repo") {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list repo labels")
break
}
}
func TestClient_CreateRepoLabel_Good(t *testing.T) { func TestClient_CreateRepoLabel_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -93,7 +42,7 @@ func TestClient_CreateRepoLabel_Good(t *testing.T) {
assert.NotNil(t, label) assert.NotNil(t, label)
} }
func TestClient_CreateRepoLabel_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateRepoLabel_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -111,7 +60,7 @@ func TestClient_GetLabelByName_Good(t *testing.T) {
assert.Equal(t, "bug", label.Name) assert.Equal(t, "bug", label.Name)
} }
func TestClient_GetLabelByName_Good_CaseInsensitive_Good(t *testing.T) { func TestClient_GetLabelByName_Good_CaseInsensitive(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -120,7 +69,7 @@ func TestClient_GetLabelByName_Good_CaseInsensitive_Good(t *testing.T) {
assert.Equal(t, "bug", label.Name) assert.Equal(t, "bug", label.Name)
} }
func TestClient_GetLabelByName_Bad_NotFound_Good(t *testing.T) { func TestClient_GetLabelByName_Bad_NotFound(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -129,7 +78,7 @@ func TestClient_GetLabelByName_Bad_NotFound_Good(t *testing.T) {
assert.Contains(t, err.Error(), "label nonexistent not found") assert.Contains(t, err.Error(), "label nonexistent not found")
} }
func TestClient_EnsureLabel_Good_Exists_Good(t *testing.T) { func TestClient_EnsureLabel_Good_Exists(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -139,7 +88,7 @@ func TestClient_EnsureLabel_Good_Exists_Good(t *testing.T) {
assert.Equal(t, "bug", label.Name) assert.Equal(t, "bug", label.Name)
} }
func TestClient_EnsureLabel_Good_Creates_Good(t *testing.T) { func TestClient_EnsureLabel_Good_Creates(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -155,38 +104,11 @@ func TestClient_ListOrgLabels_Good(t *testing.T) {
labels, err := client.ListOrgLabels("test-org") labels, err := client.ListOrgLabels("test-org")
require.NoError(t, err) require.NoError(t, err)
require.Len(t, labels, 3) // Uses first repo's labels as representative.
assert.Equal(t, "bug", labels[0].Name) assert.NotEmpty(t, labels)
assert.Equal(t, "feature", labels[1].Name)
assert.Equal(t, "documentation", labels[2].Name)
} }
func TestClient_ListOrgLabelsIter_Good(t *testing.T) { func TestClient_ListOrgLabels_Bad_ServerError(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
var names []string
for label, err := range client.ListOrgLabelsIter("test-org") {
require.NoError(t, err)
names = append(names, label.Name)
}
require.Len(t, names, 3)
assert.Equal(t, []string{"bug", "feature", "documentation"}, names)
}
func TestClient_ListOrgLabelsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListOrgLabelsIter("test-org") {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list org repos")
break
}
}
func TestClient_ListOrgLabels_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -202,7 +124,7 @@ func TestClient_AddIssueLabels_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_AddIssueLabels_Bad_ServerError_Good(t *testing.T) { func TestClient_AddIssueLabels_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -219,7 +141,7 @@ func TestClient_RemoveIssueLabel_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_RemoveIssueLabel_Bad_ServerError_Good(t *testing.T) { func TestClient_RemoveIssueLabel_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()

View file

@ -1,10 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"time" "time"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log" "dappco.re/go/core/log"
) )
@ -38,7 +38,6 @@ const commentPageSize = 50
// GetPRMeta returns structural signals for a pull request. // GetPRMeta returns structural signals for a pull request.
// This is the Forgejo side of the dual MetaReader described in the pipeline design. // This is the Forgejo side of the dual MetaReader described in the pipeline design.
// Usage: GetPRMeta(...)
func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) { func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
pull, _, err := c.api.GetPullRequest(owner, repo, pr) pull, _, err := c.api.GetPullRequest(owner, repo, pr)
if err != nil { if err != nil {
@ -76,11 +75,19 @@ func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
// Fetch comment count from the issue side (PRs are issues in Forgejo). // Fetch comment count from the issue side (PRs are issues in Forgejo).
// Paginate to get an accurate count. // Paginate to get an accurate count.
count := 0 count := 0
for _, err := range c.ListIssueCommentsIter(owner, repo, pr) { page := 1
if err != nil { for {
comments, _, listErr := c.api.ListIssueComments(owner, repo, pr, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
if listErr != nil {
break break
} }
count++ count += len(comments)
if len(comments) < commentPageSize {
break
}
page++
} }
meta.CommentCount = count meta.CommentCount = count
@ -88,31 +95,45 @@ func (c *Client) GetPRMeta(owner, repo string, pr int64) (*PRMeta, error) {
} }
// GetCommentBodies returns all comment bodies for a pull request. // GetCommentBodies returns all comment bodies for a pull request.
// Usage: GetCommentBodies(...)
func (c *Client) GetCommentBodies(owner, repo string, pr int64) ([]Comment, error) { func (c *Client) GetCommentBodies(owner, repo string, pr int64) ([]Comment, error) {
var comments []Comment var comments []Comment
for raw, err := range c.ListIssueCommentsIter(owner, repo, pr) { page := 1
for {
raw, _, err := c.api.ListIssueComments(owner, repo, pr, forgejo.ListIssueCommentOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: commentPageSize},
})
if err != nil { if err != nil {
return nil, log.E("forge.GetCommentBodies", "failed to get PR comments", err) return nil, log.E("forge.GetCommentBodies", "failed to get PR comments", err)
} }
comment := Comment{ if len(raw) == 0 {
ID: raw.ID, break
Body: raw.Body,
CreatedAt: raw.Created,
UpdatedAt: raw.Updated,
} }
if raw.Poster != nil {
comment.Author = raw.Poster.UserName for _, rc := range raw {
comment := Comment{
ID: rc.ID,
Body: rc.Body,
CreatedAt: rc.Created,
UpdatedAt: rc.Updated,
}
if rc.Poster != nil {
comment.Author = rc.Poster.UserName
} }
comments = append(comments, comment) comments = append(comments, comment)
} }
if len(raw) < commentPageSize {
break
}
page++
}
return comments, nil return comments, nil
} }
// GetIssueBody returns the body text of an issue. // GetIssueBody returns the body text of an issue.
// Usage: GetIssueBody(...)
func (c *Client) GetIssueBody(owner, repo string, issue int64) (string, error) { func (c *Client) GetIssueBody(owner, repo string, issue int64) (string, error) {
iss, _, err := c.api.GetIssue(owner, repo, issue) iss, _, err := c.api.GetIssue(owner, repo, issue)
if err != nil { if err != nil {

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
@ -25,7 +23,7 @@ func TestClient_GetPRMeta_Good(t *testing.T) {
assert.False(t, meta.IsMerged) assert.False(t, meta.IsMerged)
} }
func TestClient_GetPRMeta_Bad_ServerError_Good(t *testing.T) { func TestClient_GetPRMeta_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -47,7 +45,7 @@ func TestClient_GetCommentBodies_Good(t *testing.T) {
assert.Equal(t, "user2", comments[1].Author) assert.Equal(t, "user2", comments[1].Author)
} }
func TestClient_GetCommentBodies_Bad_ServerError_Good(t *testing.T) { func TestClient_GetCommentBodies_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -65,7 +63,7 @@ func TestClient_GetIssueBody_Good(t *testing.T) {
assert.Equal(t, "First issue body", body) assert.Equal(t, "First issue body", body)
} }
func TestClient_GetIssueBody_Bad_ServerError_Good(t *testing.T) { func TestClient_GetIssueBody_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()

View file

@ -1,17 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"iter"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log" "dappco.re/go/core/log"
) )
// ListMyOrgs returns all organisations for the authenticated user. // ListMyOrgs returns all organisations for the authenticated user.
// Usage: ListMyOrgs(...)
func (c *Client) ListMyOrgs() ([]*forgejo.Organization, error) { func (c *Client) ListMyOrgs() ([]*forgejo.Organization, error) {
var all []*forgejo.Organization var all []*forgejo.Organization
page := 1 page := 1
@ -35,37 +30,7 @@ func (c *Client) ListMyOrgs() ([]*forgejo.Organization, error) {
return all, nil return all, nil
} }
// ListMyOrgsIter returns an iterator over organisations for the authenticated user.
// Usage: ListMyOrgsIter(...)
func (c *Client) ListMyOrgsIter() iter.Seq2[*forgejo.Organization, error] {
return func(yield func(*forgejo.Organization, error) bool) {
page := 1
for {
orgs, resp, err := c.api.ListMyOrgs(forgejo.ListOrgsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListMyOrgs", "failed to list orgs", err))
return
}
for _, org := range orgs {
if !yield(org, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// GetOrg returns a single organisation by name. // GetOrg returns a single organisation by name.
// Usage: GetOrg(...)
func (c *Client) GetOrg(name string) (*forgejo.Organization, error) { func (c *Client) GetOrg(name string) (*forgejo.Organization, error) {
org, _, err := c.api.GetOrg(name) org, _, err := c.api.GetOrg(name)
if err != nil { if err != nil {
@ -76,7 +41,6 @@ func (c *Client) GetOrg(name string) (*forgejo.Organization, error) {
} }
// CreateOrg creates a new organisation. // CreateOrg creates a new organisation.
// Usage: CreateOrg(...)
func (c *Client) CreateOrg(opts forgejo.CreateOrgOption) (*forgejo.Organization, error) { func (c *Client) CreateOrg(opts forgejo.CreateOrgOption) (*forgejo.Organization, error) {
org, _, err := c.api.CreateOrg(opts) org, _, err := c.api.CreateOrg(opts)
if err != nil { if err != nil {

View file

@ -1,10 +1,6 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"net/http"
"net/http/httptest"
"testing" "testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -23,42 +19,7 @@ func TestClient_ListMyOrgs_Good(t *testing.T) {
assert.Equal(t, "test-org", orgs[0].UserName) assert.Equal(t, "test-org", orgs[0].UserName)
} }
func TestClient_ListMyOrgsIter_Good_Paginates_Good(t *testing.T) { func TestClient_ListMyOrgs_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/user/orgs", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 101, "login": "second-org", "username": "second-org", "full_name": "Second Organisation"},
})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/user/orgs?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/user/orgs?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 100, "login": "test-org", "username": "test-org", "full_name": "Test Organisation"},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var names []string
for org, err := range client.ListMyOrgsIter() {
require.NoError(t, err)
names = append(names, org.UserName)
}
require.Len(t, names, 2)
assert.Equal(t, []string{"test-org", "second-org"}, names)
}
func TestClient_ListMyOrgs_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -67,17 +28,6 @@ func TestClient_ListMyOrgs_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list orgs") assert.Contains(t, err.Error(), "failed to list orgs")
} }
func TestClient_ListMyOrgsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
for _, err := range client.ListMyOrgsIter() {
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list orgs")
break
}
}
func TestClient_GetOrg_Good(t *testing.T) { func TestClient_GetOrg_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -87,7 +37,7 @@ func TestClient_GetOrg_Good(t *testing.T) {
assert.Equal(t, "test-org", org.UserName) assert.Equal(t, "test-org", org.UserName)
} }
func TestClient_GetOrg_Bad_ServerError_Good(t *testing.T) { func TestClient_GetOrg_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -109,7 +59,7 @@ func TestClient_CreateOrg_Good(t *testing.T) {
assert.NotNil(t, org) assert.NotNil(t, org)
} }
func TestClient_CreateOrg_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateOrg_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()

View file

@ -1,12 +1,9 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"bytes" "bytes"
fmt "dappco.re/go/core/scm/internal/ax/fmtx" "encoding/json"
json "dappco.re/go/core/scm/internal/ax/jsonx" "fmt"
"iter"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
@ -18,7 +15,6 @@ import (
) )
// MergePullRequest merges a pull request with the given method ("squash", "rebase", "merge"). // MergePullRequest merges a pull request with the given method ("squash", "rebase", "merge").
// Usage: MergePullRequest(...)
func (c *Client) MergePullRequest(owner, repo string, index int64, method string) error { func (c *Client) MergePullRequest(owner, repo string, index int64, method string) error {
style := forgejo.MergeStyleMerge style := forgejo.MergeStyleMerge
switch method { switch method {
@ -44,7 +40,6 @@ func (c *Client) MergePullRequest(owner, repo string, index int64, method string
// SetPRDraft sets or clears the draft status on a pull request. // SetPRDraft sets or clears the draft status on a pull request.
// The Forgejo SDK v2.2.0 doesn't expose the draft field on EditPullRequestOption, // The Forgejo SDK v2.2.0 doesn't expose the draft field on EditPullRequestOption,
// so we use a raw HTTP PATCH request. // so we use a raw HTTP PATCH request.
// Usage: SetPRDraft(...)
func (c *Client) SetPRDraft(owner, repo string, index int64, draft bool) error { func (c *Client) SetPRDraft(owner, repo string, index int64, draft bool) error {
safeOwner, err := agentci.ValidatePathElement(owner) safeOwner, err := agentci.ValidatePathElement(owner)
if err != nil { if err != nil {
@ -86,7 +81,6 @@ func (c *Client) SetPRDraft(owner, repo string, index int64, draft bool) error {
} }
// ListPRReviews returns all reviews for a pull request. // ListPRReviews returns all reviews for a pull request.
// Usage: ListPRReviews(...)
func (c *Client) ListPRReviews(owner, repo string, index int64) ([]*forgejo.PullReview, error) { func (c *Client) ListPRReviews(owner, repo string, index int64) ([]*forgejo.PullReview, error) {
var all []*forgejo.PullReview var all []*forgejo.PullReview
page := 1 page := 1
@ -110,35 +104,7 @@ func (c *Client) ListPRReviews(owner, repo string, index int64) ([]*forgejo.Pull
return all, nil return all, nil
} }
// ListPRReviewsIter returns an iterator over reviews for a pull request.
// Usage: ListPRReviewsIter(...)
func (c *Client) ListPRReviewsIter(owner, repo string, index int64) iter.Seq2[*forgejo.PullReview, error] {
return func(yield func(*forgejo.PullReview, error) bool) {
page := 1
for {
reviews, resp, err := c.api.ListPullReviews(owner, repo, index, forgejo.ListPullReviewsOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListPRReviews", "failed to list reviews", err))
return
}
for _, review := range reviews {
if !yield(review, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}
// GetCombinedStatus returns the combined commit status for a ref (SHA or branch). // GetCombinedStatus returns the combined commit status for a ref (SHA or branch).
// Usage: GetCombinedStatus(...)
func (c *Client) GetCombinedStatus(owner, repo string, ref string) (*forgejo.CombinedStatus, error) { func (c *Client) GetCombinedStatus(owner, repo string, ref string) (*forgejo.CombinedStatus, error) {
status, _, err := c.api.GetCombinedStatus(owner, repo, ref) status, _, err := c.api.GetCombinedStatus(owner, repo, ref)
if err != nil { if err != nil {
@ -148,7 +114,6 @@ func (c *Client) GetCombinedStatus(owner, repo string, ref string) (*forgejo.Com
} }
// DismissReview dismisses a pull request review by ID. // DismissReview dismisses a pull request review by ID.
// Usage: DismissReview(...)
func (c *Client) DismissReview(owner, repo string, index, reviewID int64, message string) error { func (c *Client) DismissReview(owner, repo string, index, reviewID int64, message string) error {
_, err := c.api.DismissPullReview(owner, repo, index, reviewID, forgejo.DismissPullReviewOptions{ _, err := c.api.DismissPullReview(owner, repo, index, reviewID, forgejo.DismissPullReviewOptions{
Message: message, Message: message,
@ -158,13 +123,3 @@ func (c *Client) DismissReview(owner, repo string, index, reviewID int64, messag
} }
return nil return nil
} }
// UndismissReview removes a dismissal from a pull request review.
// Usage: UndismissReview(...)
func (c *Client) UndismissReview(owner, repo string, index, reviewID int64) error {
_, err := c.api.UnDismissPullReview(owner, repo, index, reviewID)
if err != nil {
return log.E("forge.UndismissReview", "failed to undismiss review", err)
}
return nil
}

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -21,7 +19,7 @@ func TestClient_MergePullRequest_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_MergePullRequest_Good_Squash_Good(t *testing.T) { func TestClient_MergePullRequest_Good_Squash(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -29,7 +27,7 @@ func TestClient_MergePullRequest_Good_Squash_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_MergePullRequest_Good_Rebase_Good(t *testing.T) { func TestClient_MergePullRequest_Good_Rebase(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -37,7 +35,7 @@ func TestClient_MergePullRequest_Good_Rebase_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_MergePullRequest_Bad_ServerError_Good(t *testing.T) { func TestClient_MergePullRequest_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -60,43 +58,7 @@ func TestClient_ListPRReviews_Good(t *testing.T) {
require.Len(t, reviews, 1) require.Len(t, reviews, 1)
} }
func TestClient_ListPRReviewsIter_Good_Paginates_Good(t *testing.T) { func TestClient_ListPRReviews_Bad_ServerError(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/pulls/1/reviews", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "state": "REQUEST_CHANGES", "user": map[string]any{"login": "reviewer2"}},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/pulls/1/reviews?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/pulls/1/reviews?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "state": "APPROVED", "user": map[string]any{"login": "reviewer1"}},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var states []string
for review, err := range client.ListPRReviewsIter("test-org", "org-repo", 1) {
require.NoError(t, err)
states = append(states, string(review.State))
}
require.Equal(t, []string{"APPROVED", "REQUEST_CHANGES"}, states)
}
func TestClient_ListPRReviews_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -105,19 +67,6 @@ func TestClient_ListPRReviews_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to list reviews") assert.Contains(t, err.Error(), "failed to list reviews")
} }
func TestClient_ListPRReviewsIter_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
var got bool
for _, err := range client.ListPRReviewsIter("test-org", "org-repo", 1) {
assert.Error(t, err)
got = true
}
assert.True(t, got)
}
func TestClient_GetCombinedStatus_Good(t *testing.T) { func TestClient_GetCombinedStatus_Good(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -127,7 +76,7 @@ func TestClient_GetCombinedStatus_Good(t *testing.T) {
assert.NotNil(t, status) assert.NotNil(t, status)
} }
func TestClient_GetCombinedStatus_Bad_ServerError_Good(t *testing.T) { func TestClient_GetCombinedStatus_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -144,7 +93,7 @@ func TestClient_DismissReview_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_DismissReview_Bad_ServerError_Good(t *testing.T) { func TestClient_DismissReview_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -153,24 +102,7 @@ func TestClient_DismissReview_Bad_ServerError_Good(t *testing.T) {
assert.Contains(t, err.Error(), "failed to dismiss review") assert.Contains(t, err.Error(), "failed to dismiss review")
} }
func TestClient_UndismissReview_Good(t *testing.T) { func TestClient_SetPRDraft_Good_Request(t *testing.T) {
client, srv := newTestClient(t)
defer srv.Close()
err := client.UndismissReview("test-org", "org-repo", 1, 1)
require.NoError(t, err)
}
func TestClient_UndismissReview_Bad_ServerError_Good(t *testing.T) {
client, srv := newErrorServer(t)
defer srv.Close()
err := client.UndismissReview("test-org", "org-repo", 1, 1)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to undismiss review")
}
func TestClient_SetPRDraft_Good_Request_Good(t *testing.T) {
var method, path string var method, path string
var payload map[string]any var payload map[string]any
@ -198,7 +130,7 @@ func TestClient_SetPRDraft_Good_Request_Good(t *testing.T) {
assert.Equal(t, false, payload["draft"]) assert.Equal(t, false, payload["draft"])
} }
func TestClient_SetPRDraft_Bad_PathTraversalOwner_Good(t *testing.T) { func TestClient_SetPRDraft_Bad_PathTraversalOwner(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()
@ -207,7 +139,7 @@ func TestClient_SetPRDraft_Bad_PathTraversalOwner_Good(t *testing.T) {
assert.Contains(t, err.Error(), "invalid owner") assert.Contains(t, err.Error(), "invalid owner")
} }
func TestClient_SetPRDraft_Bad_PathTraversalRepo_Good(t *testing.T) { func TestClient_SetPRDraft_Bad_PathTraversalRepo(t *testing.T) {
client, srv := newTestClient(t) client, srv := newTestClient(t)
defer srv.Close() defer srv.Close()

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
@ -11,7 +9,6 @@ import (
) )
// ListOrgRepos returns all repositories for the given organisation. // ListOrgRepos returns all repositories for the given organisation.
// Usage: ListOrgRepos(...)
func (c *Client) ListOrgRepos(org string) ([]*forgejo.Repository, error) { func (c *Client) ListOrgRepos(org string) ([]*forgejo.Repository, error) {
var all []*forgejo.Repository var all []*forgejo.Repository
page := 1 page := 1
@ -36,7 +33,6 @@ func (c *Client) ListOrgRepos(org string) ([]*forgejo.Repository, error) {
} }
// ListOrgReposIter returns an iterator over repositories for the given organisation. // ListOrgReposIter returns an iterator over repositories for the given organisation.
// Usage: ListOrgReposIter(...)
func (c *Client) ListOrgReposIter(org string) iter.Seq2[*forgejo.Repository, error] { func (c *Client) ListOrgReposIter(org string) iter.Seq2[*forgejo.Repository, error] {
return func(yield func(*forgejo.Repository, error) bool) { return func(yield func(*forgejo.Repository, error) bool) {
page := 1 page := 1
@ -62,7 +58,6 @@ func (c *Client) ListOrgReposIter(org string) iter.Seq2[*forgejo.Repository, err
} }
// ListUserRepos returns all repositories for the authenticated user. // ListUserRepos returns all repositories for the authenticated user.
// Usage: ListUserRepos(...)
func (c *Client) ListUserRepos() ([]*forgejo.Repository, error) { func (c *Client) ListUserRepos() ([]*forgejo.Repository, error) {
var all []*forgejo.Repository var all []*forgejo.Repository
page := 1 page := 1
@ -87,7 +82,6 @@ func (c *Client) ListUserRepos() ([]*forgejo.Repository, error) {
} }
// ListUserReposIter returns an iterator over repositories for the authenticated user. // ListUserReposIter returns an iterator over repositories for the authenticated user.
// Usage: ListUserReposIter(...)
func (c *Client) ListUserReposIter() iter.Seq2[*forgejo.Repository, error] { func (c *Client) ListUserReposIter() iter.Seq2[*forgejo.Repository, error] {
return func(yield func(*forgejo.Repository, error) bool) { return func(yield func(*forgejo.Repository, error) bool) {
page := 1 page := 1
@ -113,7 +107,6 @@ func (c *Client) ListUserReposIter() iter.Seq2[*forgejo.Repository, error] {
} }
// GetRepo returns a single repository by owner and name. // GetRepo returns a single repository by owner and name.
// Usage: GetRepo(...)
func (c *Client) GetRepo(owner, name string) (*forgejo.Repository, error) { func (c *Client) GetRepo(owner, name string) (*forgejo.Repository, error) {
repo, _, err := c.api.GetRepo(owner, name) repo, _, err := c.api.GetRepo(owner, name)
if err != nil { if err != nil {
@ -124,7 +117,6 @@ func (c *Client) GetRepo(owner, name string) (*forgejo.Repository, error) {
} }
// CreateOrgRepo creates a new empty repository under an organisation. // CreateOrgRepo creates a new empty repository under an organisation.
// Usage: CreateOrgRepo(...)
func (c *Client) CreateOrgRepo(org string, opts forgejo.CreateRepoOption) (*forgejo.Repository, error) { func (c *Client) CreateOrgRepo(org string, opts forgejo.CreateRepoOption) (*forgejo.Repository, error) {
repo, _, err := c.api.CreateOrgRepo(org, opts) repo, _, err := c.api.CreateOrgRepo(org, opts)
if err != nil { if err != nil {
@ -135,7 +127,6 @@ func (c *Client) CreateOrgRepo(org string, opts forgejo.CreateRepoOption) (*forg
} }
// DeleteRepo deletes a repository from Forgejo. // DeleteRepo deletes a repository from Forgejo.
// Usage: DeleteRepo(...)
func (c *Client) DeleteRepo(owner, name string) error { func (c *Client) DeleteRepo(owner, name string) error {
_, err := c.api.DeleteRepo(owner, name) _, err := c.api.DeleteRepo(owner, name)
if err != nil { if err != nil {
@ -147,7 +138,6 @@ func (c *Client) DeleteRepo(owner, name string) error {
// MigrateRepo migrates a repository from an external service using the Forgejo migration API. // MigrateRepo migrates a repository from an external service using the Forgejo migration API.
// Unlike CreateMirror, this supports importing issues, labels, PRs, and more. // Unlike CreateMirror, this supports importing issues, labels, PRs, and more.
// Usage: MigrateRepo(...)
func (c *Client) MigrateRepo(opts forgejo.MigrateRepoOption) (*forgejo.Repository, error) { func (c *Client) MigrateRepo(opts forgejo.MigrateRepoOption) (*forgejo.Repository, error) {
repo, _, err := c.api.MigrateRepo(opts) repo, _, err := c.api.MigrateRepo(opts)
if err != nil { if err != nil {

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
@ -17,12 +15,11 @@ func TestClient_ListOrgRepos_Good(t *testing.T) {
repos, err := client.ListOrgRepos("test-org") repos, err := client.ListOrgRepos("test-org")
require.NoError(t, err) require.NoError(t, err)
require.Len(t, repos, 2) require.Len(t, repos, 1)
assert.Equal(t, "org-repo", repos[0].Name) assert.Equal(t, "org-repo", repos[0].Name)
assert.Equal(t, "second-repo", repos[1].Name)
} }
func TestClient_ListOrgRepos_Bad_ServerError_Good(t *testing.T) { func TestClient_ListOrgRepos_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -42,7 +39,7 @@ func TestClient_ListUserRepos_Good(t *testing.T) {
assert.Equal(t, "repo-b", repos[1].Name) assert.Equal(t, "repo-b", repos[1].Name)
} }
func TestClient_ListUserRepos_Bad_ServerError_Good(t *testing.T) { func TestClient_ListUserRepos_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -60,7 +57,7 @@ func TestClient_GetRepo_Good(t *testing.T) {
assert.Equal(t, "org-repo", repo.Name) assert.Equal(t, "org-repo", repo.Name)
} }
func TestClient_GetRepo_Bad_ServerError_Good(t *testing.T) { func TestClient_GetRepo_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -81,7 +78,7 @@ func TestClient_CreateOrgRepo_Good(t *testing.T) {
assert.NotNil(t, repo) assert.NotNil(t, repo)
} }
func TestClient_CreateOrgRepo_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateOrgRepo_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -100,7 +97,7 @@ func TestClient_DeleteRepo_Good(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
func TestClient_DeleteRepo_Bad_ServerError_Good(t *testing.T) { func TestClient_DeleteRepo_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -122,7 +119,7 @@ func TestClient_MigrateRepo_Good(t *testing.T) {
assert.NotNil(t, repo) assert.NotNil(t, repo)
} }
func TestClient_MigrateRepo_Bad_ServerError_Good(t *testing.T) { func TestClient_MigrateRepo_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()

View file

@ -1,12 +1,10 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
json "dappco.re/go/core/scm/internal/ax/jsonx" "encoding/json"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"strings"
"testing" "testing"
) )
@ -58,7 +56,6 @@ func newForgejoMux() *http.ServeMux {
} }
jsonResponse(w, []map[string]any{ jsonResponse(w, []map[string]any{
{"id": 10, "name": "org-repo", "full_name": "test-org/org-repo", "owner": map[string]any{"login": "test-org", "id": 100}}, {"id": 10, "name": "org-repo", "full_name": "test-org/org-repo", "owner": map[string]any{"login": "test-org", "id": 100}},
{"id": 11, "name": "second-repo", "full_name": "test-org/second-repo", "owner": map[string]any{"login": "test-org", "id": 100}},
}) })
}) })
@ -84,7 +81,6 @@ func newForgejoMux() *http.ServeMux {
jsonResponse(w, map[string]any{ jsonResponse(w, map[string]any{
"id": 10, "name": "org-repo", "full_name": "test-org/org-repo", "id": 10, "name": "org-repo", "full_name": "test-org/org-repo",
"owner": map[string]any{"login": "test-org"}, "owner": map[string]any{"login": "test-org"},
"default_branch": "main",
}) })
}) })
@ -230,13 +226,6 @@ func newForgejoMux() *http.ServeMux {
}) })
}) })
mux.HandleFunc("/api/v1/repos/test-org/second-repo/labels", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, []map[string]any{
{"id": 2, "name": "feature", "color": "#0000ff"},
{"id": 3, "name": "documentation", "color": "#00aa00"},
})
})
// Webhooks. // Webhooks.
mux.HandleFunc("/api/v1/repos/test-org/org-repo/hooks", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/api/v1/repos/test-org/org-repo/hooks", func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost { if r.Method == http.MethodPost {
@ -304,13 +293,6 @@ func newForgejoMux() *http.ServeMux {
}) })
}) })
// Undismiss review.
mux.HandleFunc("/api/v1/repos/test-org/org-repo/pulls/1/reviews/1/undismissals", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]any{
"id": 1, "state": "open",
})
})
// Generic fallback — handles PATCH for SetPRDraft and other unmatched routes. // Generic fallback — handles PATCH for SetPRDraft and other unmatched routes.
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
// Handle PATCH requests (SetPRDraft). // Handle PATCH requests (SetPRDraft).

View file

@ -1,17 +1,12 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"iter"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
"dappco.re/go/core/log" "dappco.re/go/core/log"
) )
// CreateRepoWebhook creates a webhook on a repository. // CreateRepoWebhook creates a webhook on a repository.
// Usage: CreateRepoWebhook(...)
func (c *Client) CreateRepoWebhook(owner, repo string, opts forgejo.CreateHookOption) (*forgejo.Hook, error) { func (c *Client) CreateRepoWebhook(owner, repo string, opts forgejo.CreateHookOption) (*forgejo.Hook, error) {
hook, _, err := c.api.CreateRepoHook(owner, repo, opts) hook, _, err := c.api.CreateRepoHook(owner, repo, opts)
if err != nil { if err != nil {
@ -22,7 +17,6 @@ func (c *Client) CreateRepoWebhook(owner, repo string, opts forgejo.CreateHookOp
} }
// ListRepoWebhooks returns all webhooks for a repository. // ListRepoWebhooks returns all webhooks for a repository.
// Usage: ListRepoWebhooks(...)
func (c *Client) ListRepoWebhooks(owner, repo string) ([]*forgejo.Hook, error) { func (c *Client) ListRepoWebhooks(owner, repo string) ([]*forgejo.Hook, error) {
var all []*forgejo.Hook var all []*forgejo.Hook
page := 1 page := 1
@ -45,29 +39,3 @@ func (c *Client) ListRepoWebhooks(owner, repo string) ([]*forgejo.Hook, error) {
return all, nil return all, nil
} }
// ListRepoWebhooksIter returns an iterator over webhooks for a repository.
// Usage: ListRepoWebhooksIter(...)
func (c *Client) ListRepoWebhooksIter(owner, repo string) iter.Seq2[*forgejo.Hook, error] {
return func(yield func(*forgejo.Hook, error) bool) {
page := 1
for {
hooks, resp, err := c.api.ListRepoHooks(owner, repo, forgejo.ListHooksOptions{
ListOptions: forgejo.ListOptions{Page: page, PageSize: 50},
})
if err != nil {
yield(nil, log.E("forge.ListRepoWebhooks", "failed to list repo webhooks", err))
return
}
for _, hook := range hooks {
if !yield(hook, nil) {
return
}
}
if resp == nil || page >= resp.LastPage {
break
}
page++
}
}
}

View file

@ -1,10 +1,6 @@
// SPDX-License-Identifier: EUPL-1.2
package forge package forge
import ( import (
"net/http"
"net/http/httptest"
"testing" "testing"
forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2" forgejo "codeberg.org/mvdkleijn/forgejo-sdk/forgejo/v2"
@ -27,7 +23,7 @@ func TestClient_CreateRepoWebhook_Good(t *testing.T) {
assert.NotNil(t, hook) assert.NotNil(t, hook)
} }
func TestClient_CreateRepoWebhook_Bad_ServerError_Good(t *testing.T) { func TestClient_CreateRepoWebhook_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -47,7 +43,7 @@ func TestClient_ListRepoWebhooks_Good(t *testing.T) {
require.Len(t, hooks, 1) require.Len(t, hooks, 1)
} }
func TestClient_ListRepoWebhooks_Bad_ServerError_Good(t *testing.T) { func TestClient_ListRepoWebhooks_Bad_ServerError(t *testing.T) {
client, srv := newErrorServer(t) client, srv := newErrorServer(t)
defer srv.Close() defer srv.Close()
@ -55,40 +51,3 @@ func TestClient_ListRepoWebhooks_Bad_ServerError_Good(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to list repo webhooks") assert.Contains(t, err.Error(), "failed to list repo webhooks")
} }
func TestClient_ListRepoWebhooksIter_Good_Paginates_Good(t *testing.T) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/version", func(w http.ResponseWriter, r *http.Request) {
jsonResponse(w, map[string]string{"version": "1.21.0"})
})
mux.HandleFunc("/api/v1/repos/test-org/org-repo/hooks", func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Query().Get("page") {
case "2":
jsonResponse(w, []map[string]any{
{"id": 2, "type": "forgejo", "active": true, "config": map[string]any{"url": "https://example.com/second"}},
})
case "3":
jsonResponse(w, []map[string]any{})
default:
w.Header().Set("Link", "<http://"+r.Host+"/api/v1/repos/test-org/org-repo/hooks?page=2>; rel=\"next\", <http://"+r.Host+"/api/v1/repos/test-org/org-repo/hooks?page=2>; rel=\"last\"")
jsonResponse(w, []map[string]any{
{"id": 1, "type": "forgejo", "active": true, "config": map[string]any{"url": "https://example.com/hook"}},
})
}
})
srv := httptest.NewServer(mux)
defer srv.Close()
client, err := New(srv.URL, "test-token")
require.NoError(t, err)
var urls []string
for hook, err := range client.ListRepoWebhooksIter("test-org", "org-repo") {
require.NoError(t, err)
urls = append(urls, hook.Config["url"])
}
require.Len(t, urls, 2)
assert.Equal(t, []string{"https://example.com/hook", "https://example.com/second"}, urls)
}

View file

@ -1,18 +1,16 @@
// SPDX-License-Identifier: EUPL-1.2
// Package git provides utilities for git operations across multiple repositories. // Package git provides utilities for git operations across multiple repositories.
package git package git
import ( import (
"bytes" "bytes"
"context" "context"
os "dappco.re/go/core/scm/internal/ax/osx"
strings "dappco.re/go/core/scm/internal/ax/stringsx"
exec "golang.org/x/sys/execabs"
"io" "io"
"iter" "iter"
"os"
"os/exec"
"slices" "slices"
"strconv" "strconv"
"strings"
"sync" "sync"
) )
@ -30,19 +28,16 @@ type RepoStatus struct {
} }
// IsDirty returns true if there are uncommitted changes. // IsDirty returns true if there are uncommitted changes.
// Usage: IsDirty(...)
func (s *RepoStatus) IsDirty() bool { func (s *RepoStatus) IsDirty() bool {
return s.Modified > 0 || s.Untracked > 0 || s.Staged > 0 return s.Modified > 0 || s.Untracked > 0 || s.Staged > 0
} }
// HasUnpushed returns true if there are commits to push. // HasUnpushed returns true if there are commits to push.
// Usage: HasUnpushed(...)
func (s *RepoStatus) HasUnpushed() bool { func (s *RepoStatus) HasUnpushed() bool {
return s.Ahead > 0 return s.Ahead > 0
} }
// HasUnpulled returns true if there are commits to pull. // HasUnpulled returns true if there are commits to pull.
// Usage: HasUnpulled(...)
func (s *RepoStatus) HasUnpulled() bool { func (s *RepoStatus) HasUnpulled() bool {
return s.Behind > 0 return s.Behind > 0
} }
@ -56,7 +51,6 @@ type StatusOptions struct {
} }
// Status checks git status for multiple repositories in parallel. // Status checks git status for multiple repositories in parallel.
// Usage: Status(...)
func Status(ctx context.Context, opts StatusOptions) []RepoStatus { func Status(ctx context.Context, opts StatusOptions) []RepoStatus {
var wg sync.WaitGroup var wg sync.WaitGroup
results := make([]RepoStatus, len(opts.Paths)) results := make([]RepoStatus, len(opts.Paths))
@ -78,7 +72,6 @@ func Status(ctx context.Context, opts StatusOptions) []RepoStatus {
} }
// StatusIter returns an iterator over git status for multiple repositories. // StatusIter returns an iterator over git status for multiple repositories.
// Usage: StatusIter(...)
func StatusIter(ctx context.Context, opts StatusOptions) iter.Seq[RepoStatus] { func StatusIter(ctx context.Context, opts StatusOptions) iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) { return func(yield func(RepoStatus) bool) {
results := Status(ctx, opts) results := Status(ctx, opts)
@ -163,20 +156,17 @@ func getAheadBehind(ctx context.Context, path string) (ahead, behind int) {
// Push pushes commits for a single repository. // Push pushes commits for a single repository.
// Uses interactive mode to support SSH passphrase prompts. // Uses interactive mode to support SSH passphrase prompts.
// Usage: Push(...)
func Push(ctx context.Context, path string) error { func Push(ctx context.Context, path string) error {
return gitInteractive(ctx, path, "push") return gitInteractive(ctx, path, "push")
} }
// Pull pulls changes for a single repository. // Pull pulls changes for a single repository.
// Uses interactive mode to support SSH passphrase prompts. // Uses interactive mode to support SSH passphrase prompts.
// Usage: Pull(...)
func Pull(ctx context.Context, path string) error { func Pull(ctx context.Context, path string) error {
return gitInteractive(ctx, path, "pull", "--rebase") return gitInteractive(ctx, path, "pull", "--rebase")
} }
// IsNonFastForward checks if an error is a non-fast-forward rejection. // IsNonFastForward checks if an error is a non-fast-forward rejection.
// Usage: IsNonFastForward(...)
func IsNonFastForward(err error) bool { func IsNonFastForward(err error) bool {
if err == nil { if err == nil {
return false return false
@ -220,13 +210,11 @@ type PushResult struct {
// PushMultiple pushes multiple repositories sequentially. // PushMultiple pushes multiple repositories sequentially.
// Sequential because SSH passphrase prompts need user interaction. // Sequential because SSH passphrase prompts need user interaction.
// Usage: PushMultiple(...)
func PushMultiple(ctx context.Context, paths []string, names map[string]string) []PushResult { func PushMultiple(ctx context.Context, paths []string, names map[string]string) []PushResult {
return slices.Collect(PushMultipleIter(ctx, paths, names)) return slices.Collect(PushMultipleIter(ctx, paths, names))
} }
// PushMultipleIter returns an iterator that pushes repositories sequentially and yields results. // PushMultipleIter returns an iterator that pushes repositories sequentially and yields results.
// Usage: PushMultipleIter(...)
func PushMultipleIter(ctx context.Context, paths []string, names map[string]string) iter.Seq[PushResult] { func PushMultipleIter(ctx context.Context, paths []string, names map[string]string) iter.Seq[PushResult] {
return func(yield func(PushResult) bool) { return func(yield func(PushResult) bool) {
for _, path := range paths { for _, path := range paths {
@ -281,7 +269,6 @@ type GitError struct {
} }
// Error returns the git error message, preferring stderr output. // Error returns the git error message, preferring stderr output.
// Usage: Error(...)
func (e *GitError) Error() string { func (e *GitError) Error() string {
// Return just the stderr message, trimmed // Return just the stderr message, trimmed
msg := strings.TrimSpace(e.Stderr) msg := strings.TrimSpace(e.Stderr)
@ -292,7 +279,6 @@ func (e *GitError) Error() string {
} }
// Unwrap returns the underlying error for error chain inspection. // Unwrap returns the underlying error for error chain inspection.
// Usage: Unwrap(...)
func (e *GitError) Unwrap() error { func (e *GitError) Unwrap() error {
return e.Err return e.Err
} }

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
package git package git
import ( import (
@ -56,7 +54,6 @@ type Service struct {
} }
// NewService creates a git service factory. // NewService creates a git service factory.
// Usage: NewService(...)
func NewService(opts ServiceOptions) func(*core.Core) (any, error) { func NewService(opts ServiceOptions) func(*core.Core) (any, error) {
return func(c *core.Core) (any, error) { return func(c *core.Core) (any, error) {
return &Service{ return &Service{
@ -66,7 +63,6 @@ func NewService(opts ServiceOptions) func(*core.Core) (any, error) {
} }
// OnStartup registers query and task handlers. // OnStartup registers query and task handlers.
// Usage: OnStartup(...)
func (s *Service) OnStartup(ctx context.Context) error { func (s *Service) OnStartup(ctx context.Context) error {
s.Core().RegisterQuery(s.handleQuery) s.Core().RegisterQuery(s.handleQuery)
s.Core().RegisterTask(s.handleTask) s.Core().RegisterTask(s.handleTask)
@ -105,17 +101,14 @@ func (s *Service) handleTask(c *core.Core, t core.Task) core.Result {
} }
// Status returns last status result. // Status returns last status result.
// Usage: Status(...)
func (s *Service) Status() []RepoStatus { return s.lastStatus } func (s *Service) Status() []RepoStatus { return s.lastStatus }
// StatusIter returns an iterator over last status result. // StatusIter returns an iterator over last status result.
// Usage: StatusIter(...)
func (s *Service) StatusIter() iter.Seq[RepoStatus] { func (s *Service) StatusIter() iter.Seq[RepoStatus] {
return slices.Values(s.lastStatus) return slices.Values(s.lastStatus)
} }
// DirtyRepos returns repos with uncommitted changes. // DirtyRepos returns repos with uncommitted changes.
// Usage: DirtyRepos(...)
func (s *Service) DirtyRepos() []RepoStatus { func (s *Service) DirtyRepos() []RepoStatus {
var dirty []RepoStatus var dirty []RepoStatus
for _, st := range s.lastStatus { for _, st := range s.lastStatus {
@ -127,7 +120,6 @@ func (s *Service) DirtyRepos() []RepoStatus {
} }
// DirtyReposIter returns an iterator over repos with uncommitted changes. // DirtyReposIter returns an iterator over repos with uncommitted changes.
// Usage: DirtyReposIter(...)
func (s *Service) DirtyReposIter() iter.Seq[RepoStatus] { func (s *Service) DirtyReposIter() iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) { return func(yield func(RepoStatus) bool) {
for _, st := range s.lastStatus { for _, st := range s.lastStatus {
@ -141,7 +133,6 @@ func (s *Service) DirtyReposIter() iter.Seq[RepoStatus] {
} }
// AheadRepos returns repos with unpushed commits. // AheadRepos returns repos with unpushed commits.
// Usage: AheadRepos(...)
func (s *Service) AheadRepos() []RepoStatus { func (s *Service) AheadRepos() []RepoStatus {
var ahead []RepoStatus var ahead []RepoStatus
for _, st := range s.lastStatus { for _, st := range s.lastStatus {
@ -153,7 +144,6 @@ func (s *Service) AheadRepos() []RepoStatus {
} }
// AheadReposIter returns an iterator over repos with unpushed commits. // AheadReposIter returns an iterator over repos with unpushed commits.
// Usage: AheadReposIter(...)
func (s *Service) AheadReposIter() iter.Seq[RepoStatus] { func (s *Service) AheadReposIter() iter.Seq[RepoStatus] {
return func(yield func(RepoStatus) bool) { return func(yield func(RepoStatus) bool) {
for _, st := range s.lastStatus { for _, st := range s.lastStatus {

View file

@ -1,5 +1,3 @@
// SPDX-License-Identifier: EUPL-1.2
// Package gitea provides a thin wrapper around the Gitea Go SDK // Package gitea provides a thin wrapper around the Gitea Go SDK
// for managing repositories, issues, and pull requests on a Gitea instance. // for managing repositories, issues, and pull requests on a Gitea instance.
// //
@ -20,39 +18,20 @@ import (
type Client struct { type Client struct {
api *gitea.Client api *gitea.Client
url string url string
token string
} }
// New creates a new Gitea API client for the given URL and token. // New creates a new Gitea API client for the given URL and token.
// Usage: New(...)
func New(url, token string) (*Client, error) { func New(url, token string) (*Client, error) {
api, err := gitea.NewClient(url, gitea.SetToken(token)) api, err := gitea.NewClient(url, gitea.SetToken(token))
if err != nil { if err != nil {
return nil, log.E("gitea.New", "failed to create client", err) return nil, log.E("gitea.New", "failed to create client", err)
} }
return &Client{api: api, url: url, token: token}, nil return &Client{api: api, url: url}, nil
} }
// API exposes the underlying SDK client for direct access. // API exposes the underlying SDK client for direct access.
// Usage: API(...)
func (c *Client) API() *gitea.Client { return c.api } func (c *Client) API() *gitea.Client { return c.api }
// URL returns the Gitea instance URL. // URL returns the Gitea instance URL.
// Usage: URL(...)
func (c *Client) URL() string { return c.url } func (c *Client) URL() string { return c.url }
// Token returns the Gitea API token.
// Usage: Token(...)
func (c *Client) Token() string { return c.token }
// GetCurrentUser returns the authenticated user's information.
// Usage: GetCurrentUser(...)
func (c *Client) GetCurrentUser() (*gitea.User, error) {
user, _, err := c.api.GetMyUserInfo()
if err != nil {
return nil, log.E("gitea.GetCurrentUser", "failed to get current user", err)
}
return user, nil
}

Some files were not shown because too many files have changed in this diff Show more