f2bc912ebe
* ci: consolidate duplicate workflows and merge CodeQL configs Remove 17 duplicate workflow files that were split copies of the combined originals. Each family (CI, CodeQL, Coverage, PR Build, Alpha Release) had the same job duplicated across separate push/pull_request/schedule/manual trigger files. Merge codeql.yml and codescan.yml into a single codeql.yml with a language matrix covering go, javascript-typescript, python, and actions — matching the previous default setup coverage. Remaining workflows (one per family): - ci.yml (push + PR + manual) - codeql.yml (push + PR + schedule, all languages) - coverage.yml (push + PR + manual) - alpha-release.yml (push + manual) - pr-build.yml (PR + manual) - release.yml (tag push) - agent-verify.yml, auto-label.yml, auto-project.yml Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> * feat: add collect, config, crypt, plugin packages and fix all lint issues Add four new infrastructure packages with CLI commands: - pkg/config: layered configuration (defaults → file → env → flags) - pkg/crypt: crypto primitives (Argon2id, AES-GCM, ChaCha20, HMAC, checksums) - pkg/plugin: plugin system with GitHub-based install/update/remove - pkg/collect: collection subsystem (GitHub, BitcoinTalk, market, papers, excavate) Fix all golangci-lint issues across the entire codebase (~100 errcheck, staticcheck SA1012/SA1019/ST1005, unused, ineffassign fixes) so that `core go qa` passes with 0 issues. Closes #167, #168, #170, #250, #251, #252, #253, #254, #255, #256 Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
103 lines
2.7 KiB
Go
103 lines
2.7 KiB
Go
package collect
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/host-uk/core/pkg/cli"
|
|
"github.com/host-uk/core/pkg/collect"
|
|
"github.com/host-uk/core/pkg/i18n"
|
|
)
|
|
|
|
// Excavate command flags
|
|
var (
|
|
excavateScanOnly bool
|
|
excavateResume bool
|
|
)
|
|
|
|
// addExcavateCommand adds the 'excavate' subcommand to the collect parent.
|
|
func addExcavateCommand(parent *cli.Command) {
|
|
excavateCmd := &cli.Command{
|
|
Use: "excavate <project>",
|
|
Short: i18n.T("cmd.collect.excavate.short"),
|
|
Long: i18n.T("cmd.collect.excavate.long"),
|
|
Args: cli.ExactArgs(1),
|
|
RunE: func(cmd *cli.Command, args []string) error {
|
|
return runExcavate(args[0])
|
|
},
|
|
}
|
|
|
|
cli.BoolFlag(excavateCmd, &excavateScanOnly, "scan-only", "", false, i18n.T("cmd.collect.excavate.flag.scan_only"))
|
|
cli.BoolFlag(excavateCmd, &excavateResume, "resume", "r", false, i18n.T("cmd.collect.excavate.flag.resume"))
|
|
|
|
parent.AddCommand(excavateCmd)
|
|
}
|
|
|
|
func runExcavate(project string) error {
|
|
cfg := newConfig()
|
|
setupVerboseLogging(cfg)
|
|
|
|
// Load state for resume
|
|
if excavateResume {
|
|
if err := cfg.State.Load(); err != nil {
|
|
return cli.Wrap(err, "failed to load collection state")
|
|
}
|
|
}
|
|
|
|
// Build collectors for the project
|
|
collectors := buildProjectCollectors(project)
|
|
if len(collectors) == 0 {
|
|
return cli.Err("no collectors configured for project: %s", project)
|
|
}
|
|
|
|
excavator := &collect.Excavator{
|
|
Collectors: collectors,
|
|
ScanOnly: excavateScanOnly,
|
|
Resume: excavateResume,
|
|
}
|
|
|
|
if cfg.DryRun {
|
|
cli.Info(fmt.Sprintf("Dry run: would excavate project %s with %d collectors", project, len(collectors)))
|
|
for _, c := range collectors {
|
|
cli.Dim(fmt.Sprintf(" - %s", c.Name()))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
ctx := context.Background()
|
|
result, err := excavator.Run(ctx, cfg)
|
|
if err != nil {
|
|
return cli.Wrap(err, "excavation failed")
|
|
}
|
|
|
|
// Save state for future resume
|
|
if err := cfg.State.Save(); err != nil {
|
|
cli.Warnf("Failed to save state: %v", err)
|
|
}
|
|
|
|
printResult(result)
|
|
return nil
|
|
}
|
|
|
|
// buildProjectCollectors creates collectors based on the project name.
|
|
// This maps known project names to their collector configurations.
|
|
func buildProjectCollectors(project string) []collect.Collector {
|
|
switch project {
|
|
case "bitcoin":
|
|
return []collect.Collector{
|
|
&collect.GitHubCollector{Org: "bitcoin", Repo: "bitcoin"},
|
|
&collect.MarketCollector{CoinID: "bitcoin", Historical: true},
|
|
}
|
|
case "ethereum":
|
|
return []collect.Collector{
|
|
&collect.GitHubCollector{Org: "ethereum", Repo: "go-ethereum"},
|
|
&collect.MarketCollector{CoinID: "ethereum", Historical: true},
|
|
&collect.PapersCollector{Source: "all", Query: "ethereum"},
|
|
}
|
|
default:
|
|
// Treat unknown projects as GitHub org/repo
|
|
return []collect.Collector{
|
|
&collect.GitHubCollector{Org: project},
|
|
}
|
|
}
|
|
}
|