test(cli): add artifact validation harnesses

This commit is contained in:
Virgil 2026-03-30 11:45:35 +00:00
parent 4a6f59b6fc
commit 140d2b0583
61 changed files with 703 additions and 0 deletions

53
tests/cli/_lib/run.sh Executable file
View file

@ -0,0 +1,53 @@
#!/usr/bin/env bash
run_capture_stdout() {
local expected_status="$1"
local output_file="$2"
shift 2
set +e
"$@" >"$output_file"
local status=$?
set -e
if [[ "$status" -ne "$expected_status" ]]; then
printf 'expected exit %s, got %s\n' "$expected_status" "$status" >&2
if [[ -s "$output_file" ]]; then
printf 'stdout:\n' >&2
cat "$output_file" >&2
fi
return 1
fi
}
run_capture_all() {
local expected_status="$1"
local output_file="$2"
shift 2
set +e
"$@" >"$output_file" 2>&1
local status=$?
set -e
if [[ "$status" -ne "$expected_status" ]]; then
printf 'expected exit %s, got %s\n' "$expected_status" "$status" >&2
if [[ -s "$output_file" ]]; then
printf 'output:\n' >&2
cat "$output_file" >&2
fi
return 1
fi
}
assert_jq() {
local expression="$1"
local input_file="$2"
jq -e "$expression" "$input_file" >/dev/null
}
assert_contains() {
local needle="$1"
local input_file="$2"
grep -Fq "$needle" "$input_file"
}

View file

@ -0,0 +1,18 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core-lint ../../../../../cmd/core-lint
lang="$(cat fixtures/lang.txt)"
output="$(mktemp)"
run_capture_all 0 "$output" ./bin/core-lint lint catalog list --lang "$lang"
grep -Fq "go-sec-001" "$output"
grep -Fq "rule(s)" "$output"
EOF

View file

@ -0,0 +1 @@
go

View file

@ -0,0 +1,18 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core-lint ../../../../../cmd/core-lint
rule_id="$(cat fixtures/rule-id.txt)"
output="$(mktemp)"
run_capture_stdout 0 "$output" ./bin/core-lint lint catalog show "$rule_id"
jq -e '.id == "go-sec-001" and .severity == "high" and (.languages | index("go") != null)' "$output" >/dev/null
jq -e '.title == "SQL wildcard injection in LIKE clauses"' "$output" >/dev/null
EOF

View file

@ -0,0 +1 @@
go-sec-001

View file

@ -0,0 +1,17 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core-lint ../../../../cmd/core-lint
output="$(mktemp)"
run_capture_stdout 0 "$output" ./bin/core-lint lint check --format=json fixtures
jq -e 'length == 1 and .[0].rule_id == "go-cor-003" and .[0].file == "input.go"' "$output" >/dev/null
jq -e '.[0].severity == "medium" and .[0].fix != ""' "$output" >/dev/null
EOF

View file

@ -0,0 +1,12 @@
//go:build ignore
package sample
type service struct{}
func (service) Process(string) error { return nil }
func Run() {
svc := service{}
_ = svc.Process("data")
}

View file

@ -0,0 +1,11 @@
package main
import (
"forge.lthn.ai/core/cli/pkg/cli"
_ "forge.lthn.ai/core/lint/cmd/qa"
)
func main() {
cli.WithAppName("core")
cli.Main()
}

View file

@ -0,0 +1,20 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
export PATH="$(pwd)/../bin:$PATH"
run_capture_stdout 1 "$output" ../../bin/core qa audit --json
jq -e '.results[0].tool == "composer" and .results[0].vulnerabilities == 1' "$output" >/dev/null
jq -e '.has_vulnerabilities == true and .vulnerabilities == 1' "$output" >/dev/null
jq -e '.results[0].advisories[0].Package == "vendor/package-a"' "$output" >/dev/null
EOF

View file

@ -0,0 +1,17 @@
#!/usr/bin/env sh
cat <<'JSON'
{
"advisories": {
"vendor/package-a": [
{
"title": "Remote Code Execution",
"link": "https://example.com/advisory/1",
"cve": "CVE-2026-0001",
"affectedVersions": ">=1.0,<1.5"
}
]
}
}
JSON
exit 1

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return "bad";
}

View file

@ -0,0 +1,18 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
output="$(mktemp)"
run_capture_stdout 1 "$output" ./bin/core qa docblock --json --threshold 100 fixtures/src
jq -e '(.passed == false) and (.coverage < .threshold)' "$output" >/dev/null
jq -e '(.missing | length == 1) and (.missing[0].name == "Beta")' "$output" >/dev/null
jq -e '(.warnings | length == 1) and (.warnings[0].path == "fixtures/src")' "$output" >/dev/null
EOF

View file

@ -0,0 +1,6 @@
//go:build ignore
package sample
// Alpha demonstrates a documented exported function.
func Alpha() {}

View file

@ -0,0 +1,5 @@
//go:build ignore
package sample
func Beta() {}

View file

@ -0,0 +1,5 @@
//go:build ignore
package sample
func Broken(

View file

@ -0,0 +1,18 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
export PATH="../bin:$PATH"
run_capture_stdout 0 "$output" ../../bin/core qa fmt --json
jq -e '.tool == "pint" and .changed == true and .files[0].path == "src/Bad.php"' "$output" >/dev/null
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return "bad";
}

View file

@ -0,0 +1,3 @@
#!/usr/bin/env sh
printf '%s\n' '{"tool":"pint","changed":true,"files":[{"path":"src/Bad.php","fixed":1}]}'

View file

@ -0,0 +1,20 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
output="$(mktemp)"
export PATH="$(pwd)/fixtures/bin:$PATH"
run_capture_stdout 0 "$output" ./bin/core qa health --registry fixtures/repos.yaml --json
jq -e '.summary.total_repos == 2 and .summary.filtered_repos == 2' "$output" >/dev/null
jq -e '.summary.passing == 1 and .summary.errors == 1' "$output" >/dev/null
jq -e '.repos[0].status == "error" and .repos[0].name == "beta"' "$output" >/dev/null
jq -e '.repos[1].status == "passing" and .repos[1].name == "alpha"' "$output" >/dev/null
EOF

View file

@ -0,0 +1,26 @@
#!/usr/bin/env sh
case "$*" in
*"--repo forge/alpha"*)
cat <<'JSON'
[
{
"status": "completed",
"conclusion": "success",
"name": "CI",
"headSha": "abc123",
"updatedAt": "2026-03-30T00:00:00Z",
"url": "https://example.com/alpha/run/1"
}
]
JSON
;;
*"--repo forge/beta"*)
printf '%s\n' 'simulated workflow lookup failure' >&2
exit 1
;;
*)
printf '%s\n' "unexpected gh invocation: $*" >&2
exit 1
;;
esac

View file

@ -0,0 +1,8 @@
version: 1
org: forge
base_path: .
repos:
alpha:
type: module
beta:
type: module

View file

@ -0,0 +1,22 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
run_capture_all 1 "$output" ../../bin/core qa infection --min-msi 80 --min-covered-msi 90 --threads 8 --filter src --only-covered
grep -Fq "Mutation Testing" "$output"
grep -Fq -- "--min-msi=80" "$output"
grep -Fq -- "--min-covered-msi=90" "$output"
grep -Fq -- "--threads=8" "$output"
grep -Fq -- "--filter=src" "$output"
grep -Fq -- "--only-covered" "$output"
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return "bad";
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env sh
printf '%s\n' "infection args: $*"
exit 1

View file

@ -0,0 +1,20 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
output="$(mktemp)"
export PATH="$(pwd)/fixtures/bin:$PATH"
run_capture_stdout 0 "$output" ./bin/core qa issues --registry fixtures/repos.yaml --json
jq -e '.total_issues == 1 and .filtered_issues == 1' "$output" >/dev/null
jq -e '.categories[0].category == "needs_response" and .categories[0].issues[0].repo_name == "alpha"' "$output" >/dev/null
jq -e '.categories[0].issues[0].action_hint != ""' "$output" >/dev/null
jq -e '.fetch_errors[0].repo == "beta"' "$output" >/dev/null
EOF

View file

@ -0,0 +1,42 @@
#!/usr/bin/env sh
case "$*" in
*"api user"*)
printf '%s\n' 'alice'
;;
*"issue list --repo forge/alpha"*)
cat <<'JSON'
[
{
"number": 7,
"title": "Clarify agent output",
"state": "OPEN",
"body": "Explain behaviour",
"createdAt": "2026-03-30T00:00:00Z",
"updatedAt": "2026-03-30T11:00:00Z",
"author": {"login": "bob"},
"assignees": {"nodes": [{"login": "alice"}]},
"labels": {"nodes": [{"name": "agent:ready"}]},
"comments": {
"totalCount": 1,
"nodes": [
{
"author": {"login": "carol"},
"createdAt": "2026-03-30T10:30:00Z"
}
]
},
"url": "https://example.com/issues/7"
}
]
JSON
;;
*"issue list --repo forge/beta"*)
printf '%s\n' 'simulated issue query failure' >&2
exit 1
;;
*)
printf '%s\n' "unexpected gh invocation: $*" >&2
exit 1
;;
esac

View file

@ -0,0 +1,8 @@
version: 1
org: forge
base_path: .
repos:
alpha:
type: module
beta:
type: module

View file

@ -0,0 +1,17 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
run_capture_stdout 1 "$output" ../../bin/core qa psalm --json
jq -e '.tool == "psalm" and .issues[0].file == "src/Bad.php" and .issues[0].line == 3' "$output" >/dev/null
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,5 @@
<psalm>
<projectFiles>
<directory name="src" />
</projectFiles>
</psalm>

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return $anotherMissingVariable;
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env sh
printf '%s\n' '{"tool":"psalm","issues":[{"file":"src/Bad.php","line":3,"message":"Undefined variable $anotherMissingVariable"}]}'
exit 1

View file

@ -0,0 +1,19 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
run_capture_all 1 "$output" ../../bin/core qa rector
grep -Fq "Rector Refactoring" "$output"
grep -Fq "(dry-run)" "$output"
grep -Fq "1 refactoring suggestion in src/Bad.php" "$output"
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,3 @@
<?php
return [];

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return "bad";
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env sh
printf '%s\n' '1 refactoring suggestion in src/Bad.php'
exit 1

View file

@ -0,0 +1,20 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
output="$(mktemp)"
export PATH="$(pwd)/fixtures/bin:$PATH"
run_capture_stdout 0 "$output" ./bin/core qa review --repo forge/example --json
jq -e '.showing_mine == true and .showing_requested == true' "$output" >/dev/null
jq -e '.mine | length == 0 and .requested | length == 1' "$output" >/dev/null
jq -e '.requested[0].number == 42 and .requested[0].title == "Refine agent output"' "$output" >/dev/null
jq -e '.fetch_errors[0].repo == "forge/example" and .fetch_errors[0].scope == "mine"' "$output" >/dev/null
EOF

View file

@ -0,0 +1,37 @@
#!/usr/bin/env sh
case "$*" in
*"pr list --state open --search author:@me --json"*)
printf '%s\n' 'simulated author query failure' >&2
exit 1
;;
*"pr list --state open --search review-requested:@me --json"*)
cat <<'JSON'
[
{
"number": 42,
"title": "Refine agent output",
"author": {"login": "alice"},
"state": "OPEN",
"isDraft": false,
"mergeable": "MERGEABLE",
"reviewDecision": "",
"url": "https://example.com/pull/42",
"headRefName": "feature/agent-output",
"createdAt": "2026-03-30T00:00:00Z",
"updatedAt": "2026-03-30T00:00:00Z",
"additions": 12,
"deletions": 3,
"changedFiles": 2,
"statusCheckRollup": {"contexts": []},
"reviewRequests": {"nodes": []},
"reviews": []
}
]
JSON
;;
*)
printf '%s\n' "unexpected gh invocation: $*" >&2
exit 1
;;
esac

View file

@ -0,0 +1,21 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
export PATH="$(pwd)/../bin:$PATH"
run_capture_stdout 1 "$output" ../../bin/core qa security --json
jq -e '.summary.total == 4 and .summary.passed == 0' "$output" >/dev/null
jq -e '.summary.critical == 3 and .summary.high == 1' "$output" >/dev/null
jq -e '.checks[0].id == "app_key_set" and .checks[1].id == "composer_audit"' "$output" >/dev/null
jq -e '.checks[] | select(.id == "debug_mode") | .passed == false' "$output" >/dev/null
EOF

View file

@ -0,0 +1,17 @@
#!/usr/bin/env sh
cat <<'JSON'
{
"advisories": {
"vendor/package-a": [
{
"title": "Remote Code Execution",
"link": "https://example.com/advisory/1",
"cve": "CVE-2026-0001",
"affectedVersions": ">=1.0,<1.5"
}
]
}
}
JSON
exit 1

View file

@ -0,0 +1,3 @@
APP_DEBUG=true
APP_KEY=short
APP_URL=http://example.com

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return "bad";
}

View file

@ -0,0 +1,17 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
run_capture_stdout 1 "$output" ../../bin/core qa stan --json
jq -e '.tool == "phpstan" and .errors[0].file == "src/Bad.php" and .errors[0].line == 3' "$output" >/dev/null
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,2 @@
parameters:
level: 5

View file

@ -0,0 +1,5 @@
<?php
function bad_example() {
return $missingVariable;
}

View file

@ -0,0 +1,4 @@
#!/usr/bin/env sh
printf '%s\n' '{"tool":"phpstan","errors":[{"file":"src/Bad.php","line":3,"message":"Undefined variable $missingVariable"}]}'
exit 1

View file

@ -0,0 +1,17 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
cd fixtures/project
output="$(mktemp)"
run_capture_stdout 0 "$output" ../../bin/core qa test --junit
grep -Fq '<testsuite tests="1" failures="1"></testsuite>' "$output"
EOF

View file

@ -0,0 +1 @@
{}

View file

@ -0,0 +1,11 @@
<?php
use PHPUnit\Framework\TestCase;
final class ExampleTest extends TestCase
{
public function test_it_fails(): void
{
$this->assertTrue(false);
}
}

View file

@ -0,0 +1,15 @@
#!/usr/bin/env sh
junit=""
while [ $# -gt 0 ]; do
case "$1" in
--log-junit)
shift
junit="$1"
;;
esac
shift
done
printf '%s' '<testsuite tests="1" failures="1"></testsuite>' >"$junit"

View file

@ -0,0 +1,22 @@
version: "3"
tasks:
test:
cmds:
- |
bash <<'EOF'
set -euo pipefail
source ../../_lib/run.sh
go build -trimpath -ldflags="-s -w" -o bin/core ../_harness
repo="$(cat fixtures/repo.txt)"
commit="$(cat fixtures/commit.txt)"
output="$(mktemp)"
export PATH="$(pwd)/fixtures/bin:$PATH"
run_capture_all 1 "$output" ./bin/core qa watch --repo "$repo" --commit "$commit" --timeout 1s
grep -Fq "forge/example" "$output"
grep -Fq "01234567" "$output"
grep -Fq "Job: Build" "$output"
grep -Fq "Error: fatal: build failed in src/app.go:17" "$output"
EOF

View file

@ -0,0 +1,46 @@
#!/usr/bin/env sh
case "$*" in
*"run list --repo forge/example --commit 0123456789abcdef"*)
cat <<'JSON'
[
{
"databaseId": 7,
"name": "CI",
"displayTitle": "CI",
"status": "completed",
"conclusion": "failure",
"headSha": "0123456789abcdef",
"url": "https://example.com/workflows/7",
"createdAt": "2026-03-30T00:00:00Z",
"updatedAt": "2026-03-30T00:00:00Z"
}
]
JSON
;;
*"run view 7 --repo forge/example --json jobs"*)
cat <<'JSON'
{"jobs":[
{
"databaseId": 11,
"name": "Build",
"status": "completed",
"conclusion": "failure",
"url": "https://example.com/workflows/7/jobs/11",
"steps": [
{"name": "Compile", "status": "completed", "conclusion": "failure", "number": 3}
]
}
]}
JSON
;;
*"run view 7 --repo forge/example --log-failed"*)
cat <<'EOF'
fatal: build failed in src/app.go:17
EOF
;;
*)
printf '%s\n' "unexpected gh invocation: $*" >&2
exit 1
;;
esac

View file

@ -0,0 +1 @@
0123456789abcdef

View file

@ -0,0 +1 @@
forge/example