core/go-i18n
8
0
Fork 0
Code Issues 4 Pull requests 1 Projects Releases Packages Wiki Activity Actions

[agent/codex:gpt-5.4-mini] Read ~/spec/code/core/go/i18n/RFC.md fully. Find ONE feature... #102

Merged
Virgil merged 1 commit from agent/read---spec-code-core-go-i18n-rfc-md-ful into dev 2026-04-02 03:36:55 +00:00
Conversation 1 Commits 1 Files changed 2 +7 -2
2 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
reversal/tokeniser.go
View file

@ -1484,3 +1484,8 @@ func DisambiguationStatsFromTokens(tokens []Token) DisambiguationStats {
}
return s
}
// DisambiguationStats returns aggregate disambiguation stats for a token slice.
func (t *Tokeniser) DisambiguationStats(tokens []Token) DisambiguationStats {
return DisambiguationStatsFromTokens(tokens)
}

4
reversal/tokeniser_test.go
View file

@ -927,7 +927,7 @@ func TestDisambiguationStats_WithAmbiguous(t *testing.T) {
setup(t)
tok := NewTokeniser()
tokens := tok.Tokenise("The commit passed the test")
stats := DisambiguationStatsFromTokens(tokens)
stats := tok.DisambiguationStats(tokens)
if stats.AmbiguousTokens == 0 {
t.Error("expected ambiguous tokens for dual-class words")
}
@ -940,7 +940,7 @@ func TestDisambiguationStats_NoAmbiguous(t *testing.T) {
setup(t)
tok := NewTokeniser()
tokens := tok.Tokenise("Deleted the files")
stats := DisambiguationStatsFromTokens(tokens)
stats := tok.DisambiguationStats(tokens)
if stats.AmbiguousTokens != 0 {
t.Errorf("AmbiguousTokens = %d, want 0", stats.AmbiguousTokens)
}