fix(reversal): preserve Confidence on multiplier-transformed tokens

Transformed tokens get Confidence 1.0 since the transformation
is deterministic and unambiguous.

Co-Authored-By: Virgil <virgil@lethean.io>
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Snider 2026-02-19 16:19:54 +00:00
parent c1d347f079
commit 4ffe614840
2 changed files with 49 additions and 6 deletions

View file

@ -139,9 +139,10 @@ func (m *Multiplier) applyVerbTransform(tokens []Token, vi int, targetTense stri
newForm = preserveCase(tok.Raw, newForm)
result[vi] = Token{
Raw: newForm,
Lower: strings.ToLower(newForm),
Type: TokenVerb,
Raw: newForm,
Lower: strings.ToLower(newForm),
Type: TokenVerb,
Confidence: 1.0,
VerbInfo: VerbMatch{
Base: base,
Tense: targetTense,
@ -189,9 +190,10 @@ func (m *Multiplier) applyNounTransformOnTokens(tokens []Token, ni int) []Token
newForm = preserveCase(tok.Raw, newForm)
result[ni] = Token{
Raw: newForm,
Lower: strings.ToLower(newForm),
Type: TokenNoun,
Raw: newForm,
Lower: strings.ToLower(newForm),
Type: TokenNoun,
Confidence: 1.0,
NounInfo: NounMatch{
Base: base,
Plural: newPlural,

View file

@ -66,3 +66,44 @@ func TestMultiplier_Expand_Deterministic(t *testing.T) {
}
}
}
func TestMultiplier_Expand_DualClass(t *testing.T) {
svc, _ := i18n.New()
i18n.SetDefault(svc)
m := NewMultiplier()
// "the commit" — commit is noun, should still produce variants
variants := m.Expand("the commit")
if len(variants) < 2 {
t.Errorf("Expand('the commit') returned %d variants, want >= 2", len(variants))
}
// Should have at least original + plural toggle
found := false
for _, v := range variants {
if v == "the commits" {
found = true
}
}
if !found {
t.Errorf("Expected 'the commits' variant, got: %v", variants)
}
}
func TestMultiplier_TransformedTokenConfidence(t *testing.T) {
svc, _ := i18n.New()
i18n.SetDefault(svc)
m := NewMultiplier()
// Verify that transformed tokens have Confidence set
tokens := m.tokeniser.Tokenise("Delete the branch")
pastTokens := m.applyVerbTransform(tokens, 0, "past")
if pastTokens[0].Confidence == 0 {
t.Error("Verb-transformed token has zero Confidence, want 1.0")
}
pluralTokens := m.applyNounTransformOnTokens(tokens, 2)
if pluralTokens[2].Confidence == 0 {
t.Error("Noun-transformed token has zero Confidence, want 1.0")
}
}