fix(reversal): preserve Confidence on multiplier-transformed tokens
Transformed tokens get Confidence 1.0 since the transformation is deterministic and unambiguous. Co-Authored-By: Virgil <virgil@lethean.io> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
c1d347f079
commit
4ffe614840
2 changed files with 49 additions and 6 deletions
|
|
@ -139,9 +139,10 @@ func (m *Multiplier) applyVerbTransform(tokens []Token, vi int, targetTense stri
|
|||
newForm = preserveCase(tok.Raw, newForm)
|
||||
|
||||
result[vi] = Token{
|
||||
Raw: newForm,
|
||||
Lower: strings.ToLower(newForm),
|
||||
Type: TokenVerb,
|
||||
Raw: newForm,
|
||||
Lower: strings.ToLower(newForm),
|
||||
Type: TokenVerb,
|
||||
Confidence: 1.0,
|
||||
VerbInfo: VerbMatch{
|
||||
Base: base,
|
||||
Tense: targetTense,
|
||||
|
|
@ -189,9 +190,10 @@ func (m *Multiplier) applyNounTransformOnTokens(tokens []Token, ni int) []Token
|
|||
newForm = preserveCase(tok.Raw, newForm)
|
||||
|
||||
result[ni] = Token{
|
||||
Raw: newForm,
|
||||
Lower: strings.ToLower(newForm),
|
||||
Type: TokenNoun,
|
||||
Raw: newForm,
|
||||
Lower: strings.ToLower(newForm),
|
||||
Type: TokenNoun,
|
||||
Confidence: 1.0,
|
||||
NounInfo: NounMatch{
|
||||
Base: base,
|
||||
Plural: newPlural,
|
||||
|
|
|
|||
|
|
@ -66,3 +66,44 @@ func TestMultiplier_Expand_Deterministic(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiplier_Expand_DualClass(t *testing.T) {
|
||||
svc, _ := i18n.New()
|
||||
i18n.SetDefault(svc)
|
||||
m := NewMultiplier()
|
||||
|
||||
// "the commit" — commit is noun, should still produce variants
|
||||
variants := m.Expand("the commit")
|
||||
if len(variants) < 2 {
|
||||
t.Errorf("Expand('the commit') returned %d variants, want >= 2", len(variants))
|
||||
}
|
||||
|
||||
// Should have at least original + plural toggle
|
||||
found := false
|
||||
for _, v := range variants {
|
||||
if v == "the commits" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("Expected 'the commits' variant, got: %v", variants)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiplier_TransformedTokenConfidence(t *testing.T) {
|
||||
svc, _ := i18n.New()
|
||||
i18n.SetDefault(svc)
|
||||
m := NewMultiplier()
|
||||
|
||||
// Verify that transformed tokens have Confidence set
|
||||
tokens := m.tokeniser.Tokenise("Delete the branch")
|
||||
pastTokens := m.applyVerbTransform(tokens, 0, "past")
|
||||
if pastTokens[0].Confidence == 0 {
|
||||
t.Error("Verb-transformed token has zero Confidence, want 1.0")
|
||||
}
|
||||
|
||||
pluralTokens := m.applyNounTransformOnTokens(tokens, 2)
|
||||
if pluralTokens[2].Confidence == 0 {
|
||||
t.Error("Noun-transformed token has zero Confidence, want 1.0")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue