[agent/codex:gpt-5.4-mini] Read ~/spec/code/core/go/i18n/RFC.md fully. Find features de... #172
2 changed files with 32 additions and 0 deletions
|
|
@ -16,6 +16,7 @@
|
|||
package reversal
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
|
|
@ -644,6 +645,19 @@ func DefaultWeights() map[string]float64 {
|
|||
}
|
||||
}
|
||||
|
||||
// SignalWeights returns a copy of the tokeniser's configured signal weights.
|
||||
//
|
||||
// The copy keeps calibration and debugging callers from mutating the live
|
||||
// internal map by accident.
|
||||
func (t *Tokeniser) SignalWeights() map[string]float64 {
|
||||
if t == nil || len(t.weights) == 0 {
|
||||
return nil
|
||||
}
|
||||
weights := make(map[string]float64, len(t.weights))
|
||||
maps.Copy(weights, t.weights)
|
||||
return weights
|
||||
}
|
||||
|
||||
func skipDeprecatedEnglishGrammarEntry(key string) bool {
|
||||
switch core.Lower(key) {
|
||||
case "passed", "failed", "skipped":
|
||||
|
|
|
|||
|
|
@ -1305,6 +1305,24 @@ func TestDefaultWeights_ReturnsCopy(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestTokeniserSignalWeights_ReturnsCopy(t *testing.T) {
|
||||
setup(t)
|
||||
tok := NewTokeniser(WithWeights(map[string]float64{
|
||||
"noun_determiner": 0.5,
|
||||
"default_prior": 0.1,
|
||||
}))
|
||||
|
||||
weights := tok.SignalWeights()
|
||||
if weights["noun_determiner"] != 0.5 {
|
||||
t.Fatalf("SignalWeights()[noun_determiner] = %v, want 0.5", weights["noun_determiner"])
|
||||
}
|
||||
|
||||
weights["noun_determiner"] = 0
|
||||
if got := tok.SignalWeights()["noun_determiner"]; got != 0.5 {
|
||||
t.Fatalf("SignalWeights() should return a fresh copy, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTokeniser_LowInformationConfidenceFloor(t *testing.T) {
|
||||
setup(t)
|
||||
tok := NewTokeniser()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue