[agent/codex:gpt-5.4-mini] Read ~/spec/code/core/go/i18n/RFC.md fully. Find ONE feature... #99
2 changed files with 16 additions and 4 deletions
|
|
@ -113,16 +113,16 @@ func WithSignals() TokeniserOption {
|
|||
}
|
||||
|
||||
// WithWeights overrides the default signal weights for disambiguation.
|
||||
// All signal keys must be present; omitted keys silently disable those signals.
|
||||
// Omitted keys keep their default weights so partial overrides stay safe.
|
||||
func WithWeights(w map[string]float64) TokeniserOption {
|
||||
return func(t *Tokeniser) {
|
||||
if len(w) == 0 {
|
||||
t.weights = nil
|
||||
return
|
||||
}
|
||||
// Copy the map so callers can safely reuse or mutate their input after
|
||||
// constructing the tokeniser.
|
||||
copied := make(map[string]float64, len(w))
|
||||
// Start from the defaults so callers can override only the weights they
|
||||
// care about without accidentally disabling the rest of the signal set.
|
||||
copied := defaultWeights()
|
||||
for key, value := range w {
|
||||
copied[key] = value
|
||||
}
|
||||
|
|
|
|||
|
|
@ -977,6 +977,18 @@ func TestWithWeights_CopiesInputMap(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestWithWeights_PartialOverrideKeepsDefaults(t *testing.T) {
|
||||
setup(t)
|
||||
tok := NewTokeniser(WithWeights(map[string]float64{
|
||||
"verb_auxiliary": 0.25,
|
||||
}))
|
||||
|
||||
tokens := tok.Tokenise("The commit")
|
||||
if tokens[1].Type != TokenNoun {
|
||||
t.Fatalf("with partial weights, 'commit' Type = %v, want TokenNoun", tokens[1].Type)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Benchmarks ---
|
||||
|
||||
func benchSetup(b *testing.B) {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue