fix(i18n): canonicalise locale scans

Co-Authored-By: Virgil <virgil@lethean.io>

loader.go

@@ -3,6 +3,7 @@ package i18n
 import (
 	"io/fs"
 	"path"
+	"slices"
 	"strings"
 	"sync"
 
@@ -71,14 +72,23 @@ func (l *FSLoader) Languages() []string {
 			l.langErr = log.E("FSLoader.Languages", "read locale directory: "+l.dir, err)
 			return
 		}
+		seen := make(map[string]struct{}, len(entries))
 		for _, entry := range entries {
 			if entry.IsDir() || !core.HasSuffix(entry.Name(), ".json") {
 				continue
 			}
 			lang := core.TrimSuffix(entry.Name(), ".json")
-			lang = core.Replace(lang, "_", "-")
+			lang = normalizeLanguageTag(core.Replace(lang, "_", "-"))
+			if lang == "" {
+				continue
+			}
+			if _, ok := seen[lang]; ok {
+				continue
+			}
+			seen[lang] = struct{}{}
 			l.languages = append(l.languages, lang)
 		}
+		slices.Sort(l.languages)
 	})
 	return l.languages
 }

loader_test.go

@@ -19,6 +19,22 @@ func TestFSLoaderLanguages(t *testing.T) {
 	}
 }
 
+func TestFSLoaderLanguagesCanonicalAndUnique(t *testing.T) {
+	fs := fstest.MapFS{
+		"locales/en.json":    &fstest.MapFile{Data: []byte(`{}`)},
+		"locales/en_US.json": &fstest.MapFile{Data: []byte(`{}`)},
+		"locales/es-MX.json": &fstest.MapFile{Data: []byte(`{}`)},
+		"locales/fr.json":    &fstest.MapFile{Data: []byte(`{}`)},
+	}
+
+	loader := NewFSLoader(fs, "locales")
+	langs := loader.Languages()
+	want := []string{"en", "en-US", "es-MX", "fr"}
+	if !slices.Equal(langs, want) {
+		t.Fatalf("Languages() = %v, want %v", langs, want)
+	}
+}
+
 func TestFSLoaderLoad(t *testing.T) {
 	loader := NewFSLoader(localeFS, "locales")
 	messages, grammar, err := loader.Load("en")

service.go

@@ -936,6 +936,7 @@ func (s *Service) LoadFS(fsys fs.FS, dir string) error {
 	if err != nil {
 		return log.E("Service.LoadFS", "read locales directory", err)
 	}
+	seen := make(map[string]struct{}, len(entries))
 	for _, entry := range entries {
 		if entry.IsDir() || !core.HasSuffix(entry.Name(), ".json") {
 			continue
@@ -946,7 +947,14 @@ func (s *Service) LoadFS(fsys fs.FS, dir string) error {
 			return log.E("Service.LoadFS", "read locale: "+entry.Name(), err)
 		}
 		lang := core.TrimSuffix(entry.Name(), ".json")
-		lang = core.Replace(lang, "_", "-")
+		lang = normalizeLanguageTag(core.Replace(lang, "_", "-"))
+		if lang == "" {
+			continue
+		}
+		if _, ok := seen[lang]; ok {
+			continue
+		}
+		seen[lang] = struct{}{}
 		if err := s.loadJSON(lang, data); err != nil {
 			return log.E("Service.LoadFS", "parse locale: "+entry.Name(), err)
 		}