fix(search): normalization for non-NFKD Unicode letters (ø, æ, œ, ß) (#5413)

* fix(search): transliterate non-ASCII letters symmetrically in FTS5 path

Songs and artists with letters like ø, æ, œ, ß were unsearchable. The
query path in server/subsonic/searching.go transliterates with
sanitize.Accents (Øystein → Oystein), but the FTS5 tokenizer's
remove_diacritics 2 only strips NFKD-decomposable marks — atomic
letters with built-in strokes/ligatures survive tokenization, so the
query side and index side disagreed.

Apply sanitize.Accents on both sides:

- normalizeForFTS now also emits an ASCII-transliterated form for each
  word, so search_normalized contains the variant the query produces.
- buildFTS5Query transliterates the unquoted portion of the input so
  every caller (Subsonic, REST fullTextFilter) gets the same handling.
  Quoted phrases stay as typed, preserving phrase matches against the
  original title/artist columns.

Existing libraries pick up the fix as records are re-scanned; users
can trigger a manual full rescan to refresh older entries.

* fix(search): cache transliteration and add ß/quoted-phrase test coverage

Address review feedback: call sanitize.Accents once per word and reuse
the result for both the punct-stripped and accent-only paths. Add missing
test entries for ß→ss transliteration and quoted Unicode phrase preservation.

---------

Co-authored-by: Claude <noreply@anthropic.com>

persistence/sql_search_fts.go

@@ -8,6 +8,7 @@ import (
 	"unicode/utf8"
 
 	. "github.com/Masterminds/squirrel"
+	"github.com/deluan/sanitize"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 )
@@ -44,24 +45,33 @@ var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`)
 // fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries).
 var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`)
 
-// normalizeForFTS takes multiple strings, strips non-letter/non-number characters from each word,
-// and returns a space-separated string of words that changed after stripping (deduplicated).
-// This is used at index time to create concatenated forms: "R.E.M." → "REM", "AC/DC" → "ACDC".
+// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of
+// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)
+// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed
+// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —
+// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree
+// without an explicit transliterated entry here.
 func normalizeForFTS(values ...string) string {
 	seen := make(map[string]struct{})
 	var result []string
+	add := func(orig, variant string) {
+		if variant == "" || variant == orig {
+			return
+		}
+		lower := strings.ToLower(variant)
+		if _, ok := seen[lower]; ok {
+			return
+		}
+		seen[lower] = struct{}{}
+		result = append(result, variant)
+	}
 	for _, v := range values {
 		for _, word := range strings.Fields(v) {
-			stripped := fts5PunctStrip.ReplaceAllString(word, "")
-			if stripped == "" || stripped == word {
-				continue
-			}
-			lower := strings.ToLower(stripped)
-			if _, ok := seen[lower]; ok {
-				continue
-			}
-			seen[lower] = struct{}{}
-			result = append(result, stripped)
+			transliterated := sanitize.Accents(word)
+			// Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.
+			add(word, fts5PunctStrip.ReplaceAllString(transliterated, ""))
+			// Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).
+			add(word, transliterated)
 		}
 	}
 	return strings.Join(result, " ")
@@ -158,6 +168,13 @@ func buildFTS5Query(userInput string) string {
 		result = result[:start] + fmt.Sprintf("\x00PHRASE%d\x00", len(phrases)-1) + result[end+1:]
 	}
 
+	// Transliterate non-ASCII letters in the unquoted portion (ø→o, æ→ae, œ→oe, ß→ss, …)
+	// so the query matches the ASCII variants emitted by normalizeForFTS at index time.
+	// FTS5's own `remove_diacritics 2` only strips NFKD-decomposable marks, so without
+	// this step queries for words containing these letters can miss. Quoted phrases are
+	// left untouched so they continue to match the original text in title/artist columns.
+	result = sanitize.Accents(result)
+
 	// Neutralize FTS5 operators by lowercasing them (FTS5 operators are case-sensitive:
 	// AND, OR, NOT, NEAR are operators, but and, or, not, near are plain tokens)
 	result = fts5Operators.ReplaceAllStringFunc(result, strings.ToLower)

persistence/sql_search_fts_test.go

@@ -37,7 +37,13 @@ var _ = DescribeTable("buildFTS5Query",
 	Entry("handles punctuated word mixed with plain words", "best of a-ha", `best* AND of* AND ("a ha" OR aha*)`),
 	Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND got*`),
 	Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "rock* AND roll* AND vol* AND 2*"),
-	Entry("preserves unicode characters with diacritics", "Björk début", "Björk* AND début*"),
+	Entry("transliterates NFKD-decomposable diacritics", "Björk début", "Bjork* AND debut*"),
+	Entry("transliterates ø to o", "Øystein", "Oystein*"),
+	Entry("transliterates œ ligature to oe", "œuvre", "oeuvre*"),
+	Entry("transliterates æ ligature to ae", "Brennæ", "Brennae*"),
+	Entry("transliterates mixed unicode words", "Mø Sigur Rós", "Mo* AND Sigur* AND Ros*"),
+	Entry("transliterates ß to ss", "Straße", "Strasse*"),
+	Entry("preserves quoted unicode phrase verbatim", `"Björk"`, `"Björk"`),
 	Entry("collapses dotted abbreviation into phrase", "R.E.M.", `"R E M"`),
 	Entry("collapses abbreviation without trailing dot", "R.E.M", `"R E M"`),
 	Entry("collapses abbreviation mixed with words", "best of R.E.M.", `best* AND of* AND "R E M"`),
@@ -75,11 +81,19 @@ var _ = DescribeTable("normalizeForFTS",
 	Entry("strips dots and concatenates", "REM", "R.E.M."),
 	Entry("strips slash", "ACDC", "AC/DC"),
 	Entry("strips hyphen", "Aha", "A-ha"),
-	Entry("skips unchanged words", "", "The Beatles"),
+	Entry("skips unchanged ASCII words", "", "The Beatles"),
 	Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),
 	Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),
 	Entry("strips apostrophe from word", "N", "Guns N' Roses"),
 	Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),
+	Entry("transliterates ø to o", "Bjork", "Bjørk"),
+	Entry("transliterates Ø to O", "Oystein", "Øystein"),
+	Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"),
+	Entry("transliterates Latin diacritics", "cafe", "café"),
+	Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),
+	Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),
+	Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),
+	Entry("transliterates ß to ss", "Strasse", "Straße"),
 )
 
 var _ = DescribeTable("containsCJK",