diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.go index 1d4116b5d..e9b961d91 100644 --- a/persistence/sql_search_fts.go +++ b/persistence/sql_search_fts.go @@ -8,6 +8,7 @@ import ( "unicode/utf8" . "github.com/Masterminds/squirrel" + "github.com/deluan/sanitize" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" ) @@ -44,24 +45,33 @@ var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`) // fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries). var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`) -// normalizeForFTS takes multiple strings, strips non-letter/non-number characters from each word, -// and returns a space-separated string of words that changed after stripping (deduplicated). -// This is used at index time to create concatenated forms: "R.E.M." → "REM", "AC/DC" → "ACDC". +// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of +// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC) +// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed +// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics — +// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree +// without an explicit transliterated entry here. func normalizeForFTS(values ...string) string { seen := make(map[string]struct{}) var result []string + add := func(orig, variant string) { + if variant == "" || variant == orig { + return + } + lower := strings.ToLower(variant) + if _, ok := seen[lower]; ok { + return + } + seen[lower] = struct{}{} + result = append(result, variant) + } for _, v := range values { for _, word := range strings.Fields(v) { - stripped := fts5PunctStrip.ReplaceAllString(word, "") - if stripped == "" || stripped == word { - continue - } - lower := strings.ToLower(stripped) - if _, ok := seen[lower]; ok { - continue - } - seen[lower] = struct{}{} - result = append(result, stripped) + transliterated := sanitize.Accents(word) + // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne. + add(word, fts5PunctStrip.ReplaceAllString(transliterated, "")) + // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork). + add(word, transliterated) } } return strings.Join(result, " ") @@ -158,6 +168,13 @@ func buildFTS5Query(userInput string) string { result = result[:start] + fmt.Sprintf("\x00PHRASE%d\x00", len(phrases)-1) + result[end+1:] } + // Transliterate non-ASCII letters in the unquoted portion (ø→o, æ→ae, œ→oe, ß→ss, …) + // so the query matches the ASCII variants emitted by normalizeForFTS at index time. + // FTS5's own `remove_diacritics 2` only strips NFKD-decomposable marks, so without + // this step queries for words containing these letters can miss. Quoted phrases are + // left untouched so they continue to match the original text in title/artist columns. + result = sanitize.Accents(result) + // Neutralize FTS5 operators by lowercasing them (FTS5 operators are case-sensitive: // AND, OR, NOT, NEAR are operators, but and, or, not, near are plain tokens) result = fts5Operators.ReplaceAllStringFunc(result, strings.ToLower) diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.go index d0e26c8e3..b54e5856a 100644 --- a/persistence/sql_search_fts_test.go +++ b/persistence/sql_search_fts_test.go @@ -37,7 +37,13 @@ var _ = DescribeTable("buildFTS5Query", Entry("handles punctuated word mixed with plain words", "best of a-ha", `best* AND of* AND ("a ha" OR aha*)`), Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND got*`), Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "rock* AND roll* AND vol* AND 2*"), - Entry("preserves unicode characters with diacritics", "Björk début", "Björk* AND début*"), + Entry("transliterates NFKD-decomposable diacritics", "Björk début", "Bjork* AND debut*"), + Entry("transliterates ø to o", "Øystein", "Oystein*"), + Entry("transliterates œ ligature to oe", "œuvre", "oeuvre*"), + Entry("transliterates æ ligature to ae", "Brennæ", "Brennae*"), + Entry("transliterates mixed unicode words", "Mø Sigur Rós", "Mo* AND Sigur* AND Ros*"), + Entry("transliterates ß to ss", "Straße", "Strasse*"), + Entry("preserves quoted unicode phrase verbatim", `"Björk"`, `"Björk"`), Entry("collapses dotted abbreviation into phrase", "R.E.M.", `"R E M"`), Entry("collapses abbreviation without trailing dot", "R.E.M", `"R E M"`), Entry("collapses abbreviation mixed with words", "best of R.E.M.", `best* AND of* AND "R E M"`), @@ -75,11 +81,19 @@ var _ = DescribeTable("normalizeForFTS", Entry("strips dots and concatenates", "REM", "R.E.M."), Entry("strips slash", "ACDC", "AC/DC"), Entry("strips hyphen", "Aha", "A-ha"), - Entry("skips unchanged words", "", "The Beatles"), + Entry("skips unchanged ASCII words", "", "The Beatles"), Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"), Entry("deduplicates", "REM", "R.E.M.", "R.E.M."), Entry("strips apostrophe from word", "N", "Guns N' Roses"), Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"), + Entry("transliterates ø to o", "Bjork", "Bjørk"), + Entry("transliterates Ø to O", "Oystein", "Øystein"), + Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"), + Entry("transliterates Latin diacritics", "cafe", "café"), + Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"), + Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"), + Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"), + Entry("transliterates ß to ss", "Strasse", "Straße"), ) var _ = DescribeTable("containsCJK",