diff --git a/db/migrations/20260702152457_backfill_artist_search_normalized.go b/db/migrations/20260702152457_backfill_artist_search_normalized.go new file mode 100644 index 000000000..93902e230 --- /dev/null +++ b/db/migrations/20260702152457_backfill_artist_search_normalized.go @@ -0,0 +1,58 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + + "github.com/navidrome/navidrome/utils/str" + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(upBackfillArtistSearchNormalized, downBackfillArtistSearchNormalized) +} + +// The FTS5 migration back-filled artist.search_normalized with a SQL approximation that +// cannot transliterate atomic letters (Ø, æ, ß, ...), and the scanner never rewrote the +// column, leaving artists like "GØGGS" unfindable by ASCII searches. Recompute it in Go; +// the artist_fts update trigger re-indexes every row that changes. +func upBackfillArtistSearchNormalized(ctx context.Context, tx *sql.Tx) error { + notice(ctx, tx, "Rebuilding artist search index data. This may take a moment on large libraries.") + + rows, err := tx.QueryContext(ctx, "SELECT id, name, search_normalized FROM artist") + if err != nil { + return fmt.Errorf("querying artists: %w", err) + } + defer rows.Close() + + updates := map[string]string{} + for rows.Next() { + var id, name, current string + if err := rows.Scan(&id, &name, ¤t); err != nil { + return fmt.Errorf("scanning artist: %w", err) + } + if expected := str.NormalizeForFTS(name); expected != current { + updates[id] = expected + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating artists: %w", err) + } + + stmt, err := tx.PrepareContext(ctx, "UPDATE artist SET search_normalized = ? WHERE id = ?") + if err != nil { + return fmt.Errorf("preparing update: %w", err) + } + defer stmt.Close() + for id, normalized := range updates { + if _, err := stmt.ExecContext(ctx, normalized, id); err != nil { + return fmt.Errorf("updating artist %s: %w", id, err) + } + } + return nil +} + +func downBackfillArtistSearchNormalized(context.Context, *sql.Tx) error { + return nil +} diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 31a0f5c91..34845be15 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -17,6 +17,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/slice" + "github.com/navidrome/navidrome/utils/str" "github.com/pocketbase/dbx" ) @@ -69,7 +70,7 @@ func (a *dbAlbum) PostMapArgs(args map[string]any) error { fullText = append(fullText, a.Album.Tags[model.TagCatalogNumber]...) args["full_text"] = formatFullText(fullText...) args["search_participants"] = strings.Join(participantNames, " ") - args["search_normalized"] = normalizeForFTS(a.Name, a.AlbumArtist) + args["search_normalized"] = str.NormalizeForFTS(a.Name, a.AlbumArtist) args["tags"] = marshalTags(a.Album.Tags) args["participants"] = marshalParticipants(a.Album.Participants) diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index 5152e774f..f84f410e9 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -19,6 +19,7 @@ import ( "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils" "github.com/navidrome/navidrome/utils/slice" + "github.com/navidrome/navidrome/utils/str" "github.com/pocketbase/dbx" ) @@ -102,8 +103,10 @@ func (a *dbArtist) PostMapArgs(m map[string]any) error { } similarArtists, _ := json.Marshal(sa) m["similar_artists"] = string(similarArtists) + // When adding a derived column here, also add it to the scanner's artist Put column list + // in phase_1_folders.go, or rescans will never update it (how search_normalized went stale). m["full_text"] = formatFullText(a.Name, a.SortArtistName) - m["search_normalized"] = normalizeForFTS(a.Name) + m["search_normalized"] = str.NormalizeForFTS(a.Name) // Do not override the sort_artist_name and mbz_artist_id fields if they are empty // TODO: Better way to handle this? diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index d7d892ed1..094268783 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -16,6 +16,7 @@ import ( "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/utils/slice" + "github.com/navidrome/navidrome/utils/str" "github.com/pocketbase/dbx" ) @@ -62,7 +63,7 @@ func (m *dbMediaFile) PostMapArgs(args map[string]any) error { fullText = append(fullText, participantNames...) args["full_text"] = formatFullText(fullText...) args["search_participants"] = strings.Join(participantNames, " ") - args["search_normalized"] = normalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist) + args["search_normalized"] = str.NormalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist) args["tags"] = marshalTags(m.MediaFile.Tags) args["participants"] = marshalParticipants(m.MediaFile.Participants) return nil diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.go index bbae47fe8..999c904af 100644 --- a/persistence/sql_search_fts.go +++ b/persistence/sql_search_fts.go @@ -11,6 +11,7 @@ import ( "github.com/deluan/sanitize" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/utils/str" ) // containsCJK returns true if the string contains any CJK (Chinese/Japanese/Korean) characters. @@ -35,48 +36,12 @@ func containsCJK(s string) bool { // as unbalanced string delimiters. var fts5SpecialChars = regexp.MustCompile(`[^\p{L}\p{N}\s*"\x00]`) -// fts5PunctStrip strips everything except letters and numbers (no whitespace, wildcards, or quotes). -// Used for normalizing words at index time to create concatenated forms (e.g., "R.E.M." → "REM"). -var fts5PunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`) - // fts5Operators matches FTS5 boolean operators as whole words (case-insensitive). var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`) // fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries). var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`) -// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of -// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC) -// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed -// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics — -// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree -// without an explicit transliterated entry here. -func normalizeForFTS(values ...string) string { - seen := make(map[string]struct{}) - var result []string - add := func(orig, variant string) { - if variant == "" || variant == orig { - return - } - lower := strings.ToLower(variant) - if _, ok := seen[lower]; ok { - return - } - seen[lower] = struct{}{} - result = append(result, variant) - } - for _, v := range values { - for word := range strings.FieldsSeq(v) { - transliterated := sanitize.Accents(word) - // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne. - add(word, fts5PunctStrip.ReplaceAllString(transliterated, "")) - // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork). - add(word, transliterated) - } - } - return strings.Join(result, " ") -} - // isSingleUnicodeLetter returns true if token is exactly one Unicode letter. func isSingleUnicodeLetter(token string) bool { r, size := utf8.DecodeRuneInString(token) @@ -100,7 +65,7 @@ func processPunctuatedWords(input string, phrases []string) (string, []string) { result = append(result, w) continue } - concat := fts5PunctStrip.ReplaceAllString(w, "") + concat := str.FTSPunctStrip.ReplaceAllString(w, "") if concat == "" || concat == w { result = append(result, w) continue @@ -329,7 +294,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool { // Strip quotes from original for comparison — we want the raw content stripped := strings.ReplaceAll(original, `"`, "") // Extract the alphanumeric content from the original query - alphaNum := fts5PunctStrip.ReplaceAllString(stripped, "") + alphaNum := str.FTSPunctStrip.ReplaceAllString(stripped, "") // If the original is entirely alphanumeric, nothing was stripped — not degraded if len(alphaNum) == len(stripped) { return false @@ -353,7 +318,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool { if strings.HasPrefix(t, `"`) { // Extract content between quotes inner := strings.Trim(t, `"`) - innerAlpha := fts5PunctStrip.ReplaceAllString(inner, " ") + innerAlpha := str.FTSPunctStrip.ReplaceAllString(inner, " ") for it := range strings.FieldsSeq(innerAlpha) { if len(it) > 2 { return false diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.go index b54e5856a..6c975c601 100644 --- a/persistence/sql_search_fts_test.go +++ b/persistence/sql_search_fts_test.go @@ -74,28 +74,6 @@ var _ = DescribeTable("ftsQueryDegraded", Entry("not degraded for OR groups from processPunctuatedWords", "AC/DC", `("AC DC" OR ACDC*)`, false), ) -var _ = DescribeTable("normalizeForFTS", - func(expected string, values ...string) { - Expect(normalizeForFTS(values...)).To(Equal(expected)) - }, - Entry("strips dots and concatenates", "REM", "R.E.M."), - Entry("strips slash", "ACDC", "AC/DC"), - Entry("strips hyphen", "Aha", "A-ha"), - Entry("skips unchanged ASCII words", "", "The Beatles"), - Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"), - Entry("deduplicates", "REM", "R.E.M.", "R.E.M."), - Entry("strips apostrophe from word", "N", "Guns N' Roses"), - Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"), - Entry("transliterates ø to o", "Bjork", "Bjørk"), - Entry("transliterates Ø to O", "Oystein", "Øystein"), - Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"), - Entry("transliterates Latin diacritics", "cafe", "café"), - Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"), - Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"), - Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"), - Entry("transliterates ß to ss", "Strasse", "Straße"), -) - var _ = DescribeTable("containsCJK", func(input string, expected bool) { Expect(containsCJK(input)).To(Equal(expected)) diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go index 38967832c..5e898590b 100644 --- a/scanner/phase_1_folders.go +++ b/scanner/phase_1_folders.go @@ -360,7 +360,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error) // Save all new/modified artists to DB. Their information will be incomplete, but they will be refreshed later for i := range entry.artists { err = artistRepo.Put(&entry.artists[i], "name", - "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "updated_at") + "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "search_normalized", "updated_at") if err != nil { log.Error(p.ctx, "Scanner: Error persisting artist to DB", "folder", entry.path, "artist", entry.artists[i].Name, err) return err diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index cc3732bc3..7f3dca775 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -189,6 +189,34 @@ var _ = Describe("Scanner", Ordered, func() { }) }) + Context("Artist with atomic non-ASCII letters, 'GØGGS'", func() { + BeforeEach(func() { + goggs := template(_t{"albumartist": "GØGGS", "album": "Pre Strike Sweep", "year": 2018}) + createFS(fstest.MapFS{ + "GØGGS/Pre Strike Sweep/01 - Falling For You.mp3": goggs(track(1, "Falling For You")), + }) + }) + + searchNormalized := func() string { + var sn string + Expect(db.Db().QueryRowContext(ctx, + "SELECT search_normalized FROM artist WHERE name = 'GØGGS'").Scan(&sn)).To(Succeed()) + return sn + } + + It("repopulates a stale search_normalized on a full rescan", func() { + Expect(runScanner(ctx, true)).To(Succeed()) + Expect(searchNormalized()).To(Equal("GOGGS")) + + // Simulate the stale value left by the FTS5 migration's SQL back-fill + _, err := db.Db().ExecContext(ctx, "UPDATE artist SET search_normalized = '' WHERE name = 'GØGGS'") + Expect(err).ToNot(HaveOccurred()) + + Expect(runScanner(ctx, true)).To(Succeed()) + Expect(searchNormalized()).To(Equal("GOGGS")) + }) + }) + Context("Ignored entries", func() { BeforeEach(func() { revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966}) diff --git a/utils/str/normalize_fts.go b/utils/str/normalize_fts.go new file mode 100644 index 000000000..994f77cc9 --- /dev/null +++ b/utils/str/normalize_fts.go @@ -0,0 +1,45 @@ +package str + +import ( + "regexp" + "strings" + + "github.com/deluan/sanitize" +) + +// FTSPunctStrip matches any character that is not a letter or number. Index-time +// normalization (NormalizeForFTS) and query-time processing in persistence share it +// so both sides produce matching tokens. +var FTSPunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`) + +// NormalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of +// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC) +// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed +// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics — +// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree +// without an explicit transliterated entry here. +func NormalizeForFTS(values ...string) string { + seen := make(map[string]struct{}) + var result []string + add := func(orig, variant string) { + if variant == "" || variant == orig { + return + } + lower := strings.ToLower(variant) + if _, ok := seen[lower]; ok { + return + } + seen[lower] = struct{}{} + result = append(result, variant) + } + for _, v := range values { + for word := range strings.FieldsSeq(v) { + transliterated := sanitize.Accents(word) + // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne. + add(word, FTSPunctStrip.ReplaceAllString(transliterated, "")) + // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork). + add(word, transliterated) + } + } + return strings.Join(result, " ") +} diff --git a/utils/str/normalize_fts_test.go b/utils/str/normalize_fts_test.go new file mode 100644 index 000000000..52387cb1d --- /dev/null +++ b/utils/str/normalize_fts_test.go @@ -0,0 +1,29 @@ +package str_test + +import ( + "github.com/navidrome/navidrome/utils/str" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = DescribeTable("NormalizeForFTS", + func(expected string, values ...string) { + Expect(str.NormalizeForFTS(values...)).To(Equal(expected)) + }, + Entry("strips dots and concatenates", "REM", "R.E.M."), + Entry("strips slash", "ACDC", "AC/DC"), + Entry("strips hyphen", "Aha", "A-ha"), + Entry("skips unchanged ASCII words", "", "The Beatles"), + Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"), + Entry("deduplicates", "REM", "R.E.M.", "R.E.M."), + Entry("strips apostrophe from word", "N", "Guns N' Roses"), + Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"), + Entry("transliterates ø to o", "Bjork", "Bjørk"), + Entry("transliterates Ø to O", "Oystein", "Øystein"), + Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"), + Entry("transliterates Latin diacritics", "cafe", "café"), + Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"), + Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"), + Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"), + Entry("transliterates ß to ss", "Strasse", "Straße"), +)