fix(search): artists with atomic non-ASCII names unfindable after FTS5 migration (#5703)

* fix(scanner): update artist search_normalized when rescanning

The FTS5 migration back-fills artist.search_normalized with a SQL
punctuation-strip approximation, relying on the next scan to compute the
precise value in Go (normalizeForFTS transliterates atomic letters like
Ø/æ/ß that FTS5's remove_diacritics cannot fold). But the scanner
persisted artists with an explicit column list that omitted
search_normalized, so not even a full scan ever repaired it: an artist
migrated from a pre-FTS database (e.g. "GØGGS") stayed unfindable by
any ASCII search, while their albums and songs, which are saved with all
columns, were fixed by a full scan. Add search_normalized to the column
list so a full scan re-indexes the artist via the artist_fts trigger.

* refactor(persistence): move normalizeForFTS to utils/str

Export it as str.NormalizeForFTS so the upcoming migration can reuse the
exact index-time normalization. Migrations cannot import the persistence
package (persistence -> db -> db/migrations would be an import cycle).

* fix(persistence): backfill artist search_normalized via migration

Recompute artist.search_normalized with the precise Go normalization for
databases migrated from pre-FTS5 versions, where the SQL back-fill could
not transliterate atomic letters (Ø/æ/ß) and the scanner never rewrote
the column. Only changed rows are updated, so the artist_fts update
trigger re-indexes exactly the affected artists, making artists like
GØGGS or MØ findable again without requiring a full scan.

* refactor(persistence): share FTS punctuation-strip regex via utils/str

Index-time normalization (NormalizeForFTS) and query-time processing
(buildFTS5Query/ftsQueryDegraded) must produce matching tokens, so keep
the punctuation-strip pattern in a single exported symbol instead of two
identical private copies that could drift. Also document that derived
columns computed in dbArtist.PostMapArgs must be listed in the scanner's
artist Put, which is how search_normalized went stale in the first place.

* chore(migrations): announce artist search backfill in the log

Match the FTS5 migration's notice() pattern so startup isn't silent
while the backfill runs on large libraries.

* docs: tighten comments added in this branch

* docs: describe FTSPunctStrip by what it matches, not one replacement

db/migrations/20260702152457_backfill_artist_search_normalized.go

@@ -0,0 +1,58 @@
+package migrations
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+
+	"github.com/navidrome/navidrome/utils/str"
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigrationContext(upBackfillArtistSearchNormalized, downBackfillArtistSearchNormalized)
+}
+
+// The FTS5 migration back-filled artist.search_normalized with a SQL approximation that
+// cannot transliterate atomic letters (Ø, æ, ß, ...), and the scanner never rewrote the
+// column, leaving artists like "GØGGS" unfindable by ASCII searches. Recompute it in Go;
+// the artist_fts update trigger re-indexes every row that changes.
+func upBackfillArtistSearchNormalized(ctx context.Context, tx *sql.Tx) error {
+	notice(ctx, tx, "Rebuilding artist search index data. This may take a moment on large libraries.")
+
+	rows, err := tx.QueryContext(ctx, "SELECT id, name, search_normalized FROM artist")
+	if err != nil {
+		return fmt.Errorf("querying artists: %w", err)
+	}
+	defer rows.Close()
+
+	updates := map[string]string{}
+	for rows.Next() {
+		var id, name, current string
+		if err := rows.Scan(&id, &name, &current); err != nil {
+			return fmt.Errorf("scanning artist: %w", err)
+		}
+		if expected := str.NormalizeForFTS(name); expected != current {
+			updates[id] = expected
+		}
+	}
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("iterating artists: %w", err)
+	}
+
+	stmt, err := tx.PrepareContext(ctx, "UPDATE artist SET search_normalized = ? WHERE id = ?")
+	if err != nil {
+		return fmt.Errorf("preparing update: %w", err)
+	}
+	defer stmt.Close()
+	for id, normalized := range updates {
+		if _, err := stmt.ExecContext(ctx, normalized, id); err != nil {
+			return fmt.Errorf("updating artist %s: %w", id, err)
+		}
+	}
+	return nil
+}
+
+func downBackfillArtistSearchNormalized(context.Context, *sql.Tx) error {
+	return nil
+}

persistence/album_repository.go

@@ -17,6 +17,7 @@ import (
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/utils/slice"
+	"github.com/navidrome/navidrome/utils/str"
 	"github.com/pocketbase/dbx"
 )
 
@@ -69,7 +70,7 @@ func (a *dbAlbum) PostMapArgs(args map[string]any) error {
 	fullText = append(fullText, a.Album.Tags[model.TagCatalogNumber]...)
 	args["full_text"] = formatFullText(fullText...)
 	args["search_participants"] = strings.Join(participantNames, " ")
-	args["search_normalized"] = normalizeForFTS(a.Name, a.AlbumArtist)
+	args["search_normalized"] = str.NormalizeForFTS(a.Name, a.AlbumArtist)
 
 	args["tags"] = marshalTags(a.Album.Tags)
 	args["participants"] = marshalParticipants(a.Album.Participants)

persistence/artist_repository.go

@@ -19,6 +19,7 @@ import (
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/utils"
 	"github.com/navidrome/navidrome/utils/slice"
+	"github.com/navidrome/navidrome/utils/str"
 	"github.com/pocketbase/dbx"
 )
 
@@ -102,8 +103,10 @@ func (a *dbArtist) PostMapArgs(m map[string]any) error {
 	}
 	similarArtists, _ := json.Marshal(sa)
 	m["similar_artists"] = string(similarArtists)
+	// When adding a derived column here, also add it to the scanner's artist Put column list
+	// in phase_1_folders.go, or rescans will never update it (how search_normalized went stale).
 	m["full_text"] = formatFullText(a.Name, a.SortArtistName)
-	m["search_normalized"] = normalizeForFTS(a.Name)
+	m["search_normalized"] = str.NormalizeForFTS(a.Name)
 
 	// Do not override the sort_artist_name and mbz_artist_id fields if they are empty
 	// TODO: Better way to handle this?

persistence/mediafile_repository.go

@@ -16,6 +16,7 @@ import (
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
 	"github.com/navidrome/navidrome/utils/slice"
+	"github.com/navidrome/navidrome/utils/str"
 	"github.com/pocketbase/dbx"
 )
 
@@ -62,7 +63,7 @@ func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
 	fullText = append(fullText, participantNames...)
 	args["full_text"] = formatFullText(fullText...)
 	args["search_participants"] = strings.Join(participantNames, " ")
-	args["search_normalized"] = normalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)
+	args["search_normalized"] = str.NormalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)
 	args["tags"] = marshalTags(m.MediaFile.Tags)
 	args["participants"] = marshalParticipants(m.MediaFile.Participants)
 	return nil

persistence/sql_search_fts.go

@@ -11,6 +11,7 @@ import (
 	"github.com/deluan/sanitize"
 	"github.com/navidrome/navidrome/log"
 	"github.com/navidrome/navidrome/model"
+	"github.com/navidrome/navidrome/utils/str"
 )
 
 // containsCJK returns true if the string contains any CJK (Chinese/Japanese/Korean) characters.
@@ -35,48 +36,12 @@ func containsCJK(s string) bool {
 // as unbalanced string delimiters.
 var fts5SpecialChars = regexp.MustCompile(`[^\p{L}\p{N}\s*"\x00]`)
 
-// fts5PunctStrip strips everything except letters and numbers (no whitespace, wildcards, or quotes).
-// Used for normalizing words at index time to create concatenated forms (e.g., "R.E.M." → "REM").
-var fts5PunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)
-
 // fts5Operators matches FTS5 boolean operators as whole words (case-insensitive).
 var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`)
 
 // fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries).
 var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`)
 
-// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of
-// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)
-// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed
-// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —
-// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree
-// without an explicit transliterated entry here.
-func normalizeForFTS(values ...string) string {
-	seen := make(map[string]struct{})
-	var result []string
-	add := func(orig, variant string) {
-		if variant == "" || variant == orig {
-			return
-		}
-		lower := strings.ToLower(variant)
-		if _, ok := seen[lower]; ok {
-			return
-		}
-		seen[lower] = struct{}{}
-		result = append(result, variant)
-	}
-	for _, v := range values {
-		for word := range strings.FieldsSeq(v) {
-			transliterated := sanitize.Accents(word)
-			// Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.
-			add(word, fts5PunctStrip.ReplaceAllString(transliterated, ""))
-			// Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).
-			add(word, transliterated)
-		}
-	}
-	return strings.Join(result, " ")
-}
-
 // isSingleUnicodeLetter returns true if token is exactly one Unicode letter.
 func isSingleUnicodeLetter(token string) bool {
 	r, size := utf8.DecodeRuneInString(token)
@@ -100,7 +65,7 @@ func processPunctuatedWords(input string, phrases []string) (string, []string) {
 			result = append(result, w)
 			continue
 		}
-		concat := fts5PunctStrip.ReplaceAllString(w, "")
+		concat := str.FTSPunctStrip.ReplaceAllString(w, "")
 		if concat == "" || concat == w {
 			result = append(result, w)
 			continue
@@ -329,7 +294,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {
 	// Strip quotes from original for comparison — we want the raw content
 	stripped := strings.ReplaceAll(original, `"`, "")
 	// Extract the alphanumeric content from the original query
-	alphaNum := fts5PunctStrip.ReplaceAllString(stripped, "")
+	alphaNum := str.FTSPunctStrip.ReplaceAllString(stripped, "")
 	// If the original is entirely alphanumeric, nothing was stripped — not degraded
 	if len(alphaNum) == len(stripped) {
 		return false
@@ -353,7 +318,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {
 		if strings.HasPrefix(t, `"`) {
 			// Extract content between quotes
 			inner := strings.Trim(t, `"`)
-			innerAlpha := fts5PunctStrip.ReplaceAllString(inner, " ")
+			innerAlpha := str.FTSPunctStrip.ReplaceAllString(inner, " ")
 			for it := range strings.FieldsSeq(innerAlpha) {
 				if len(it) > 2 {
 					return false

persistence/sql_search_fts_test.go

@@ -74,28 +74,6 @@ var _ = DescribeTable("ftsQueryDegraded",
 	Entry("not degraded for OR groups from processPunctuatedWords", "AC/DC", `("AC DC" OR ACDC*)`, false),
 )
 
-var _ = DescribeTable("normalizeForFTS",
-	func(expected string, values ...string) {
-		Expect(normalizeForFTS(values...)).To(Equal(expected))
-	},
-	Entry("strips dots and concatenates", "REM", "R.E.M."),
-	Entry("strips slash", "ACDC", "AC/DC"),
-	Entry("strips hyphen", "Aha", "A-ha"),
-	Entry("skips unchanged ASCII words", "", "The Beatles"),
-	Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),
-	Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),
-	Entry("strips apostrophe from word", "N", "Guns N' Roses"),
-	Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),
-	Entry("transliterates ø to o", "Bjork", "Bjørk"),
-	Entry("transliterates Ø to O", "Oystein", "Øystein"),
-	Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"),
-	Entry("transliterates Latin diacritics", "cafe", "café"),
-	Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),
-	Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),
-	Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),
-	Entry("transliterates ß to ss", "Strasse", "Straße"),
-)
-
 var _ = DescribeTable("containsCJK",
 	func(input string, expected bool) {
 		Expect(containsCJK(input)).To(Equal(expected))

scanner/phase_1_folders.go

@@ -360,7 +360,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
 		// Save all new/modified artists to DB. Their information will be incomplete, but they will be refreshed later
 		for i := range entry.artists {
 			err = artistRepo.Put(&entry.artists[i], "name",
-				"mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "updated_at")
+				"mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "search_normalized", "updated_at")
 			if err != nil {
 				log.Error(p.ctx, "Scanner: Error persisting artist to DB", "folder", entry.path, "artist", entry.artists[i].Name, err)
 				return err

scanner/scanner_test.go

@@ -189,6 +189,34 @@ var _ = Describe("Scanner", Ordered, func() {
 		})
 	})
 
+	Context("Artist with atomic non-ASCII letters, 'GØGGS'", func() {
+		BeforeEach(func() {
+			goggs := template(_t{"albumartist": "GØGGS", "album": "Pre Strike Sweep", "year": 2018})
+			createFS(fstest.MapFS{
+				"GØGGS/Pre Strike Sweep/01 - Falling For You.mp3": goggs(track(1, "Falling For You")),
+			})
+		})
+
+		searchNormalized := func() string {
+			var sn string
+			Expect(db.Db().QueryRowContext(ctx,
+				"SELECT search_normalized FROM artist WHERE name = 'GØGGS'").Scan(&sn)).To(Succeed())
+			return sn
+		}
+
+		It("repopulates a stale search_normalized on a full rescan", func() {
+			Expect(runScanner(ctx, true)).To(Succeed())
+			Expect(searchNormalized()).To(Equal("GOGGS"))
+
+			// Simulate the stale value left by the FTS5 migration's SQL back-fill
+			_, err := db.Db().ExecContext(ctx, "UPDATE artist SET search_normalized = '' WHERE name = 'GØGGS'")
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(runScanner(ctx, true)).To(Succeed())
+			Expect(searchNormalized()).To(Equal("GOGGS"))
+		})
+	})
+
 	Context("Ignored entries", func() {
 		BeforeEach(func() {
 			revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})

utils/str/normalize_fts.go

@@ -0,0 +1,45 @@
+package str
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/deluan/sanitize"
+)
+
+// FTSPunctStrip matches any character that is not a letter or number. Index-time
+// normalization (NormalizeForFTS) and query-time processing in persistence share it
+// so both sides produce matching tokens.
+var FTSPunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)
+
+// NormalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of
+// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)
+// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed
+// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —
+// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree
+// without an explicit transliterated entry here.
+func NormalizeForFTS(values ...string) string {
+	seen := make(map[string]struct{})
+	var result []string
+	add := func(orig, variant string) {
+		if variant == "" || variant == orig {
+			return
+		}
+		lower := strings.ToLower(variant)
+		if _, ok := seen[lower]; ok {
+			return
+		}
+		seen[lower] = struct{}{}
+		result = append(result, variant)
+	}
+	for _, v := range values {
+		for word := range strings.FieldsSeq(v) {
+			transliterated := sanitize.Accents(word)
+			// Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.
+			add(word, FTSPunctStrip.ReplaceAllString(transliterated, ""))
+			// Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).
+			add(word, transliterated)
+		}
+	}
+	return strings.Join(result, " ")
+}

utils/str/normalize_fts_test.go

@@ -0,0 +1,29 @@
+package str_test
+
+import (
+	"github.com/navidrome/navidrome/utils/str"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = DescribeTable("NormalizeForFTS",
+	func(expected string, values ...string) {
+		Expect(str.NormalizeForFTS(values...)).To(Equal(expected))
+	},
+	Entry("strips dots and concatenates", "REM", "R.E.M."),
+	Entry("strips slash", "ACDC", "AC/DC"),
+	Entry("strips hyphen", "Aha", "A-ha"),
+	Entry("skips unchanged ASCII words", "", "The Beatles"),
+	Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),
+	Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),
+	Entry("strips apostrophe from word", "N", "Guns N' Roses"),
+	Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),
+	Entry("transliterates ø to o", "Bjork", "Bjørk"),
+	Entry("transliterates Ø to O", "Oystein", "Øystein"),
+	Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"),
+	Entry("transliterates Latin diacritics", "cafe", "café"),
+	Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),
+	Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),
+	Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),
+	Entry("transliterates ß to ss", "Strasse", "Straße"),
+)