perf(search): speed up CJK search with NOCASE covering indexes

Queries containing CJK (and punctuation-only) text bypass FTS5 and fall back to
LIKE-based search, because the unicode61 tokenizer indexes whole space-free CJK
runs as single tokens and cannot substring-match them. That fallback had no usable
index, so each query did a full scan of the wide media_file table across four
columns — roughly 4 seconds on a one-million-track library.

Add COLLATE NOCASE covering indexes on exactly the columns the LIKE fallback
searches (likeSearchColumns), so SQLite scans the narrow per-column indexes instead
of the full table. A plain LIKE uses them because the server runs with
case_sensitive_like = OFF. Results are unchanged; only latency improves
(~4s to ~0.3s end-to-end). The FTS path for Latin/numeric queries is untouched.

A new test asserts every likeSearchColumns column has a matching NOCASE index in
the migrated schema, so the column set and the migration cannot silently drift.
This commit is contained in:
Deluan 2026-06-14 23:54:52 -04:00
parent 08a027dbcc
commit 7bb2c0968f
3 changed files with 63 additions and 0 deletions

View file

@ -0,0 +1,23 @@
-- +goose Up
-- COLLATE NOCASE covering indexes for the LIKE-based search fallback (CJK and
-- punctuation-only queries route to it; see persistence/sql_search_like.go
-- likeSearchColumns). Without these, a CJK search3 is a full scan of the wide
-- media_file table across 4 columns (~4s on 1M songs); with them SQLite scans the
-- narrow per-column indexes instead (~0.3s). Columns MUST match likeSearchColumns.
-- A plain LIKE uses these because Navidrome runs with case_sensitive_like = OFF.
CREATE INDEX IF NOT EXISTS idx_media_file_title_nocase ON media_file (title COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_file_album_nocase ON media_file (album COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_file_artist_nocase ON media_file (artist COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_media_file_album_artist_nocase ON media_file (album_artist COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_album_name_nocase ON album (name COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_album_album_artist_nocase ON album (album_artist COLLATE NOCASE);
CREATE INDEX IF NOT EXISTS idx_artist_name_nocase ON artist (name COLLATE NOCASE);
-- +goose Down
DROP INDEX IF EXISTS idx_media_file_title_nocase;
DROP INDEX IF EXISTS idx_media_file_album_nocase;
DROP INDEX IF EXISTS idx_media_file_artist_nocase;
DROP INDEX IF EXISTS idx_media_file_album_artist_nocase;
DROP INDEX IF EXISTS idx_album_name_nocase;
DROP INDEX IF EXISTS idx_album_album_artist_nocase;
DROP INDEX IF EXISTS idx_artist_name_nocase;

View file

@ -71,6 +71,10 @@ func legacySearchExpr(tableName string, s string) Sqlizer {
// likeSearchColumns defines the core columns to search with LIKE queries.
// These are the primary user-visible fields for each entity type.
// Used as a fallback when FTS5 cannot handle the query (e.g., CJK text, punctuation-only input).
//
// Each column needs a matching COLLATE NOCASE index (migration
// add_like_search_covering_indexes) so these LIKE scans hit a narrow covering index
// instead of the full table. The "likeSearchColumns covering indexes" test enforces this.
var likeSearchColumns = map[string][]string{
"media_file": {"title", "album", "artist", "album_artist"},
"album": {"name", "album_artist"},

View file

@ -2,6 +2,8 @@ package persistence
import (
"context"
"fmt"
"regexp"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
@ -132,3 +134,37 @@ var _ = Describe("Legacy Integration Search", func() {
Expect(results).ToNot(BeEmpty(), "Max=0 should mean no limit, not LIMIT 0")
})
})
// Guards the invariant that every column searched by the LIKE fallback has a COLLATE NOCASE
// covering index (migration add_like_search_covering_indexes). Without it a CJK/punctuation
// search does a full table scan. If you add a column to likeSearchColumns without an index,
// this test fails — keep the two in sync.
var _ = Describe("likeSearchColumns covering indexes", func() {
It("has a COLLATE NOCASE index for every searched column", func() {
var indexSQLs []string
err := GetDBXBuilder().
NewQuery("SELECT sql FROM sqlite_master WHERE type='index' AND sql IS NOT NULL").
Column(&indexSQLs)
Expect(err).ToNot(HaveOccurred())
for table, columns := range likeSearchColumns {
// Match the index's target table precisely: "ON <table> (" (case-insensitive).
tableRe := regexp.MustCompile(`(?i)\bon\s+` + regexp.QuoteMeta(table) + `\s*\(`)
for _, col := range columns {
// Match e.g. "(title COLLATE NOCASE)" — column immediately followed by the
// NOCASE collation, case-insensitive (SQLite emits both "COLLATE" and "collate").
colRe := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(col) + `\s+collate\s+nocase\b`)
found := false
for _, sql := range indexSQLs {
if tableRe.MatchString(sql) && colRe.MatchString(sql) {
found = true
break
}
}
Expect(found).To(BeTrue(),
fmt.Sprintf("missing COLLATE NOCASE index for %s.%s — add it to the "+
"add_like_search_covering_indexes migration to keep LIKE search fast", table, col))
}
}
})
})