mirror of
https://github.com/navidrome/navidrome.git
synced 2026-07-09 17:18:45 +00:00
perf(search): speed up CJK search with NOCASE covering indexes
Queries containing CJK (and punctuation-only) text bypass FTS5 and fall back to LIKE-based search, because the unicode61 tokenizer indexes whole space-free CJK runs as single tokens and cannot substring-match them. That fallback had no usable index, so each query did a full scan of the wide media_file table across four columns — roughly 4 seconds on a one-million-track library. Add COLLATE NOCASE covering indexes on exactly the columns the LIKE fallback searches (likeSearchColumns), so SQLite scans the narrow per-column indexes instead of the full table. A plain LIKE uses them because the server runs with case_sensitive_like = OFF. Results are unchanged; only latency improves (~4s to ~0.3s end-to-end). The FTS path for Latin/numeric queries is untouched. A new test asserts every likeSearchColumns column has a matching NOCASE index in the migrated schema, so the column set and the migration cannot silently drift.
This commit is contained in:
parent
08a027dbcc
commit
7bb2c0968f
3 changed files with 63 additions and 0 deletions
|
|
@ -0,0 +1,23 @@
|
|||
-- +goose Up
|
||||
-- COLLATE NOCASE covering indexes for the LIKE-based search fallback (CJK and
|
||||
-- punctuation-only queries route to it; see persistence/sql_search_like.go
|
||||
-- likeSearchColumns). Without these, a CJK search3 is a full scan of the wide
|
||||
-- media_file table across 4 columns (~4s on 1M songs); with them SQLite scans the
|
||||
-- narrow per-column indexes instead (~0.3s). Columns MUST match likeSearchColumns.
|
||||
-- A plain LIKE uses these because Navidrome runs with case_sensitive_like = OFF.
|
||||
CREATE INDEX IF NOT EXISTS idx_media_file_title_nocase ON media_file (title COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_file_album_nocase ON media_file (album COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_file_artist_nocase ON media_file (artist COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_media_file_album_artist_nocase ON media_file (album_artist COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_album_name_nocase ON album (name COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_album_album_artist_nocase ON album (album_artist COLLATE NOCASE);
|
||||
CREATE INDEX IF NOT EXISTS idx_artist_name_nocase ON artist (name COLLATE NOCASE);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS idx_media_file_title_nocase;
|
||||
DROP INDEX IF EXISTS idx_media_file_album_nocase;
|
||||
DROP INDEX IF EXISTS idx_media_file_artist_nocase;
|
||||
DROP INDEX IF EXISTS idx_media_file_album_artist_nocase;
|
||||
DROP INDEX IF EXISTS idx_album_name_nocase;
|
||||
DROP INDEX IF EXISTS idx_album_album_artist_nocase;
|
||||
DROP INDEX IF EXISTS idx_artist_name_nocase;
|
||||
|
|
@ -71,6 +71,10 @@ func legacySearchExpr(tableName string, s string) Sqlizer {
|
|||
// likeSearchColumns defines the core columns to search with LIKE queries.
|
||||
// These are the primary user-visible fields for each entity type.
|
||||
// Used as a fallback when FTS5 cannot handle the query (e.g., CJK text, punctuation-only input).
|
||||
//
|
||||
// Each column needs a matching COLLATE NOCASE index (migration
|
||||
// add_like_search_covering_indexes) so these LIKE scans hit a narrow covering index
|
||||
// instead of the full table. The "likeSearchColumns covering indexes" test enforces this.
|
||||
var likeSearchColumns = map[string][]string{
|
||||
"media_file": {"title", "album", "artist", "album_artist"},
|
||||
"album": {"name", "album_artist"},
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ package persistence
|
|||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"github.com/navidrome/navidrome/conf"
|
||||
"github.com/navidrome/navidrome/conf/configtest"
|
||||
|
|
@ -132,3 +134,37 @@ var _ = Describe("Legacy Integration Search", func() {
|
|||
Expect(results).ToNot(BeEmpty(), "Max=0 should mean no limit, not LIMIT 0")
|
||||
})
|
||||
})
|
||||
|
||||
// Guards the invariant that every column searched by the LIKE fallback has a COLLATE NOCASE
|
||||
// covering index (migration add_like_search_covering_indexes). Without it a CJK/punctuation
|
||||
// search does a full table scan. If you add a column to likeSearchColumns without an index,
|
||||
// this test fails — keep the two in sync.
|
||||
var _ = Describe("likeSearchColumns covering indexes", func() {
|
||||
It("has a COLLATE NOCASE index for every searched column", func() {
|
||||
var indexSQLs []string
|
||||
err := GetDBXBuilder().
|
||||
NewQuery("SELECT sql FROM sqlite_master WHERE type='index' AND sql IS NOT NULL").
|
||||
Column(&indexSQLs)
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
|
||||
for table, columns := range likeSearchColumns {
|
||||
// Match the index's target table precisely: "ON <table> (" (case-insensitive).
|
||||
tableRe := regexp.MustCompile(`(?i)\bon\s+` + regexp.QuoteMeta(table) + `\s*\(`)
|
||||
for _, col := range columns {
|
||||
// Match e.g. "(title COLLATE NOCASE)" — column immediately followed by the
|
||||
// NOCASE collation, case-insensitive (SQLite emits both "COLLATE" and "collate").
|
||||
colRe := regexp.MustCompile(`(?i)\b` + regexp.QuoteMeta(col) + `\s+collate\s+nocase\b`)
|
||||
found := false
|
||||
for _, sql := range indexSQLs {
|
||||
if tableRe.MatchString(sql) && colRe.MatchString(sql) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
Expect(found).To(BeTrue(),
|
||||
fmt.Sprintf("missing COLLATE NOCASE index for %s.%s — add it to the "+
|
||||
"add_like_search_covering_indexes migration to keep LIKE search fast", table, col))
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue