mirror of
https://github.com/austinried/subtracks.git
synced 2026-11-10 15:11:58 +01:00
search the library through FTS5 trigram tables
This commit is contained in:
austinried
+1335
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,11 @@ internal const val ALBUMS_FILTER =
|
||||
"AND (:downloadedFilter = 0 OR EXISTS (SELECT 1 FROM songs " +
|
||||
"JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
|
||||
"WHERE songs.sourceId = albums.sourceId AND songs.albumId = albums.id AND sd.status = 'Completed')) " +
|
||||
"AND (:search = '' OR instr(lower(name), lower(:search)) > 0 OR instr(lower(albumArtist), lower(:search)) > 0) "
|
||||
"AND (:search = '' " +
|
||||
"OR (length(:search) >= 3 AND rowid IN (SELECT rowid FROM album_search " +
|
||||
"WHERE album_search MATCH '\"' || replace(:search, '\"', '\"\"') || '\"')) " +
|
||||
"OR (length(:search) < 3 AND (instr(lower(name), lower(:search)) > 0 " +
|
||||
"OR instr(lower(albumArtist), lower(:search)) > 0))) "
|
||||
|
||||
internal const val ALBUM_ORDER_BY_NAME = "name COLLATE NOCASE, id"
|
||||
internal const val ALBUM_ORDER_BY_NAME_REVERSED = "name COLLATE NOCASE DESC, id DESC"
|
||||
@@ -55,7 +59,10 @@ internal const val ARTISTS_FILTER =
|
||||
"JOIN albums dl ON dl.sourceId = songs.sourceId AND dl.id = songs.albumId " +
|
||||
"JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
|
||||
"WHERE songs.sourceId = artists.sourceId AND dl.artistId = artists.id AND sd.status = 'Completed')) " +
|
||||
"AND (:search = '' OR instr(lower(name), lower(:search)) > 0) "
|
||||
"AND (:search = '' " +
|
||||
"OR (length(:search) >= 3 AND rowid IN (SELECT rowid FROM artist_search " +
|
||||
"WHERE artist_search MATCH '\"' || replace(:search, '\"', '\"\"') || '\"')) " +
|
||||
"OR (length(:search) < 3 AND instr(lower(name), lower(:search)) > 0)) "
|
||||
|
||||
internal const val ARTIST_ORDER_BY_NAME = "name COLLATE NOCASE, id"
|
||||
internal const val ARTIST_ORDER_BY_NAME_REVERSED = "name COLLATE NOCASE DESC, id DESC"
|
||||
@@ -69,7 +76,10 @@ internal const val PLAYLISTS_FILTER =
|
||||
"AND (:downloadedFilter = 0 OR EXISTS (SELECT 1 FROM playlist_songs ps " +
|
||||
"JOIN song_downloads sd ON sd.sourceId = ps.sourceId AND sd.songId = ps.songId " +
|
||||
"WHERE ps.sourceId = playlists.sourceId AND ps.playlistId = playlists.id AND sd.status = 'Completed')) " +
|
||||
"AND (:search = '' OR instr(lower(name), lower(:search)) > 0) "
|
||||
"AND (:search = '' " +
|
||||
"OR (length(:search) >= 3 AND rowid IN (SELECT rowid FROM playlist_search " +
|
||||
"WHERE playlist_search MATCH '\"' || replace(:search, '\"', '\"\"') || '\"')) " +
|
||||
"OR (length(:search) < 3 AND instr(lower(name), lower(:search)) > 0)) "
|
||||
|
||||
internal const val PLAYLIST_ORDER_BY_NAME = "name COLLATE NOCASE, id"
|
||||
internal const val PLAYLIST_ORDER_BY_NAME_REVERSED = "name COLLATE NOCASE DESC, id DESC"
|
||||
|
||||
@@ -271,6 +271,47 @@ val MIGRATION_18_19 =
|
||||
}
|
||||
}
|
||||
|
||||
val MIGRATION_19_20 =
|
||||
object : Migration(19, 20) {
|
||||
override suspend fun migrate(connection: SQLiteConnection) {
|
||||
createSearchIndex(connection, "album_search", "albums", listOf("name", "albumArtist"))
|
||||
createSearchIndex(connection, "artist_search", "artists", listOf("name"))
|
||||
createSearchIndex(connection, "playlist_search", "playlists", listOf("name"))
|
||||
}
|
||||
}
|
||||
|
||||
// Mirrors the FTS5 table and content-sync triggers Room generates for the @Fts5 entities.
|
||||
private suspend fun createSearchIndex(
|
||||
connection: SQLiteConnection,
|
||||
table: String,
|
||||
content: String,
|
||||
columns: List<String>,
|
||||
) {
|
||||
connection.execSQL(
|
||||
"CREATE VIRTUAL TABLE IF NOT EXISTS `$table` USING FTS5(" +
|
||||
columns.joinToString(", ") { "`$it`" } + ", tokenize=`trigram`, content=`$content`)",
|
||||
)
|
||||
val inserted = (listOf("rowid") + columns).joinToString(", ") { "`$it`" }
|
||||
val values = columns.joinToString(", ") { "NEW.`$it`" }
|
||||
connection.execSQL(
|
||||
"CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_${table}_BEFORE_UPDATE BEFORE UPDATE ON `$content` " +
|
||||
"BEGIN DELETE FROM `$table` WHERE `rowid`=OLD.`rowid`; END",
|
||||
)
|
||||
connection.execSQL(
|
||||
"CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_${table}_BEFORE_DELETE BEFORE DELETE ON `$content` " +
|
||||
"BEGIN DELETE FROM `$table` WHERE `rowid`=OLD.`rowid`; END",
|
||||
)
|
||||
connection.execSQL(
|
||||
"CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_${table}_AFTER_UPDATE AFTER UPDATE ON `$content` " +
|
||||
"BEGIN INSERT INTO `$table`($inserted) VALUES (NEW.`rowid`, $values); END",
|
||||
)
|
||||
connection.execSQL(
|
||||
"CREATE TRIGGER IF NOT EXISTS room_fts_content_sync_${table}_AFTER_INSERT AFTER INSERT ON `$content` " +
|
||||
"BEGIN INSERT INTO `$table`($inserted) VALUES (NEW.`rowid`, $values); END",
|
||||
)
|
||||
connection.execSQL("INSERT INTO `$table`(`$table`) VALUES('rebuild')")
|
||||
}
|
||||
|
||||
val MIGRATIONS: Array<Migration> =
|
||||
arrayOf(
|
||||
MIGRATION_1_2,
|
||||
@@ -291,4 +332,5 @@ val MIGRATIONS: Array<Migration> =
|
||||
MIGRATION_16_17,
|
||||
MIGRATION_17_18,
|
||||
MIGRATION_18_19,
|
||||
MIGRATION_19_20,
|
||||
)
|
||||
|
||||
@@ -4,12 +4,15 @@ import androidx.room3.ColumnTypeConverters
|
||||
import androidx.room3.Database
|
||||
import androidx.room3.RoomDatabase
|
||||
import com.subtracks.data.model.Album
|
||||
import com.subtracks.data.model.AlbumSearch
|
||||
import com.subtracks.data.model.Artist
|
||||
import com.subtracks.data.model.ArtistSearch
|
||||
import com.subtracks.data.model.ArtworkSeed
|
||||
import com.subtracks.data.model.Disc
|
||||
import com.subtracks.data.model.DownloadStatusConverter
|
||||
import com.subtracks.data.model.PlaybackCursor
|
||||
import com.subtracks.data.model.Playlist
|
||||
import com.subtracks.data.model.PlaylistSearch
|
||||
import com.subtracks.data.model.PlaylistSong
|
||||
import com.subtracks.data.model.QueueEntry
|
||||
import com.subtracks.data.model.QueueKindConverter
|
||||
@@ -24,9 +27,12 @@ import com.subtracks.data.model.UpNextEntry
|
||||
Source::class,
|
||||
SubsonicSource::class,
|
||||
Artist::class,
|
||||
ArtistSearch::class,
|
||||
Album::class,
|
||||
AlbumSearch::class,
|
||||
Disc::class,
|
||||
Playlist::class,
|
||||
PlaylistSearch::class,
|
||||
PlaylistSong::class,
|
||||
Song::class,
|
||||
QueueEntry::class,
|
||||
@@ -35,7 +41,7 @@ import com.subtracks.data.model.UpNextEntry
|
||||
ArtworkSeed::class,
|
||||
SongDownload::class,
|
||||
],
|
||||
version = 19,
|
||||
version = 20,
|
||||
exportSchema = true,
|
||||
)
|
||||
@ColumnTypeConverters(QueueKindConverter::class, DownloadStatusConverter::class)
|
||||
|
||||
@@ -5,6 +5,8 @@ import androidx.room3.ColumnTypeConverter
|
||||
import androidx.room3.Embedded
|
||||
import androidx.room3.Entity
|
||||
import androidx.room3.ForeignKey
|
||||
import androidx.room3.Fts5
|
||||
import androidx.room3.FtsOptions
|
||||
import androidx.room3.Ignore
|
||||
import androidx.room3.Index
|
||||
import androidx.room3.PrimaryKey
|
||||
@@ -351,6 +353,28 @@ data class ArtworkSeed(
|
||||
@ColumnInfo(defaultValue = "0") val storedAt: Long = 0,
|
||||
)
|
||||
|
||||
@Entity(tableName = "album_search")
|
||||
@Fts5(contentEntity = Album::class, tokenizer = FtsOptions.TOKENIZER_TRIGRAM)
|
||||
data class AlbumSearch(
|
||||
@PrimaryKey @ColumnInfo(name = "rowid") val rowId: Long = 0,
|
||||
val name: String,
|
||||
val albumArtist: String?,
|
||||
)
|
||||
|
||||
@Entity(tableName = "artist_search")
|
||||
@Fts5(contentEntity = Artist::class, tokenizer = FtsOptions.TOKENIZER_TRIGRAM)
|
||||
data class ArtistSearch(
|
||||
@PrimaryKey @ColumnInfo(name = "rowid") val rowId: Long = 0,
|
||||
val name: String,
|
||||
)
|
||||
|
||||
@Entity(tableName = "playlist_search")
|
||||
@Fts5(contentEntity = Playlist::class, tokenizer = FtsOptions.TOKENIZER_TRIGRAM)
|
||||
data class PlaylistSearch(
|
||||
@PrimaryKey @ColumnInfo(name = "rowid") val rowId: Long = 0,
|
||||
val name: String,
|
||||
)
|
||||
|
||||
class QueueKindConverter {
|
||||
@ColumnTypeConverter
|
||||
fun fromQueueKind(kind: QueueKind): String = kind.name
|
||||
|
||||
@@ -347,6 +347,106 @@ class LibraryDaoTest {
|
||||
assertEquals(3, dao.albumsByName(sourceId, 0, "").page().size)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun searchMatchesInfixCaseInsensitivelyAndFallsBackToAScanBelowThreeCharacters() =
|
||||
runTest {
|
||||
val sourceId = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertAlbums(
|
||||
listOf(
|
||||
album(sourceId, "a", "Gamma", year = null, starred = null).copy(albumArtist = "The Artist"),
|
||||
album(sourceId, "b", "Alpha", year = null, starred = null).copy(albumArtist = "Someone Else"),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "amm").page().map { it.name })
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "GAM").page().map { it.name })
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "ga").page().map { it.name })
|
||||
assertEquals(listOf("Alpha"), dao.albumsByName(sourceId, 0, "Else").page().map { it.name })
|
||||
assertTrue(dao.albumsByName(sourceId, 0, "zzz").page().isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun searchIndexFollowsRenamesUpdatesAndDeletes() =
|
||||
runTest {
|
||||
val sourceId = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertAlbums(listOf(album(sourceId, "a", "Gamma", year = null, starred = null)))
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "gam").page().map { it.name })
|
||||
|
||||
dao.setAlbumStar(sourceId, "a", 7)
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "gam").page().map { it.name })
|
||||
|
||||
dao.upsertAlbums(listOf(album(sourceId, "a", "Delta", year = null, starred = null)))
|
||||
assertTrue(dao.albumsByName(sourceId, 0, "gam").page().isEmpty())
|
||||
assertEquals(listOf("Delta"), dao.albumsByName(sourceId, 0, "elt").page().map { it.name })
|
||||
|
||||
dao.deleteAlbums(sourceId, listOf("a"))
|
||||
assertTrue(dao.albumsByName(sourceId, 0, "elt").page().isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun deletingASourceClearsItsSearchRows() =
|
||||
runTest {
|
||||
val first = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertAlbums(listOf(album(first, "a", "Gamma", year = null, starred = null)))
|
||||
db.sourcesDao().deleteSource(first)
|
||||
|
||||
val second = source()
|
||||
dao.upsertAlbums(listOf(album(second, "a", "Beta", year = null, starred = null)))
|
||||
|
||||
assertEquals(listOf("Beta"), dao.albumsByName(second, 0, "eta").page().map { it.name })
|
||||
assertTrue(dao.albumsByName(second, 0, "gam").page().isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun artistAndPlaylistSearchUseTheirOwnIndices() =
|
||||
runTest {
|
||||
val sourceId = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertArtists(listOf(artist(sourceId, "r1", "The Beatles", albumCount = 1)))
|
||||
dao.upsertPlaylists(listOf(playlist(sourceId, "l1", "Road Trip", created = 0, changed = 0)))
|
||||
|
||||
assertEquals(listOf("r1"), dao.artistsByName(sourceId, 0, "eat").page().map { it.id })
|
||||
assertEquals(listOf("l1"), dao.playlistsByName(sourceId, "rip").page().map { it.id })
|
||||
assertTrue(dao.artistsByName(sourceId, 0, "rip").page().isEmpty())
|
||||
assertTrue(dao.playlistsByName(sourceId, "eat").page().isEmpty())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun aQuoteInTheSearchTermIsTreatedLiterally() =
|
||||
runTest {
|
||||
val sourceId = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertAlbums(
|
||||
listOf(
|
||||
album(sourceId, "a", "Say \"Hello\"", year = null, starred = null),
|
||||
album(sourceId, "b", "Plain", year = null, starred = null),
|
||||
),
|
||||
)
|
||||
|
||||
assertEquals(listOf("Say \"Hello\""), dao.albumsByName(sourceId, 0, "\"Hel").page().map { it.name })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun searchHandlesANullAlbumArtist() =
|
||||
runTest {
|
||||
val sourceId = source()
|
||||
val dao = db.libraryDao()
|
||||
dao.upsertAlbums(listOf(album(sourceId, "a", "Gamma", year = null, starred = null).copy(albumArtist = null)))
|
||||
|
||||
assertEquals(listOf("Gamma"), dao.albumsByName(sourceId, 0, "amm").page().map { it.name })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun searchRunsThroughTheTrigramIndex() =
|
||||
runTest {
|
||||
val plan = plan("SELECT * $ALBUMS_FILTER ORDER BY $ALBUM_ORDER_BY_NAME LIMIT 20 OFFSET 0")
|
||||
|
||||
assertTrue(plan, plan.contains("album_search"))
|
||||
}
|
||||
|
||||
private suspend fun plan(sql: String): String =
|
||||
db.useReaderConnection { connection ->
|
||||
connection.usePrepared("EXPLAIN QUERY PLAN $sql") { statement ->
|
||||
|
||||
@@ -122,7 +122,11 @@ Sync is not atomic. Each batch commits on its own, so a failure leaves the batch
|
||||
|
||||
## Search
|
||||
|
||||
Search is a case-insensitive substring filter on the library queries (`instr(lower(column), lower(:search)) > 0` in `LibraryDao`), scoped to the active tab and with the matching rows ordered by the tab's sort. There is no separate search index; the filter runs directly against the library tables. An FTS5 trigram shadow table was trialled and dropped: it was wired to a separate search screen that went unused, and it indexed only `title`, so it did not cover the per-field substring the inline filter does. The decision is reversible if infix search over a very large library ever needs an index, because `tokenize='trigram'` is the only SQLite built-in that can index an arbitrary substring match. A revival must index every field the filter scans (`name`/`albumArtist` for albums, `name` for artists and playlists), be kept current incrementally per sync batch rather than cleared and rebuilt, and accept a three-character minimum (shorter queries fall back to the scan); sorting and paging would still come from the normal indexed library queries.
|
||||
Search is a case-insensitive substring filter on the library queries, scoped to the active tab and ordered by the tab's sort. A query of three characters or more goes through a per-entity FTS5 trigram table: `album_search` indexes `name` and `albumArtist`, `artist_search` and `playlist_search` index `name`, and the filter is `rowid IN (SELECT rowid FROM <table> WHERE <table> MATCH '"' || replace(:search, '"', '""') || '"')`. Two characters or fewer cannot form a trigram, so they fall back to the `instr(lower(column), lower(:search))` scan; an empty query skips both.
|
||||
|
||||
The trigram tables are external content (`content=` the base table, keyed by its rowid), so they store no second copy of the text. Room declares them as `@Fts5(contentEntity = ...)` entities and generates the insert/update/delete triggers that keep the index current per row, so every write path — batch sync upserts, star toggles, prune deletes and source cascades — maintains it with no extra code. Sorting and paging still come from the normal indexed library queries; the FTS match only narrows which rows they return, and `MIGRATION_19_20` creates the tables, triggers and a `'rebuild'` for existing libraries.
|
||||
|
||||
The two branches do not fold case identically: the trigram tokenizer lowercases all of Unicode, while SQLite's `lower()` (used by the fallback) is ASCII-only. So an accented query of three or more characters matches case-insensitively where its two-character prefix would not. This only ever widens the result set, and is not worth normalising in SQL.
|
||||
|
||||
## Networking and auth
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ Not done yet in this slice (deliberately): cover-art tonal colour extraction, lo
|
||||
|
||||
## Phase 4 - Search, playlists, offline
|
||||
|
||||
- (done) Search: a substring filter on the library queries, scoped to the active tab, with a docked search field and per-tab sort/filter controls.
|
||||
- (done) Search: a substring filter on the library queries, scoped to the active tab, with a docked search field and per-tab sort/filter controls. Queries of three or more characters run through per-entity FTS5 trigram tables (external content, trigger-maintained), with the `instr` scan as the below-three-character fallback.
|
||||
- Playlist browsing and editing (create/add/remove), syncing changes back to the server.
|
||||
- Offline mode and downloads.
|
||||
|
||||
@@ -61,7 +61,7 @@ Follow-up work for very large libraries, playlists and long sessions, grouped so
|
||||
- **Sync pipeline.** Prune with a memory-bounded id set, remove `fetchRanks` and the write-only rank columns, fetch playlists/albums with bounded concurrency and skip unchanged ones, and raise/make configurable the page cap with a clear "library too large" failure. Independent of the read path.
|
||||
- **Queue view and edits.** Bound the queue view's loaded rows (or page over the queue ordinal) and scope its ViewModel to the overlay; compact adjacent same-ref ranges after an edit and cache resolved entry lengths. The view half is independent; the edit half shares `QueueRepository` with the read-path PR.
|
||||
- **Artwork.** Give `artwork_seeds` a `sourceId` with `ON DELETE CASCADE` and an LRU/TTL (or drop the table and rely on Coil's disk cache); prefetch the thumbnail, not the original, on now-playing transitions. Independent.
|
||||
- **FTS5 search.** Deferred; the revival requirements are recorded in `architecture.md` (per-entity trigram tables indexing every scanned field, incremental maintenance, three-character floor).
|
||||
- **FTS5 search.** Landed: per-entity external-content trigram tables (`album_search`, `artist_search`, `playlist_search`) with Room-generated content-sync triggers, a three-character floor and an `instr` fallback below it.
|
||||
- **UI/flow overhead.** Subscribe only the active tab's paging flow; move the position ticker off the shared `PlaybackState`. Independent.
|
||||
|
||||
A `syncGen` column was considered for the sync prune and rejected: it is part of the upsert's update set, so its always-changing value defeats `upsertChanged`'s changed-only `WHERE` predicate and rewrites every row on every full sync. The prune should instead keep the two-phase diff with a memory-bounded id set (primitive long hashes) or diff a staging `seen` table in SQL.
|
||||
|
||||
Reference in New Issue
Block a user