library: keep artist queries index-driven on large libraries

Rewrite artist queue membership as a UNION of indexed song-id branches instead of an OR that scanned every song, and recompute artist play data from an explicit credited-album set instead of scanning all albums per artist. Run PRAGMA optimize after syncing songs and before play aggregation, and derive downloaded artist lists and filters from downloads rather than all songs. Count downloaded albums per artist with artist-first branches. Extend the query timing harness with the artist playback, download and status queries plus a partial-download fixture.
2026-11-10 15:11:58 +01:00 · 2026-10-18 03:40:16 +00:00
parent 97d33a781a
commit ef0c34d973
6 changed files with 150 additions and 47 deletions
@@ -13,6 +13,7 @@ import kotlinx.coroutines.flow.Flow

 // A song is attributed to its album artist and to its credited track artist, so a guest artist's
 // downloads are visible even though they lead no albums of their own.
+// Starts from songs so status totals include every song the artist is credited on.
 internal const val ARTIST_DOWNLOAD_ROWS =
    "SELECT albums.artistId AS id, songs.id AS songId, song_downloads.status AS status, " +
        "song_downloads.downloadedAt AS downloadedAt " +
@@ -44,6 +45,38 @@ internal const val ARTIST_DOWNLOAD_ROWS =
        "LEFT JOIN song_downloads ON song_downloads.sourceId = songs.sourceId AND song_downloads.songId = songs.id " +
        "WHERE songs.sourceId = :sourceId"

+// Starts from downloads so lists and filters over partially downloaded libraries do not scan songs.
+internal const val DOWNLOADED_ARTIST_ROWS =
+    "SELECT albums.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " +
+        "song_downloads.status AS status " +
+        "FROM song_downloads " +
+        "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " +
+        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
+        "WHERE song_downloads.sourceId = :sourceId AND albums.artistId IS NOT NULL " +
+        "UNION ALL " +
+        "SELECT album_artists.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " +
+        "song_downloads.status AS status " +
+        "FROM song_downloads " +
+        "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " +
+        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
+        "JOIN album_artists ON album_artists.sourceId = albums.sourceId AND album_artists.albumId = albums.id " +
+        "WHERE song_downloads.sourceId = :sourceId " +
+        "UNION ALL " +
+        "SELECT songs.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " +
+        "song_downloads.status AS status " +
+        "FROM song_downloads " +
+        "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " +
+        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
+        "WHERE song_downloads.sourceId = :sourceId AND songs.artistId IS NOT NULL " +
+        "UNION ALL " +
+        "SELECT song_artists.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " +
+        "song_downloads.status AS status " +
+        "FROM song_downloads " +
+        "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " +
+        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
+        "JOIN song_artists ON song_artists.sourceId = songs.sourceId AND song_artists.songId = songs.id " +
+        "WHERE song_downloads.sourceId = :sourceId"
+
@Dao
 interface DownloadDao {
    @Query("SELECT * FROM song_downloads WHERE sourceId = :sourceId")
@@ -111,7 +144,10 @@ interface DownloadDao {
            "COUNT(DISTINCT songId) AS total, " +
            "COUNT(DISTINCT CASE WHEN status = 'Completed' THEN songId END) AS downloaded, " +
            "COUNT(DISTINCT CASE WHEN status IN ('Queued', 'Running') THEN songId END) AS downloading " +
-            "FROM (" + ARTIST_DOWNLOAD_ROWS + ") GROUP BY id " +
+            "FROM (" + ARTIST_DOWNLOAD_ROWS + ") " +
+            "WHERE id IN (SELECT id FROM (" + DOWNLOADED_ARTIST_ROWS + ") " +
+            "WHERE status IN ('Completed', 'Queued', 'Running')) " +
+            "GROUP BY id " +
            "HAVING downloaded > 0 OR downloading > 0",
    )
    fun artistStatuses(sourceId: Long): Flow<List<EntityDownloadStatus>>
@@ -65,29 +65,37 @@ internal const val ALBUM_ORDER_BY_FREQUENT_REVERSED = "playCount ASC, name COLLA
 internal const val ALBUM_ORDER_BY_RECENT = "played DESC NULLS LAST, name COLLATE NOCASE, id"
 internal const val ALBUM_ORDER_BY_RECENT_REVERSED = "played ASC NULLS LAST, name COLLATE NOCASE DESC, id DESC"

-internal const val SONG_MATCHES_ARTIST =
-    "(songs.artistId = :artistId OR songs.id IN (SELECT songId FROM song_artists " +
-        "WHERE sourceId = songs.sourceId AND artistId = :artistId))"
-
-internal const val ALBUM_MATCHES_ARTIST =
-    "(albums.artistId = :artistId OR albums.id IN (SELECT albumId FROM album_artists " +
-        "WHERE sourceId = albums.sourceId AND artistId = :artistId))"
-
-private const val ARTIST_ALBUM_MATCH =
-    "(a.artistId = artists.id OR a.id IN (SELECT albumId FROM album_artists " +
-        "WHERE sourceId = artists.sourceId AND artistId = artists.id))"
+private const val DOWNLOADED_ARTIST_ALBUMS =
+    "SELECT albums.id FROM albums " +
+        "WHERE albums.sourceId = :sourceId AND albums.artistId = artists.id " +
+        "AND EXISTS (SELECT 1 FROM songs " +
+        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
+        "WHERE songs.sourceId = albums.sourceId AND songs.albumId = albums.id AND sd.status = 'Completed') " +
+        "UNION " +
+        "SELECT albums.id FROM album_artists " +
+        "JOIN albums ON albums.sourceId = album_artists.sourceId AND albums.id = album_artists.albumId " +
+        "WHERE album_artists.sourceId = :sourceId AND album_artists.artistId = artists.id " +
+        "AND EXISTS (SELECT 1 FROM songs " +
+        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
+        "WHERE songs.sourceId = albums.sourceId AND songs.albumId = albums.id AND sd.status = 'Completed') " +
+        "UNION " +
+        "SELECT songs.albumId FROM songs " +
+        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
+        "WHERE songs.sourceId = :sourceId AND songs.artistId = artists.id " +
+        "AND sd.status = 'Completed' AND songs.albumId IS NOT NULL " +
+        "UNION " +
+        "SELECT songs.albumId FROM song_artists " +
+        "JOIN songs ON songs.sourceId = song_artists.sourceId AND songs.id = song_artists.songId " +
+        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
+        "WHERE song_artists.sourceId = :sourceId AND song_artists.artistId = artists.id " +
+        "AND sd.status = 'Completed' AND songs.albumId IS NOT NULL"

 internal const val ARTISTS_FILTER =
    "FROM artists WHERE sourceId = :sourceId " +
        "AND (:starredFilter = 0 OR (:starredFilter = 1 AND starred IS NOT NULL) OR (:starredFilter = 2 AND starred IS NULL)) " +
-        "AND (:downloadedFilter = 0 OR EXISTS (SELECT 1 FROM songs " +
-        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
-        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
-        "WHERE songs.sourceId = artists.sourceId AND sd.status = 'Completed' " +
-        "AND (albums.artistId = artists.id " +
-        "OR albums.id IN (SELECT albumId FROM album_artists aa WHERE aa.sourceId = albums.sourceId AND aa.artistId = artists.id) " +
-        "OR songs.artistId = artists.id " +
-        "OR songs.id IN (SELECT songId FROM song_artists sa WHERE sa.sourceId = songs.sourceId AND sa.artistId = artists.id)))) " +
+        "AND (:downloadedFilter = 0 OR EXISTS (" +
+        "SELECT 1 FROM ($ARTIST_DOWNLOAD_ROWS) dl " +
+        "WHERE dl.id = artists.id AND dl.status = 'Completed')) " +
        "AND (:search = '' " +
        "OR (length(:search) >= 3 AND rowid IN (SELECT rowid FROM artist_search " +
        "WHERE artist_search MATCH '\"' || replace(:search, '\"', '\"\"') || '\"')) " +
@@ -98,14 +106,7 @@ internal const val ARTISTS_FILTER =
 internal const val ARTISTS_SELECT =
    "SELECT artists.sourceId, artists.id, artists.name, " +
        "CASE WHEN :downloadedFilter = 1 THEN (" +
-        "SELECT COUNT(DISTINCT albums.id) FROM songs " +
-        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
-        "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " +
-        "WHERE songs.sourceId = artists.sourceId AND sd.status = 'Completed' " +
-        "AND (albums.artistId = artists.id " +
-        "OR albums.id IN (SELECT albumId FROM album_artists aa WHERE aa.sourceId = albums.sourceId AND aa.artistId = artists.id) " +
-        "OR songs.artistId = artists.id " +
-        "OR songs.id IN (SELECT songId FROM song_artists sa WHERE sa.sourceId = songs.sourceId AND sa.artistId = artists.id))" +
+        "SELECT COUNT(*) FROM ($DOWNLOADED_ARTIST_ALBUMS)" +
        ") ELSE artists.albumCount END AS albumCount, " +
        "artists.starred, artists.coverArt, artists.playCount, artists.played "

@@ -200,19 +201,27 @@ interface LibraryDao {
    )
    suspend fun recomputeAlbumPlayData(sourceId: Long)

-    // INDEXED BY is required: the planner otherwise answers MAX(played) from index_albums_recent
-    // (sourceId, played), which cannot apply the artistId equality.
+    // The credited albums are the union of the lead id and the relation table, so a shared album
+    // counts once per artist rather than twice.
    @Query(
        "UPDATE artists SET " +
-            "playCount = COALESCE((SELECT SUM(a.playCount) FROM albums a INDEXED BY index_albums_sourceId_artistId " +
-            "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH), 0), " +
-            "played = (SELECT MAX(a.played) FROM albums a INDEXED BY index_albums_sourceId_artistId " +
-            "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH) " +
+            "playCount = COALESCE((SELECT SUM(a.playCount) FROM albums a " +
+            "WHERE a.sourceId = artists.sourceId AND a.id IN (" +
+            "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " +
+            "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)), 0), " +
+            "played = (SELECT MAX(a.played) FROM albums a " +
+            "WHERE a.sourceId = artists.sourceId AND a.id IN (" +
+            "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " +
+            "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)) " +
            "WHERE sourceId = :sourceId AND (" +
-            "playCount IS NOT COALESCE((SELECT SUM(a.playCount) FROM albums a INDEXED BY index_albums_sourceId_artistId " +
-            "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH), 0) " +
-            "OR played IS NOT (SELECT MAX(a.played) FROM albums a INDEXED BY index_albums_sourceId_artistId " +
-            "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH))",
+            "playCount IS NOT COALESCE((SELECT SUM(a.playCount) FROM albums a " +
+            "WHERE a.sourceId = artists.sourceId AND a.id IN (" +
+            "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " +
+            "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)), 0) " +
+            "OR played IS NOT (SELECT MAX(a.played) FROM albums a " +
+            "WHERE a.sourceId = artists.sourceId AND a.id IN (" +
+            "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " +
+            "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)))",
    )
    suspend fun recomputeArtistPlayData(sourceId: Long)

@@ -831,10 +840,10 @@ interface LibraryDao {

    @Query(
        "SELECT artists.* FROM artists " +
-            "JOIN ($ARTIST_DOWNLOAD_ROWS) dl ON dl.id = artists.id " +
-            "WHERE artists.sourceId = :sourceId AND dl.status = 'Completed' " +
-            "GROUP BY artists.sourceId, artists.id " +
-            "ORDER BY MAX(dl.downloadedAt) DESC, artists.name COLLATE NOCASE, artists.id",
+            "JOIN (SELECT id, MAX(downloadedAt) AS lastDownloadedAt FROM ($DOWNLOADED_ARTIST_ROWS) " +
+            "WHERE status = 'Completed' GROUP BY id) dl ON dl.id = artists.id " +
+            "WHERE artists.sourceId = :sourceId " +
+            "ORDER BY dl.lastDownloadedAt DESC, artists.name COLLATE NOCASE, artists.id",
    )
    fun downloadedArtists(sourceId: Long): PagingSource<Int, Artist>

@@ -19,12 +19,23 @@ private const val ALBUM_SONGS_ORDER = " ORDER BY songs.disc, songs.track, songs.

 private const val ALBUM_SONGS_SQL = ALBUM_SONGS_SELECT + ALBUM_SONGS_ORDER

-private const val ARTIST_SONGS_MATCH = "($ALBUM_MATCHES_ARTIST OR $SONG_MATCHES_ARTIST)"
+private const val ARTIST_SONG_IDS =
+    "(SELECT id FROM songs WHERE sourceId = :sourceId AND artistId = :artistId " +
+        "UNION " +
+        "SELECT songId FROM song_artists WHERE sourceId = :sourceId AND artistId = :artistId " +
+        "UNION " +
+        "SELECT songs.id FROM albums " +
+        "JOIN songs ON songs.sourceId = albums.sourceId AND songs.albumId = albums.id " +
+        "WHERE albums.sourceId = :sourceId AND albums.artistId = :artistId " +
+        "UNION " +
+        "SELECT songs.id FROM album_artists " +
+        "JOIN songs ON songs.sourceId = album_artists.sourceId AND songs.albumId = album_artists.albumId " +
+        "WHERE album_artists.sourceId = :sourceId AND album_artists.artistId = :artistId)"

 private const val ARTIST_SONGS_SELECT =
    "SELECT songs.*, albums.coverArt AS coverArt FROM songs " +
        "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
-        "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH"
+        "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS"

 private const val ARTIST_SONGS_ORDER =
    " ORDER BY albums.year DESC, albums.name COLLATE NOCASE, albums.id, songs.disc, songs.track, songs.id"
@@ -85,7 +96,7 @@ interface QueueDao {
    @Query(
        "SELECT COUNT(*) FROM songs " +
            "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
-            "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH",
+            "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS",
    )
    suspend fun artistLength(
        sourceId: Long,
@@ -193,7 +204,7 @@ interface QueueDao {
    @Query(
        "SELECT songs.id FROM songs " +
            "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " +
-            "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH" +
+            "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS" +
            ARTIST_SONGS_ORDER,
    )
    suspend fun artistSongIds(
@@ -35,6 +35,7 @@ class SyncService(
        syncArtists()
        syncAlbums()
        syncSongs()
+        optimize()
        aggregatePlayData()
        syncPlaylists()
        syncPlaylistSongs()
@@ -137,6 +138,13 @@ class SyncService(
        db.libraryDao().recomputePlayData(source.id)
    }

+    // Without sqlite_stat1 the artist and relation queries can fall back to whole-table scans.
+    private suspend fun optimize() {
+        db.useWriterConnection { connection ->
+            connection.usePrepared("PRAGMA optimize") { it.step() }
+        }
+    }
+
    private suspend fun syncPlaylists() {
        val library = db.libraryDao()
        val seen = HashedIds()
@@ -3,6 +3,8 @@ package com.subtracks.data.db
 import android.content.Context
 import androidx.paging.PagingSource
 import androidx.room3.Room
+import androidx.room3.immediateTransaction
+import androidx.room3.useWriterConnection
 import androidx.sqlite.driver.bundled.BundledSQLiteDriver
 import androidx.test.core.app.ApplicationProvider
 import androidx.test.ext.junit.runners.AndroidJUnit4
@@ -61,6 +63,27 @@ class QueryTimingMeasurementTest {
            val secondSyncMs = wall { SyncService(db, source).sync() }
            val secondSyncExecutions = executions.toList()

+            db.useWriterConnection { connection ->
+                connection.immediateTransaction {
+                    connection.usePrepared(
+                        "INSERT INTO song_downloads (sourceId, songId, status, downloadedAt) VALUES (?, ?, ?, ?)",
+                    ) { statement ->
+                        (0 until songCount step 10).forEach { i ->
+                            statement.bindLong(1, 1L)
+                            statement.bindText(2, "s$i")
+                            statement.bindText(3, "Completed")
+                            statement.bindLong(4, i.toLong())
+                            statement.step()
+                            statement.reset()
+                            statement.clearBindings()
+                        }
+                    }
+                }
+            }
+            db.useWriterConnection { connection ->
+                connection.usePrepared("ANALYZE") { it.step() }
+            }
+
            executions.clear()
            measureReads()
            val readExecutions = executions.toList()
@@ -93,6 +116,13 @@ class QueryTimingMeasurementTest {
        timed("artists first page (frequent)") { dao.artistsByFrequent(1, 0, "").firstPage() }
        timed("artist albums") { dao.albumsForArtist(1, "ar0").first() }
        timed("artist name") { dao.artistName(1, "ar0").first() }
+        timed("artist songs") { db.queueDao().artistSongs(1, "ar0", 0, 100) }
+        timed("artist length") { db.queueDao().artistLength(1, "ar0") }
+        timed("artist song ids") { db.queueDao().artistSongIds(1, "ar0") }
+        timed("downloaded artists") { dao.downloadedArtists(1).firstPage() }
+        timed("artists downloaded filter") { dao.artistsByName(1, 0, "", downloadedFilter = 1).firstPage() }
+        timed("artist statuses") { db.downloadDao().artistStatuses(1).first() }
+        timed("artist status") { db.downloadDao().artistStatus(1, "ar0").first() }
        timed("playlists first page (name)") { dao.playlistsByName(1, "").firstPage() }
        timed("albums deep page") { dao.albumsByName(1, 0, "").deepPage(1000) }
        timed("albums search FTS") { dao.albumsByName(1, 0, "Album 1").firstPage() }
@@ -1026,7 +1026,7 @@ class SyncServiceTest {
            insertSource()
            val source =
                FakeMusicSource(
-                    artists = listOf(artist("a1", "A"), artist("a2", "B")),
+                    artists = listOf(artist("a1", "A"), artist("a2", "B"), artist("a3", "C")),
                    albums =
                        listOf(
                            album("al1").copy(
@@ -1054,6 +1054,15 @@ class SyncServiceTest {
                    .map { it.name }
                    .toSet(),
            )
+
+            val statuses =
+                db
+                    .downloadDao()
+                    .artistStatuses(1)
+                    .first()
+                    .associateBy { it.id }
+            assertEquals(setOf("a1", "a2"), statuses.keys)
+            assertEquals(1L, statuses.getValue("a2").downloaded)
        }

    @Test