diff --git a/app/src/main/kotlin/com/subtracks/data/db/DownloadDao.kt b/app/src/main/kotlin/com/subtracks/data/db/DownloadDao.kt index ef5d943c..2bc32ac8 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/DownloadDao.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/DownloadDao.kt @@ -13,6 +13,7 @@ import kotlinx.coroutines.flow.Flow // A song is attributed to its album artist and to its credited track artist, so a guest artist's // downloads are visible even though they lead no albums of their own. +// Starts from songs so status totals include every song the artist is credited on. internal const val ARTIST_DOWNLOAD_ROWS = "SELECT albums.artistId AS id, songs.id AS songId, song_downloads.status AS status, " + "song_downloads.downloadedAt AS downloadedAt " + @@ -44,6 +45,38 @@ internal const val ARTIST_DOWNLOAD_ROWS = "LEFT JOIN song_downloads ON song_downloads.sourceId = songs.sourceId AND song_downloads.songId = songs.id " + "WHERE songs.sourceId = :sourceId" +// Starts from downloads so lists and filters over partially downloaded libraries do not scan songs. +internal const val DOWNLOADED_ARTIST_ROWS = + "SELECT albums.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " + + "song_downloads.status AS status " + + "FROM song_downloads " + + "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " + + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + + "WHERE song_downloads.sourceId = :sourceId AND albums.artistId IS NOT NULL " + + "UNION ALL " + + "SELECT album_artists.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " + + "song_downloads.status AS status " + + "FROM song_downloads " + + "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " + + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + + "JOIN album_artists ON album_artists.sourceId = albums.sourceId AND album_artists.albumId = albums.id " + + "WHERE song_downloads.sourceId = :sourceId " + + "UNION ALL " + + "SELECT songs.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " + + "song_downloads.status AS status " + + "FROM song_downloads " + + "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " + + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + + "WHERE song_downloads.sourceId = :sourceId AND songs.artistId IS NOT NULL " + + "UNION ALL " + + "SELECT song_artists.artistId AS id, song_downloads.downloadedAt AS downloadedAt, " + + "song_downloads.status AS status " + + "FROM song_downloads " + + "JOIN songs ON songs.sourceId = song_downloads.sourceId AND songs.id = song_downloads.songId " + + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + + "JOIN song_artists ON song_artists.sourceId = songs.sourceId AND song_artists.songId = songs.id " + + "WHERE song_downloads.sourceId = :sourceId" + @Dao interface DownloadDao { @Query("SELECT * FROM song_downloads WHERE sourceId = :sourceId") @@ -111,7 +144,10 @@ interface DownloadDao { "COUNT(DISTINCT songId) AS total, " + "COUNT(DISTINCT CASE WHEN status = 'Completed' THEN songId END) AS downloaded, " + "COUNT(DISTINCT CASE WHEN status IN ('Queued', 'Running') THEN songId END) AS downloading " + - "FROM (" + ARTIST_DOWNLOAD_ROWS + ") GROUP BY id " + + "FROM (" + ARTIST_DOWNLOAD_ROWS + ") " + + "WHERE id IN (SELECT id FROM (" + DOWNLOADED_ARTIST_ROWS + ") " + + "WHERE status IN ('Completed', 'Queued', 'Running')) " + + "GROUP BY id " + "HAVING downloaded > 0 OR downloading > 0", ) fun artistStatuses(sourceId: Long): Flow> diff --git a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt index d0bd7543..876041be 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt @@ -65,29 +65,37 @@ internal const val ALBUM_ORDER_BY_FREQUENT_REVERSED = "playCount ASC, name COLLA internal const val ALBUM_ORDER_BY_RECENT = "played DESC NULLS LAST, name COLLATE NOCASE, id" internal const val ALBUM_ORDER_BY_RECENT_REVERSED = "played ASC NULLS LAST, name COLLATE NOCASE DESC, id DESC" -internal const val SONG_MATCHES_ARTIST = - "(songs.artistId = :artistId OR songs.id IN (SELECT songId FROM song_artists " + - "WHERE sourceId = songs.sourceId AND artistId = :artistId))" - -internal const val ALBUM_MATCHES_ARTIST = - "(albums.artistId = :artistId OR albums.id IN (SELECT albumId FROM album_artists " + - "WHERE sourceId = albums.sourceId AND artistId = :artistId))" - -private const val ARTIST_ALBUM_MATCH = - "(a.artistId = artists.id OR a.id IN (SELECT albumId FROM album_artists " + - "WHERE sourceId = artists.sourceId AND artistId = artists.id))" +private const val DOWNLOADED_ARTIST_ALBUMS = + "SELECT albums.id FROM albums " + + "WHERE albums.sourceId = :sourceId AND albums.artistId = artists.id " + + "AND EXISTS (SELECT 1 FROM songs " + + "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + + "WHERE songs.sourceId = albums.sourceId AND songs.albumId = albums.id AND sd.status = 'Completed') " + + "UNION " + + "SELECT albums.id FROM album_artists " + + "JOIN albums ON albums.sourceId = album_artists.sourceId AND albums.id = album_artists.albumId " + + "WHERE album_artists.sourceId = :sourceId AND album_artists.artistId = artists.id " + + "AND EXISTS (SELECT 1 FROM songs " + + "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + + "WHERE songs.sourceId = albums.sourceId AND songs.albumId = albums.id AND sd.status = 'Completed') " + + "UNION " + + "SELECT songs.albumId FROM songs " + + "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + + "WHERE songs.sourceId = :sourceId AND songs.artistId = artists.id " + + "AND sd.status = 'Completed' AND songs.albumId IS NOT NULL " + + "UNION " + + "SELECT songs.albumId FROM song_artists " + + "JOIN songs ON songs.sourceId = song_artists.sourceId AND songs.id = song_artists.songId " + + "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + + "WHERE song_artists.sourceId = :sourceId AND song_artists.artistId = artists.id " + + "AND sd.status = 'Completed' AND songs.albumId IS NOT NULL" internal const val ARTISTS_FILTER = "FROM artists WHERE sourceId = :sourceId " + "AND (:starredFilter = 0 OR (:starredFilter = 1 AND starred IS NOT NULL) OR (:starredFilter = 2 AND starred IS NULL)) " + - "AND (:downloadedFilter = 0 OR EXISTS (SELECT 1 FROM songs " + - "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + - "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + - "WHERE songs.sourceId = artists.sourceId AND sd.status = 'Completed' " + - "AND (albums.artistId = artists.id " + - "OR albums.id IN (SELECT albumId FROM album_artists aa WHERE aa.sourceId = albums.sourceId AND aa.artistId = artists.id) " + - "OR songs.artistId = artists.id " + - "OR songs.id IN (SELECT songId FROM song_artists sa WHERE sa.sourceId = songs.sourceId AND sa.artistId = artists.id)))) " + + "AND (:downloadedFilter = 0 OR EXISTS (" + + "SELECT 1 FROM ($ARTIST_DOWNLOAD_ROWS) dl " + + "WHERE dl.id = artists.id AND dl.status = 'Completed')) " + "AND (:search = '' " + "OR (length(:search) >= 3 AND rowid IN (SELECT rowid FROM artist_search " + "WHERE artist_search MATCH '\"' || replace(:search, '\"', '\"\"') || '\"')) " + @@ -98,14 +106,7 @@ internal const val ARTISTS_FILTER = internal const val ARTISTS_SELECT = "SELECT artists.sourceId, artists.id, artists.name, " + "CASE WHEN :downloadedFilter = 1 THEN (" + - "SELECT COUNT(DISTINCT albums.id) FROM songs " + - "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + - "JOIN song_downloads sd ON sd.sourceId = songs.sourceId AND sd.songId = songs.id " + - "WHERE songs.sourceId = artists.sourceId AND sd.status = 'Completed' " + - "AND (albums.artistId = artists.id " + - "OR albums.id IN (SELECT albumId FROM album_artists aa WHERE aa.sourceId = albums.sourceId AND aa.artistId = artists.id) " + - "OR songs.artistId = artists.id " + - "OR songs.id IN (SELECT songId FROM song_artists sa WHERE sa.sourceId = songs.sourceId AND sa.artistId = artists.id))" + + "SELECT COUNT(*) FROM ($DOWNLOADED_ARTIST_ALBUMS)" + ") ELSE artists.albumCount END AS albumCount, " + "artists.starred, artists.coverArt, artists.playCount, artists.played " @@ -200,19 +201,27 @@ interface LibraryDao { ) suspend fun recomputeAlbumPlayData(sourceId: Long) - // INDEXED BY is required: the planner otherwise answers MAX(played) from index_albums_recent - // (sourceId, played), which cannot apply the artistId equality. + // The credited albums are the union of the lead id and the relation table, so a shared album + // counts once per artist rather than twice. @Query( "UPDATE artists SET " + - "playCount = COALESCE((SELECT SUM(a.playCount) FROM albums a INDEXED BY index_albums_sourceId_artistId " + - "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH), 0), " + - "played = (SELECT MAX(a.played) FROM albums a INDEXED BY index_albums_sourceId_artistId " + - "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH) " + + "playCount = COALESCE((SELECT SUM(a.playCount) FROM albums a " + + "WHERE a.sourceId = artists.sourceId AND a.id IN (" + + "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " + + "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)), 0), " + + "played = (SELECT MAX(a.played) FROM albums a " + + "WHERE a.sourceId = artists.sourceId AND a.id IN (" + + "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " + + "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)) " + "WHERE sourceId = :sourceId AND (" + - "playCount IS NOT COALESCE((SELECT SUM(a.playCount) FROM albums a INDEXED BY index_albums_sourceId_artistId " + - "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH), 0) " + - "OR played IS NOT (SELECT MAX(a.played) FROM albums a INDEXED BY index_albums_sourceId_artistId " + - "WHERE a.sourceId = artists.sourceId AND $ARTIST_ALBUM_MATCH))", + "playCount IS NOT COALESCE((SELECT SUM(a.playCount) FROM albums a " + + "WHERE a.sourceId = artists.sourceId AND a.id IN (" + + "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " + + "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)), 0) " + + "OR played IS NOT (SELECT MAX(a.played) FROM albums a " + + "WHERE a.sourceId = artists.sourceId AND a.id IN (" + + "SELECT albumId FROM album_artists WHERE sourceId = artists.sourceId AND artistId = artists.id " + + "UNION SELECT id FROM albums WHERE sourceId = artists.sourceId AND artistId = artists.id)))", ) suspend fun recomputeArtistPlayData(sourceId: Long) @@ -831,10 +840,10 @@ interface LibraryDao { @Query( "SELECT artists.* FROM artists " + - "JOIN ($ARTIST_DOWNLOAD_ROWS) dl ON dl.id = artists.id " + - "WHERE artists.sourceId = :sourceId AND dl.status = 'Completed' " + - "GROUP BY artists.sourceId, artists.id " + - "ORDER BY MAX(dl.downloadedAt) DESC, artists.name COLLATE NOCASE, artists.id", + "JOIN (SELECT id, MAX(downloadedAt) AS lastDownloadedAt FROM ($DOWNLOADED_ARTIST_ROWS) " + + "WHERE status = 'Completed' GROUP BY id) dl ON dl.id = artists.id " + + "WHERE artists.sourceId = :sourceId " + + "ORDER BY dl.lastDownloadedAt DESC, artists.name COLLATE NOCASE, artists.id", ) fun downloadedArtists(sourceId: Long): PagingSource diff --git a/app/src/main/kotlin/com/subtracks/data/db/QueueDao.kt b/app/src/main/kotlin/com/subtracks/data/db/QueueDao.kt index 9688f72e..8d0f381f 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/QueueDao.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/QueueDao.kt @@ -19,12 +19,23 @@ private const val ALBUM_SONGS_ORDER = " ORDER BY songs.disc, songs.track, songs. private const val ALBUM_SONGS_SQL = ALBUM_SONGS_SELECT + ALBUM_SONGS_ORDER -private const val ARTIST_SONGS_MATCH = "($ALBUM_MATCHES_ARTIST OR $SONG_MATCHES_ARTIST)" +private const val ARTIST_SONG_IDS = + "(SELECT id FROM songs WHERE sourceId = :sourceId AND artistId = :artistId " + + "UNION " + + "SELECT songId FROM song_artists WHERE sourceId = :sourceId AND artistId = :artistId " + + "UNION " + + "SELECT songs.id FROM albums " + + "JOIN songs ON songs.sourceId = albums.sourceId AND songs.albumId = albums.id " + + "WHERE albums.sourceId = :sourceId AND albums.artistId = :artistId " + + "UNION " + + "SELECT songs.id FROM album_artists " + + "JOIN songs ON songs.sourceId = album_artists.sourceId AND songs.albumId = album_artists.albumId " + + "WHERE album_artists.sourceId = :sourceId AND album_artists.artistId = :artistId)" private const val ARTIST_SONGS_SELECT = "SELECT songs.*, albums.coverArt AS coverArt FROM songs " + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + - "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH" + "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS" private const val ARTIST_SONGS_ORDER = " ORDER BY albums.year DESC, albums.name COLLATE NOCASE, albums.id, songs.disc, songs.track, songs.id" @@ -85,7 +96,7 @@ interface QueueDao { @Query( "SELECT COUNT(*) FROM songs " + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + - "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH", + "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS", ) suspend fun artistLength( sourceId: Long, @@ -193,7 +204,7 @@ interface QueueDao { @Query( "SELECT songs.id FROM songs " + "JOIN albums ON albums.sourceId = songs.sourceId AND albums.id = songs.albumId " + - "WHERE songs.sourceId = :sourceId AND $ARTIST_SONGS_MATCH" + + "WHERE songs.sourceId = :sourceId AND songs.id IN $ARTIST_SONG_IDS" + ARTIST_SONGS_ORDER, ) suspend fun artistSongIds( diff --git a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt index cc828992..f87145da 100644 --- a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt +++ b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt @@ -35,6 +35,7 @@ class SyncService( syncArtists() syncAlbums() syncSongs() + optimize() aggregatePlayData() syncPlaylists() syncPlaylistSongs() @@ -137,6 +138,13 @@ class SyncService( db.libraryDao().recomputePlayData(source.id) } + // Without sqlite_stat1 the artist and relation queries can fall back to whole-table scans. + private suspend fun optimize() { + db.useWriterConnection { connection -> + connection.usePrepared("PRAGMA optimize") { it.step() } + } + } + private suspend fun syncPlaylists() { val library = db.libraryDao() val seen = HashedIds() diff --git a/app/src/test/kotlin/com/subtracks/data/db/QueryTimingMeasurementTest.kt b/app/src/test/kotlin/com/subtracks/data/db/QueryTimingMeasurementTest.kt index 60419a78..6b8d7936 100644 --- a/app/src/test/kotlin/com/subtracks/data/db/QueryTimingMeasurementTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/db/QueryTimingMeasurementTest.kt @@ -3,6 +3,8 @@ package com.subtracks.data.db import android.content.Context import androidx.paging.PagingSource import androidx.room3.Room +import androidx.room3.immediateTransaction +import androidx.room3.useWriterConnection import androidx.sqlite.driver.bundled.BundledSQLiteDriver import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 @@ -61,6 +63,27 @@ class QueryTimingMeasurementTest { val secondSyncMs = wall { SyncService(db, source).sync() } val secondSyncExecutions = executions.toList() + db.useWriterConnection { connection -> + connection.immediateTransaction { + connection.usePrepared( + "INSERT INTO song_downloads (sourceId, songId, status, downloadedAt) VALUES (?, ?, ?, ?)", + ) { statement -> + (0 until songCount step 10).forEach { i -> + statement.bindLong(1, 1L) + statement.bindText(2, "s$i") + statement.bindText(3, "Completed") + statement.bindLong(4, i.toLong()) + statement.step() + statement.reset() + statement.clearBindings() + } + } + } + } + db.useWriterConnection { connection -> + connection.usePrepared("ANALYZE") { it.step() } + } + executions.clear() measureReads() val readExecutions = executions.toList() @@ -93,6 +116,13 @@ class QueryTimingMeasurementTest { timed("artists first page (frequent)") { dao.artistsByFrequent(1, 0, "").firstPage() } timed("artist albums") { dao.albumsForArtist(1, "ar0").first() } timed("artist name") { dao.artistName(1, "ar0").first() } + timed("artist songs") { db.queueDao().artistSongs(1, "ar0", 0, 100) } + timed("artist length") { db.queueDao().artistLength(1, "ar0") } + timed("artist song ids") { db.queueDao().artistSongIds(1, "ar0") } + timed("downloaded artists") { dao.downloadedArtists(1).firstPage() } + timed("artists downloaded filter") { dao.artistsByName(1, 0, "", downloadedFilter = 1).firstPage() } + timed("artist statuses") { db.downloadDao().artistStatuses(1).first() } + timed("artist status") { db.downloadDao().artistStatus(1, "ar0").first() } timed("playlists first page (name)") { dao.playlistsByName(1, "").firstPage() } timed("albums deep page") { dao.albumsByName(1, 0, "").deepPage(1000) } timed("albums search FTS") { dao.albumsByName(1, 0, "Album 1").firstPage() } diff --git a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt index 03b1bc7d..f5b169dd 100644 --- a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt @@ -1026,7 +1026,7 @@ class SyncServiceTest { insertSource() val source = FakeMusicSource( - artists = listOf(artist("a1", "A"), artist("a2", "B")), + artists = listOf(artist("a1", "A"), artist("a2", "B"), artist("a3", "C")), albums = listOf( album("al1").copy( @@ -1054,6 +1054,15 @@ class SyncServiceTest { .map { it.name } .toSet(), ) + + val statuses = + db + .downloadDao() + .artistStatuses(1) + .first() + .associateBy { it.id } + assertEquals(setOf("a1", "a2"), statuses.keys) + assertEquals(1L, statuses.getValue("a2").downloaded) } @Test