harden sync for large libraries: bounded prune, concurrent fetches, higher page cap

app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt

@@ -65,7 +65,6 @@ class SubsonicSourceIntegrationTest(
             assertNotNull(kosmo.coverArt)
             assertNotNull(kosmo.starred)
             assertTrue(kosmo.created <= System.currentTimeMillis() / 1000)
-            assertTrue(kosmo.frequentRank != null || kosmo.recentRank != null)
 
             assertNull(albums.first { it.name == "Retroconnaissance EP" }.starred)
             assertNull(albums.first { it.name == "I Don't Know What I'm Doing" }.starred)

app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt

@@ -65,18 +65,58 @@ interface LibraryDao {
     @Query("SELECT id FROM artists WHERE sourceId = :sourceId")
     suspend fun artistIds(sourceId: Long): List<String>
 
+    @Query("SELECT id FROM artists WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit")
+    suspend fun artistIdsAfter(
+        sourceId: Long,
+        afterId: String,
+        limit: Int,
+    ): List<String>
+
     @Query("SELECT id FROM albums WHERE sourceId = :sourceId")
     suspend fun albumIds(sourceId: Long): List<String>
 
+    @Query("SELECT id FROM albums WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit")
+    suspend fun albumIdsAfter(
+        sourceId: Long,
+        afterId: String,
+        limit: Int,
+    ): List<String>
+
     @Query("SELECT albumId, disc FROM discs WHERE sourceId = :sourceId")
     suspend fun discKeys(sourceId: Long): List<DiscKey>
 
+    @Query(
+        "SELECT albumId, disc FROM discs WHERE sourceId = :sourceId " +
+            "AND (albumId > :afterAlbumId OR (albumId = :afterAlbumId AND disc > :afterDisc)) " +
+            "ORDER BY albumId, disc LIMIT :limit",
+    )
+    suspend fun discKeysAfter(
+        sourceId: Long,
+        afterAlbumId: String,
+        afterDisc: Long,
+        limit: Int,
+    ): List<DiscKey>
+
     @Query("SELECT id FROM playlists WHERE sourceId = :sourceId")
     suspend fun playlistIds(sourceId: Long): List<String>
 
+    @Query("SELECT id FROM playlists WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit")
+    suspend fun playlistIdsAfter(
+        sourceId: Long,
+        afterId: String,
+        limit: Int,
+    ): List<String>
+
     @Query("SELECT id FROM songs WHERE sourceId = :sourceId")
     suspend fun songIds(sourceId: Long): List<String>
 
+    @Query("SELECT id FROM songs WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit")
+    suspend fun songIdsAfter(
+        sourceId: Long,
+        afterId: String,
+        limit: Int,
+    ): List<String>
+
     @Query("DELETE FROM artists WHERE sourceId = :sourceId AND id IN (:ids)")
     suspend fun deleteArtists(
         sourceId: Long,

app/src/main/kotlin/com/subtracks/data/db/Migrations.kt

@@ -127,6 +127,14 @@ val MIGRATION_12_13 =
         }
     }
 
+val MIGRATION_13_14 =
+    object : Migration(13, 14) {
+        override suspend fun migrate(connection: SQLiteConnection) {
+            connection.execSQL("ALTER TABLE `albums` DROP COLUMN `frequentRank`")
+            connection.execSQL("ALTER TABLE `albums` DROP COLUMN `recentRank`")
+        }
+    }
+
 val MIGRATIONS: Array<Migration> =
     arrayOf(
         MIGRATION_1_2,
@@ -141,4 +149,5 @@ val MIGRATIONS: Array<Migration> =
         MIGRATION_10_11,
         MIGRATION_11_12,
         MIGRATION_12_13,
+        MIGRATION_13_14,
     )

app/src/main/kotlin/com/subtracks/data/db/SubtracksDatabase.kt

@@ -32,7 +32,7 @@ import com.subtracks.data.model.SubsonicSource
         ShuffleOrder::class,
         ArtworkSeed::class,
     ],
-    version = 13,
+    version = 14,
     exportSchema = true,
 )
 @ColumnTypeConverters(QueueKindConverter::class)

app/src/main/kotlin/com/subtracks/data/model/Models.kt

@@ -84,8 +84,6 @@ data class Album(
     val year: Long?,
     val starred: Long?,
     val songCount: Long,
-    val frequentRank: Long?,
-    val recentRank: Long?,
     @Ignore val discTitles: Map<Long, String> = emptyMap(),
 )
 

app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt

@@ -9,7 +9,10 @@ import com.subtracks.data.source.MusicSource
 import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.ExperimentalCoroutinesApi
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
 import kotlinx.coroutines.channels.trySendBlocking
+import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.flow.Flow
 import kotlinx.coroutines.flow.buffer
 import kotlinx.coroutines.flow.channelFlow
@@ -17,6 +20,9 @@ import kotlinx.coroutines.flow.emitAll
 import kotlinx.coroutines.flow.flow
 import kotlinx.coroutines.flow.flowOn
 import kotlinx.coroutines.flow.map
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import kotlinx.coroutines.withContext
 import org.xml.sax.Attributes
 
@@ -43,7 +49,6 @@ class SubsonicSource(
 
     override fun albums(): Flow<List<Album>> =
         flow {
-            val (frequent, recent) = fetchRanks()
             var offset = 0
             var pages = 0
             while (true) {
@@ -57,16 +62,7 @@ class SubsonicSource(
                 ).collect { batch ->
                     raw += batch.size
                     val accepted = batch.filter { it.id.isNotEmpty() }
-                    if (accepted.isNotEmpty()) {
+                    if (accepted.isNotEmpty()) emit(accepted.map { it.toAlbum() })
-                        emit(
-                            accepted.map {
-                                it.toAlbum().copy(
-                                    frequentRank = frequent[it.id],
-                                    recentRank = recent[it.id],
-                                )
-                            },
-                        )
-                    }
                 }
                 if (raw < PAGE_SIZE) {
                     if (raw > 0 && pages >= maxPages) throw pageCapExceeded()
@@ -95,20 +91,25 @@ class SubsonicSource(
         ).map { batch -> batch.filter { it.id.isNotEmpty() } }
 
     override fun playlistSongs(playlistIds: List<String>): Flow<List<PlaylistSong>> =
-        flow {
+        channelFlow {
-            for (playlistId in playlistIds) {
+            val permits = Semaphore(MAX_CONCURRENT_FETCHES)
-                var position = 0L
+            playlistIds.forEach { playlistId ->
-                entityBatches(
+                launch {
-                    method = "getPlaylist",
+                    permits.withPermit {
-                    params = mapOf("id" to playlistId),
+                        var position = 0L
-                    tag = "entry",
+                        entityBatches(
-                    create = { attrs ->
+                            method = "getPlaylist",
-                        val entryId = attrs.attr("id")
+                            params = mapOf("id" to playlistId),
-                        SubsonicXml.playlistSong(id, playlistId, if (entryId.isEmpty()) -1L else position++, attrs)
+                            tag = "entry",
-                    },
+                            create = { attrs ->
-                ).collect { batch ->
+                                val entryId = attrs.attr("id")
-                    val accepted = batch.filter { it.songId.isNotEmpty() }
+                                SubsonicXml.playlistSong(id, playlistId, if (entryId.isEmpty()) -1L else position++, attrs)
-                    if (accepted.isNotEmpty()) emit(accepted)
+                            },
+                        ).collect { batch ->
+                            val accepted = batch.filter { it.songId.isNotEmpty() }
+                            if (accepted.isNotEmpty()) send(accepted)
+                        }
+                    }
                 }
             }
         }.flowOn(Dispatchers.IO).buffer(1)
@@ -174,7 +175,8 @@ class SubsonicSource(
         }
 
     private fun albumSongs(): Flow<List<Song>> =
-        flow {
+        channelFlow {
+            val permits = Semaphore(MAX_CONCURRENT_FETCHES)
             var offset = 0
             var pages = 0
             while (true) {
@@ -189,16 +191,23 @@ class SubsonicSource(
                     raw += batch.size
                     batch.forEach { if (it.isNotEmpty()) albums += it }
                 }
-                for (albumId in albums) {
+                coroutineScope {
-                    entityBatches(
+                    albums
-                        method = "getAlbum",
+                        .map { albumId ->
-                        params = mapOf("id" to albumId),
+                            async {
-                        tag = "song",
+                                permits.withPermit {
-                        create = { SubsonicXml.song(id, it) },
+                                    entityBatches(
-                    ).collect { batch ->
+                                        method = "getAlbum",
-                        val accepted = batch.filter { it.id.isNotEmpty() }
+                                        params = mapOf("id" to albumId),
-                        if (accepted.isNotEmpty()) emit(accepted)
+                                        tag = "song",
-                    }
+                                        create = { SubsonicXml.song(id, it) },
+                                    ).collect { batch ->
+                                        val accepted = batch.filter { it.id.isNotEmpty() }
+                                        if (accepted.isNotEmpty()) send(accepted)
+                                    }
+                                }
+                            }
+                        }.awaitAll()
                 }
                 if (raw < PAGE_SIZE) {
                     if (raw > 0 && pages >= maxPages) throw pageCapExceeded()
@@ -210,44 +219,6 @@ class SubsonicSource(
             }
         }
 
-    private suspend fun fetchRanks(): Pair<Map<String, Long>, Map<String, Long>> = fetchRank("frequent") to fetchRank("recent")
-
-    private suspend fun fetchRank(type: String): Map<String, Long> =
-        try {
-            val ranks = HashMap<String, Long>()
-            var offset = 0
-            var pages = 0
-            while (true) {
-                var raw = 0
-                var index = 0
-                entityBatches(
-                    method = "getAlbumList2",
-                    params = page(type, offset),
-                    tag = "album",
-                    create = { it.attr("id") },
-                ).collect { batch ->
-                    raw += batch.size
-                    batch.forEach { albumId ->
-                        if (albumId.isNotEmpty()) {
-                            ranks[albumId] = (offset + index).toLong()
-                            index++
-                        }
-                    }
-                }
-                if (raw < PAGE_SIZE) {
-                    if (raw > 0 && pages >= maxPages) throw pageCapExceeded()
-                    break
-                }
-                pages++
-                if (pages > maxPages) throw pageCapExceeded()
-                offset += PAGE_SIZE
-            }
-            ranks
-        } catch (failure: SubsonicException) {
-            if (failure is PageCapExceeded) throw failure
-            emptyMap()
-        }
-
     private fun supportsEmptyQuerySearch(): Boolean {
         emptyQuerySearchSupported?.let { return it }
         val supported =
@@ -263,7 +234,7 @@ class SubsonicSource(
         return supported
     }
 
-    private fun pageCapExceeded() = PageCapExceeded("Library exceeds the ${maxPages * PAGE_SIZE}-row page cap")
+    private fun pageCapExceeded() = PageCapExceeded("Library is too large to sync: it exceeds ${maxPages.toLong() * PAGE_SIZE} rows")
 
     private fun <T> entityBatches(
         method: String,
@@ -310,7 +281,8 @@ class SubsonicSource(
 
     private companion object {
         const val PAGE_SIZE = 500
-        const val MAX_PAGES = 1000
+        const val MAX_PAGES = 20_000
+        const val MAX_CONCURRENT_FETCHES = 4
         const val THUMBNAIL_SIZE = 256
     }
 }

app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt

@@ -155,8 +155,6 @@ internal class AlbumDraft(
             year = year,
             starred = starred,
             songCount = songCount,
-            frequentRank = null,
-            recentRank = null,
             discTitles = discTitles.toMap(),
         )
 }

app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt

@@ -13,6 +13,7 @@ import com.subtracks.data.model.Playlist
 import com.subtracks.data.model.PlaylistSong
 import com.subtracks.data.model.Song
 import com.subtracks.data.source.MusicSource
+import java.util.Arrays
 
 class SyncService(
     private val db: SubtracksDatabase,
@@ -28,19 +29,19 @@ class SyncService(
 
     private suspend fun syncArtists() {
         val library = db.libraryDao()
-        val stale = library.artistIds(source.id).toHashSet()
+        val seen = HashedIds()
         source.artists().collect { batch ->
             if (batch.isEmpty()) return@collect
             write("artists", artistColumns, primaryKey, batch) { it.values() }
-            stale -= batch.mapTo(HashSet()) { it.id }
+            batch.forEach { seen.add(idHash(it.id)) }
         }
-        deleteStale(stale) { library.deleteArtists(source.id, it) }
+        pruneStaleIds(seen, { library.artistIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteArtists(source.id, it) })
     }
 
     private suspend fun syncAlbums() {
         val library = db.libraryDao()
-        val staleAlbums = library.albumIds(source.id).toHashSet()
+        val seenAlbums = HashedIds()
-        val staleDiscs = library.discKeys(source.id).toHashSet()
+        val seenDiscs = HashedIds()
         source.albums().collect { batch ->
             if (batch.isEmpty()) return@collect
             val discs =
@@ -48,33 +49,35 @@ class SyncService(
                     album.discTitles.map { (disc, title) -> Disc(album.sourceId, album.id, disc, title) }
                 }
             writeAlbumsAndDiscs(batch, discs)
-            staleAlbums -= batch.mapTo(HashSet()) { it.id }
+            batch.forEach { seenAlbums.add(idHash(it.id)) }
-            staleDiscs -= discs.mapTo(HashSet()) { DiscKey(it.albumId, it.disc) }
+            discs.forEach { seenDiscs.add(discHash(it.albumId, it.disc)) }
+        }
+        pruneStaleIds(seenAlbums, { library.albumIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteAlbums(source.id, it) })
+        pruneStaleDiscs(seenDiscs, { albumId, disc -> library.discKeysAfter(source.id, albumId, disc, PRUNE_PAGE) }) {
+            deleteStaleDiscs(it)
         }
-        deleteStale(staleAlbums) { library.deleteAlbums(source.id, it) }
-        deleteStaleDiscs(staleDiscs)
     }
 
     private suspend fun syncSongs() {
         val library = db.libraryDao()
-        val stale = library.songIds(source.id).toHashSet()
+        val seen = HashedIds()
         source.songs().collect { batch ->
             if (batch.isEmpty()) return@collect
             write("songs", songColumns, primaryKey, batch) { it.values() }
-            stale -= batch.mapTo(HashSet()) { it.id }
+            batch.forEach { seen.add(idHash(it.id)) }
         }
-        deleteStale(stale) { library.deleteSongs(source.id, it) }
+        pruneStaleIds(seen, { library.songIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteSongs(source.id, it) })
     }
 
     private suspend fun syncPlaylists() {
         val library = db.libraryDao()
-        val stale = library.playlistIds(source.id).toHashSet()
+        val seen = HashedIds()
         source.playlists().collect { batch ->
             if (batch.isEmpty()) return@collect
             write("playlists", playlistColumns, primaryKey, batch) { it.values() }
-            stale -= batch.mapTo(HashSet()) { it.id }
+            batch.forEach { seen.add(idHash(it.id)) }
         }
-        deleteStale(stale) { library.deletePlaylists(source.id, it) }
+        pruneStaleIds(seen, { library.playlistIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deletePlaylists(source.id, it) })
     }
 
     private suspend fun syncPlaylistSongs() {
@@ -118,7 +121,7 @@ class SyncService(
         }
     }
 
-    private suspend fun deleteStaleDiscs(stale: Set<DiscKey>) {
+    private suspend fun deleteStaleDiscs(stale: Collection<DiscKey>) {
         stale.chunked(DELETE_CHUNK).forEach { chunk ->
             db.useWriterConnection { connection ->
                 connection.immediateTransaction {
@@ -137,15 +140,77 @@ class SyncService(
         }
     }
 
-    private suspend fun deleteStale(
+    private suspend fun pruneStaleIds(
-        stale: Set<String>,
+        seen: HashedIds,
+        pageOf: suspend (afterId: String) -> List<String>,
         delete: suspend (Collection<String>) -> Unit,
     ) {
-        stale.chunked(DELETE_CHUNK).forEach { delete(it) }
+        seen.freeze()
+        var afterId = ""
+        while (true) {
+            val ids = pageOf(afterId)
+            if (ids.isEmpty()) return
+            val stale = ids.filterNot { seen.contains(idHash(it)) }
+            stale.chunked(DELETE_CHUNK).forEach { delete(it) }
+            if (ids.size < PRUNE_PAGE) return
+            afterId = ids.last()
+        }
+    }
+
+    private suspend fun pruneStaleDiscs(
+        seen: HashedIds,
+        pageOf: suspend (afterAlbumId: String, afterDisc: Long) -> List<DiscKey>,
+        delete: suspend (Collection<DiscKey>) -> Unit,
+    ) {
+        seen.freeze()
+        var afterAlbumId = ""
+        var afterDisc = 0L
+        while (true) {
+            val keys = pageOf(afterAlbumId, afterDisc)
+            if (keys.isEmpty()) return
+            val stale = keys.filterNot { seen.contains(discHash(it.albumId, it.disc)) }
+            stale.chunked(DELETE_CHUNK).forEach { delete(it) }
+            if (keys.size < PRUNE_PAGE) return
+            afterAlbumId = keys.last().albumId
+            afterDisc = keys.last().disc
+        }
     }
 }
 
 private const val DELETE_CHUNK = 500
+private const val PRUNE_PAGE = 500
+private const val INITIAL_HASH_CAPACITY = 256
+private const val FNV_OFFSET = -3750763034362895579L
+private const val FNV_PRIME = 1099511628211L
+
+private class HashedIds {
+    private var values = LongArray(0)
+    private var count = 0
+
+    fun add(value: Long) {
+        if (count == values.size) {
+            values = values.copyOf(if (values.isEmpty()) INITIAL_HASH_CAPACITY else values.size * 2)
+        }
+        values[count++] = value
+    }
+
+    fun freeze() {
+        Arrays.sort(values, 0, count)
+    }
+
+    operator fun contains(value: Long): Boolean = Arrays.binarySearch(values, 0, count, value) >= 0
+}
+
+private fun idHash(id: String): Long {
+    var hash = FNV_OFFSET
+    id.forEach { hash = (hash xor it.code.toLong()) * FNV_PRIME }
+    return hash
+}
+
+private fun discHash(
+    albumId: String,
+    disc: Long,
+): Long = idHash(albumId) * 31 + disc
 
 private val primaryKey = listOf("sourceId", "id")
 
@@ -164,8 +229,6 @@ private val albumColumns =
         "year",
         "starred",
         "songCount",
-        "frequentRank",
-        "recentRank",
     )
 
 private val discKey = listOf("sourceId", "albumId", "disc")
@@ -197,8 +260,7 @@ private val playlistSongKey = listOf("sourceId", "playlistId", "position")
 
 private fun Artist.values() = listOf<Any?>(sourceId, id, name, albumCount, starred, coverArt)
 
-private fun Album.values() =
+private fun Album.values() = listOf<Any?>(sourceId, id, artistId, name, albumArtist, created, coverArt, genre, year, starred, songCount)
-    listOf<Any?>(sourceId, id, artistId, name, albumArtist, created, coverArt, genre, year, starred, songCount, frequentRank, recentRank)
 
 private fun Disc.values() = listOf<Any?>(sourceId, albumId, disc, title)
 

app/src/test/kotlin/com/subtracks/data/db/LibraryDaoTest.kt

@@ -162,8 +162,6 @@ class LibraryDaoTest {
         year = year,
         starred = starred,
         songCount = 1,
-        frequentRank = null,
-        recentRank = null,
     )
 
     private fun song(

app/src/test/kotlin/com/subtracks/data/repo/QueueRepositoryTest.kt

@@ -405,8 +405,6 @@ class QueueRepositoryTest {
         year = null,
         starred = null,
         songCount = 1,
-        frequentRank = null,
-        recentRank = null,
     )
 
     private fun song(

app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt

@@ -16,6 +16,8 @@ import org.junit.Assert.assertTrue
 import org.junit.Before
 import org.junit.Test
 import java.util.concurrent.CopyOnWriteArrayList
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
 import java.util.concurrent.atomic.AtomicInteger
 
 class SubsonicSourceTest {
@@ -91,12 +93,15 @@ class SubsonicSourceTest {
     fun albumsPaginateUntilAShortPage() =
         runBlocking {
             val requestedOffsets = CopyOnWriteArrayList<Int>()
+            val requestedTypes = CopyOnWriteArrayList<String>()
             server.dispatcher =
                 object : Dispatcher() {
                     override fun dispatch(request: RecordedRequest): MockResponse {
                         val url = request.requestUrl!!
                         if (url.encodedPath != "/rest/getAlbumList2.view") return MockResponse().setResponseCode(404)
-                        if (url.queryParameter("type") != "newest") {
+                        val type = url.queryParameter("type")!!
+                        requestedTypes += type
+                        if (type != "newest") {
                             return MockResponse().setBody(emptyAlbumList())
                         }
                         val offset = url.queryParameter("offset")!!.toInt()
@@ -110,6 +115,7 @@ class SubsonicSourceTest {
 
             assertEquals(1003, albums.size)
             assertEquals(listOf(0, 500, 1000), requestedOffsets.toList())
+            assertEquals(listOf("newest"), requestedTypes.distinct())
             assertEquals("al-0", albums.first().id)
             assertEquals("al-1002", albums.last().id)
         }
@@ -200,7 +206,10 @@ class SubsonicSourceTest {
                     SubsonicSource(1, client(), maxPages = 2).albums().toList()
                 }
 
-            assertEquals(true, failure.exceptionOrNull() is SubsonicException)
+            val thrown = failure.exceptionOrNull()
+            assertEquals(true, thrown is SubsonicException)
+            assertTrue(thrown?.message.orEmpty(), thrown?.message.orEmpty().contains("too large"))
+            assertTrue(thrown?.message.orEmpty(), thrown?.message.orEmpty().contains("1000 rows"))
         }
 
     @Test
@@ -270,6 +279,119 @@ class SubsonicSourceTest {
             assertEquals("al-502", albums.last().id)
         }
 
+    @Test
+    fun playlistSongsFetchesPlaylistsConcurrently() =
+        runBlocking {
+            val inFlight = AtomicInteger()
+            val maxInFlight = AtomicInteger()
+            val secondStarted = CountDownLatch(1)
+            server.dispatcher =
+                object : Dispatcher() {
+                    override fun dispatch(request: RecordedRequest): MockResponse {
+                        val url = request.requestUrl!!
+                        if (url.encodedPath != "/rest/getPlaylist.view") return MockResponse().setResponseCode(404)
+                        val current = inFlight.incrementAndGet()
+                        maxInFlight.updateAndGet { maxOf(it, current) }
+                        if (current >= 2) secondStarted.countDown()
+                        secondStarted.await(2, TimeUnit.SECONDS)
+                        inFlight.decrementAndGet()
+                        val playlistId = url.queryParameter("id")!!
+                        return MockResponse().setBody(
+                            "<subsonic-response status=\"ok\"><playlist id=\"$playlistId\">" +
+                                (1..3).joinToString("") { "<entry id=\"$playlistId-s$it\" title=\"Title $it\"/>" } +
+                                "</playlist></subsonic-response>",
+                        )
+                    }
+                }
+
+            val entries = SubsonicSource(1, client()).playlistSongs(listOf("p1", "p2", "p3", "p4")).toList().flatten()
+
+            assertEquals(12, entries.size)
+            assertEquals(4, entries.map { it.playlistId }.distinct().size)
+            entries.groupBy { it.playlistId }.forEach { (playlistId, batch) ->
+                assertEquals(listOf(0L, 1L, 2L), batch.sortedBy { it.position }.map { it.position })
+                assertEquals((1..3).map { "$playlistId-s$it" }, batch.sortedBy { it.position }.map { it.songId })
+            }
+            assertTrue(maxInFlight.get() >= 2)
+        }
+
+    @Test
+    fun playlistSongsPropagatesFailuresFromAnyPlaylist() =
+        runBlocking {
+            server.dispatcher =
+                object : Dispatcher() {
+                    override fun dispatch(request: RecordedRequest): MockResponse {
+                        val url = request.requestUrl!!
+                        if (url.encodedPath != "/rest/getPlaylist.view") return MockResponse().setResponseCode(404)
+                        return if (url.queryParameter("id") == "p2") {
+                            MockResponse().setBody(failed(70, "Playlist not found"))
+                        } else {
+                            MockResponse().setBody(
+                                "<subsonic-response status=\"ok\"><playlist id=\"p1\">" +
+                                    "<entry id=\"s1\" title=\"One\"/>" +
+                                    "</playlist></subsonic-response>",
+                            )
+                        }
+                    }
+                }
+
+            val failure = runCatching { SubsonicSource(1, client()).playlistSongs(listOf("p1", "p2", "p3")).toList() }
+
+            assertEquals(true, failure.exceptionOrNull() is SubsonicException)
+        }
+
+    @Test
+    fun albumSongsFetchesAlbumsConcurrentlyWhenSearchIsUnsupported() =
+        runBlocking {
+            val inFlight = AtomicInteger()
+            val maxInFlight = AtomicInteger()
+            val secondStarted = CountDownLatch(1)
+            server.dispatcher =
+                object : Dispatcher() {
+                    override fun dispatch(request: RecordedRequest): MockResponse {
+                        val url = request.requestUrl!!
+                        return when {
+                            url.encodedPath == "/rest/search3.view" -> {
+                                MockResponse().setBody(failed(10, "Required parameter is missing"))
+                            }
+
+                            url.encodedPath == "/rest/getAlbumList2.view" -> {
+                                MockResponse().setBody(
+                                    "<subsonic-response status=\"ok\"><albumList2>" +
+                                        (0 until 4).joinToString("") {
+                                            "<album id=\"al$it\" name=\"Album $it\" artist=\"Artist\" artistId=\"ar1\" songCount=\"1\"/>"
+                                        } +
+                                        "</albumList2></subsonic-response>",
+                                )
+                            }
+
+                            url.encodedPath == "/rest/getAlbum.view" -> {
+                                val current = inFlight.incrementAndGet()
+                                maxInFlight.updateAndGet { maxOf(it, current) }
+                                if (current >= 2) secondStarted.countDown()
+                                secondStarted.await(2, TimeUnit.SECONDS)
+                                inFlight.decrementAndGet()
+                                val albumId = url.queryParameter("id")!!
+                                MockResponse().setBody(
+                                    "<subsonic-response status=\"ok\"><album id=\"$albumId\">" +
+                                        "<song id=\"$albumId-s1\" title=\"One\"/>" +
+                                        "</album></subsonic-response>",
+                                )
+                            }
+
+                            else -> {
+                                MockResponse().setResponseCode(404)
+                            }
+                        }
+                    }
+                }
+
+            val songs = SubsonicSource(1, client()).songs().toList().flatten()
+
+            assertEquals(4, songs.size)
+            assertTrue(maxInFlight.get() >= 2)
+        }
+
     @Test
     fun playlistEntriesWithoutIdsAreSkipped() =
         runBlocking {

app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt

@@ -525,6 +525,23 @@ class SyncServiceTest {
             assertEquals(0, db.libraryDao().artistIds(1).size)
         }
 
+    @Test
+    fun partialPruneAcrossManyPagesKeepsTheSurvivors() =
+        runTest {
+            insertSource()
+            val source = FakeMusicSource(artists = (1..1100).map { artist("a$it") })
+
+            SyncService(db, source).sync()
+            assertEquals(1100, db.libraryDao().artistIds(1).size)
+
+            val survivors = (1..1100).filter { it % 2 == 0 }.map { "a$it" }
+            source.artists = survivors.map { artist(it) }
+
+            SyncService(db, source).sync()
+
+            assertEquals(survivors.sorted(), db.libraryDao().artistIds(1).sorted())
+        }
+
     @Test
     fun anIdenticalSyncDoesNotRewriteRows() =
         runTest {
@@ -593,8 +610,6 @@ class SyncServiceTest {
             year = null,
             starred = null,
             songCount = 1,
-            frequentRank = null,
-            recentRank = null,
         )
 
     private fun song(id: String) =

app/src/test/kotlin/com/subtracks/ui/AlbumDiscsTest.kt

@@ -150,7 +150,5 @@ class AlbumDiscsTest {
             year = null,
             starred = null,
             songCount = 3,
-            frequentRank = null,
-            recentRank = null,
         )
 }

app/src/test/kotlin/com/subtracks/ui/AppScreenshotTest.kt

@@ -603,8 +603,6 @@ private object Fixtures {
         year = year,
         starred = null,
         songCount = songCount,
-        frequentRank = null,
-        recentRank = null,
     )
 
     fun song(

app/src/test/kotlin/com/subtracks/ui/LibraryOptionsScreenshotTest.kt

@@ -82,8 +82,6 @@ class LibraryOptionsScreenshotTest {
                 year = null,
                 starred = null,
                 songCount = 1,
-                frequentRank = null,
-                recentRank = null,
             )
         }
 }

app/src/test/kotlin/com/subtracks/ui/LibraryOptionsTest.kt

@@ -113,8 +113,6 @@ class LibraryOptionsTest {
                 year = null,
                 starred = null,
                 songCount = 1,
-                frequentRank = null,
-                recentRank = null,
             )
         }
 }

app/src/test/kotlin/com/subtracks/ui/LibraryScreenTest.kt

@@ -279,8 +279,6 @@ class LibraryScreenTest {
                 year = null,
                 starred = null,
                 songCount = 1,
-                frequentRank = null,
-                recentRank = null,
             )
         }
 

app/src/test/kotlin/com/subtracks/ui/LibrarySearchScreenshotTest.kt

@@ -86,8 +86,6 @@ class LibrarySearchScreenshotTest {
                 year = null,
                 starred = null,
                 songCount = 1,
-                frequentRank = null,
-                recentRank = null,
             )
         }
 }