diff --git a/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt b/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt index 809a13f2..9e93ea04 100644 --- a/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt +++ b/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt @@ -65,7 +65,6 @@ class SubsonicSourceIntegrationTest( assertNotNull(kosmo.coverArt) assertNotNull(kosmo.starred) assertTrue(kosmo.created <= System.currentTimeMillis() / 1000) - assertTrue(kosmo.frequentRank != null || kosmo.recentRank != null) assertNull(albums.first { it.name == "Retroconnaissance EP" }.starred) assertNull(albums.first { it.name == "I Don't Know What I'm Doing" }.starred) diff --git a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt index b539fafd..532b9e73 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt @@ -65,18 +65,58 @@ interface LibraryDao { @Query("SELECT id FROM artists WHERE sourceId = :sourceId") suspend fun artistIds(sourceId: Long): List + @Query("SELECT id FROM artists WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit") + suspend fun artistIdsAfter( + sourceId: Long, + afterId: String, + limit: Int, + ): List + @Query("SELECT id FROM albums WHERE sourceId = :sourceId") suspend fun albumIds(sourceId: Long): List + @Query("SELECT id FROM albums WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit") + suspend fun albumIdsAfter( + sourceId: Long, + afterId: String, + limit: Int, + ): List + @Query("SELECT albumId, disc FROM discs WHERE sourceId = :sourceId") suspend fun discKeys(sourceId: Long): List + @Query( + "SELECT albumId, disc FROM discs WHERE sourceId = :sourceId " + + "AND (albumId > :afterAlbumId OR (albumId = :afterAlbumId AND disc > :afterDisc)) " + + "ORDER BY albumId, disc LIMIT :limit", + ) + suspend fun discKeysAfter( + sourceId: Long, + afterAlbumId: String, + afterDisc: Long, + limit: Int, + ): List + @Query("SELECT id FROM playlists WHERE sourceId = :sourceId") suspend fun playlistIds(sourceId: Long): List + @Query("SELECT id FROM playlists WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit") + suspend fun playlistIdsAfter( + sourceId: Long, + afterId: String, + limit: Int, + ): List + @Query("SELECT id FROM songs WHERE sourceId = :sourceId") suspend fun songIds(sourceId: Long): List + @Query("SELECT id FROM songs WHERE sourceId = :sourceId AND id > :afterId ORDER BY id LIMIT :limit") + suspend fun songIdsAfter( + sourceId: Long, + afterId: String, + limit: Int, + ): List + @Query("DELETE FROM artists WHERE sourceId = :sourceId AND id IN (:ids)") suspend fun deleteArtists( sourceId: Long, diff --git a/app/src/main/kotlin/com/subtracks/data/db/Migrations.kt b/app/src/main/kotlin/com/subtracks/data/db/Migrations.kt index c3eeba20..301365d6 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/Migrations.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/Migrations.kt @@ -127,6 +127,14 @@ val MIGRATION_12_13 = } } +val MIGRATION_13_14 = + object : Migration(13, 14) { + override suspend fun migrate(connection: SQLiteConnection) { + connection.execSQL("ALTER TABLE `albums` DROP COLUMN `frequentRank`") + connection.execSQL("ALTER TABLE `albums` DROP COLUMN `recentRank`") + } + } + val MIGRATIONS: Array = arrayOf( MIGRATION_1_2, @@ -141,4 +149,5 @@ val MIGRATIONS: Array = MIGRATION_10_11, MIGRATION_11_12, MIGRATION_12_13, + MIGRATION_13_14, ) diff --git a/app/src/main/kotlin/com/subtracks/data/db/SubtracksDatabase.kt b/app/src/main/kotlin/com/subtracks/data/db/SubtracksDatabase.kt index 9c3cb186..a1e05130 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/SubtracksDatabase.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/SubtracksDatabase.kt @@ -32,7 +32,7 @@ import com.subtracks.data.model.SubsonicSource ShuffleOrder::class, ArtworkSeed::class, ], - version = 13, + version = 14, exportSchema = true, ) @ColumnTypeConverters(QueueKindConverter::class) diff --git a/app/src/main/kotlin/com/subtracks/data/model/Models.kt b/app/src/main/kotlin/com/subtracks/data/model/Models.kt index 84d5e35c..293897d5 100644 --- a/app/src/main/kotlin/com/subtracks/data/model/Models.kt +++ b/app/src/main/kotlin/com/subtracks/data/model/Models.kt @@ -84,8 +84,6 @@ data class Album( val year: Long?, val starred: Long?, val songCount: Long, - val frequentRank: Long?, - val recentRank: Long?, @Ignore val discTitles: Map = emptyMap(), ) diff --git a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt index 22888fed..6a1f8086 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt @@ -9,7 +9,10 @@ import com.subtracks.data.source.MusicSource import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.channels.trySendBlocking +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.buffer import kotlinx.coroutines.flow.channelFlow @@ -17,6 +20,9 @@ import kotlinx.coroutines.flow.emitAll import kotlinx.coroutines.flow.flow import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.map +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit import kotlinx.coroutines.withContext import org.xml.sax.Attributes @@ -43,7 +49,6 @@ class SubsonicSource( override fun albums(): Flow> = flow { - val (frequent, recent) = fetchRanks() var offset = 0 var pages = 0 while (true) { @@ -57,16 +62,7 @@ class SubsonicSource( ).collect { batch -> raw += batch.size val accepted = batch.filter { it.id.isNotEmpty() } - if (accepted.isNotEmpty()) { - emit( - accepted.map { - it.toAlbum().copy( - frequentRank = frequent[it.id], - recentRank = recent[it.id], - ) - }, - ) - } + if (accepted.isNotEmpty()) emit(accepted.map { it.toAlbum() }) } if (raw < PAGE_SIZE) { if (raw > 0 && pages >= maxPages) throw pageCapExceeded() @@ -95,20 +91,25 @@ class SubsonicSource( ).map { batch -> batch.filter { it.id.isNotEmpty() } } override fun playlistSongs(playlistIds: List): Flow> = - flow { - for (playlistId in playlistIds) { - var position = 0L - entityBatches( - method = "getPlaylist", - params = mapOf("id" to playlistId), - tag = "entry", - create = { attrs -> - val entryId = attrs.attr("id") - SubsonicXml.playlistSong(id, playlistId, if (entryId.isEmpty()) -1L else position++, attrs) - }, - ).collect { batch -> - val accepted = batch.filter { it.songId.isNotEmpty() } - if (accepted.isNotEmpty()) emit(accepted) + channelFlow { + val permits = Semaphore(MAX_CONCURRENT_FETCHES) + playlistIds.forEach { playlistId -> + launch { + permits.withPermit { + var position = 0L + entityBatches( + method = "getPlaylist", + params = mapOf("id" to playlistId), + tag = "entry", + create = { attrs -> + val entryId = attrs.attr("id") + SubsonicXml.playlistSong(id, playlistId, if (entryId.isEmpty()) -1L else position++, attrs) + }, + ).collect { batch -> + val accepted = batch.filter { it.songId.isNotEmpty() } + if (accepted.isNotEmpty()) send(accepted) + } + } } } }.flowOn(Dispatchers.IO).buffer(1) @@ -174,7 +175,8 @@ class SubsonicSource( } private fun albumSongs(): Flow> = - flow { + channelFlow { + val permits = Semaphore(MAX_CONCURRENT_FETCHES) var offset = 0 var pages = 0 while (true) { @@ -189,16 +191,23 @@ class SubsonicSource( raw += batch.size batch.forEach { if (it.isNotEmpty()) albums += it } } - for (albumId in albums) { - entityBatches( - method = "getAlbum", - params = mapOf("id" to albumId), - tag = "song", - create = { SubsonicXml.song(id, it) }, - ).collect { batch -> - val accepted = batch.filter { it.id.isNotEmpty() } - if (accepted.isNotEmpty()) emit(accepted) - } + coroutineScope { + albums + .map { albumId -> + async { + permits.withPermit { + entityBatches( + method = "getAlbum", + params = mapOf("id" to albumId), + tag = "song", + create = { SubsonicXml.song(id, it) }, + ).collect { batch -> + val accepted = batch.filter { it.id.isNotEmpty() } + if (accepted.isNotEmpty()) send(accepted) + } + } + } + }.awaitAll() } if (raw < PAGE_SIZE) { if (raw > 0 && pages >= maxPages) throw pageCapExceeded() @@ -210,44 +219,6 @@ class SubsonicSource( } } - private suspend fun fetchRanks(): Pair, Map> = fetchRank("frequent") to fetchRank("recent") - - private suspend fun fetchRank(type: String): Map = - try { - val ranks = HashMap() - var offset = 0 - var pages = 0 - while (true) { - var raw = 0 - var index = 0 - entityBatches( - method = "getAlbumList2", - params = page(type, offset), - tag = "album", - create = { it.attr("id") }, - ).collect { batch -> - raw += batch.size - batch.forEach { albumId -> - if (albumId.isNotEmpty()) { - ranks[albumId] = (offset + index).toLong() - index++ - } - } - } - if (raw < PAGE_SIZE) { - if (raw > 0 && pages >= maxPages) throw pageCapExceeded() - break - } - pages++ - if (pages > maxPages) throw pageCapExceeded() - offset += PAGE_SIZE - } - ranks - } catch (failure: SubsonicException) { - if (failure is PageCapExceeded) throw failure - emptyMap() - } - private fun supportsEmptyQuerySearch(): Boolean { emptyQuerySearchSupported?.let { return it } val supported = @@ -263,7 +234,7 @@ class SubsonicSource( return supported } - private fun pageCapExceeded() = PageCapExceeded("Library exceeds the ${maxPages * PAGE_SIZE}-row page cap") + private fun pageCapExceeded() = PageCapExceeded("Library is too large to sync: it exceeds ${maxPages.toLong() * PAGE_SIZE} rows") private fun entityBatches( method: String, @@ -310,7 +281,8 @@ class SubsonicSource( private companion object { const val PAGE_SIZE = 500 - const val MAX_PAGES = 1000 + const val MAX_PAGES = 20_000 + const val MAX_CONCURRENT_FETCHES = 4 const val THUMBNAIL_SIZE = 256 } } diff --git a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt index fb5c740b..c8eed8d7 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt @@ -155,8 +155,6 @@ internal class AlbumDraft( year = year, starred = starred, songCount = songCount, - frequentRank = null, - recentRank = null, discTitles = discTitles.toMap(), ) } diff --git a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt index ca137f4d..9ccb4fe8 100644 --- a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt +++ b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt @@ -13,6 +13,7 @@ import com.subtracks.data.model.Playlist import com.subtracks.data.model.PlaylistSong import com.subtracks.data.model.Song import com.subtracks.data.source.MusicSource +import java.util.Arrays class SyncService( private val db: SubtracksDatabase, @@ -28,19 +29,19 @@ class SyncService( private suspend fun syncArtists() { val library = db.libraryDao() - val stale = library.artistIds(source.id).toHashSet() + val seen = HashedIds() source.artists().collect { batch -> if (batch.isEmpty()) return@collect write("artists", artistColumns, primaryKey, batch) { it.values() } - stale -= batch.mapTo(HashSet()) { it.id } + batch.forEach { seen.add(idHash(it.id)) } } - deleteStale(stale) { library.deleteArtists(source.id, it) } + pruneStaleIds(seen, { library.artistIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteArtists(source.id, it) }) } private suspend fun syncAlbums() { val library = db.libraryDao() - val staleAlbums = library.albumIds(source.id).toHashSet() - val staleDiscs = library.discKeys(source.id).toHashSet() + val seenAlbums = HashedIds() + val seenDiscs = HashedIds() source.albums().collect { batch -> if (batch.isEmpty()) return@collect val discs = @@ -48,33 +49,35 @@ class SyncService( album.discTitles.map { (disc, title) -> Disc(album.sourceId, album.id, disc, title) } } writeAlbumsAndDiscs(batch, discs) - staleAlbums -= batch.mapTo(HashSet()) { it.id } - staleDiscs -= discs.mapTo(HashSet()) { DiscKey(it.albumId, it.disc) } + batch.forEach { seenAlbums.add(idHash(it.id)) } + discs.forEach { seenDiscs.add(discHash(it.albumId, it.disc)) } + } + pruneStaleIds(seenAlbums, { library.albumIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteAlbums(source.id, it) }) + pruneStaleDiscs(seenDiscs, { albumId, disc -> library.discKeysAfter(source.id, albumId, disc, PRUNE_PAGE) }) { + deleteStaleDiscs(it) } - deleteStale(staleAlbums) { library.deleteAlbums(source.id, it) } - deleteStaleDiscs(staleDiscs) } private suspend fun syncSongs() { val library = db.libraryDao() - val stale = library.songIds(source.id).toHashSet() + val seen = HashedIds() source.songs().collect { batch -> if (batch.isEmpty()) return@collect write("songs", songColumns, primaryKey, batch) { it.values() } - stale -= batch.mapTo(HashSet()) { it.id } + batch.forEach { seen.add(idHash(it.id)) } } - deleteStale(stale) { library.deleteSongs(source.id, it) } + pruneStaleIds(seen, { library.songIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deleteSongs(source.id, it) }) } private suspend fun syncPlaylists() { val library = db.libraryDao() - val stale = library.playlistIds(source.id).toHashSet() + val seen = HashedIds() source.playlists().collect { batch -> if (batch.isEmpty()) return@collect write("playlists", playlistColumns, primaryKey, batch) { it.values() } - stale -= batch.mapTo(HashSet()) { it.id } + batch.forEach { seen.add(idHash(it.id)) } } - deleteStale(stale) { library.deletePlaylists(source.id, it) } + pruneStaleIds(seen, { library.playlistIdsAfter(source.id, it, PRUNE_PAGE) }, { library.deletePlaylists(source.id, it) }) } private suspend fun syncPlaylistSongs() { @@ -118,7 +121,7 @@ class SyncService( } } - private suspend fun deleteStaleDiscs(stale: Set) { + private suspend fun deleteStaleDiscs(stale: Collection) { stale.chunked(DELETE_CHUNK).forEach { chunk -> db.useWriterConnection { connection -> connection.immediateTransaction { @@ -137,15 +140,77 @@ class SyncService( } } - private suspend fun deleteStale( - stale: Set, + private suspend fun pruneStaleIds( + seen: HashedIds, + pageOf: suspend (afterId: String) -> List, delete: suspend (Collection) -> Unit, ) { - stale.chunked(DELETE_CHUNK).forEach { delete(it) } + seen.freeze() + var afterId = "" + while (true) { + val ids = pageOf(afterId) + if (ids.isEmpty()) return + val stale = ids.filterNot { seen.contains(idHash(it)) } + stale.chunked(DELETE_CHUNK).forEach { delete(it) } + if (ids.size < PRUNE_PAGE) return + afterId = ids.last() + } + } + + private suspend fun pruneStaleDiscs( + seen: HashedIds, + pageOf: suspend (afterAlbumId: String, afterDisc: Long) -> List, + delete: suspend (Collection) -> Unit, + ) { + seen.freeze() + var afterAlbumId = "" + var afterDisc = 0L + while (true) { + val keys = pageOf(afterAlbumId, afterDisc) + if (keys.isEmpty()) return + val stale = keys.filterNot { seen.contains(discHash(it.albumId, it.disc)) } + stale.chunked(DELETE_CHUNK).forEach { delete(it) } + if (keys.size < PRUNE_PAGE) return + afterAlbumId = keys.last().albumId + afterDisc = keys.last().disc + } } } private const val DELETE_CHUNK = 500 +private const val PRUNE_PAGE = 500 +private const val INITIAL_HASH_CAPACITY = 256 +private const val FNV_OFFSET = -3750763034362895579L +private const val FNV_PRIME = 1099511628211L + +private class HashedIds { + private var values = LongArray(0) + private var count = 0 + + fun add(value: Long) { + if (count == values.size) { + values = values.copyOf(if (values.isEmpty()) INITIAL_HASH_CAPACITY else values.size * 2) + } + values[count++] = value + } + + fun freeze() { + Arrays.sort(values, 0, count) + } + + operator fun contains(value: Long): Boolean = Arrays.binarySearch(values, 0, count, value) >= 0 +} + +private fun idHash(id: String): Long { + var hash = FNV_OFFSET + id.forEach { hash = (hash xor it.code.toLong()) * FNV_PRIME } + return hash +} + +private fun discHash( + albumId: String, + disc: Long, +): Long = idHash(albumId) * 31 + disc private val primaryKey = listOf("sourceId", "id") @@ -164,8 +229,6 @@ private val albumColumns = "year", "starred", "songCount", - "frequentRank", - "recentRank", ) private val discKey = listOf("sourceId", "albumId", "disc") @@ -197,8 +260,7 @@ private val playlistSongKey = listOf("sourceId", "playlistId", "position") private fun Artist.values() = listOf(sourceId, id, name, albumCount, starred, coverArt) -private fun Album.values() = - listOf(sourceId, id, artistId, name, albumArtist, created, coverArt, genre, year, starred, songCount, frequentRank, recentRank) +private fun Album.values() = listOf(sourceId, id, artistId, name, albumArtist, created, coverArt, genre, year, starred, songCount) private fun Disc.values() = listOf(sourceId, albumId, disc, title) diff --git a/app/src/test/kotlin/com/subtracks/data/db/LibraryDaoTest.kt b/app/src/test/kotlin/com/subtracks/data/db/LibraryDaoTest.kt index 6cd81431..4ddfa508 100644 --- a/app/src/test/kotlin/com/subtracks/data/db/LibraryDaoTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/db/LibraryDaoTest.kt @@ -162,8 +162,6 @@ class LibraryDaoTest { year = year, starred = starred, songCount = 1, - frequentRank = null, - recentRank = null, ) private fun song( diff --git a/app/src/test/kotlin/com/subtracks/data/repo/QueueRepositoryTest.kt b/app/src/test/kotlin/com/subtracks/data/repo/QueueRepositoryTest.kt index f9cf231e..7e5cff10 100644 --- a/app/src/test/kotlin/com/subtracks/data/repo/QueueRepositoryTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/repo/QueueRepositoryTest.kt @@ -405,8 +405,6 @@ class QueueRepositoryTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) private fun song( diff --git a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt index 87e780ea..0ce3dbfe 100644 --- a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt @@ -16,6 +16,8 @@ import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicInteger class SubsonicSourceTest { @@ -91,12 +93,15 @@ class SubsonicSourceTest { fun albumsPaginateUntilAShortPage() = runBlocking { val requestedOffsets = CopyOnWriteArrayList() + val requestedTypes = CopyOnWriteArrayList() server.dispatcher = object : Dispatcher() { override fun dispatch(request: RecordedRequest): MockResponse { val url = request.requestUrl!! if (url.encodedPath != "/rest/getAlbumList2.view") return MockResponse().setResponseCode(404) - if (url.queryParameter("type") != "newest") { + val type = url.queryParameter("type")!! + requestedTypes += type + if (type != "newest") { return MockResponse().setBody(emptyAlbumList()) } val offset = url.queryParameter("offset")!!.toInt() @@ -110,6 +115,7 @@ class SubsonicSourceTest { assertEquals(1003, albums.size) assertEquals(listOf(0, 500, 1000), requestedOffsets.toList()) + assertEquals(listOf("newest"), requestedTypes.distinct()) assertEquals("al-0", albums.first().id) assertEquals("al-1002", albums.last().id) } @@ -200,7 +206,10 @@ class SubsonicSourceTest { SubsonicSource(1, client(), maxPages = 2).albums().toList() } - assertEquals(true, failure.exceptionOrNull() is SubsonicException) + val thrown = failure.exceptionOrNull() + assertEquals(true, thrown is SubsonicException) + assertTrue(thrown?.message.orEmpty(), thrown?.message.orEmpty().contains("too large")) + assertTrue(thrown?.message.orEmpty(), thrown?.message.orEmpty().contains("1000 rows")) } @Test @@ -270,6 +279,119 @@ class SubsonicSourceTest { assertEquals("al-502", albums.last().id) } + @Test + fun playlistSongsFetchesPlaylistsConcurrently() = + runBlocking { + val inFlight = AtomicInteger() + val maxInFlight = AtomicInteger() + val secondStarted = CountDownLatch(1) + server.dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + val url = request.requestUrl!! + if (url.encodedPath != "/rest/getPlaylist.view") return MockResponse().setResponseCode(404) + val current = inFlight.incrementAndGet() + maxInFlight.updateAndGet { maxOf(it, current) } + if (current >= 2) secondStarted.countDown() + secondStarted.await(2, TimeUnit.SECONDS) + inFlight.decrementAndGet() + val playlistId = url.queryParameter("id")!! + return MockResponse().setBody( + "" + + (1..3).joinToString("") { "" } + + "", + ) + } + } + + val entries = SubsonicSource(1, client()).playlistSongs(listOf("p1", "p2", "p3", "p4")).toList().flatten() + + assertEquals(12, entries.size) + assertEquals(4, entries.map { it.playlistId }.distinct().size) + entries.groupBy { it.playlistId }.forEach { (playlistId, batch) -> + assertEquals(listOf(0L, 1L, 2L), batch.sortedBy { it.position }.map { it.position }) + assertEquals((1..3).map { "$playlistId-s$it" }, batch.sortedBy { it.position }.map { it.songId }) + } + assertTrue(maxInFlight.get() >= 2) + } + + @Test + fun playlistSongsPropagatesFailuresFromAnyPlaylist() = + runBlocking { + server.dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + val url = request.requestUrl!! + if (url.encodedPath != "/rest/getPlaylist.view") return MockResponse().setResponseCode(404) + return if (url.queryParameter("id") == "p2") { + MockResponse().setBody(failed(70, "Playlist not found")) + } else { + MockResponse().setBody( + "" + + "" + + "", + ) + } + } + } + + val failure = runCatching { SubsonicSource(1, client()).playlistSongs(listOf("p1", "p2", "p3")).toList() } + + assertEquals(true, failure.exceptionOrNull() is SubsonicException) + } + + @Test + fun albumSongsFetchesAlbumsConcurrentlyWhenSearchIsUnsupported() = + runBlocking { + val inFlight = AtomicInteger() + val maxInFlight = AtomicInteger() + val secondStarted = CountDownLatch(1) + server.dispatcher = + object : Dispatcher() { + override fun dispatch(request: RecordedRequest): MockResponse { + val url = request.requestUrl!! + return when { + url.encodedPath == "/rest/search3.view" -> { + MockResponse().setBody(failed(10, "Required parameter is missing")) + } + + url.encodedPath == "/rest/getAlbumList2.view" -> { + MockResponse().setBody( + "" + + (0 until 4).joinToString("") { + "" + } + + "", + ) + } + + url.encodedPath == "/rest/getAlbum.view" -> { + val current = inFlight.incrementAndGet() + maxInFlight.updateAndGet { maxOf(it, current) } + if (current >= 2) secondStarted.countDown() + secondStarted.await(2, TimeUnit.SECONDS) + inFlight.decrementAndGet() + val albumId = url.queryParameter("id")!! + MockResponse().setBody( + "" + + "" + + "", + ) + } + + else -> { + MockResponse().setResponseCode(404) + } + } + } + } + + val songs = SubsonicSource(1, client()).songs().toList().flatten() + + assertEquals(4, songs.size) + assertTrue(maxInFlight.get() >= 2) + } + @Test fun playlistEntriesWithoutIdsAreSkipped() = runBlocking { diff --git a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt index ed6ca389..f848c8cd 100644 --- a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt @@ -525,6 +525,23 @@ class SyncServiceTest { assertEquals(0, db.libraryDao().artistIds(1).size) } + @Test + fun partialPruneAcrossManyPagesKeepsTheSurvivors() = + runTest { + insertSource() + val source = FakeMusicSource(artists = (1..1100).map { artist("a$it") }) + + SyncService(db, source).sync() + assertEquals(1100, db.libraryDao().artistIds(1).size) + + val survivors = (1..1100).filter { it % 2 == 0 }.map { "a$it" } + source.artists = survivors.map { artist(it) } + + SyncService(db, source).sync() + + assertEquals(survivors.sorted(), db.libraryDao().artistIds(1).sorted()) + } + @Test fun anIdenticalSyncDoesNotRewriteRows() = runTest { @@ -593,8 +610,6 @@ class SyncServiceTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) private fun song(id: String) = diff --git a/app/src/test/kotlin/com/subtracks/ui/AlbumDiscsTest.kt b/app/src/test/kotlin/com/subtracks/ui/AlbumDiscsTest.kt index 34821551..8da4445d 100644 --- a/app/src/test/kotlin/com/subtracks/ui/AlbumDiscsTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/AlbumDiscsTest.kt @@ -150,7 +150,5 @@ class AlbumDiscsTest { year = null, starred = null, songCount = 3, - frequentRank = null, - recentRank = null, ) } diff --git a/app/src/test/kotlin/com/subtracks/ui/AppScreenshotTest.kt b/app/src/test/kotlin/com/subtracks/ui/AppScreenshotTest.kt index d3798da1..daa4f491 100644 --- a/app/src/test/kotlin/com/subtracks/ui/AppScreenshotTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/AppScreenshotTest.kt @@ -603,8 +603,6 @@ private object Fixtures { year = year, starred = null, songCount = songCount, - frequentRank = null, - recentRank = null, ) fun song( diff --git a/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsScreenshotTest.kt b/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsScreenshotTest.kt index 6ea366ec..6662628c 100644 --- a/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsScreenshotTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsScreenshotTest.kt @@ -82,8 +82,6 @@ class LibraryOptionsScreenshotTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) } } diff --git a/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsTest.kt b/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsTest.kt index ba8fd8d4..fde389c5 100644 --- a/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/LibraryOptionsTest.kt @@ -113,8 +113,6 @@ class LibraryOptionsTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) } } diff --git a/app/src/test/kotlin/com/subtracks/ui/LibraryScreenTest.kt b/app/src/test/kotlin/com/subtracks/ui/LibraryScreenTest.kt index 3a929d90..afa8a370 100644 --- a/app/src/test/kotlin/com/subtracks/ui/LibraryScreenTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/LibraryScreenTest.kt @@ -279,8 +279,6 @@ class LibraryScreenTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) } diff --git a/app/src/test/kotlin/com/subtracks/ui/LibrarySearchScreenshotTest.kt b/app/src/test/kotlin/com/subtracks/ui/LibrarySearchScreenshotTest.kt index 3344f089..4aed3489 100644 --- a/app/src/test/kotlin/com/subtracks/ui/LibrarySearchScreenshotTest.kt +++ b/app/src/test/kotlin/com/subtracks/ui/LibrarySearchScreenshotTest.kt @@ -86,8 +86,6 @@ class LibrarySearchScreenshotTest { year = null, starred = null, songCount = 1, - frequentRank = null, - recentRank = null, ) } }