diff --git a/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt b/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt index 620d176a..809a13f2 100644 --- a/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt +++ b/app/src/integrationTest/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceIntegrationTest.kt @@ -1,5 +1,7 @@ package com.subtracks.data.source.subsonic +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.OkHttpClient @@ -40,7 +42,7 @@ class SubsonicSourceIntegrationTest( @Test fun artists() = runBlocking { - val artists = source.getArtists() + val artists = source.artists().collectAll() assertEquals(2, artists.size) assertNotNull(artists.first { it.name == "Ugress" }.starred) @@ -50,7 +52,7 @@ class SubsonicSourceIntegrationTest( @Test fun albums() = runBlocking { - val albums = source.getAlbums() + val albums = source.albums().collectAll() assertEquals(3, albums.size) @@ -72,23 +74,23 @@ class SubsonicSourceIntegrationTest( @Test fun songs() = runBlocking { - assertEquals(20, source.getSongs().size) + assertEquals(20, source.songs().collectAll().size) } @Test fun playlists() = runBlocking { - val playlists = source.getPlaylists() + val playlists = source.playlists().collectAll() assertEquals(1, playlists.size) - assertEquals(7, source.getPlaylistSongs(playlists).size) + assertEquals(7, source.playlistSongs(playlists.map { it.id }).collectAll().size) } @Test fun albumArtistRelation() = runBlocking { - val artists = source.getArtists() - val albums = source.getAlbums() + val artists = source.artists().collectAll() + val albums = source.albums().collectAll() val ugressAlbums = albums @@ -104,6 +106,8 @@ class SubsonicSourceIntegrationTest( assertEquals(listOf("I Don't Know What I'm Doing"), bradAlbums) } + private suspend fun Flow>.collectAll(): List = toList().flatten() + data class Server( val name: String, val baseUrl: String, diff --git a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt index a70d301f..b539fafd 100644 --- a/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt +++ b/app/src/main/kotlin/com/subtracks/data/db/LibraryDao.kt @@ -115,6 +115,12 @@ interface LibraryDao { position: Long, ) + @Query( + "DELETE FROM playlist_songs WHERE sourceId = :sourceId " + + "AND playlistId NOT IN (SELECT id FROM playlists WHERE sourceId = :sourceId)", + ) + suspend fun deleteOrphanPlaylistSongs(sourceId: Long) + @Query( "SELECT * FROM albums WHERE sourceId = :sourceId " + "AND (:starredFilter = 0 OR (:starredFilter = 1 AND starred IS NOT NULL) OR (:starredFilter = 2 AND starred IS NULL)) " + diff --git a/app/src/main/kotlin/com/subtracks/data/repo/SourceRepository.kt b/app/src/main/kotlin/com/subtracks/data/repo/SourceRepository.kt index 26d07a6c..68609af6 100644 --- a/app/src/main/kotlin/com/subtracks/data/repo/SourceRepository.kt +++ b/app/src/main/kotlin/com/subtracks/data/repo/SourceRepository.kt @@ -112,7 +112,7 @@ class SourceRepository( useTokenAuth: Boolean, ): Result = withContext(Dispatchers.IO) { - runCatching { client(address, username, password, useTokenAuth).get("ping") }.map { } + runCatching { client(address, username, password, useTokenAuth).check("ping") } } suspend fun sync(): Result = diff --git a/app/src/main/kotlin/com/subtracks/data/source/MusicSource.kt b/app/src/main/kotlin/com/subtracks/data/source/MusicSource.kt index 0f16ec5e..a0fb3856 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/MusicSource.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/MusicSource.kt @@ -5,19 +5,20 @@ import com.subtracks.data.model.Artist import com.subtracks.data.model.Playlist import com.subtracks.data.model.PlaylistSong import com.subtracks.data.model.Song +import kotlinx.coroutines.flow.Flow interface MusicSource { val id: Long suspend fun ping() - suspend fun getArtists(): List + fun artists(): Flow> - suspend fun getAlbums(): List + fun albums(): Flow> - suspend fun getSongs(): List + fun songs(): Flow> - suspend fun getPlaylists(): List + fun playlists(): Flow> - suspend fun getPlaylistSongs(playlists: List): List + fun playlistSongs(playlistIds: List): Flow> } diff --git a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicClient.kt b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicClient.kt index ef4d902e..289ec10d 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicClient.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicClient.kt @@ -3,11 +3,8 @@ package com.subtracks.data.source.subsonic import okhttp3.HttpUrl import okhttp3.OkHttpClient import okhttp3.Request -import org.w3c.dom.Document -import org.w3c.dom.Element +import java.io.InputStream import java.security.MessageDigest -import javax.xml.parsers.DocumentBuilderFactory -import javax.xml.parsers.ParserConfigurationException class SubsonicException( val code: Int, @@ -41,10 +38,18 @@ class SubsonicClient( return builder.build() } - fun get( + fun check( method: String, params: Map = emptyMap(), - ): Document { + ) { + stream(method, params) { SubsonicXml.readStatus(it) } + } + + fun stream( + method: String, + params: Map = emptyMap(), + body: (InputStream) -> T, + ): T { val request = Request .Builder() @@ -55,20 +60,7 @@ class SubsonicClient( if (!response.isSuccessful) { throw SubsonicException(-1, "HTTP ${response.code}") } - val stream = response.body.byteStream() - val document = secureDocumentBuilderFactory().newDocumentBuilder().parse(stream) - val root = document.documentElement - if (root == null || root.tagName != "subsonic-response") { - throw SubsonicException(-1, "Unexpected response from the server") - } - if (root.getAttribute("status") == "failed") { - val error = root.getElementsByTagName("error").item(0) as? Element - throw SubsonicException( - error?.getAttribute("code")?.toIntOrNull() ?: -1, - error?.getAttribute("message") ?: "Unknown error", - ) - } - return document + return body(response.body.byteStream()) } } @@ -76,27 +68,6 @@ class SubsonicClient( const val API_VERSION = "1.13.0" const val CLIENT = "subtracks" - private fun secureDocumentBuilderFactory(): DocumentBuilderFactory = - DocumentBuilderFactory.newInstance().apply { - isNamespaceAware = false - isExpandEntityReferences = false - setFeatureQuietly("http://apache.org/xml/features/disallow-doctype-decl", true) - setFeatureQuietly("http://xml.org/sax/features/external-general-entities", false) - setFeatureQuietly("http://xml.org/sax/features/external-parameter-entities", false) - setFeatureQuietly("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) - } - - private fun DocumentBuilderFactory.setFeatureQuietly( - name: String, - enabled: Boolean, - ) { - try { - setFeature(name, enabled) - } catch (_: ParserConfigurationException) { - // Android's parser does not implement every hardening flag; the root-tag check in get() is the backstop. - } - } - private fun randomSalt(): String = (1..4).map { ('a'..'z').random() }.joinToString("") private fun md5Hex(input: String): String = diff --git a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt index ab9cb5a0..bb4b4888 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicSource.kt @@ -6,9 +6,20 @@ import com.subtracks.data.model.Playlist import com.subtracks.data.model.PlaylistSong import com.subtracks.data.model.Song import com.subtracks.data.source.MusicSource +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.channels.trySendBlocking +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.buffer +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.emitAll +import kotlinx.coroutines.flow.flow +import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.withContext +import org.xml.sax.Attributes +@OptIn(ExperimentalCoroutinesApi::class) class SubsonicSource( override val id: Long, private val client: SubsonicClient, @@ -18,53 +29,77 @@ class SubsonicSource( private var emptyQuerySearchSupported: Boolean? = null override suspend fun ping() { - withContext(Dispatchers.IO) { client.get("ping") } + withContext(Dispatchers.IO) { client.check("ping") } } - override suspend fun getArtists(): List = - withContext(Dispatchers.IO) { - SubsonicXml.artists(id, client.get("getArtists")) - } + override fun artists(): Flow> = + entityBatches( + method = "getArtists", + tag = "artist", + create = { SubsonicXml.artist(id, it) }, + accept = { it.id.isNotEmpty() }, + ) - override suspend fun getAlbums(): List = - withContext(Dispatchers.IO) { + override fun albums(): Flow> = + flow { val (frequent, recent) = fetchRanks() - val albums = mutableListOf() var offset = 0 var pages = 0 while (pages < MAX_PAGES) { - val document = client.get("getAlbumList2", page("newest", offset)) - val batch = SubsonicXml.albums(id, document) - albums += - batch.map { - it.copy(frequentRank = frequent[it.id], recentRank = recent[it.id]) - } - if (batch.size < PAGE_SIZE) break + var count = 0 + entityBatches( + method = "getAlbumList2", + params = page("newest", offset), + tag = "album", + create = { SubsonicXml.albumDraft(id, it) }, + onChild = { draft, name, attrs -> if (name == "discTitles") draft.addDiscTitle(attrs) }, + accept = { it.id.isNotEmpty() }, + ).collect { batch -> + count += batch.size + emit( + batch.map { + it.toAlbum().copy( + frequentRank = frequent[it.id], + recentRank = recent[it.id], + ) + }, + ) + } + if (count < PAGE_SIZE) break offset += PAGE_SIZE pages++ } - albums - } + }.flowOn(Dispatchers.IO) - override suspend fun getSongs(): List = - withContext(Dispatchers.IO) { - if (supportsEmptyQuerySearch()) searchSongs() else albumSongs() - } - - override suspend fun getPlaylists(): List = - withContext(Dispatchers.IO) { - SubsonicXml.playlists(id, client.get("getPlaylists")) - } - - override suspend fun getPlaylistSongs(playlists: List): List = - withContext(Dispatchers.IO) { - val result = mutableListOf() - for (playlist in playlists) { - val document = client.get("getPlaylist", mapOf("id" to playlist.id)) - result += SubsonicXml.playlistSongs(id, playlist.id, document) + override fun songs(): Flow> = + flow { + if (supportsEmptyQuerySearch()) { + emitAll(searchSongs()) + } else { + emitAll(albumSongs()) } - result - } + }.flowOn(Dispatchers.IO) + + override fun playlists(): Flow> = + entityBatches( + method = "getPlaylists", + tag = "playlist", + create = { SubsonicXml.playlist(id, it) }, + accept = { it.id.isNotEmpty() }, + ) + + override fun playlistSongs(playlistIds: List): Flow> = + flow { + for (playlistId in playlistIds) { + var position = 0L + entityBatches( + method = "getPlaylist", + params = mapOf("id" to playlistId), + tag = "entry", + create = { attrs -> SubsonicXml.playlistSong(id, playlistId, position++, attrs) }, + ).collect { emit(it) } + } + }.flowOn(Dispatchers.IO) fun streamUri(songId: String) = client.uri( @@ -93,16 +128,84 @@ class SubsonicSource( ) } - private fun fetchRanks(): Pair, Map> = fetchRank("frequent") to fetchRank("recent") + private fun searchSongs(): Flow> = + flow { + var offset = 0 + var pages = 0 + while (pages < MAX_PAGES) { + var count = 0 + entityBatches( + method = "search3", + params = + mapOf( + "query" to "\"\"", + "songCount" to PAGE_SIZE.toString(), + "songOffset" to offset.toString(), + "artistCount" to "0", + "albumCount" to "0", + ), + tag = "song", + create = { SubsonicXml.song(id, it) }, + accept = { it.id.isNotEmpty() }, + ).collect { batch -> + count += batch.size + emit(batch) + } + if (count < PAGE_SIZE) break + offset += PAGE_SIZE + pages++ + } + } - private fun fetchRank(type: String): Map { - val ranks = mutableMapOf() + private fun albumSongs(): Flow> = + flow { + var offset = 0 + var pages = 0 + while (pages < MAX_PAGES) { + val albums = ArrayList(PAGE_SIZE) + entityBatches( + method = "getAlbumList2", + params = page("alphabeticalByName", offset), + tag = "album", + create = { it.attr("id") }, + accept = { it.isNotEmpty() }, + ).collect { albums += it } + for (albumId in albums) { + entityBatches( + method = "getAlbum", + params = mapOf("id" to albumId), + tag = "song", + create = { SubsonicXml.song(id, it) }, + accept = { it.id.isNotEmpty() }, + ).collect { emit(it) } + } + if (albums.size < PAGE_SIZE) break + offset += PAGE_SIZE + pages++ + } + } + + private suspend fun fetchRanks(): Pair, Map> = fetchRank("frequent") to fetchRank("recent") + + private suspend fun fetchRank(type: String): Map { + val ranks = HashMap() var offset = 0 var pages = 0 while (pages < MAX_PAGES) { - val batch = SubsonicXml.albums(id, client.get("getAlbumList2", page(type, offset))) - batch.forEachIndexed { index, album -> ranks[album.id] = (offset + index).toLong() } - if (batch.size < PAGE_SIZE) break + var index = 0 + entityBatches( + method = "getAlbumList2", + params = page(type, offset), + tag = "album", + create = { it.attr("id") }, + accept = { it.isNotEmpty() }, + ).collect { batch -> + batch.forEach { albumId -> + ranks[albumId] = (offset + index).toLong() + index++ + } + } + if (index < PAGE_SIZE) break offset += PAGE_SIZE pages++ } @@ -113,12 +216,10 @@ class SubsonicSource( emptyQuerySearchSupported?.let { return it } val supported = try { - val document = - client.get( - "search3", - mapOf("query" to "\"\"", "songCount" to "1", "artistCount" to "0", "albumCount" to "0"), - ) - document.getElementsByTagName("song").length > 0 + client.stream( + "search3", + mapOf("query" to "\"\"", "songCount" to "1", "artistCount" to "0", "albumCount" to "0"), + ) { input -> SubsonicXml.containsEntity(input, "song") } } catch (_: SubsonicException) { false } @@ -126,48 +227,41 @@ class SubsonicSource( return supported } - private fun searchSongs(): List { - val songs = mutableListOf() - var offset = 0 - var pages = 0 - while (pages < MAX_PAGES) { - val document = - client.get( - "search3", - mapOf( - "query" to "\"\"", - "songCount" to PAGE_SIZE.toString(), - "songOffset" to offset.toString(), - "artistCount" to "0", - "albumCount" to "0", - ), - ) - val batch = SubsonicXml.songs(id, document) - songs += batch - if (batch.size < PAGE_SIZE) break - offset += PAGE_SIZE - pages++ - } - return songs - } + private fun entityBatches( + method: String, + params: Map = emptyMap(), + tag: String, + create: (Attributes) -> T, + onChild: (T, String, Attributes) -> Unit = { _, _, _ -> }, + accept: (T) -> Boolean = { true }, + ): Flow> = + channelFlow { + val buffer = ArrayList(PAGE_SIZE) - private fun albumSongs(): List { - val songs = mutableListOf() - var offset = 0 - var pages = 0 - while (pages < MAX_PAGES) { - val document = client.get("getAlbumList2", page("alphabeticalByName", offset)) - val albums = SubsonicXml.albums(id, document) - for (album in albums) { - val albumDocument = client.get("getAlbum", mapOf("id" to album.id)) - songs += SubsonicXml.songs(id, albumDocument) + fun flush() { + if (buffer.isEmpty()) return + if (trySendBlocking(buffer.toList()).isFailure) { + throw CancellationException("Library sync cancelled") + } + buffer.clear() } - if (albums.size < PAGE_SIZE) break - offset += PAGE_SIZE - pages++ - } - return songs - } + + client.stream(method, params) { input -> + SubsonicXml.readEntities( + input = input, + entityTag = tag, + create = create, + onChild = onChild, + onEnd = { entity -> + if (accept(entity)) { + buffer += entity + if (buffer.size >= PAGE_SIZE) flush() + } + }, + ) + } + flush() + }.flowOn(Dispatchers.IO).buffer(1) private fun page( type: String, diff --git a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt index 14b8fdea..15e4b4a2 100644 --- a/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt +++ b/app/src/main/kotlin/com/subtracks/data/source/subsonic/SubsonicXml.kt @@ -5,147 +5,305 @@ import com.subtracks.data.model.Artist import com.subtracks.data.model.Playlist import com.subtracks.data.model.PlaylistSong import com.subtracks.data.model.Song -import org.w3c.dom.Document -import org.w3c.dom.Element +import org.xml.sax.Attributes +import org.xml.sax.SAXException +import org.xml.sax.helpers.DefaultHandler +import java.io.InputStream import java.time.LocalDate import java.time.OffsetDateTime import java.time.ZoneOffset import java.time.format.DateTimeParseException +import javax.xml.parsers.ParserConfigurationException +import javax.xml.parsers.SAXParserFactory object SubsonicXml { - fun artists( + fun artist( sourceId: Long, - document: Document, - ): List = - document - .elements("artist") - .map { element -> - Artist( - sourceId = sourceId, - id = element.attr("id"), - name = element.attr("name"), - albumCount = element.longAttr("albumCount") ?: 0L, - starred = element.dateAttr("starred"), - coverArt = element.attr("coverArt").ifEmpty { null }, - ) - }.filter { it.id.isNotEmpty() } - - fun albums( - sourceId: Long, - document: Document, - ): List = document.elements("album").map { album(sourceId, it) }.filter { it.id.isNotEmpty() } - - fun album( - sourceId: Long, - element: Element, - ): Album = - Album( + attrs: Attributes, + ): Artist = + Artist( sourceId = sourceId, - id = element.attr("id"), - artistId = element.attr("artistId").ifEmpty { null }, - name = element.attr("name"), - albumArtist = element.attr("artist").ifEmpty { null }, - created = element.dateAttr("created") ?: 0L, - coverArt = element.attr("coverArt").ifEmpty { null }, - genre = element.attr("genre").ifEmpty { null }, - year = element.longAttr("year"), - starred = element.dateAttr("starred"), - songCount = element.longAttr("songCount") ?: 0L, - frequentRank = null, - recentRank = null, - discTitles = element.discTitles(), + id = attrs.attr("id"), + name = attrs.attr("name"), + albumCount = attrs.longAttr("albumCount") ?: 0L, + starred = attrs.dateAttr("starred"), + coverArt = attrs.attr("coverArt").ifEmpty { null }, ) - fun songs( + internal fun albumDraft( sourceId: Long, - document: Document, - ): List = document.elements("song").map { song(sourceId, it) }.filter { it.id.isNotEmpty() } + attrs: Attributes, + ): AlbumDraft = + AlbumDraft( + sourceId = sourceId, + id = attrs.attr("id"), + artistId = attrs.attr("artistId").ifEmpty { null }, + name = attrs.attr("name"), + albumArtist = attrs.attr("artist").ifEmpty { null }, + created = attrs.dateAttr("created") ?: 0L, + coverArt = attrs.attr("coverArt").ifEmpty { null }, + genre = attrs.attr("genre").ifEmpty { null }, + year = attrs.longAttr("year"), + starred = attrs.dateAttr("starred"), + songCount = attrs.longAttr("songCount") ?: 0L, + ) fun song( sourceId: Long, - element: Element, + attrs: Attributes, ): Song = Song( sourceId = sourceId, - id = element.attr("id"), - albumId = element.attr("albumId").ifEmpty { null }, - artistId = element.attr("artistId").ifEmpty { null }, - title = element.attr("title"), - album = element.attr("album").ifEmpty { null }, - artist = element.attr("artist").ifEmpty { null }, - duration = element.longAttr("duration"), - track = element.longAttr("track"), - disc = element.longAttr("discNumber"), - starred = element.dateAttr("starred"), - genre = element.attr("genre").ifEmpty { null }, - created = element.dateAttr("created") ?: 0L, + id = attrs.attr("id"), + albumId = attrs.attr("albumId").ifEmpty { null }, + artistId = attrs.attr("artistId").ifEmpty { null }, + title = attrs.attr("title"), + album = attrs.attr("album").ifEmpty { null }, + artist = attrs.attr("artist").ifEmpty { null }, + duration = attrs.longAttr("duration"), + track = attrs.longAttr("track"), + disc = attrs.longAttr("discNumber"), + starred = attrs.dateAttr("starred"), + genre = attrs.attr("genre").ifEmpty { null }, + created = attrs.dateAttr("created") ?: 0L, ) - fun playlists( - sourceId: Long, - document: Document, - ): List = document.elements("playlist").map { playlist(sourceId, it) }.filter { it.id.isNotEmpty() } - fun playlist( sourceId: Long, - element: Element, + attrs: Attributes, ): Playlist = Playlist( sourceId = sourceId, - id = element.attr("id"), - name = element.attr("name"), - comment = element.attr("comment").ifEmpty { null }, - coverArt = element.attr("coverArt").ifEmpty { null }, - songCount = element.longAttr("songCount") ?: 0L, - created = element.dateAttr("created") ?: 0L, - changed = element.dateAttr("changed") ?: 0L, - duration = element.longAttr("duration") ?: 0L, + id = attrs.attr("id"), + name = attrs.attr("name"), + comment = attrs.attr("comment").ifEmpty { null }, + coverArt = attrs.attr("coverArt").ifEmpty { null }, + songCount = attrs.longAttr("songCount") ?: 0L, + created = attrs.dateAttr("created") ?: 0L, + changed = attrs.dateAttr("changed") ?: 0L, + duration = attrs.longAttr("duration") ?: 0L, ) - fun playlistSongs( + fun playlistSong( sourceId: Long, playlistId: String, - document: Document, - ): List = - document.elements("entry").mapIndexed { index, element -> - PlaylistSong( - sourceId = sourceId, - playlistId = playlistId, - songId = element.attr("id"), - position = index.toLong(), - ) - } -} + position: Long, + attrs: Attributes, + ): PlaylistSong = + PlaylistSong( + sourceId = sourceId, + playlistId = playlistId, + songId = attrs.attr("id"), + position = position, + ) -private fun Element.discTitles(): Map = - childElements("discTitles") - .mapNotNull { title -> - val disc = title.longAttr("disc") ?: return@mapNotNull null - title.attr("title").takeIf { it.isNotBlank() }?.let { disc to it } - }.toMap() + internal fun readStatus(input: InputStream) { + parse(input, StatusHandler()) + } -private fun Element.childElements(tag: String): List { - val children = childNodes - return (0 until children.length).mapNotNull { index -> - (children.item(index) as? Element)?.takeIf { it.tagName == tag } + internal fun readEntities( + input: InputStream, + entityTag: String, + create: (Attributes) -> T, + onChild: (T, String, Attributes) -> Unit = { _, _, _ -> }, + onEnd: (T) -> Unit, + ) { + parse(input, EntityHandler(entityTag, create, onChild, onEnd, stopAfterFirst = false)) + } + + internal fun containsEntity( + input: InputStream, + entityTag: String, + ): Boolean { + val handler = EntityHandler(entityTag, { Unit }, { _, _, _ -> }, { }, stopAfterFirst = true) + parse(input, handler) + return handler.found } } -private fun Element.elements(tag: String): List { - val nodes = getElementsByTagName(tag) - return (0 until nodes.length).mapNotNull { nodes.item(it) as? Element } +internal class AlbumDraft( + val sourceId: Long, + var id: String = "", + var artistId: String? = null, + var name: String = "", + var albumArtist: String? = null, + var created: Long = 0L, + var coverArt: String? = null, + var genre: String? = null, + var year: Long? = null, + var starred: Long? = null, + var songCount: Long = 0L, + val discTitles: MutableMap = mutableMapOf(), +) { + fun addDiscTitle(attrs: Attributes) { + val disc = attrs.longAttr("disc") ?: return + val title = attrs.attr("title").takeIf { it.isNotBlank() } ?: return + discTitles[disc] = title + } + + fun toAlbum(): Album = + Album( + sourceId = sourceId, + id = id, + artistId = artistId, + name = name, + albumArtist = albumArtist, + created = created, + coverArt = coverArt, + genre = genre, + year = year, + starred = starred, + songCount = songCount, + frequentRank = null, + recentRank = null, + discTitles = discTitles.toMap(), + ) } -private fun Document.elements(tag: String): List { - val nodes = getElementsByTagName(tag) - return (0 until nodes.length).mapNotNull { nodes.item(it) as? Element } +internal fun Attributes.attr(name: String): String = getValue(name) ?: "" + +internal fun Attributes.longAttr(name: String): Long? = attr(name).takeIf { it.isNotEmpty() }?.toLongOrNull() + +internal fun Attributes.dateAttr(name: String): Long? = attr(name).takeIf { it.isNotEmpty() }?.let(IsoDate::parse) + +private const val ROOT_TAG = "subsonic-response" + +private class XmlFailure( + val code: Int, + message: String, +) : RuntimeException(message) + +private class StopReading : RuntimeException() + +private abstract class SubsonicResponseHandler : DefaultHandler() { + private var depth = 0 + private var sawRoot = false + private var statusFailed = false + + override fun startElement( + uri: String?, + localName: String?, + qName: String, + attributes: Attributes, + ) { + depth++ + if (!sawRoot) { + sawRoot = true + if (qName != ROOT_TAG) throw XmlFailure(-1, "Unexpected response from the server") + if (attributes.getValue("status") != "ok") statusFailed = true + } + if (qName == "error") { + throw XmlFailure( + attributes.getValue("code")?.toIntOrNull() ?: -1, + attributes.getValue("message") ?: "Unknown error", + ) + } + onElementStart(qName, attributes, depth) + } + + override fun endElement( + uri: String?, + localName: String?, + qName: String, + ) { + onElementEnd(qName, depth) + depth-- + } + + override fun endDocument() { + if (statusFailed) throw XmlFailure(-1, "Unknown error") + } + + protected open fun onElementStart( + name: String, + attributes: Attributes, + depth: Int, + ) = Unit + + protected open fun onElementEnd( + name: String, + depth: Int, + ) = Unit } -private fun Element.attr(name: String): String = getAttribute(name) +private class StatusHandler : SubsonicResponseHandler() -private fun Element.longAttr(name: String): Long? = getAttribute(name).takeIf { it.isNotEmpty() }?.toLongOrNull() +private class EntityHandler( + private val entityTag: String, + private val create: (Attributes) -> T, + private val onChild: (T, String, Attributes) -> Unit, + private val onEnd: (T) -> Unit, + private val stopAfterFirst: Boolean, +) : SubsonicResponseHandler() { + private var current: T? = null + private var entityDepth = -1 -private fun Element.dateAttr(name: String): Long? = getAttribute(name).takeIf { it.isNotEmpty() }?.let(IsoDate::parse) + var found = false + private set + + override fun onElementStart( + name: String, + attributes: Attributes, + depth: Int, + ) { + val active = current + if (active == null) { + if (name == entityTag) { + current = create(attributes) + entityDepth = depth + } + } else if (depth == entityDepth + 1) { + onChild(active, name, attributes) + } + } + + override fun onElementEnd( + name: String, + depth: Int, + ) { + val active = current ?: return + if (name == entityTag && depth == entityDepth) { + onEnd(active) + found = true + current = null + entityDepth = -1 + if (stopAfterFirst) throw StopReading() + } + } +} + +private fun parse( + input: InputStream, + handler: DefaultHandler, +) { + try { + secureParserFactory().newSAXParser().parse(input, handler) + } catch (failure: XmlFailure) { + throw SubsonicException(failure.code, failure.message ?: "Unknown error") + } catch (_: StopReading) { + } +} + +private fun secureParserFactory(): SAXParserFactory = + SAXParserFactory.newInstance().apply { + isNamespaceAware = false + isValidating = false + setFeatureQuietly("http://apache.org/xml/features/disallow-doctype-decl", true) + setFeatureQuietly("http://xml.org/sax/features/external-general-entities", false) + setFeatureQuietly("http://xml.org/sax/features/external-parameter-entities", false) + setFeatureQuietly("http://apache.org/xml/features/nonvalidating/load-external-dtd", false) + } + +private fun SAXParserFactory.setFeatureQuietly( + name: String, + enabled: Boolean, +) { + try { + setFeature(name, enabled) + } catch (_: ParserConfigurationException) { + } catch (_: SAXException) { + } +} internal object IsoDate { private val timezone = Regex("(Z|[+-]\\d{2}:?\\d{2})$", RegexOption.IGNORE_CASE) diff --git a/app/src/main/kotlin/com/subtracks/data/sync/SyncManager.kt b/app/src/main/kotlin/com/subtracks/data/sync/SyncManager.kt index 9f1bf0c4..0d27bb34 100644 --- a/app/src/main/kotlin/com/subtracks/data/sync/SyncManager.kt +++ b/app/src/main/kotlin/com/subtracks/data/sync/SyncManager.kt @@ -36,12 +36,11 @@ class SyncManager( private suspend fun runSync() { _status.value = SyncStatus.Running + val result = sourceRepository.sync() + queueRepository.invalidateLibraryCache() _status.value = - sourceRepository.sync().fold( - onSuccess = { - queueRepository.invalidateLibraryCache() - SyncStatus.Success - }, + result.fold( + onSuccess = { SyncStatus.Success }, onFailure = { error -> Log.w(TAG, "Sync failed", error) SyncStatus.Failed(error.message ?: "Sync failed") diff --git a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt index e0820cf8..2eef267b 100644 --- a/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt +++ b/app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt @@ -24,62 +24,132 @@ class SyncService( suspend fun sync() = lock.withLock { - val artists = source.getArtists() - val albums = source.getAlbums() - val songs = source.getSongs() + syncArtists() + syncAlbums() + syncSongs() + syncPlaylists() + syncPlaylistSongs() + } + + private suspend fun syncArtists() { + val library = db.libraryDao() + val stale = library.artistIds(source.id).toHashSet() + source.artists().collect { batch -> + if (batch.isEmpty()) return@collect + write("artists", artistColumns, primaryKey, batch) { it.values() } + stale -= batch.mapTo(HashSet()) { it.id } + } + deleteStale(stale) { library.deleteArtists(source.id, it) } + } + + private suspend fun syncAlbums() { + val library = db.libraryDao() + val staleAlbums = library.albumIds(source.id).toHashSet() + val staleDiscs = library.discKeys(source.id).toHashSet() + source.albums().collect { batch -> + if (batch.isEmpty()) return@collect val discs = - albums.flatMap { album -> + batch.flatMap { album -> album.discTitles.map { (disc, title) -> Disc(album.sourceId, album.id, disc, title) } } - val playlists = source.getPlaylists() - val playlistSongs = source.getPlaylistSongs(playlists) + writeAlbumsAndDiscs(batch, discs) + staleAlbums -= batch.mapTo(HashSet()) { it.id } + staleDiscs -= discs.mapTo(HashSet()) { DiscKey(it.albumId, it.disc) } + } + deleteStale(staleAlbums) { library.deleteAlbums(source.id, it) } + deleteStaleDiscs(staleDiscs) + } - db.useWriterConnection { transactor -> - transactor.immediateTransaction { - upsertChanged("artists", artistColumns, primaryKey, artists) { it.values() } - upsertChanged("albums", albumColumns, primaryKey, albums) { it.values() } - upsertChanged("discs", discColumns, discKey, discs) { it.values() } - upsertChanged("songs", songColumns, primaryKey, songs) { it.values() } - upsertChanged("playlists", playlistColumns, primaryKey, playlists) { it.values() } - upsertChanged("playlist_songs", playlistSongColumns, playlistSongKey, playlistSongs) { it.values() } + private suspend fun syncSongs() { + val library = db.libraryDao() + val stale = library.songIds(source.id).toHashSet() + source.songs().collect { batch -> + if (batch.isEmpty()) return@collect + write("songs", songColumns, primaryKey, batch) { it.values() } + stale -= batch.mapTo(HashSet()) { it.id } + } + deleteStale(stale) { library.deleteSongs(source.id, it) } + } - val library = db.libraryDao() - val existingArtistIds = library.artistIds(source.id) - val existingAlbumIds = library.albumIds(source.id) - val existingSongIds = library.songIds(source.id) - val existingPlaylistIds = library.playlistIds(source.id) - val existingDiscKeys = library.discKeys(source.id) + private suspend fun syncPlaylists() { + val library = db.libraryDao() + val stale = library.playlistIds(source.id).toHashSet() + source.playlists().collect { batch -> + if (batch.isEmpty()) return@collect + write("playlists", playlistColumns, primaryKey, batch) { it.values() } + stale -= batch.mapTo(HashSet()) { it.id } + } + deleteStale(stale) { library.deletePlaylists(source.id, it) } + } - prune(existingArtistIds, artists.map { it.id }) { library.deleteArtists(source.id, it) } - prune(existingAlbumIds, albums.map { it.id }) { library.deleteAlbums(source.id, it) } - prune(existingSongIds, songs.map { it.id }) { library.deleteSongs(source.id, it) } - prune(existingPlaylistIds, playlists.map { it.id }) { library.deletePlaylists(source.id, it) } + private suspend fun syncPlaylistSongs() { + val library = db.libraryDao() + val playlistIds = library.playlistIds(source.id) + val counts = HashMap() + source.playlistSongs(playlistIds).collect { batch -> + if (batch.isEmpty()) return@collect + write("playlist_songs", playlistSongColumns, playlistSongKey, batch) { it.values() } + batch.forEach { counts[it.playlistId] = (counts[it.playlistId] ?: 0L) + 1L } + } + playlistIds.forEach { library.deletePlaylistSongsFrom(source.id, it, counts[it] ?: 0L) } + library.deleteOrphanPlaylistSongs(source.id) + } - val currentDiscKeys = discs.mapTo(HashSet()) { DiscKey(it.albumId, it.disc) } - existingDiscKeys.filterNot { it in currentDiscKeys }.forEach { library.deleteDisc(source.id, it.albumId, it.disc) } + private suspend fun write( + table: String, + columns: List, + key: List, + rows: List, + values: (T) -> List, + ) { + if (rows.isEmpty()) return + db.useWriterConnection { connection -> + connection.immediateTransaction { + connection.upsertChanged(table, columns, key, rows, values) + } + } + } - val playlistSongCounts = playlistSongs.groupingBy { it.playlistId }.eachCount() - playlists.forEach { playlist -> - library.deletePlaylistSongsFrom(source.id, playlist.id, (playlistSongCounts[playlist.id] ?: 0).toLong()) + private suspend fun writeAlbumsAndDiscs( + albums: List, + discs: List, + ) { + db.useWriterConnection { connection -> + connection.immediateTransaction { + connection.upsertChanged("albums", albumColumns, primaryKey, albums) { it.values() } + connection.upsertChanged("discs", discColumns, discKey, discs) { it.values() } + } + } + } + + private suspend fun deleteStaleDiscs(stale: Set) { + if (stale.isEmpty()) return + db.useWriterConnection { connection -> + connection.immediateTransaction { + connection.usePrepared("DELETE FROM discs WHERE sourceId = ? AND albumId = ? AND disc = ?") { statement -> + stale.forEach { key -> + statement.bind(1, source.id) + statement.bind(2, key.albumId) + statement.bind(3, key.disc) + statement.step() + statement.reset() + statement.clearBindings() } - (existingPlaylistIds - playlists.map { it.id }).forEach { library.deletePlaylistSongsFrom(source.id, it, 0) } } } } + } - private suspend fun prune( - existing: List, - current: List, + private suspend fun deleteStale( + stale: Set, delete: suspend (Collection) -> Unit, ) { - val currentIds = current.toHashSet() - existing - .filterNot { it in currentIds } - .chunked(500) - .forEach { delete(it) } + stale.chunked(DELETE_CHUNK).forEach { delete(it) } } } +private const val DELETE_CHUNK = 500 + private val primaryKey = listOf("sourceId", "id") private val artistColumns = listOf("sourceId", "id", "name", "albumCount", "starred", "coverArt") diff --git a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicClientTest.kt b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicClientTest.kt index 0557cf2e..2fe4c353 100644 --- a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicClientTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicClientTest.kt @@ -38,7 +38,7 @@ class SubsonicClientTest { fun tokenAuthSendsSaltAndHash() { server.enqueue(MockResponse().setBody("")) - client(tokenAuth = true).get("ping") + client(tokenAuth = true).check("ping") val url = server.takeRequest().requestUrl!! assertEquals("/rest/ping.view", url.encodedPath) @@ -53,7 +53,7 @@ class SubsonicClientTest { fun plaintextAuthSendsPassword() { server.enqueue(MockResponse().setBody("")) - client(tokenAuth = false).get("ping") + client(tokenAuth = false).check("ping") val url = server.takeRequest().requestUrl!! assertEquals("secret", url.queryParameter("p")) @@ -64,7 +64,7 @@ class SubsonicClientTest { fun paramsAreForwarded() { server.enqueue(MockResponse().setBody("")) - client(tokenAuth = true).get("getCoverArt", mapOf("id" to "cov", "size" to "256")) + client(tokenAuth = true).check("getCoverArt", mapOf("id" to "cov", "size" to "256")) val url = server.takeRequest().requestUrl!! assertEquals("cov", url.queryParameter("id")) @@ -79,7 +79,7 @@ class SubsonicClientTest { ), ) - val error = assertThrows(SubsonicException::class.java) { client(tokenAuth = true).get("ping") } + val error = assertThrows(SubsonicException::class.java) { client(tokenAuth = true).check("ping") } assertEquals(40, error.code) assertEquals("Wrong username", error.message) } @@ -88,6 +88,6 @@ class SubsonicClientTest { fun nonSubsonicResponseThrows() { server.enqueue(MockResponse().setBody("Sign in")) - assertThrows(SubsonicException::class.java) { client(tokenAuth = true).get("ping") } + assertThrows(SubsonicException::class.java) { client(tokenAuth = true).check("ping") } } } diff --git a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt index e47a86bb..e59120c5 100644 --- a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicSourceTest.kt @@ -1,5 +1,6 @@ package com.subtracks.data.source.subsonic +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking import okhttp3.OkHttpClient import okhttp3.mockwebserver.Dispatcher @@ -53,7 +54,33 @@ class SubsonicSourceTest { } val source = SubsonicSource(1, client()) - assertEquals(listOf("s1", "s2"), source.getSongs().map { it.id }) + assertEquals( + listOf("s1", "s2"), + source + .songs() + .toList() + .flatten() + .map { it.id }, + ) + } + + @Test + fun artistsAreEmittedInBoundedBatches() = + runBlocking { + val count = 1200 + val body = + buildString { + append("") + repeat(count) { index -> + append("") + } + append("") + } + server.enqueue(MockResponse().setBody(body)) + + val sizes = SubsonicSource(1, client()).artists().toList().map { it.size } + + assertEquals(listOf(500, 500, 200), sizes) } @Test diff --git a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicXmlTest.kt b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicXmlTest.kt index 2f362ec4..2a223323 100644 --- a/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicXmlTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/source/subsonic/SubsonicXmlTest.kt @@ -2,19 +2,37 @@ package com.subtracks.data.source.subsonic import org.junit.Assert.assertEquals import org.junit.Test -import org.w3c.dom.Document -import javax.xml.parsers.DocumentBuilderFactory +import org.xml.sax.Attributes class SubsonicXmlTest { - private fun parse(xml: String): Document = - DocumentBuilderFactory - .newInstance() - .newDocumentBuilder() - .parse(xml.byteInputStream(Charsets.UTF_8)) + private fun parse( + xml: String, + tag: String, + create: (Attributes) -> T, + onChild: (T, String, Attributes) -> Unit = { _, _, _ -> }, + ): List { + val result = mutableListOf() + SubsonicXml.readEntities( + input = xml.byteInputStream(Charsets.UTF_8), + entityTag = tag, + create = create, + onChild = onChild, + onEnd = { result += it }, + ) + return result + } + + private fun albums(xml: String) = + parse( + xml, + tag = "album", + create = { SubsonicXml.albumDraft(1, it) }, + onChild = { draft, name, attrs -> if (name == "discTitles") draft.addDiscTitle(attrs) }, + ).map { it.toAlbum() } @Test fun mapsArtists() { - val document = + val artists = parse( """ @@ -24,10 +42,10 @@ class SubsonicXmlTest { """.trimIndent(), + tag = "artist", + create = { SubsonicXml.artist(7, it) }, ) - val artists = SubsonicXml.artists(7, document) - assertEquals(2, artists.size) assertEquals("Radiohead", artists[0].name) assertEquals(7L, artists[0].sourceId) @@ -38,8 +56,8 @@ class SubsonicXmlTest { @Test fun mapsAlbumList() { - val document = - parse( + val albums = + albums( """ @@ -50,8 +68,6 @@ class SubsonicXmlTest { """.trimIndent(), ) - val albums = SubsonicXml.albums(1, document) - assertEquals(2, albums.size) assertEquals("OK Computer", albums[0].name) assertEquals("Kid A", albums[1].name) @@ -62,8 +78,8 @@ class SubsonicXmlTest { @Test fun mapsAlbumDiscTitles() { - val document = - parse( + val albums = + albums( """ @@ -77,8 +93,6 @@ class SubsonicXmlTest { """.trimIndent(), ) - val albums = SubsonicXml.albums(1, document) - assertEquals(mapOf(1L to "The Calm", 2L to "The Storm"), albums.single().discTitles) } @@ -94,28 +108,52 @@ class SubsonicXmlTest { @Test fun mapsSongsAndPlaylistEntries() { - val document = - parse( - """ - - - - - - - - - """.trimIndent(), - ) + val xml = + """ + + + + + + + + + """.trimIndent() - val songs = SubsonicXml.songs(1, document) - val entries = SubsonicXml.playlistSongs(1, "pl1", document) + val songs = parse(xml, tag = "song", create = { SubsonicXml.song(1, it) }) + var position = 0L + val entries = parse(xml, tag = "entry", create = { SubsonicXml.playlistSong(1, "pl1", position++, it) }) assertEquals(1, songs.size) assertEquals("Idioteque", songs[0].title) assertEquals(300L, songs[0].duration) assertEquals(0L, entries[0].position) assertEquals("sg1", entries[0].songId) - assertEquals(251L, SubsonicXml.song(1, document.getElementsByTagName("entry").item(0) as org.w3c.dom.Element).duration) + } + + @Test + fun statusOnlyReadRejectsAnHtmlLoginPage() { + val failure = + runCatching { + SubsonicXml.readStatus("Sign in".byteInputStream(Charsets.UTF_8)) + } + + assertEquals(true, failure.exceptionOrNull() is SubsonicException) + } + + @Test + fun containsEntityStopsAtTheFirstMatch() { + val xml = + """ + + + + + + + """.trimIndent() + + assertEquals(true, SubsonicXml.containsEntity(xml.byteInputStream(Charsets.UTF_8), "song")) + assertEquals(false, SubsonicXml.containsEntity(xml.byteInputStream(Charsets.UTF_8), "artist")) } } diff --git a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt index 3296d598..8e3f11d2 100644 --- a/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt +++ b/app/src/test/kotlin/com/subtracks/data/sync/SyncServiceTest.kt @@ -14,7 +14,9 @@ import com.subtracks.data.model.PlaylistSong import com.subtracks.data.model.Song import com.subtracks.data.model.Source import com.subtracks.data.source.MusicSource +import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.test.runTest import org.junit.After import org.junit.Assert.assertEquals @@ -293,6 +295,103 @@ class SyncServiceTest { assertEquals(mapOf(1L to "New"), storedDiscs("al1")) } + @Test + fun syncCommitsMultipleBatchesAndPrunes() = + runTest { + insertSource() + val source = + FakeMusicSource( + artists = (1..5).map { artist("a$it") }, + songs = (1..5).map { song("s$it") }, + batchSize = 2, + ) + + SyncService(db, source).sync() + + assertEquals((1..5).map { "a$it" }, db.libraryDao().artistIds(1).sorted()) + assertEquals((1..5).map { "s$it" }, db.libraryDao().songIds(1).sorted()) + + source.artists = (1..3).map { artist("a$it") } + source.songs = (1..3).map { song("s$it") } + + SyncService(db, source).sync() + + assertEquals((1..3).map { "a$it" }, db.libraryDao().artistIds(1).sorted()) + assertEquals((1..3).map { "s$it" }, db.libraryDao().songIds(1).sorted()) + } + + @Test + fun playlistSongsSplitAcrossBatchesKeepTheirTail() = + runTest { + insertSource() + val source = + FakeMusicSource( + songs = (1..5).map { song("s$it") }, + playlists = listOf(playlist("p1")), + playlistSongs = (1..5).map { PlaylistSong(1, "p1", "s$it", (it - 1).toLong()) }, + batchSize = 2, + ) + + SyncService(db, source).sync() + assertEquals( + 5, + db + .libraryDao() + .playlistSongs(1, "p1") + .allRows() + .size, + ) + + source.playlistSongs = listOf(PlaylistSong(1, "p1", "s1", 0), PlaylistSong(1, "p1", "s2", 1)) + + SyncService(db, source).sync() + assertEquals( + 2, + db + .libraryDao() + .playlistSongs(1, "p1") + .allRows() + .size, + ) + } + + @Test + fun removingAPlaylistRemovesItsSongs() = + runTest { + insertSource() + val source = + FakeMusicSource( + songs = listOf(song("s1")), + playlists = listOf(playlist("p1")), + playlistSongs = listOf(PlaylistSong(1, "p1", "s1", 0)), + ) + + SyncService(db, source).sync() + assertEquals( + 1, + db + .libraryDao() + .playlistSongs(1, "p1") + .allRows() + .size, + ) + + source.playlists = emptyList() + source.playlistSongs = emptyList() + + SyncService(db, source).sync() + + assertEquals(0, db.libraryDao().playlistIds(1).size) + assertEquals( + 0, + db + .libraryDao() + .playlistSongs(1, "p1") + .allRows() + .size, + ) + } + private suspend fun storedDiscs(albumId: String): Map = db .libraryDao() @@ -359,18 +458,24 @@ private class FakeMusicSource( var songs: List = emptyList(), var playlists: List = emptyList(), var playlistSongs: List = emptyList(), + private val batchSize: Int = 1000, ) : MusicSource { override val id: Long = 1 override suspend fun ping() = Unit - override suspend fun getArtists(): List = artists + override fun artists(): Flow> = batches(artists) - override suspend fun getAlbums(): List = albums + override fun albums(): Flow> = batches(albums) - override suspend fun getSongs(): List = songs + override fun songs(): Flow> = batches(songs) - override suspend fun getPlaylists(): List = playlists + override fun playlists(): Flow> = batches(playlists) - override suspend fun getPlaylistSongs(playlists: List): List = playlistSongs + override fun playlistSongs(playlistIds: List): Flow> = batches(playlistSongs) + + private fun batches(items: List): Flow> = + flow { + items.chunked(batchSize).forEach { emit(it) } + } }