Makussu/subtracks
1
0
Fork 0
mirror of https://github.com/austinried/subtracks.git synced 2026-11-11 15:29:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

sync: declare upsert columns and values together

Browse Source
This commit is contained in:
austinried
2026-11-03 01:21:28 +00:00
parent b18e0891f6
commit 446a35d26e
1 changed files with 118 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/src/main/kotlin/com/subtracks/data/sync/SyncService.kt
+118 -123
View File
@@ -61,7 +61,7 @@ class SyncService(
source.artists().collect { batch -> source.artists().collect { batch ->
if (batch.isEmpty()) return@collect if (batch.isEmpty()) return@collect
changed = changed =
write("artists", artistColumns, primaryKey, batch, preserved = setOf("albumCount"), markStale = true) { it.values() } || write("artists", primaryKey, batch, preserved = setOf("albumCount"), markStale = true) { it.columns() } ||
changed changed
batch.forEach { seen.add(idHash(it.id)) } batch.forEach { seen.add(idHash(it.id)) }
artistsSynced += batch.size artistsSynced += batch.size
@@ -137,7 +137,7 @@ class SyncService(
connection.immediateTransaction { connection.immediateTransaction {
val before = connection.totalChanges() val before = connection.totalChanges()
val resolved = connection.resolveSongSortNames(batch) val resolved = connection.resolveSongSortNames(batch)
connection.upsertChanged("songs", songColumns, primaryKey, resolved) { it.values() } connection.upsertChanged("songs", primaryKey, resolved) { it.columns() }
connection.usePrepared("DELETE FROM song_genres WHERE sourceId = ? AND songId = ? AND position >= ?") { statement -> connection.usePrepared("DELETE FROM song_genres WHERE sourceId = ? AND songId = ? AND position >= ?") { statement ->
batch.forEach { song -> batch.forEach { song ->
statement.bind(1, song.sourceId) statement.bind(1, song.sourceId)
@@ -148,7 +148,7 @@ class SyncService(
statement.clearBindings() statement.clearBindings()
} }
} }
connection.upsertChanged("song_genres", songGenreColumns, songGenreKey, genres) { it.values() } connection.upsertChanged("song_genres", songGenreKey, genres) { it.columns() }
connection.usePrepared("DELETE FROM song_artists WHERE sourceId = ? AND songId = ? AND position >= ?") { statement -> connection.usePrepared("DELETE FROM song_artists WHERE sourceId = ? AND songId = ? AND position >= ?") { statement ->
batch.forEach { song -> batch.forEach { song ->
statement.bind(1, song.sourceId) statement.bind(1, song.sourceId)
@@ -159,7 +159,7 @@ class SyncService(
statement.clearBindings() statement.clearBindings()
} }
} }
connection.upsertChanged("song_artists", songArtistColumns, songArtistKey, artists) { it.values() } connection.upsertChanged("song_artists", songArtistKey, artists) { it.columns() }
(connection.totalChanges() != before).also { changed -> (connection.totalChanges() != before).also { changed ->
if (changed) connection.markCreditsStale(batch.first().sourceId) if (changed) connection.markCreditsStale(batch.first().sourceId)
} }
@@ -256,7 +256,7 @@ class SyncService(
val seen = HashedIds() val seen = HashedIds()
source.playlists().collect { batch -> source.playlists().collect { batch ->
if (batch.isEmpty()) return@collect if (batch.isEmpty()) return@collect
write("playlists", playlistColumns, primaryKey, batch) { it.values() } write("playlists", primaryKey, batch) { it.columns() }
batch.forEach { seen.add(idHash(it.id)) } batch.forEach { seen.add(idHash(it.id)) }
playlistsSynced += batch.size playlistsSynced += batch.size
} }
@@ -269,7 +269,7 @@ class SyncService(
val counts = HashMap<String, Long>() val counts = HashMap<String, Long>()
source.playlistSongs(playlistIds).collect { batch -> source.playlistSongs(playlistIds).collect { batch ->
if (batch.isEmpty()) return@collect if (batch.isEmpty()) return@collect
write("playlist_songs", playlistSongColumns, playlistSongKey, batch) { it.values() } write("playlist_songs", playlistSongKey, batch) { it.columns() }
batch.forEach { counts[it.playlistId] = (counts[it.playlistId] ?: 0L) + 1L } batch.forEach { counts[it.playlistId] = (counts[it.playlistId] ?: 0L) + 1L }
playlistSongsSynced += batch.size playlistSongsSynced += batch.size
} }
@@ -280,17 +280,16 @@ class SyncService(
// One commit per batch, so Room notifies paged observers once per batch rather than per row. // One commit per batch, so Room notifies paged observers once per batch rather than per row.
private suspend fun <T> write( private suspend fun <T> write(
table: String, table: String,
columns: List<String>,
key: List<String>, key: List<String>,
rows: List<T>, rows: List<T>,
preserved: Set<String> = emptySet(), preserved: Set<String> = emptySet(),
markStale: Boolean = false, markStale: Boolean = false,
values: (T) -> List<Any?>, columns: (T) -> List<Pair<String, Any?>>,
): Boolean = ): Boolean =
db.useWriterConnection { connection -> db.useWriterConnection { connection ->
connection.immediateTransaction { connection.immediateTransaction {
val before = connection.totalChanges() val before = connection.totalChanges()
connection.upsertChanged(table, columns, key, rows, preserved, values) connection.upsertChanged(table, key, rows, preserved, columns)
(connection.totalChanges() != before).also { changed -> (connection.totalChanges() != before).also { changed ->
if (changed && markStale) connection.markCreditsStale(source.id) if (changed && markStale) connection.markCreditsStale(source.id)
} }
@@ -309,8 +308,8 @@ class SyncService(
connection.immediateTransaction { connection.immediateTransaction {
val before = connection.totalChanges() val before = connection.totalChanges()
val resolved = connection.resolveAlbumSortNames(albums) val resolved = connection.resolveAlbumSortNames(albums)
connection.upsertChanged("albums", albumColumns, primaryKey, resolved) { it.values() } connection.upsertChanged("albums", primaryKey, resolved) { it.columns() }
connection.upsertChanged("discs", discColumns, discKey, discs) { it.values() } connection.upsertChanged("discs", discKey, discs) { it.columns() }
connection.usePrepared("DELETE FROM album_artists WHERE sourceId = ? AND albumId = ? AND position >= ?") { statement -> connection.usePrepared("DELETE FROM album_artists WHERE sourceId = ? AND albumId = ? AND position >= ?") { statement ->
albums.forEach { album -> albums.forEach { album ->
statement.bind(1, album.sourceId) statement.bind(1, album.sourceId)
@@ -321,7 +320,7 @@ class SyncService(
statement.clearBindings() statement.clearBindings()
} }
} }
connection.upsertChanged("album_artists", albumArtistColumns, albumArtistKey, artists) { it.values() } connection.upsertChanged("album_artists", albumArtistKey, artists) { it.columns() }
(connection.totalChanges() != before).also { changed -> (connection.totalChanges() != before).also { changed ->
if (changed) connection.markCreditsStale(albums.first().sourceId) if (changed) connection.markCreditsStale(albums.first().sourceId)
} }
@@ -438,58 +437,8 @@ private fun discHash(
private val primaryKey = listOf("sourceId", "id") private val primaryKey = listOf("sourceId", "id")
private val artistColumns = listOf("sourceId", "id", "name", "albumCount", "starred", "coverArt", "artistImageUrl", "sortName")
private val albumColumns =
listOf(
"sourceId",
"id",
"artistId",
"name",
"albumArtist",
"created",
"coverArt",
"genre",
"year",
"starred",
"songCount",
"artists",
"sortName",
"albumArtistSort",
)
private val discKey = listOf("sourceId", "albumId", "disc") private val discKey = listOf("sourceId", "albumId", "disc")
private val discColumns = listOf("sourceId", "albumId", "disc", "title")
private val songColumns =
listOf(
"sourceId",
"id",
"albumId",
"artistId",
"title",
"album",
"artist",
"duration",
"track",
"disc",
"starred",
"created",
"playCount",
"played",
"trackGain",
"albumGain",
"trackPeak",
"albumPeak",
"artists",
"sortName",
"artistSort",
)
private val playlistColumns =
listOf("sourceId", "id", "name", "comment", "coverArt", "songCount", "created", "changed", "duration", "owner", "shared")
private fun Song.genreNames(): List<String> = private fun Song.genreNames(): List<String> =
genres genres
.flatMap { it.genreParts() } .flatMap { it.genreParts() }
@@ -500,71 +449,93 @@ private fun Song.genreNames(): List<String> =
// Navidrome's default genre tag split (resources/mappings.yaml): semicolon, slash, comma. // Navidrome's default genre tag split (resources/mappings.yaml): semicolon, slash, comma.
private fun String.genreParts(): List<String> = split(';', '/', ',').map { it.trim() }.filter { it.isNotEmpty() } private fun String.genreParts(): List<String> = split(';', '/', ',').map { it.trim() }.filter { it.isNotEmpty() }
private val songGenreColumns = listOf("sourceId", "songId", "position", "genre")
private val songGenreKey = listOf("sourceId", "songId", "position") private val songGenreKey = listOf("sourceId", "songId", "position")
private val albumArtistColumns = listOf("sourceId", "albumId", "position", "artistId", "name")
private val albumArtistKey = listOf("sourceId", "albumId", "position") private val albumArtistKey = listOf("sourceId", "albumId", "position")
private val songArtistColumns = listOf("sourceId", "songId", "position", "artistId", "name")
private val songArtistKey = listOf("sourceId", "songId", "position") private val songArtistKey = listOf("sourceId", "songId", "position")
private val playlistSongColumns = listOf("sourceId", "playlistId", "songId", "position")
private val playlistSongKey = listOf("sourceId", "playlistId", "position") private val playlistSongKey = listOf("sourceId", "playlistId", "position")
private fun Artist.values() = listOf<Any?>(sourceId, id, name, albumCount, starred, coverArt, artistImageUrl, sortName) private fun Artist.columns(): List<Pair<String, Any?>> =
listOf(
private fun Album.values() = "sourceId" to sourceId,
listOf<Any?>( "id" to id,
sourceId, "name" to name,
id, "albumCount" to albumCount,
artistId, "starred" to starred,
name, "coverArt" to coverArt,
albumArtist, "artistImageUrl" to artistImageUrl,
created, "sortName" to sortName,
coverArt,
genre,
year,
starred,
songCount,
ArtistRefsCodec.encode(artistRefs),
sortName,
albumArtistSort,
) )
private fun AlbumArtistRef.values() = listOf<Any?>(sourceId, albumId, position, artistId, name) private fun Album.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"id" to id,
"artistId" to artistId,
"name" to name,
"albumArtist" to albumArtist,
"created" to created,
"coverArt" to coverArt,
"genre" to genre,
"year" to year,
"starred" to starred,
"songCount" to songCount,
"artists" to ArtistRefsCodec.encode(artistRefs),
"sortName" to sortName,
"albumArtistSort" to albumArtistSort,
)
private fun SongArtistRef.values() = listOf<Any?>(sourceId, songId, position, artistId, name) private fun AlbumArtistRef.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"albumId" to albumId,
"position" to position,
"artistId" to artistId,
"name" to name,
)
private fun Disc.values() = listOf<Any?>(sourceId, albumId, disc, title) private fun SongArtistRef.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"songId" to songId,
"position" to position,
"artistId" to artistId,
"name" to name,
)
private fun Song.values() = private fun Disc.columns(): List<Pair<String, Any?>> =
listOf<Any?>( listOf(
sourceId, "sourceId" to sourceId,
id, "albumId" to albumId,
albumId, "disc" to disc,
artistId, "title" to title,
title, )
album,
artist, private fun Song.columns(): List<Pair<String, Any?>> =
duration, listOf(
track, "sourceId" to sourceId,
disc, "id" to id,
starred, "albumId" to albumId,
created, "artistId" to artistId,
playCount, "title" to title,
played, "album" to album,
trackGain, "artist" to artist,
albumGain, "duration" to duration,
trackPeak, "track" to track,
albumPeak, "disc" to disc,
ArtistRefsCodec.encode(artistRefs), "starred" to starred,
sortName, "created" to created,
artistSort, "playCount" to playCount,
"played" to played,
"trackGain" to trackGain,
"albumGain" to albumGain,
"trackPeak" to trackPeak,
"albumPeak" to albumPeak,
"artists" to ArtistRefsCodec.encode(artistRefs),
"sortName" to sortName,
"artistSort" to artistSort,
) )
private fun List<ArtistRef>.positions(): List<Pair<Long, ArtistRef>> = private fun List<ArtistRef>.positions(): List<Pair<Long, ArtistRef>> =
@@ -576,32 +547,56 @@ private fun Album.artistPositions(): List<Pair<Long, ArtistRef>> = artistRefs.po
private fun Song.artistPositions(): List<Pair<Long, ArtistRef>> = artistRefs.positions() private fun Song.artistPositions(): List<Pair<Long, ArtistRef>> = artistRefs.positions()
private fun SongGenre.values() = listOf<Any?>(sourceId, songId, position, genre) private fun SongGenre.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"songId" to songId,
"position" to position,
"genre" to genre,
)
private fun Playlist.values() = listOf<Any?>(sourceId, id, name, comment, coverArt, songCount, created, changed, duration, owner, shared) private fun Playlist.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"id" to id,
"name" to name,
"comment" to comment,
"coverArt" to coverArt,
"songCount" to songCount,
"created" to created,
"changed" to changed,
"duration" to duration,
"owner" to owner,
"shared" to shared,
)
private fun PlaylistSong.values() = listOf<Any?>(sourceId, playlistId, songId, position) private fun PlaylistSong.columns(): List<Pair<String, Any?>> =
listOf(
"sourceId" to sourceId,
"playlistId" to playlistId,
"songId" to songId,
"position" to position,
)
private suspend fun <T> PooledConnection.upsertChanged( private suspend fun <T> PooledConnection.upsertChanged(
table: String, table: String,
columns: List<String>,
key: List<String>, key: List<String>,
rows: List<T>, rows: List<T>,
preserved: Set<String> = emptySet(), preserved: Set<String> = emptySet(),
values: (T) -> List<Any?>, columns: (T) -> List<Pair<String, Any?>>,
) { ) {
if (rows.isEmpty()) return if (rows.isEmpty()) return
check(values(rows.first()).size == columns.size) { "column/value mismatch for $table" } val columnNames = columns(rows.first()).map { it.first }
val updates = columns.filterNot { it in key || it in preserved } val updates = columnNames.filterNot { it in key || it in preserved }
val sql = val sql =
"INSERT INTO $table (${columns.joinToString()}) VALUES (${columns.joinToString { "?" }}) " + "INSERT INTO $table (${columnNames.joinToString()}) VALUES (${columnNames.joinToString { "?" }}) " +
"ON CONFLICT(${key.joinToString()}) DO UPDATE SET " + "ON CONFLICT(${key.joinToString()}) DO UPDATE SET " +
updates.joinToString { "$it = excluded.$it" } + updates.joinToString { "$it = excluded.$it" } +
" WHERE " + " WHERE " +
updates.joinToString(" OR ") { "$table.$it COLLATE BINARY IS NOT excluded.$it" } updates.joinToString(" OR ") { "$table.$it COLLATE BINARY IS NOT excluded.$it" }
usePrepared(sql) { statement -> usePrepared(sql) { statement ->
rows.forEach { row -> rows.forEach { row ->
values(row).forEachIndexed { index, value -> statement.bind(index + 1, value) } columns(row).forEachIndexed { index, (_, value) -> statement.bind(index + 1, value) }
statement.step() statement.step()
statement.reset() statement.reset()
statement.clearBindings() statement.clearBindings()