From 7f83204b24d7948874a5cad79419d655bb53423f Mon Sep 17 00:00:00 2001 From: austinried <4966622+austinried@users.noreply.github.com> Date: Sat, 6 May 2023 10:51:18 +0900 Subject: [PATCH] don't pass all ids as params instead, only pass ids to delete and chunk those by the param limit --- lib/database/database.dart | 146 ++++++++++++++++++--------- lib/database/database.g.dart | 28 ++++- lib/database/tables.drift | 22 +++- lib/services/download_service.dart | 2 +- lib/services/download_service.g.dart | 2 +- lib/services/sync_service.dart | 8 +- lib/services/sync_service.g.dart | 2 +- 7 files changed, 152 insertions(+), 58 deletions(-) diff --git a/lib/database/database.dart b/lib/database/database.dart index 26ae3f4..2d3bc2b 100644 --- a/lib/database/database.dart +++ b/lib/database/database.dart @@ -17,6 +17,10 @@ import 'converters.dart'; part 'database.g.dart'; +// don't exceed SQLITE_MAX_VARIABLE_NUMBER (32766 for version >= 3.32.0) +// https://www.sqlite.org/limits.html +const kSqliteMaxVariableNumber = 32766; + @DriftDatabase(include: {'tables.drift'}) class SubtracksDatabase extends _$SubtracksDatabase { SubtracksDatabase() : super(_openConnection()); @@ -169,12 +173,27 @@ class SubtracksDatabase extends _$SubtracksDatabase { }); } - Future deleteArtistsNotIn(int sourceId, Iterable ids) async { - await (delete(artists) - ..where( - (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isNotIn(ids), - )) - .go(); + Future deleteArtistsNotIn(int sourceId, Set ids) { + return transaction(() async { + final allIds = (await (selectOnly(artists) + ..addColumns([artists.id]) + ..where(artists.sourceId.equals(sourceId))) + .map((row) => row.read(artists.id)) + .get()) + .whereNotNull() + .toSet(); + final downloadIds = (await artistIdsWithDownloadStatus(sourceId).get()) + .whereNotNull() + .toSet(); + + final diff = allIds.difference(downloadIds).difference(ids); + for (var slice in diff.slices(kSqliteMaxVariableNumber)) { + await (delete(artists) + ..where( + (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isIn(slice))) + .go(); + } + }); } Future saveAlbums(Iterable albums) async { @@ -183,15 +202,27 @@ class SubtracksDatabase extends _$SubtracksDatabase { }); } - Future deleteAlbumsNotIn(int sourceId, Iterable ids) async { - final alsoKeep = (await albumIdsWithDownloaded(sourceId).get()).toSet(); + Future deleteAlbumsNotIn(int sourceId, Set ids) { + return transaction(() async { + final allIds = (await (selectOnly(albums) + ..addColumns([albums.id]) + ..where(albums.sourceId.equals(sourceId))) + .map((row) => row.read(albums.id)) + .get()) + .whereNotNull() + .toSet(); + final downloadIds = (await albumIdsWithDownloadStatus(sourceId).get()) + .whereNotNull() + .toSet(); - ids = ids.toList()..addAll(alsoKeep); - await (delete(albums) - ..where( - (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isNotIn(ids), - )) - .go(); + final diff = allIds.difference(downloadIds).difference(ids); + for (var slice in diff.slices(kSqliteMaxVariableNumber)) { + await (delete(albums) + ..where( + (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isIn(slice))) + .go(); + } + }); } Future savePlaylists( @@ -215,18 +246,31 @@ class SubtracksDatabase extends _$SubtracksDatabase { }); } - Future deletePlaylistsNotIn(int sourceId, Iterable ids) async { - await (delete(playlists) - ..where( - (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isNotIn(ids), - )) - .go(); - await (delete(playlistSongs) - ..where( - (tbl) => - tbl.sourceId.equals(sourceId) & tbl.playlistId.isNotIn(ids), - )) - .go(); + Future deletePlaylistsNotIn(int sourceId, Set ids) { + return transaction(() async { + final allIds = (await (selectOnly(playlists) + ..addColumns([playlists.id]) + ..where(playlists.sourceId.equals(sourceId))) + .map((row) => row.read(playlists.id)) + .get()) + .whereNotNull() + .toSet(); + final downloadIds = (await playlistIdsWithDownloadStatus(sourceId).get()) + .whereNotNull() + .toSet(); + + final diff = allIds.difference(downloadIds).difference(ids); + for (var slice in diff.slices(kSqliteMaxVariableNumber)) { + await (delete(playlists) + ..where( + (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isIn(slice))) + .go(); + await (delete(playlistSongs) + ..where((tbl) => + tbl.sourceId.equals(sourceId) & tbl.playlistId.isIn(slice))) + .go(); + } + }); } Future savePlaylistSongs( @@ -250,29 +294,33 @@ class SubtracksDatabase extends _$SubtracksDatabase { }); } - Future deleteSongsNotIn(int sourceId, Iterable ids) async { - await (delete(songs) - ..where( - (tbl) => - tbl.sourceId.equals(sourceId) & - tbl.id.isNotIn(ids) & - tbl.downloadFilePath.isNull() & - tbl.downloadTaskId.isNull(), - )) - .go(); - final remainingIds = (await (selectOnly(songs) - ..addColumns([songs.id]) - ..where(songs.sourceId.equals(sourceId))) - .map((row) => row.read(songs.id)) - .get()) - .whereNotNull(); - await (delete(playlistSongs) - ..where( - (tbl) => - tbl.sourceId.equals(sourceId) & - tbl.songId.isNotIn(remainingIds), - )) - .go(); + Future deleteSongsNotIn(int sourceId, Set ids) { + return transaction(() async { + final allIds = (await (selectOnly(songs) + ..addColumns([songs.id]) + ..where( + songs.sourceId.equals(sourceId) & + songs.downloadFilePath.isNull() & + songs.downloadTaskId.isNull(), + )) + .map((row) => row.read(songs.id)) + .get()) + .whereNotNull() + .toSet(); + + final diff = allIds.difference(ids); + for (var slice in diff.slices(kSqliteMaxVariableNumber)) { + await (delete(songs) + ..where( + (tbl) => tbl.sourceId.equals(sourceId) & tbl.id.isIn(slice))) + .go(); + await (delete(playlistSongs) + ..where( + (tbl) => tbl.sourceId.equals(sourceId) & tbl.songId.isIn(slice), + )) + .go(); + } + }); } Selectable getLastBottomNavState() { diff --git a/lib/database/database.g.dart b/lib/database/database.g.dart index 8740b4d..a379bea 100644 --- a/lib/database/database.g.dart +++ b/lib/database/database.g.dart @@ -4596,7 +4596,7 @@ abstract class _$SubtracksDatabase extends GeneratedDatabase { )); } - Selectable albumIdsWithDownloaded(int sourceId) { + Selectable albumIdsWithDownloadStatus(int sourceId) { return customSelect( 'SELECT albums.id FROM albums JOIN songs ON songs.source_id = albums.source_id AND songs.album_id = albums.id WHERE albums.source_id = ?1 AND(songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL)GROUP BY albums.id', variables: [ @@ -4608,6 +4608,32 @@ abstract class _$SubtracksDatabase extends GeneratedDatabase { }).map((QueryRow row) => row.read('id')); } + Selectable artistIdsWithDownloadStatus(int sourceId) { + return customSelect( + 'SELECT artists.id FROM artists LEFT JOIN albums ON artists.source_id = albums.source_id AND artists.id = albums.artist_id LEFT JOIN songs ON albums.source_id = songs.source_id AND albums.id = songs.album_id WHERE artists.source_id = ?1 AND(songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL)GROUP BY artists.id', + variables: [ + Variable(sourceId) + ], + readsFrom: { + artists, + albums, + songs, + }).map((QueryRow row) => row.read('id')); + } + + Selectable playlistIdsWithDownloadStatus(int sourceId) { + return customSelect( + 'SELECT playlists.id FROM playlists LEFT JOIN playlist_songs ON playlist_songs.source_id = playlists.source_id AND playlist_songs.playlist_id = playlists.id LEFT JOIN songs ON playlist_songs.source_id = songs.source_id AND playlist_songs.song_id = songs.id WHERE playlists.source_id = ?1 AND(songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL)GROUP BY playlists.id', + variables: [ + Variable(sourceId) + ], + readsFrom: { + playlists, + playlistSongs, + songs, + }).map((QueryRow row) => row.read('id')); + } + Selectable searchArtists(String query, int limit, int offset) { return customSelect( 'SELECT "rowid" FROM artists_fts WHERE artists_fts MATCH ?1 ORDER BY rank LIMIT ?2 OFFSET ?3', diff --git a/lib/database/tables.drift b/lib/database/tables.drift index b6d533e..d013027 100644 --- a/lib/database/tables.drift +++ b/lib/database/tables.drift @@ -244,7 +244,7 @@ allSubsonicSources WITH SubsonicSettings: FROM sources JOIN subsonic_sources ON subsonic_sources.source_id = sources.id; -albumIdsWithDownloaded: +albumIdsWithDownloadStatus: SELECT albums.id FROM albums JOIN songs on songs.source_id = albums.source_id AND songs.album_id = albums.id @@ -253,6 +253,26 @@ albumIdsWithDownloaded: AND (songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL) GROUP BY albums.id; +artistIdsWithDownloadStatus: + SELECT artists.id + FROM artists + LEFT JOIN albums ON artists.source_id = albums.source_id AND artists.id = albums.artist_id + LEFT JOIN songs ON albums.source_id = songs.source_id AND albums.id = songs.album_id + WHERE + artists.source_id = :source_id + AND (songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL) + GROUP BY artists.id; + +playlistIdsWithDownloadStatus: + SELECT playlists.id + FROM playlists + LEFT JOIN playlist_songs ON playlist_songs.source_id = playlists.source_id AND playlist_songs.playlist_id = playlists.id + LEFT JOIN songs ON playlist_songs.source_id = songs.source_id AND playlist_songs.song_id = songs.id + WHERE + playlists.source_id = :source_id + AND (songs.download_file_path IS NOT NULL OR songs.download_task_id IS NOT NULL) + GROUP BY playlists.id; + searchArtists: SELECT rowid FROM artists_fts diff --git a/lib/services/download_service.dart b/lib/services/download_service.dart index 51d094e..7955944 100644 --- a/lib/services/download_service.dart +++ b/lib/services/download_service.dart @@ -214,7 +214,7 @@ class DownloadService extends _$DownloadService { Future deleteAll(int sourceId) async { final db = ref.read(databaseProvider); - final albumIds = await db.albumIdsWithDownloaded(sourceId).get(); + final albumIds = await db.albumIdsWithDownloadStatus(sourceId).get(); for (var id in albumIds) { await deleteAlbum(await (db.albumById(sourceId, id)).getSingle()); } diff --git a/lib/services/download_service.g.dart b/lib/services/download_service.g.dart index d09e50b..6a2026e 100644 --- a/lib/services/download_service.g.dart +++ b/lib/services/download_service.g.dart @@ -6,7 +6,7 @@ part of 'download_service.dart'; // RiverpodGenerator // ************************************************************************** -String _$downloadServiceHash() => r'92e963b5c070f4d1edb0cd81899b16393c2b9a70'; +String _$downloadServiceHash() => r'c72c49f980e307f3013467e76b6564d14a34a736'; /// See also [DownloadService]. @ProviderFor(DownloadService) diff --git a/lib/services/sync_service.dart b/lib/services/sync_service.dart index 4bdf255..7a05133 100644 --- a/lib/services/sync_service.dart +++ b/lib/services/sync_service.dart @@ -31,7 +31,7 @@ class SyncService extends _$SyncService { final source = ref.read(musicSourceProvider); final db = ref.read(databaseProvider); - final ids = []; + final ids = {}; await for (var artists in source.allArtists()) { ids.addAll(artists.map((e) => e.id.value)); await db.saveArtists(artists); @@ -44,7 +44,7 @@ class SyncService extends _$SyncService { final source = ref.read(musicSourceProvider); final db = ref.read(databaseProvider); - final ids = []; + final ids = {}; await for (var albums in source.allAlbums()) { ids.addAll(albums.map((e) => e.id.value)); await db.saveAlbums(albums); @@ -57,7 +57,7 @@ class SyncService extends _$SyncService { final source = ref.read(musicSourceProvider); final db = ref.read(databaseProvider); - final ids = []; + final ids = {}; await for (var playlists in source.allPlaylists()) { ids.addAll(playlists.map((e) => e.playist.id.value)); await db.savePlaylists(playlists); @@ -70,7 +70,7 @@ class SyncService extends _$SyncService { final source = ref.read(musicSourceProvider); final db = ref.read(databaseProvider); - final ids = []; + final ids = {}; await for (var songs in source.allSongs()) { ids.addAll(songs.map((e) => e.id.value)); await db.saveSongs(songs); diff --git a/lib/services/sync_service.g.dart b/lib/services/sync_service.g.dart index 857a65e..8288d77 100644 --- a/lib/services/sync_service.g.dart +++ b/lib/services/sync_service.g.dart @@ -6,7 +6,7 @@ part of 'sync_service.dart'; // RiverpodGenerator // ************************************************************************** -String _$syncServiceHash() => r'2b8da374c3143bc56f17115440d57bc70468a17e'; +String _$syncServiceHash() => r'58ebee4e6f055b64ee6789ae43d63c0e15c679e0'; /// See also [SyncService]. @ProviderFor(SyncService)