From 332ddd76cfd0fef8e7a8292e55bb16d55ba61f07 Mon Sep 17 00:00:00 2001 From: Daniel Bulant Date: Mon, 20 Apr 2026 17:58:50 +0200 Subject: [PATCH] continue fixing up sync --- api/src/db/spotify.ts | 389 +++++++++++++++++++++----------------- api/src/workflows/sync.ts | 4 +- 2 files changed, 222 insertions(+), 171 deletions(-) diff --git a/api/src/db/spotify.ts b/api/src/db/spotify.ts index 7d9e96d..9d45009 100644 --- a/api/src/db/spotify.ts +++ b/api/src/db/spotify.ts @@ -43,30 +43,39 @@ type DbTransaction = Parameters[0] extends ( type DbLike = DbClient | DbTransaction; export async function upsertImages(images: Image[], dbClient: DbLike = db) { - await dbClient.insert(platformImage).values( - images.map(({ url, height, width }) => ({ - platform: PLATFORM_SPOTIFY, - url, - height, - width, - })), - ); + await dbClient + .insert(platformImage) + .values( + images.map(({ url, height, width }) => ({ + platform: PLATFORM_SPOTIFY, + url, + height, + width, + })), + ) + .onConflictDoNothing(); } export async function upsertGenres(genres: string[], dbClient: DbLike = db) { - await dbClient.insert(genre).values(genres.map((name) => ({ name }))); + await dbClient + .insert(genre) + .values(genres.map((name) => ({ name }))) + .onConflictDoNothing(); } export async function upsertArtists(artists: Artist[], dbClient: DbLike = db) { - await dbClient.insert(artist).values( - artists.map(({ id, name, images, genres, popularity, type }) => ({ - platform: PLATFORM_SPOTIFY, - platform_id: id, - name, - popularity, - type, - })), - ); + await dbClient + .insert(artist) + .values( + artists.map(({ id, name, images, genres, popularity, type }) => ({ + platform: PLATFORM_SPOTIFY, + platform_id: id, + name, + popularity, + type, + })), + ) + .onConflictDoNothing(); await upsertImages( artists.flatMap((a) => a.images), dbClient, @@ -76,34 +85,40 @@ export async function upsertArtists(artists: Artist[], dbClient: DbLike = db) { dbClient, ); for (const spotifyArtist of artists) { - await dbClient.insert(artistImage).select( - dbClient - .select({ - artistId: artist.id, - imageId: platformImage.id, - }) - .from(platformImage) - .where( - and( - eq(platformImage.platform, PLATFORM_SPOTIFY), - inArray( - platformImage.url, - spotifyArtist.images.map((t) => t.url), + await dbClient + .insert(artistImage) + .select( + dbClient + .select({ + artistId: artist.id, + imageId: platformImage.id, + }) + .from(platformImage) + .where( + and( + eq(platformImage.platform, PLATFORM_SPOTIFY), + inArray( + platformImage.url, + spotifyArtist.images.map((t) => t.url), + ), ), - ), - ) - .innerJoin(artist, eq(artist.platform_id, spotifyArtist.id)), - ); - await dbClient.insert(artistGenre).select( - dbClient - .select({ - artistId: artist.id, - genreId: genre.id, - }) - .from(genre) - .where(inArray(genre.name, spotifyArtist.genres)) - .innerJoin(artist, eq(artist.platform_id, spotifyArtist.id)), - ); + ) + .innerJoin(artist, eq(artist.platform_id, spotifyArtist.id)), + ) + .onConflictDoNothing(); + await dbClient + .insert(artistGenre) + .select( + dbClient + .select({ + artistId: artist.id, + genreId: genre.id, + }) + .from(genre) + .where(inArray(genre.name, spotifyArtist.genres)) + .innerJoin(artist, eq(artist.platform_id, spotifyArtist.id)), + ) + .onConflictDoNothing(); } } @@ -191,17 +206,20 @@ export async function upsertAlbums( albums.flatMap((a) => a.artists), dbClient, ); - await dbClient.insert(album).values( - albums.map(({ id, name, type, popularity, release_date, label }) => ({ - platform: PLATFORM_SPOTIFY, - platform_id: id, - name, - type, - popularity, - release_date: new Date(release_date), - label, - })), - ); + await dbClient + .insert(album) + .values( + albums.map(({ id, name, type, popularity, release_date, label }) => ({ + platform: PLATFORM_SPOTIFY, + platform_id: id, + name, + type, + popularity, + release_date: new Date(release_date), + label, + })), + ) + .onConflictDoNothing(); await upsertImages( albums.flatMap((a) => a.images), dbClient, @@ -211,49 +229,58 @@ export async function upsertAlbums( dbClient, ); for (const spotifyAlbum of albums) { - await dbClient.insert(albumImage).select( - dbClient - .select({ - albumId: album.id, - imageId: platformImage.id, - }) - .from(platformImage) - .where( - and( - eq(platformImage.platform, PLATFORM_SPOTIFY), - inArray( - platformImage.url, - spotifyAlbum.images.map((t) => t.url), + await dbClient + .insert(albumImage) + .select( + dbClient + .select({ + albumId: album.id, + imageId: platformImage.id, + }) + .from(platformImage) + .where( + and( + eq(platformImage.platform, PLATFORM_SPOTIFY), + inArray( + platformImage.url, + spotifyAlbum.images.map((t) => t.url), + ), ), - ), - ) - .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), - ); - await dbClient.insert(albumArtist).select( - dbClient - .select({ - albumId: album.id, - artistId: artist.id, - }) - .from(artist) - .where( - inArray( - artist.platform_id, - spotifyAlbum.artists.map((t) => t.id), - ), - ) - .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), - ); - await dbClient.insert(albumGenre).select( - dbClient - .select({ - albumId: album.id, - genreId: genre.id, - }) - .from(genre) - .where(inArray(genre.name, spotifyAlbum.genres)) - .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), - ); + ) + .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), + ) + .onConflictDoNothing(); + await dbClient + .insert(albumArtist) + .select( + dbClient + .select({ + albumId: album.id, + artistId: artist.id, + }) + .from(artist) + .where( + inArray( + artist.platform_id, + spotifyAlbum.artists.map((t) => t.id), + ), + ) + .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), + ) + .onConflictDoNothing(); + await dbClient + .insert(albumGenre) + .select( + dbClient + .select({ + albumId: album.id, + genreId: genre.id, + }) + .from(genre) + .where(inArray(genre.name, spotifyAlbum.genres)) + .innerJoin(album, eq(album.platform_id, spotifyAlbum.id)), + ) + .onConflictDoNothing(); } } @@ -271,35 +298,41 @@ export async function upsertTracks(tracks: Track[], dbClient: DbLike = db) { tracks.map((t) => t.album.id), dbClient, ); - await dbClient.insert(track).values( - tracks.map((spotifyTrack) => ({ - albumId: albumIdMap.get(spotifyTrack.album.id)!, - name: spotifyTrack.name, - platform: PLATFORM_SPOTIFY, - platform_id: spotifyTrack.id, - popularity: spotifyTrack.popularity, - duration: spotifyTrack.duration_ms, - explicit: spotifyTrack.explicit, - disc_number: spotifyTrack.disc_number, - track_number: spotifyTrack.track_number, - })), - ); + await dbClient + .insert(track) + .values( + tracks.map((spotifyTrack) => ({ + albumId: albumIdMap.get(spotifyTrack.album.id)!, + name: spotifyTrack.name, + platform: PLATFORM_SPOTIFY, + platform_id: spotifyTrack.id, + popularity: spotifyTrack.popularity, + duration: spotifyTrack.duration_ms, + explicit: spotifyTrack.explicit, + disc_number: spotifyTrack.disc_number, + track_number: spotifyTrack.track_number, + })), + ) + .onConflictDoNothing(); for (const spotifyTrack of tracks) { - await dbClient.insert(trackArtist).select( - dbClient - .select({ - trackId: track.id, - artistId: artist.id, - }) - .from(artist) - .where( - inArray( - artist.platform_id, - spotifyTrack.artists.map((t) => t.id), - ), - ) - .innerJoin(track, eq(track.platform_id, spotifyTrack.id)), - ); + await dbClient + .insert(trackArtist) + .select( + dbClient + .select({ + trackId: track.id, + artistId: artist.id, + }) + .from(artist) + .where( + inArray( + artist.platform_id, + spotifyTrack.artists.map((t) => t.id), + ), + ) + .innerJoin(track, eq(track.platform_id, spotifyTrack.id)), + ) + .onConflictDoNothing(); } } @@ -315,14 +348,17 @@ export async function upsertTopArtists( artists.map((t) => t.id), dbClient, ); - await dbClient.insert(topArtist).values( - artists.map((spotifyArtist, index) => ({ - artistId: artistIdMap.get(spotifyArtist.id)!, - position: index + 1, - userId, - timeline, - })), - ); + await dbClient + .insert(topArtist) + .values( + artists.map((spotifyArtist, index) => ({ + artistId: artistIdMap.get(spotifyArtist.id)!, + position: index + 1, + userId, + timeline, + })), + ) + .onConflictDoNothing(); } export async function upsertTopTracks( @@ -337,14 +373,17 @@ export async function upsertTopTracks( tracks.map((t) => t.id), dbClient, ); - await dbClient.insert(topTrack).values( - tracks.map((spotifyTrack, index) => ({ - trackId: trackIdMap.get(spotifyTrack.id)!, - position: index + 1, - userId, - timeline, - })), - ); + await dbClient + .insert(topTrack) + .values( + tracks.map((spotifyTrack, index) => ({ + trackId: trackIdMap.get(spotifyTrack.id)!, + position: index + 1, + userId, + timeline, + })), + ) + .onConflictDoNothing(); } export async function upsertSavedAlbums( @@ -359,13 +398,16 @@ export async function upsertSavedAlbums( albums.map((t) => t.id), dbClient, ); - await dbClient.insert(savedAlbum).values( - saved.map((item) => ({ - albumId: albumIdMap.get(item.album.id)!, - userId, - saved_at: new Date(item.added_at), - })), - ); + await dbClient + .insert(savedAlbum) + .values( + saved.map((item) => ({ + albumId: albumIdMap.get(item.album.id)!, + userId, + saved_at: new Date(item.added_at), + })), + ) + .onConflictDoNothing(); } export async function upsertSavedTracks( @@ -380,13 +422,16 @@ export async function upsertSavedTracks( tracks.map((t) => t.id), dbClient, ); - await dbClient.insert(savedTrack).values( - saved.map((item) => ({ - trackId: trackIdMap.get(item.track.id)!, - userId, - saved_at: new Date(item.added_at), - })), - ); + await dbClient + .insert(savedTrack) + .values( + saved.map((item) => ({ + trackId: trackIdMap.get(item.track.id)!, + userId, + saved_at: new Date(item.added_at), + })), + ) + .onConflictDoNothing(); } export async function upsertFollowedArtists( @@ -400,12 +445,15 @@ export async function upsertFollowedArtists( artists.map((t) => t.id), dbClient, ); - await dbClient.insert(followedArtist).values( - artists.map((spotifyArtist) => ({ - artistId: artistIdMap.get(spotifyArtist.id)!, - userId, - })), - ); + await dbClient + .insert(followedArtist) + .values( + artists.map((spotifyArtist) => ({ + artistId: artistIdMap.get(spotifyArtist.id)!, + userId, + })), + ) + .onConflictDoNothing(); } export async function upsertPlaybackHistory( @@ -420,11 +468,14 @@ export async function upsertPlaybackHistory( tracks.map((t) => t.id), dbClient, ); - await dbClient.insert(playbackHistory).values( - items.map((item) => ({ - trackId: trackIdMap.get(item.track.id)!, - userId, - played_at: new Date(item.played_at), - })), - ); + await dbClient + .insert(playbackHistory) + .values( + items.map((item) => ({ + trackId: trackIdMap.get(item.track.id)!, + userId, + played_at: new Date(item.played_at), + })), + ) + .onConflictDoNothing(); } diff --git a/api/src/workflows/sync.ts b/api/src/workflows/sync.ts index 7f35d37..dcdf613 100644 --- a/api/src/workflows/sync.ts +++ b/api/src/workflows/sync.ts @@ -140,8 +140,8 @@ export class SpotifySyncWorkflow extends ConfiguredInstance { const page = await sdk.currentUser.followedArtists(after, 50); const artists = page.artists; followed.push(...artists.items); - if (!artists.next) break; - after = artists.next; + if (!artists.next || artists.items.length === 0) break; + after = artists.items[artists.items.length - 1]!.id; } return followed; }