Migrate stately to postgres

api/db/item-annotations-queries.ts

@@ -82,7 +82,7 @@ function convertItemAnnotation(row: ItemAnnotationRow): ItemAnnotation {
  */
 export async function updateItemAnnotation(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   destinyVersion: DestinyVersion,
   itemAnnotation: ItemAnnotation,
@@ -135,7 +135,7 @@ export async function updateItemAnnotation(
         END)
     `,
     values: [
-      bungieMembershipId, // $1
+      bungieMembershipId ?? null, // $1
       platformMembershipId, // $2
       destinyVersion, // $3
       itemAnnotation.id, // $4

api/db/item-hash-tags-queries.ts

@@ -66,7 +66,7 @@ function convertItemHashTag(row: ItemHashTagRow): ItemHashTag {
  */
 export async function updateItemHashTag(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   itemHashTag: ItemHashTag,
 ): Promise<QueryResult> {
@@ -114,7 +114,7 @@ export async function updateItemHashTag(
         END)
     `,
     values: [
-      bungieMembershipId,
+      bungieMembershipId ?? null,
       platformMembershipId,
       itemHashTag.hash,
       tagValue === null ? null : TagValueEnum[tagValue],

api/db/loadout-share-queries.ts

@@ -75,7 +75,7 @@ export async function addLoadoutShareIgnoring(
   shareId: string,
   loadout: Loadout,
   viewCount = 0,
-): Promise<QueryResult> {
+): Promise<boolean> {
   const response = await client.query({
     name: 'add_loadout_share_ignoring',
     text: `insert into loadout_shares (id, membership_id, platform_membership_id, name, notes, class_type, items, parameters, view_count)
@@ -96,7 +96,7 @@ values ($1, $2, $3, $4, $5, $6, $7, $8, $9) on conflict (id) do nothing`,
     ],
   });
 
-  return response;
+  return (response.rowCount ?? 0) > 1;
 }
 
 /**

api/db/loadouts-queries.ts

@@ -79,7 +79,7 @@ export function convertLoadout(row: LoadoutRow): Loadout {
  */
 export async function updateLoadout(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   destinyVersion: DestinyVersion,
   loadout: Loadout,
@@ -101,7 +101,7 @@ do update set
   created_at = CASE WHEN loadouts.deleted_at IS NOT NULL THEN now() ELSE loadouts.created_at END`,
     values: [
       loadout.id,
-      bungieMembershipId,
+      bungieMembershipId ?? null,
       platformMembershipId,
       destinyVersion,
       loadout.name,

api/db/migration-state-queries.ts

@@ -13,15 +13,21 @@ export const enum MigrationState {
 
 export interface MigrationStateInfo {
   platformMembershipId: string;
-  bungieMembershipId: number;
+  bungieMembershipId: number | undefined;
   state: MigrationState;
   lastStateChangeAt: number;
   attemptCount: number;
   lastError?: string;
 }
 
+export interface MigrationWorkItem {
+  platformMembershipId: string;
+  bungieMembershipId: number | undefined;
+  attemptCount: number;
+}
+
 interface MigrationStateRow {
-  membership_id: number;
+  membership_id: number | null;
   platform_membership_id: string;
   state: number;
   last_state_change_at: Date;
@@ -45,12 +51,54 @@ on conflict (platform_membership_id) do update set state = migration_state.state
   return result.rows[0].state;
 }
 
-export async function getUsersToMigrate(client: ClientBase): Promise<number[]> {
+export async function getUsersToMigrate(client: ClientBase): Promise<string[]> {
   const results = await client.query<MigrationStateRow>({
     name: 'get_users_to_migrate',
-    text: 'select membership_id from migration_state where state != 3 limit 1000',
+    text: 'select platform_membership_id from migration_state where state != 3 limit 1000',
   });
-  return results.rows.map((row) => row.membership_id);
+  return results.rows.map((row) => row.platform_membership_id);
+}
+
+export async function claimMigrationWork(
+  client: ClientBase,
+  batchSize: number,
+): Promise<MigrationWorkItem[]> {
+  const results = await client.query<{
+    platform_membership_id: string;
+    membership_id: number | null;
+    attempt_count: number;
+  }>({
+    name: 'claim_migration_work',
+    text: `with candidates as (
+      select platform_membership_id
+      from migration_state
+      where state = $1
+        and attempt_count < $2
+      order by last_state_change_at asc
+      limit $3
+    )
+    update migration_state
+    set state = $4,
+        attempt_count = migration_state.attempt_count + 1,
+        last_state_change_at = current_timestamp
+    from candidates
+    where migration_state.platform_membership_id = candidates.platform_membership_id
+      and migration_state.state = $1
+      and migration_state.attempt_count < $2
+    returning migration_state.platform_membership_id, migration_state.membership_id, migration_state.attempt_count`,
+    values: [
+      MigrationState.Stately,
+      MAX_MIGRATION_ATTEMPTS,
+      batchSize,
+      MigrationState.MigratingToPostgres,
+    ],
+  });
+
+  return results.rows.map((row) => ({
+    platformMembershipId: row.platform_membership_id,
+    bungieMembershipId: row.membership_id ?? undefined,
+    attemptCount: row.attempt_count,
+  }));
 }
 
 export async function getMigrationState(
@@ -66,7 +114,7 @@ export async function getMigrationState(
     return convert(results.rows[0]);
   } else {
     return {
-      bungieMembershipId: 0,
+      bungieMembershipId: undefined,
       platformMembershipId,
       state: MigrationState.Stately,
       lastStateChangeAt: 0,
@@ -77,7 +125,7 @@ export async function getMigrationState(
 
 function convert(row: MigrationStateRow): MigrationStateInfo {
   return {
-    bungieMembershipId: row.membership_id,
+    bungieMembershipId: row.membership_id ?? undefined,
     platformMembershipId: row.platform_membership_id,
     state: row.state,
     lastStateChangeAt: row.last_state_change_at.getTime(),
@@ -88,7 +136,7 @@ function convert(row: MigrationStateRow): MigrationStateInfo {
 
 export function startMigrationToPostgres(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
 ): Promise<void> {
   return updateMigrationState(
@@ -103,7 +151,7 @@ export function startMigrationToPostgres(
 
 export function finishMigrationToPostgres(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
 ): Promise<void> {
   return updateMigrationState(
@@ -118,7 +166,7 @@ export function finishMigrationToPostgres(
 
 export function abortMigrationToPostgres(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   err: string,
 ): Promise<void> {
@@ -135,7 +183,7 @@ export function abortMigrationToPostgres(
 
 async function updateMigrationState(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   state: MigrationState,
   expectedState: MigrationState,
@@ -149,13 +197,14 @@ async function updateMigrationState(
 on conflict (platform_membership_id)
 do update set
   state = $3,
+  membership_id = coalesce($2, migration_state.membership_id),
   last_state_change_at = current_timestamp,
   attempt_count = migration_state.attempt_count + $4,
   last_error = $5
 where migration_state.state = $6`,
     values: [
       platformMembershipId,
-      bungieMembershipId,
+      bungieMembershipId ?? null,
       state,
       incrementAttempt ? 1 : 0,
       err ?? null,
@@ -186,15 +235,15 @@ export async function deleteMigrationState(
 export async function setMigrationStateForTest(
   client: ClientBase,
   platformMembershipId: string,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   state: MigrationState,
 ): Promise<void> {
   await client.query({
     text: `INSERT INTO migration_state (platform_membership_id, membership_id, state)
            VALUES ($1, $2, $3)
            ON CONFLICT (platform_membership_id)
            DO UPDATE SET state = $3, last_state_change_at = NOW()`,
-    values: [platformMembershipId, bungieMembershipId, state],
+    values: [platformMembershipId, bungieMembershipId ?? null, state],
   });
 }
 

api/db/searches-queries.ts

@@ -75,7 +75,7 @@ function convertSearch(row: SearchRow): Search {
  */
 export async function updateUsedSearch(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   destinyVersion: DestinyVersion,
   query: string,
@@ -90,7 +90,7 @@ do update set
   usage_count = CASE WHEN searches.deleted_at IS NOT NULL THEN 1 ELSE searches.usage_count + 1 END,
   last_used = current_timestamp,
   deleted_at = null`,
-    values: [bungieMembershipId, platformMembershipId, destinyVersion, query, type],
+    values: [bungieMembershipId ?? null, platformMembershipId, destinyVersion, query, type],
   });
 
   if (response.rowCount! < 1) {
@@ -107,7 +107,7 @@ do update set
  */
 export async function saveSearch(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   destinyVersion: DestinyVersion,
   query: string,
@@ -123,7 +123,7 @@ DO UPDATE SET
   saved = $6,
   usage_count = CASE WHEN searches.deleted_at IS NOT NULL THEN 1 ELSE searches.usage_count END,
   deleted_at = null`,
-    values: [bungieMembershipId, platformMembershipId, destinyVersion, query, type, saved],
+    values: [bungieMembershipId ?? null, platformMembershipId, destinyVersion, query, type, saved],
   });
 }
 
@@ -132,7 +132,7 @@ DO UPDATE SET
  */
 export async function importSearch(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   destinyVersion: DestinyVersion,
   query: string,
@@ -148,7 +148,7 @@ values ($1, $2, $3, $4, $5, $6, $7, $8)
 ON CONFLICT (platform_membership_id, destiny_version, qhash)
 DO UPDATE SET saved = $5, usage_count = $7, last_used = $8, deleted_at = null`,
     values: [
-      bungieMembershipId,
+      bungieMembershipId ?? null,
       platformMembershipId,
       destinyVersion,
       query,

api/db/triumphs-queries.ts

@@ -44,7 +44,7 @@ export async function syncTrackedTriumphsForProfile(
  */
 export async function trackTriumph(
   client: ClientBase,
-  bungieMembershipId: number,
+  bungieMembershipId: number | undefined,
   platformMembershipId: string,
   recordHash: number,
 ): Promise<QueryResult> {
@@ -53,7 +53,7 @@ export async function trackTriumph(
     text: `insert INTO tracked_triumphs (membership_id, platform_membership_id, record_hash)
 values ($1, $2, $3)
 on conflict (platform_membership_id, record_hash) do update set deleted_at = null, membership_id = $1`,
-    values: [bungieMembershipId, platformMembershipId, recordHash],
+    values: [bungieMembershipId ?? null, platformMembershipId, recordHash],
   });
 
   return response;

api/migrations/20260426062237-nullable-membership.js

@@ -0,0 +1,45 @@
+'use strict';
+
+var dbm;
+var type;
+var seed;
+
+/**
+ * We receive the dbmigrate dependency from dbmigrate initially.
+ * This enables us to not have to rely on NODE_PATH.
+ */
+exports.setup = function (options, seedLink) {
+  dbm = options.dbmigrate;
+  type = dbm.dataType;
+  seed = seedLink;
+};
+
+exports.up = function (db, callback) {
+  db.runSql(
+    `alter table loadouts alter column membership_id drop not null;
+alter table item_annotations alter column membership_id drop not null;
+alter table item_hash_tags alter column membership_id drop not null;
+alter table tracked_triumphs alter column membership_id drop not null;
+alter table searches alter column membership_id drop not null;
+alter table loadout_shares alter column membership_id drop not null;
+alter table migration_state alter column membership_id drop not null;`,
+    callback,
+  );
+};
+
+exports.down = function (db, callback) {
+  db.runSql(
+    `alter table loadouts alter column membership_id set not null;
+alter table item_annotations alter column membership_id set not null;
+alter table item_hash_tags alter column membership_id set not null;
+alter table tracked_triumphs alter column membership_id set not null;
+alter table searches alter column membership_id set not null;
+alter table loadout_shares alter column membership_id set not null;
+alter table migration_state alter column membership_id set not null;`,
+    callback,
+  );
+};
+
+exports._meta = {
+  version: 1,
+};