Skip to content

Commit ec98297

Browse files
eabdelmoneimclaude
eabdelmoneim
and
claude
authored
Feature/tx backfill fallback (#933)
* feat: add transaction backfill fallback for queue ID lookups - Add ENABLE_TX_BACKFILL_FALLBACK env var (default: false) - Add backfill methods to TransactionDB (getBackfillHash, setBackfill, bulkSetBackfill) - Add fallback lookup in /transaction/logs endpoint - Add POST /admin/backfill endpoint for loading backfill data When enabled, the /transaction/logs endpoint will check a fallback Redis table (backfill:<queueId>) when the primary transaction cache misses. This allows recovering transaction logs for queue IDs that were pruned from the main cache. Co-Authored-By: Claude Opus 4.5 <[email protected]> * feat: add DELETE /admin/backfill endpoint to clear backfill table Co-Authored-By: Claude Opus 4.5 <[email protected]> * docs: add comments explaining AMEX backfill logic Co-Authored-By: Claude Opus 4.5 <[email protected]> * fix: validate backfill hash format before use Add isHex validation to prevent unexpected behavior if malformed data exists in the backfill table. Co-Authored-By: Claude Opus 4.5 <[email protected]> * feat: extend backfill fallback to support /transaction/status endpoint - Add BackfillEntry interface with status field ("mined" | "errored") - Update TransactionDB to store/retrieve JSON format for backfill entries - Add backfill fallback lookup to /transaction/status routes - Update /transaction/logs to use new getBackfill method - Update admin backfill schema to accept status field - Maintain backwards compatibility for plain string tx hash entries Co-Authored-By: Claude Opus 4.5 <[email protected]> * docs: add AMEX backfill comments to /transaction/status routes Co-Authored-By: Claude Opus 4.5 <[email protected]> * fix: use discriminated union for backfill schema and remove duplicate comment - Use discriminated union so transactionHash is required for mined entries - Remove duplicated AMEX comment block Co-Authored-By: Claude Opus 4.5 <[email protected]> --------- Co-authored-by: Claude Opus 4.5 <[email protected]>
1 parent c50bc44 commit ec98297

File tree

6 files changed

+339
-0
lines changed

6 files changed

+339
-0
lines changed

‎src/server/routes/admin/backfill.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
import { type Static, Type } from "@sinclair/typebox";
2+
import type { FastifyInstance } from "fastify";
3+
import { StatusCodes } from "http-status-codes";
4+
import { TransactionDB } from "../../../shared/db/transactions/db";
5+
import { standardResponseSchema } from "../../schemas/shared-api-schemas";
6+
7+
// SPECIAL LOGIC FOR AMEX
8+
// Two admin routes to backfill transaction data:
9+
// - loadBackfillRoute: Load queueId to status/transactionHash mappings
10+
// - clearBackfillRoute: Clear all backfill entries
11+
// See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts
12+
13+
const MinedEntrySchema = Type.Object({
14+
queueId: Type.String({ description: "Queue ID (UUID)" }),
15+
status: Type.Literal("mined"),
16+
transactionHash: Type.String({ description: "Transaction hash (0x...)" }),
17+
});
18+
19+
const ErroredEntrySchema = Type.Object({
20+
queueId: Type.String({ description: "Queue ID (UUID)" }),
21+
status: Type.Literal("errored"),
22+
});
23+
24+
const loadRequestBodySchema = Type.Object({
25+
entries: Type.Array(
26+
Type.Union([MinedEntrySchema, ErroredEntrySchema], {
27+
description: "Entry with status 'mined' requires transactionHash; status 'errored' does not",
28+
}),
29+
{
30+
description: "Array of queueId to status/transactionHash mappings",
31+
maxItems: 10000,
32+
},
33+
),
34+
});
35+
36+
const loadResponseBodySchema = Type.Object({
37+
result: Type.Object({
38+
inserted: Type.Integer({ description: "Number of entries inserted" }),
39+
skipped: Type.Integer({
40+
description: "Number of entries skipped (already exist)",
41+
}),
42+
}),
43+
});
44+
45+
const clearResponseBodySchema = Type.Object({
46+
result: Type.Object({
47+
deleted: Type.Integer({ description: "Number of entries deleted" }),
48+
}),
49+
});
50+
51+
export async function loadBackfillRoute(fastify: FastifyInstance) {
52+
fastify.route<{
53+
Body: Static<typeof loadRequestBodySchema>;
54+
Reply: Static<typeof loadResponseBodySchema>;
55+
}>({
56+
method: "POST",
57+
url: "/admin/backfill",
58+
schema: {
59+
summary: "Load backfill entries",
60+
description:
61+
"Load queueId to transactionHash mappings into the backfill table. Uses SETNX to never overwrite existing entries.",
62+
tags: ["Admin"],
63+
operationId: "loadBackfill",
64+
body: loadRequestBodySchema,
65+
response: {
66+
...standardResponseSchema,
67+
[StatusCodes.OK]: loadResponseBodySchema,
68+
},
69+
hide: true,
70+
},
71+
handler: async (request, reply) => {
72+
const { entries } = request.body;
73+
74+
const { inserted, skipped } =
75+
await TransactionDB.bulkSetBackfill(entries);
76+
77+
reply.status(StatusCodes.OK).send({
78+
result: { inserted, skipped },
79+
});
80+
},
81+
});
82+
}
83+
84+
export async function clearBackfillRoute(fastify: FastifyInstance) {
85+
fastify.route<{
86+
Reply: Static<typeof clearResponseBodySchema>;
87+
}>({
88+
method: "DELETE",
89+
url: "/admin/backfill",
90+
schema: {
91+
summary: "Clear backfill table",
92+
description:
93+
"Delete all entries from the backfill table. This action cannot be undone.",
94+
tags: ["Admin"],
95+
operationId: "clearBackfill",
96+
response: {
97+
...standardResponseSchema,
98+
[StatusCodes.OK]: clearResponseBodySchema,
99+
},
100+
hide: true,
101+
},
102+
handler: async (_request, reply) => {
103+
const deleted = await TransactionDB.clearBackfill();
104+
105+
reply.status(StatusCodes.OK).send({
106+
result: { deleted },
107+
});
108+
},
109+
});
110+
}

‎src/server/routes/index.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { FastifyInstance } from "fastify";
2+
import { clearBackfillRoute, loadBackfillRoute } from "./admin/backfill";
23
import { getNonceDetailsRoute } from "./admin/nonces";
34
import { getTransactionDetails } from "./admin/transaction";
45
import { createAccessToken } from "./auth/access-tokens/create";
@@ -297,4 +298,6 @@ export async function withRoutes(fastify: FastifyInstance) {
297298
// Admin
298299
await fastify.register(getTransactionDetails);
299300
await fastify.register(getNonceDetailsRoute);
301+
await fastify.register(loadBackfillRoute);
302+
await fastify.register(clearBackfillRoute);
300303
}

‎src/server/routes/transaction/blockchain/get-logs.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,15 @@ import {
88
eth_getTransactionReceipt,
99
getContract,
1010
getRpcClient,
11+
isHex,
1112
parseEventLogs,
1213
prepareEvent,
1314
} from "thirdweb";
1415
import { resolveContractAbi } from "thirdweb/contract";
1516
import type { TransactionReceipt } from "thirdweb/transaction";
1617
import { TransactionDB } from "../../../../shared/db/transactions/db";
1718
import { getChain } from "../../../../shared/utils/chain";
19+
import { env } from "../../../../shared/utils/env";
1820
import { thirdwebClient } from "../../../../shared/utils/sdk";
1921
import { createCustomError } from "../../../middleware/error";
2022
import { AddressSchema, TransactionHashSchema } from "../../../schemas/address";
@@ -153,10 +155,23 @@ export async function getTransactionLogs(fastify: FastifyInstance) {
153155
// Get the transaction hash from the provided input.
154156
let hash: Hex | undefined;
155157
if (queueId) {
158+
// Primary lookup
156159
const transaction = await TransactionDB.get(queueId);
157160
if (transaction?.status === "mined") {
158161
hash = transaction.transactionHash;
159162
}
163+
164+
// SPECIAL LOGIC FOR AMEX
165+
// AMEX uses this endpoint to get logs for transactions they didn't receive webhooks for
166+
// the queue ID's were cleaned out of REDIS so we backfilled tx hashes to this backfill table
167+
// see https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts
168+
// Fallback to backfill table if enabled and not found
169+
if (!hash && env.ENABLE_TX_BACKFILL_FALLBACK) {
170+
const backfill = await TransactionDB.getBackfill(queueId);
171+
if (backfill?.status === "mined" && backfill.transactionHash && isHex(backfill.transactionHash)) {
172+
hash = backfill.transactionHash as Hex;
173+
}
174+
}
160175
} else if (transactionHash) {
161176
hash = transactionHash as Hex;
162177
}

‎src/server/routes/transaction/status.ts‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,85 @@ import { type Static, Type } from "@sinclair/typebox";
22
import type { FastifyInstance } from "fastify";
33
import { StatusCodes } from "http-status-codes";
44
import { TransactionDB } from "../../../shared/db/transactions/db";
5+
import { env } from "../../../shared/utils/env";
56
import { createCustomError } from "../../middleware/error";
67
import { standardResponseSchema } from "../../schemas/shared-api-schemas";
78
import {
89
TransactionSchema,
910
toTransactionSchema,
1011
} from "../../schemas/transaction";
1112

13+
/**
14+
* Creates a minimal transaction response from backfill data.
15+
* Used when the transaction is not found in Redis but exists in the backfill table.
16+
*/
17+
const createBackfillResponse = (
18+
queueId: string,
19+
backfill: { status: "mined" | "errored"; transactionHash?: string },
20+
): Static<typeof TransactionSchema> => {
21+
const baseResponse: Static<typeof TransactionSchema> = {
22+
queueId,
23+
status: backfill.status,
24+
chainId: null,
25+
fromAddress: null,
26+
toAddress: null,
27+
data: null,
28+
extension: null,
29+
value: null,
30+
nonce: null,
31+
gasLimit: null,
32+
gasPrice: null,
33+
maxFeePerGas: null,
34+
maxPriorityFeePerGas: null,
35+
transactionType: null,
36+
transactionHash: null,
37+
queuedAt: null,
38+
sentAt: null,
39+
minedAt: null,
40+
cancelledAt: null,
41+
deployedContractAddress: null,
42+
deployedContractType: null,
43+
errorMessage: null,
44+
sentAtBlockNumber: null,
45+
blockNumber: null,
46+
retryCount: 0,
47+
retryGasValues: null,
48+
retryMaxFeePerGas: null,
49+
retryMaxPriorityFeePerGas: null,
50+
signerAddress: null,
51+
accountAddress: null,
52+
accountSalt: null,
53+
accountFactoryAddress: null,
54+
target: null,
55+
sender: null,
56+
initCode: null,
57+
callData: null,
58+
callGasLimit: null,
59+
verificationGasLimit: null,
60+
preVerificationGas: null,
61+
paymasterAndData: null,
62+
userOpHash: null,
63+
functionName: null,
64+
functionArgs: null,
65+
onChainTxStatus: null,
66+
onchainStatus: null,
67+
effectiveGasPrice: null,
68+
cumulativeGasUsed: null,
69+
batchOperations: null,
70+
};
71+
72+
if (backfill.status === "mined" && backfill.transactionHash) {
73+
return {
74+
...baseResponse,
75+
transactionHash: backfill.transactionHash,
76+
onchainStatus: "success",
77+
onChainTxStatus: 1,
78+
};
79+
}
80+
81+
return baseResponse;
82+
};
83+
1284
// INPUT
1385
const requestSchema = Type.Object({
1486
queueId: Type.String({
@@ -75,6 +147,20 @@ export async function getTransactionStatusRoute(fastify: FastifyInstance) {
75147

76148
const transaction = await TransactionDB.get(queueId);
77149
if (!transaction) {
150+
// SPECIAL LOGIC FOR AMEX
151+
// AMEX uses this endpoint to check transaction status for queue IDs they didn't receive webhooks for.
152+
// The queue ID's were cleaned out of Redis so we backfilled tx data to this backfill table.
153+
// See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts
154+
// Fallback to backfill table if enabled and not found
155+
if (env.ENABLE_TX_BACKFILL_FALLBACK) {
156+
const backfill = await TransactionDB.getBackfill(queueId);
157+
if (backfill) {
158+
return reply.status(StatusCodes.OK).send({
159+
result: createBackfillResponse(queueId, backfill),
160+
});
161+
}
162+
}
163+
78164
throw createCustomError(
79165
"Transaction not found.",
80166
StatusCodes.BAD_REQUEST,
@@ -122,6 +208,20 @@ export async function getTransactionStatusQueryParamRoute(
122208

123209
const transaction = await TransactionDB.get(queueId);
124210
if (!transaction) {
211+
// SPECIAL LOGIC FOR AMEX
212+
// AMEX uses this endpoint to check transaction status for queue IDs they didn't receive webhooks for.
213+
// The queue ID's were cleaned out of Redis so we backfilled tx data to this backfill table.
214+
// See https://github.com/thirdweb-dev/solutions-customer-scripts/blob/main/amex/scripts/load-backfill-via-api.ts
215+
// Fallback to backfill table if enabled and not found
216+
if (env.ENABLE_TX_BACKFILL_FALLBACK) {
217+
const backfill = await TransactionDB.getBackfill(queueId);
218+
if (backfill) {
219+
return reply.status(StatusCodes.OK).send({
220+
result: createBackfillResponse(queueId, backfill),
221+
});
222+
}
223+
}
224+
125225
throw createCustomError(
126226
"Transaction not found.",
127227
StatusCodes.BAD_REQUEST,

0 commit comments

Comments
 (0)