Skip to content

Commit 048bb47

Browse files
authored
Merge pull request #3636 from tangly1024/feat/deploy-ratelimiter
Feat/deploy ratelimiter 接口限流相关
2 parents e141933 + 33debf6 commit 048bb47

File tree

7 files changed

+180
-29
lines changed

7 files changed

+180
-29
lines changed

.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,3 +50,5 @@ yarn-error.log*
5050
# yarn
5151
package-lock.json
5252
# yarn.lock
53+
54+
.notion-api-lock

lib/cache/cache_manager.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ export async function getOrSetDataWithCustomCache(
4545
}
4646
const data = await getDataFunction(...getDataArgs)
4747
if (data) {
48-
console.log('[API-->>缓存]:', key)
48+
// console.log('[API-->>缓存]:', key)
4949
await setDataToCache(key, data, customCacheTime)
5050
}
5151
return data || null

lib/notion/RateLimiter.ts

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
import fs from 'fs'
2+
import path from 'path'
3+
4+
interface QueueItem<T> {
5+
requestFunc: () => Promise<T>
6+
resolve: (value: T) => void
7+
reject: (err: any) => void
8+
}
9+
10+
export class RateLimiter {
11+
private queue: QueueItem<any>[] = []
12+
private inflight = new Set<string>()
13+
private isProcessing = false
14+
private lastRequestTime = 0
15+
private requestCount = 0
16+
private windowStart = Date.now()
17+
18+
constructor(
19+
private maxRequestsPerMinute = 200,
20+
private lockFilePath?: string
21+
) { }
22+
23+
private async acquireLock() {
24+
if (!this.lockFilePath) return
25+
// 如果锁文件存在且创建时间过久(比如 >5分钟),认为是陈旧锁,直接删除
26+
if (fs.existsSync(this.lockFilePath)) {
27+
const stats = fs.statSync(this.lockFilePath)
28+
const age = Date.now() - stats.ctimeMs
29+
if (age > 30 * 1000) { // 30秒
30+
try {
31+
fs.unlinkSync(this.lockFilePath)
32+
console.warn('[限流] 删除陈旧锁文件:', this.lockFilePath)
33+
} catch (err) {
34+
console.error('[限流] 删除陈旧锁失败:', err)
35+
}
36+
}
37+
}
38+
while (true) {
39+
try {
40+
fs.writeFileSync(this.lockFilePath, process.pid.toString(), { flag: 'wx' })
41+
return
42+
} catch (err: any) {
43+
if (err.code === 'EEXIST') await new Promise(res => setTimeout(res, 100))
44+
else throw err
45+
}
46+
}
47+
}
48+
49+
private releaseLock() {
50+
if (!this.lockFilePath) return
51+
try { if (fs.existsSync(this.lockFilePath)) fs.unlinkSync(this.lockFilePath) }
52+
catch (err) { console.error('释放锁失败', err) }
53+
}
54+
55+
public enqueue<T>(key: string, requestFunc: () => Promise<T>): Promise<T> {
56+
if (this.inflight.has(key)) {
57+
return new Promise((resolve, reject) => {
58+
const interval = setInterval(() => {
59+
if (!this.inflight.has(key)) {
60+
clearInterval(interval)
61+
this.enqueue(key, requestFunc).then(resolve).catch(reject)
62+
}
63+
}, 50)
64+
})
65+
}
66+
67+
return new Promise((resolve, reject) => {
68+
this.queue.push({ requestFunc, resolve, reject })
69+
if (!this.isProcessing) this.processQueue()
70+
})
71+
}
72+
73+
private async processQueue() {
74+
if (this.queue.length === 0) { this.isProcessing = false; return }
75+
this.isProcessing = true
76+
77+
try {
78+
await this.acquireLock()
79+
const now = Date.now()
80+
const elapsed = now - this.windowStart
81+
82+
if (elapsed > 60_000) { this.requestCount = 0; this.windowStart = now }
83+
if (this.requestCount >= this.maxRequestsPerMinute) {
84+
const waitTime = 60_000 - elapsed + 100
85+
await new Promise(res => setTimeout(res, waitTime))
86+
this.requestCount = 0
87+
this.windowStart = Date.now()
88+
}
89+
90+
const minInterval = 300
91+
const waitTime = Math.max(0, minInterval - (now - this.lastRequestTime))
92+
if (waitTime > 0) await new Promise(res => setTimeout(res, waitTime))
93+
94+
const { requestFunc, resolve, reject } = this.queue.shift()!
95+
const key = crypto.randomUUID()
96+
this.inflight.add(key)
97+
98+
try {
99+
const result = await requestFunc()
100+
this.lastRequestTime = Date.now()
101+
this.requestCount++
102+
resolve(result)
103+
} catch (err) { reject(err) }
104+
finally { this.inflight.delete(key) }
105+
106+
} catch (err) {
107+
console.error('限流队列异常', err)
108+
} finally {
109+
this.releaseLock()
110+
setTimeout(() => this.processQueue(), 0)
111+
}
112+
}
113+
}

lib/notion/getNotionAPI.js

Lines changed: 60 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,68 @@
11
import { NotionAPI as NotionLibrary } from 'notion-client'
22
import BLOG from '@/blog.config'
3+
import path from 'path'
4+
import { RateLimiter } from './RateLimiter'
35

4-
const notionAPI = getNotionAPI()
5-
6-
function getNotionAPI() {
7-
return new NotionLibrary({
8-
apiBaseUrl: BLOG.API_BASE_URL || 'https://www.notion.so/api/v3', // https://[xxxxx].notion.site/api/v3
9-
activeUser: BLOG.NOTION_ACTIVE_USER || null,
10-
authToken: BLOG.NOTION_TOKEN_V2 || null,
11-
userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
12-
kyOptions: {
13-
mode:'cors',
14-
hooks: {
15-
beforeRequest: [
16-
(request) => {
17-
const url = request.url.toString()
18-
if (url.includes('/api/v3/syncRecordValues')) {
19-
return new Request(
20-
url.replace('/api/v3/syncRecordValues', '/api/v3/syncRecordValuesMain'),
21-
request
22-
)
6+
// 限流配置,打包编译阶段避免接口频繁,限制频率
7+
const useRateLimiter = process.env.BUILD_MODE || process.env.EXPORT
8+
const lockFilePath = path.resolve(process.cwd(), '.notion-api-lock')
9+
const rateLimiter = new RateLimiter(200, lockFilePath)
10+
11+
const globalStore = { notion: null, inflight: new Map() }
12+
13+
function getRawNotion() {
14+
if (!globalStore.notion) {
15+
globalStore.notion = new NotionLibrary({
16+
apiBaseUrl: BLOG.API_BASE_URL || 'https://www.notion.so/api/v3',
17+
activeUser: BLOG.NOTION_ACTIVE_USER || null,
18+
authToken: BLOG.NOTION_TOKEN_V2 || null,
19+
userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
20+
kyOptions: {
21+
mode: 'cors',
22+
hooks: {
23+
beforeRequest: [
24+
(request) => {
25+
const url = request.url.toString()
26+
if (url.includes('/api/v3/syncRecordValues')) {
27+
return new Request(
28+
url.replace('/api/v3/syncRecordValues', '/api/v3/syncRecordValuesMain'),
29+
request
30+
)
31+
}
32+
return request
2333
}
24-
return request
25-
}
26-
]
34+
]
35+
}
2736
}
28-
}
29-
})
37+
})
38+
}
39+
return globalStore.notion
40+
}
41+
42+
async function callNotion(methodName, ...args) {
43+
const notion = getRawNotion()
44+
const original = notion[methodName]
45+
if (typeof original !== 'function') throw new Error(`${methodName} is not a function`)
46+
47+
const key = `${methodName}-${JSON.stringify(args)}`
48+
49+
if (globalStore.inflight.has(key)) return globalStore.inflight.get(key)
50+
51+
const execute = async () => original.apply(notion, args)
52+
const promise = useRateLimiter
53+
? rateLimiter.enqueue(key, execute)
54+
: execute()
55+
56+
globalStore.inflight.set(key, promise)
57+
promise.finally(() => globalStore.inflight.delete(key))
58+
return promise
59+
}
60+
61+
export const notionAPI = {
62+
getPage: (...args) => callNotion('getPage', ...args),
63+
getBlocks: (...args) => callNotion('getBlocks', ...args),
64+
getUsers: (...args) => callNotion('getUsers', ...args),
65+
__call: callNotion
3066
}
3167

3268
export default notionAPI

lib/notion/getPostBlocks.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,7 @@ export const fetchInBatches = async (ids, batchSize = 100) => {
173173
let fetchedBlocks = {}
174174
for (let i = 0; i < ids.length; i += batchSize) {
175175
const batch = ids.slice(i, i + batchSize)
176-
console.log('[API-->>请求] Fetching missing blocks', batch, ids.length)
176+
console.log('[API-->>请求] Fetching missing blocks', ids.length)
177177
const start = new Date().getTime()
178178
const pageChunk = await notionAPI.getBlocks(batch)
179179
const end = new Date().getTime()

lib/plugins/algolia.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const initAlgolia = () => {
1313
!BLOG.ALGOLIA_ADMIN_APP_KEY ||
1414
!BLOG.ALGOLIA_INDEX
1515
) {
16-
console.warn('Algolia configuration is missing')
16+
// console.warn('Algolia configuration is missing')
1717
}
1818
algoliaClient = algoliasearch(
1919
BLOG.ALGOLIA_APP_ID,

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@
1717
},
1818
"scripts": {
1919
"dev": "next dev",
20-
"build": "next build",
20+
"build": "cross-env BUILD_MODE=true next build",
2121
"start": "next start",
2222
"post-build": "next-sitemap --config next-sitemap.config.js",
23-
"export": "cross-env EXPORT=true next build && next-sitemap --config next-sitemap.config.js",
23+
"export": "cross-env BUILD_MODE=true EXPORT=true next build && next-sitemap --config next-sitemap.config.js",
2424
"bundle-report": "cross-env ANALYZE=true next build",
2525
"build-all-in-dev": "cross-env VERCEL_ENV=production next build",
2626
"version": "echo $npm_package_version",

0 commit comments

Comments
 (0)