Add proxy option / Rename the data object raw of fetchHTML return value to html

Author: coder-hxl

src/index.ts

@@ -38,22 +38,29 @@ export default class XCrawl {
 
   private mergeConfig<T extends IFetchBaseConifg>(rawConfig: T): T {
     const baseConfig = this.baseConfig
-    const newConfig: T = structuredClone(rawConfig)
+    const newConfig = structuredClone(rawConfig)
 
     // 1.处理 requestConifg
     const requestConifgArr = isArray(newConfig.requestConifg)
       ? newConfig.requestConifg
       : [newConfig.requestConifg]
     for (const requestItem of requestConifgArr) {
-      const { url, timeout } = requestItem
+      const { url, timeout, proxy } = requestItem
 
+      // 1.1.baseUrl
       if (!isUndefined(baseConfig.baseUrl)) {
         requestItem.url = baseConfig.baseUrl + url
       }
 
+      // 1.2.timeout
       if (isUndefined(timeout)) {
         requestItem.timeout = baseConfig.timeout
       }
+
+      // 1.3.porxy
+      if (isUndefined(proxy)) {
+        requestItem.proxy = baseConfig.proxy
+      }
     }
 
     // 2.处理 intervalTime
@@ -88,13 +95,13 @@ export default class XCrawl {
     })
 
     const requestRes = await request(requestConifg)
-    const rawData = requestRes.data.toString()
+    const html = requestRes.data.toString()
 
     const res: IFetchHTML = {
       ...requestRes,
       data: {
-        raw: rawData,
-        jsdom: new JSDOM(rawData)
+        html,
+        jsdom: new JSDOM(html)
       }
     }
 

src/request.ts

@@ -1,6 +1,7 @@
-import http, { Agent, RequestOptions } from 'node:http'
-import { Agent as httpsAgent } from 'https'
+import http, { RequestOptions, IncomingMessage, ClientRequest } from 'node:http'
+import https from 'node:https'
 import Url, { URL } from 'node:url'
+import HttpsProxyAgent from 'https-proxy-agent'
 
 import {
   isNumber,
@@ -62,8 +63,15 @@ function handleRequestConfig(
   const { protocol, hostname, port, pathname, search } = new Url.URL(
     rawConfig.url
   )
+  const isHttp = protocol === 'http:'
 
   const config: RequestOptions & IMapTypeEmptyObject<URL> = {
+    agent: rawConfig.proxy
+      ? HttpsProxyAgent(rawConfig.proxy)
+      : isHttp
+      ? new http.Agent()
+      : new https.Agent(),
+
     protocol,
     hostname,
     port,
@@ -77,46 +85,17 @@ function handleRequestConfig(
 
   config.headers = parseHeaders(rawConfig, config)
 
-  if (protocol === 'http:') {
-    config.agent = new Agent()
-  } else {
-    config.agent = new httpsAgent()
-  }
-
   return config
 }
 
-async function useSleepByBatch(
-  isHaveIntervalTime: boolean,
-  isNumberIntervalTime: boolean,
-  intervalTime: any,
-  id: number
-) {
-  if (isHaveIntervalTime && id > 1) {
-    const timeout: number = isNumberIntervalTime
-      ? intervalTime
-      : random(intervalTime.max, intervalTime.min)
-
-    log(
-      `Request ${logNumber(id)} needs to sleep for ${logNumber(
-        timeout + 'ms'
-      )} milliseconds before sending`
-    )
-
-    await sleep(timeout)
-  } else {
-    log(`Request ${logNumber(id)} does not need to sleep, send immediately`)
-  }
-}
-
 export function request(config: IRequestConfig) {
   return new Promise<IRequest>((resolve, reject) => {
     const isDataUndefine = isUndefined(config.data)
     config.data = !isDataUndefine ? JSON.stringify(config.data) : config.data
 
     const requestConfig = handleRequestConfig(config)
 
-    const req = http.request(requestConfig, (res) => {
+    function handleRes(res: IncomingMessage) {
       const { statusCode, headers } = res
 
       const container: Buffer[] = []
@@ -133,7 +112,14 @@ export function request(config: IRequestConfig) {
 
         resolve(resolveRes)
       })
-    })
+    }
+
+    let req: ClientRequest
+    if (requestConfig.protocol === 'http:') {
+      req = http.request(requestConfig, handleRes)
+    } else {
+      req = https.request(requestConfig, handleRes)
+    }
 
     req.on('timeout', () => {
       reject(new Error(`Timeout ${config.timeout}ms`))
@@ -152,6 +138,29 @@ export function request(config: IRequestConfig) {
   })
 }
 
+async function useSleepByBatch(
+  isHaveIntervalTime: boolean,
+  isNumberIntervalTime: boolean,
+  intervalTime: any,
+  id: number
+) {
+  if (isHaveIntervalTime && id > 1) {
+    const timeout: number = isNumberIntervalTime
+      ? intervalTime
+      : random(intervalTime.max, intervalTime.min)
+
+    log(
+      `Request ${logNumber(id)} needs to sleep for ${logNumber(
+        timeout + 'ms'
+      )} milliseconds before sending`
+    )
+
+    await sleep(timeout)
+  } else {
+    log(`Request ${logNumber(id)} does not need to sleep, send immediately`)
+  }
+}
+
 export async function batchRequest(
   requestConifgs: IRequestConfig[],
   intervalTime: IIntervalTime | undefined

src/types.ts

@@ -52,6 +52,7 @@ export interface IRequestConfig {
   params?: IAnyObject
   data?: any
   timeout?: number
+  proxy?: string
 }
 
 export type IIntervalTime = number | { max: number; min?: number }
@@ -61,6 +62,7 @@ export interface IXCrawlBaseConifg {
   timeout?: number
   intervalTime?: IIntervalTime
   mode?: 'async' | 'sync'
+  proxy?: string
 }
 
 export interface IFetchBaseConifg {
@@ -104,7 +106,7 @@ export interface IFetchHTML {
   statusCode: number | undefined
   headers: IncomingHttpHeaders
   data: {
-    raw: string
+    html: string
     jsdom: JSDOM
   }
 }