fetchHTML API remove content option

Author: coder-hxl

src/api.ts

@@ -29,11 +29,11 @@ import {
   IntervalTime,
   StartPollingConfig
 } from './types/api'
-import { XCrawlBaseConifg } from './types'
+import { LoaderXCrawlBaseConifg } from './types'
 import { RequestConfig, RequestResItem } from './types/request'
 
 function mergeConfig<T extends FetchBaseConifgV1>(
-  baseConfig: XCrawlBaseConifg,
+  baseConfig: LoaderXCrawlBaseConifg,
   rawConfig: T
 ): T {
   const newConfig = structuredClone(rawConfig)
@@ -70,7 +70,7 @@ function mergeConfig<T extends FetchBaseConifgV1>(
 }
 
 async function useBatchRequestByMode(
-  mode: 'async' | 'sync' | undefined,
+  mode: 'async' | 'sync',
   requestConifg: RequestConfig | RequestConfig[],
   intervalTime: IntervalTime | undefined,
   callback: (requestRestem: RequestResItem) => void
@@ -79,14 +79,14 @@ async function useBatchRequestByMode(
     ? requestConifg
     : [requestConifg]
 
-  if (mode !== 'sync') {
+  if (mode === 'async') {
     await batchRequest(requestConfigQueue, intervalTime, callback)
   } else {
     await syncBatchRequest(requestConfigQueue, intervalTime, callback)
   }
 }
 
-export function createFetchHTML(baseConfig: XCrawlBaseConifg) {
+export function createFetchHTML(baseConfig: LoaderXCrawlBaseConifg) {
   let browser: Browser | null = null
   let createBrowserState: Promise<void> | null = null
   let callTotal = 0
@@ -95,7 +95,7 @@ export function createFetchHTML(baseConfig: XCrawlBaseConifg) {
     config: FetchHTMLConfig,
     callback?: (res: FetchHTML) => void
   ): Promise<FetchHTML> {
-    // 记录调用次数, 为关闭浏览器
+    // 记录调用次数, 目的: 关闭浏览器
     callTotal++
 
     // 只创建一次浏览器
@@ -129,22 +129,20 @@ export function createFetchHTML(baseConfig: XCrawlBaseConifg) {
       })
     }
 
-    const httpResponse = await page!.goto(requestConifg.url)
+    const httpResponse = await page!.goto(requestConifg.url, {
+      timeout: requestConifg.timeout
+    })
 
     const content = await page!.content()
 
     // 关闭浏览器
     if (--callTotal === 0) {
-      await browser!.close()
+      browser!.close()
     }
 
     const res: FetchHTML = {
       httpResponse,
-      data: {
-        page,
-        content,
-        jsdom: new JSDOM(content)
-      }
+      data: { page, jsdom: new JSDOM(content) }
     }
 
     if (callback) {
@@ -157,7 +155,7 @@ export function createFetchHTML(baseConfig: XCrawlBaseConifg) {
   return fetchHTML
 }
 
-export function createFetchData(baseConfig: XCrawlBaseConifg) {
+export function createFetchData(baseConfig: LoaderXCrawlBaseConifg) {
   async function fetchData<T = any>(
     config: FetchDataConfig,
     callback?: (res: FetchResCommonV1<T>) => void
@@ -198,7 +196,7 @@ export function createFetchData(baseConfig: XCrawlBaseConifg) {
   return fetchData
 }
 
-export function createFetchFile(baseConfig: XCrawlBaseConifg) {
+export function createFetchFile(baseConfig: LoaderXCrawlBaseConifg) {
   async function fetchFile(
     config: FetchFileConfig,
     callback?: (res: FetchResCommonV1<FileInfo>) => void

src/index.ts

@@ -5,9 +5,25 @@ import {
   startPolling
 } from './api'
 
-import { XCrawlBaseConifg, XCrawlInstance } from './types'
+import {
+  LoaderXCrawlBaseConifg,
+  XCrawlBaseConifg,
+  XCrawlInstance
+} from './types'
+
+function loaderBaseConifg(
+  baseConfig: XCrawlBaseConifg | undefined
+): LoaderXCrawlBaseConifg {
+  const loaderBaseConfig = baseConfig ? baseConfig : {}
+
+  if (!loaderBaseConfig.mode) {
+    loaderBaseConfig.mode = 'async'
+  }
 
-function createnstance(baseConfig: XCrawlBaseConifg): XCrawlInstance {
+  return loaderBaseConfig as LoaderXCrawlBaseConifg
+}
+
+function createnInstance(baseConfig: LoaderXCrawlBaseConifg): XCrawlInstance {
   const instance: XCrawlInstance = {
     fetchHTML: createFetchHTML(baseConfig),
     fetchData: createFetchData(baseConfig),
@@ -18,10 +34,10 @@ function createnstance(baseConfig: XCrawlBaseConifg): XCrawlInstance {
   return instance
 }
 
-export default function xCrawl(
-  baseConfig: XCrawlBaseConifg = {}
-): XCrawlInstance {
-  const instance = createnstance(baseConfig)
+export default function xCrawl(baseConfig?: XCrawlBaseConifg): XCrawlInstance {
+  const newBaseConfig = loaderBaseConifg(baseConfig)
+
+  const instance = createnInstance(newBaseConfig)
 
   return instance
 }

src/request.ts

@@ -11,6 +11,7 @@ import {
   logError,
   logNumber,
   logSuccess,
+  logWarn,
   random,
   sleep
 } from './utils'
@@ -166,7 +167,9 @@ export async function batchRequest(
   const isNumberntervalTime = isNumber(intervalTime)
 
   log(
-    `Begin execution, mode: async, total: ${logNumber(requestConifgs.length)} `
+    `Begin execution, mode: ${logWarn('async')}, total: ${logNumber(
+      requestConifgs.length
+    )} `
   )
 
   let index = 0
@@ -227,7 +230,9 @@ export async function syncBatchRequest(
   const isNumberntervalTime = isNumber(intervalTime)
 
   log(
-    `Begin execution, mode: sync, total: ${logNumber(requestConifgs.length)} `
+    `Begin execution, mode: ${logWarn('sync')}, total: ${logNumber(
+      requestConifgs.length
+    )} `
   )
 
   let id = 0

src/types/api.ts

@@ -3,7 +3,6 @@ import { HTTPResponse, Page } from 'puppeteer'
 import { JSDOM } from 'jsdom'
 
 import { RequestConfig } from './request'
-import { AnyObject } from './common'
 
 export type IntervalTime = number | { max: number; min?: number }
 
@@ -14,7 +13,6 @@ export interface FetchBaseConifgV1 {
 
 export interface FetchBaseConifgV2 {
   url: string
-  header?: AnyObject
   timeout?: number
   proxy?: string
 }
@@ -56,7 +54,6 @@ export interface FetchHTML {
   httpResponse: HTTPResponse | null
   data: {
     page: Page
-    content: string
     jsdom: JSDOM
   }
 }

src/types/index.ts

@@ -9,6 +9,7 @@ import {
   StartPollingConfig,
   IntervalTime
 } from './api'
+import { MapTypeObject } from './common'
 
 export interface XCrawlBaseConifg {
   baseUrl?: string
@@ -18,6 +19,13 @@ export interface XCrawlBaseConifg {
   proxy?: string
 }
 
+interface LoaderXCrawlBaseConifgValue {
+  mode: 'async' | 'sync'
+}
+
+export type LoaderXCrawlBaseConifg = XCrawlBaseConifg &
+  MapTypeObject<LoaderXCrawlBaseConifgValue>
+
 export interface XCrawlInstance {
   fetchHTML: (
     config: FetchHTMLConfig,