fetchHTML API exposes more content

Author: coder-hxl

src/index.ts

@@ -11,8 +11,9 @@ import {
   IFetchDataConfig,
   IFetchFileConfig,
   IFetchBaseConifg,
-  IFileInfo,
   IFetchCommon,
+  IFileInfo,
+  IFetchHTML,
   IRequestResItem,
   IRequestConfig,
   IIntervalTime
@@ -77,7 +78,7 @@ export default class XCrawl {
     return requestRes
   }
 
-  async fetchHTML(config: string | IFetchHTMLConfig): Promise<JSDOM> {
+  async fetchHTML(config: IFetchHTMLConfig): Promise<IFetchHTML> {
     const rawRequestConifg: IFetchHTMLConfig = isString(config)
       ? { url: config }
       : config
@@ -86,11 +87,18 @@ export default class XCrawl {
       requestConifg: rawRequestConifg
     })
 
-    const requestResItem = await request(requestConifg)
+    const requestRes = await request(requestConifg)
+    const rawData = requestRes.data.toString()
 
-    const dom = new JSDOM(requestResItem.data)
+    const res: IFetchHTML = {
+      ...requestRes,
+      data: {
+        raw: rawData,
+        jsdom: new JSDOM(rawData)
+      }
+    }
 
-    return dom
+    return res
   }
 
   async fetchData<T = any>(config: IFetchDataConfig): Promise<IFetchCommon<T>> {

src/types.ts

@@ -1,4 +1,5 @@
 import { IncomingHttpHeaders } from 'node:http'
+import { JSDOM } from 'jsdom'
 
 export interface IAnyObject extends Object {
   [key: string | number | symbol]: any
@@ -67,7 +68,7 @@ export interface IFetchBaseConifg {
   intervalTime?: IIntervalTime
 }
 
-export interface IFetchHTMLConfig extends IRequestConfig {}
+export type IFetchHTMLConfig = string | IRequestConfig
 
 export interface IFetchDataConfig extends IFetchBaseConifg {}
 
@@ -77,16 +78,25 @@ export interface IFetchFileConfig extends IFetchBaseConifg {
   }
 }
 
+export type IFetchCommon<T> = {
+  id: number
+  statusCode: number | undefined
+  headers: IncomingHttpHeaders
+  data: T
+}[]
+
 export interface IFileInfo {
   fileName: string
   mimeType: string
   size: number
   filePath: string
 }
 
-export type IFetchCommon<T> = {
-  id: number
+export interface IFetchHTML {
   statusCode: number | undefined
   headers: IncomingHttpHeaders
-  data: T
-}[]
+  data: {
+    raw: string
+    jsdom: JSDOM
+  }
+}

test/start/index.js

@@ -10,19 +10,19 @@ const testXCrawl = new XCrawl({
   mode: 'sync'
 })
 
-testXCrawl
-  .fetchData({
-    requestConifg: [
-      { url: 'http://localhost:3001/home' },
-      { url: 'http://localhost:9001/api/home/wonderfulplace' },
-      { url: 'http://localhost:9001/api/home/goodprice' },
-      { url: 'http://localhost:3001/home' },
-      { url: 'http://localhost:9001/ai/home/goodprice' }
-    ]
-  })
-  .then((res) => {
-    console.log(res)
-  })
+// testXCrawl
+//   .fetchData({
+//     requestConifg: [
+//       { url: 'http://localhost:3001/home' },
+//       { url: 'http://localhost:9001/api/home/wonderfulplace' },
+//       { url: 'http://localhost:9001/api/home/goodprice' },
+//       { url: 'http://localhost:3001/home' },
+//       { url: 'http://localhost:9001/ai/home/goodprice' }
+//     ]
+//   })
+//   .then((res) => {
+//     console.log(res)
+//   })
 
 // testXCrawl.fetchHTML({ url: 'https://www.bilibili.com/' }).then((jsdom) => {
 //   const document = jsdom.window.document
@@ -52,3 +52,12 @@ testXCrawl
 //       console.log(res)
 //     })
 // })
+
+testXCrawl.fetchHTML('https://cn.bing.com').then((res) => {
+  const { jsdom } = res.data
+})
+
+testXCrawl.fetchHTML('https://docs.github.com/zh/get-started').then((res) => {
+  const { jsdom } = res.data
+  console.log(jsdom.window.document.querySelector('title')?.textContent)
+})