other

Author: coder-hxl

README.md

@@ -2,14 +2,20 @@
 
 English | [简体中文](https://github.com/coder-hxl/x-crawl/blob/main/document/cn.md)
 
-XCrawl is a Nodejs multifunctional crawler library. Crawl HTML, JSON, file resources, etc. through simple configuration.
+XCrawl is a Nodejs multifunctional crawler library. 
 
-## highlights
+## Feature
 
-- Simple configuration to grab HTML, JSON, file resources, etc
-- Batch requests can choose mode asynchronous or synchronous
-- polling function
+- Crawl HTML, JSON, file resources, etc. with simple configuration
+- Use the JSDOM library to parse HTML, or parse HTML by yourself
+- Optional mode asynchronous/synchronous for batch requests
+- Polling function
 - Anthropomorphic request interval
+- Written in TypeScript
+
+## Catalog
+
+[TOC]
 
 ## Install
 
@@ -174,7 +180,7 @@ myXCrawl.fetchFile({
 
 fetchPolling is a method of the [myXCrawl](https://github.com/coder-hxl/x-crawl#Example-1) instance, typically used to perform polling operations, such as getting news every once in a while.
 
-#### 类型
+#### Type
 
 ```ts
 function fetchPolling(
@@ -183,7 +189,7 @@ function fetchPolling(
 ): void
 ```
 
-#### 示例
+#### Example
 
 ```js
 myXCrawl.fetchPolling({ h: 1, m: 30 }, () => {
@@ -194,21 +200,21 @@ myXCrawl.fetchPolling({ h: 1, m: 30 }, () => {
 
 ## Types
 
-#### IAnyObject
+### IAnyObject
 
 ```ts
 interface IAnyObject extends Object {
   [key: string | number | symbol]: any
 }
 ```
 
-#### IMethod
+### IMethod
 
 ```ts
 type IMethod = 'get' | 'GET' | 'delete' | 'DELETE' | 'head' | 'HEAD' | 'options' | 'OPTIONS' | 'post' | 'POST' | 'put' | 'PUT' | 'patch' | 'PATCH' | 'purge' | 'PURGE' | 'link' | 'LINK' | 'unlink' | 'UNLINK'
 ```
 
-#### IRequestConfig
+### IRequestConfig
 
 ```ts 
 interface IRequestConfig {
@@ -218,10 +224,11 @@ interface IRequestConfig {
   params?: IAnyObject
   data?: any
   timeout?: number
+  proxy?: string
 }
 ```
 
-#### IIntervalTime
+### IIntervalTime
 
 ```ts
 type IIntervalTime = number | {
@@ -230,7 +237,7 @@ type IIntervalTime = number | {
 }
 ```
 
-#### IFetchBaseConifg
+### IFetchBaseConifg
 
 ```ts
 interface IFetchBaseConifg {
@@ -239,31 +246,32 @@ interface IFetchBaseConifg {
 }
 ```
 
-#### IXCrawlBaseConifg
+### IXCrawlBaseConifg
 
 ```ts
 interface IXCrawlBaseConifg {
   baseUrl?: string
   timeout?: number
   intervalTime?: IIntervalTime
   mode?: 'async' | 'sync'
+  proxy?: string
 }
 ```
 
-#### IFetchHTMLConfig
+### IFetchHTMLConfig
 
 ```ts
 type IFetchHTMLConfig = string | IRequestConfig
 ```
 
-#### IFetchDataConfig
+### IFetchDataConfig
 
 ```ts
 interface IFetchDataConfig extends IFetchBaseConifg {
 }
 ```
 
-#### IFetchFileConfig
+### IFetchFileConfig
 
 ```ts
 interface IFetchFileConfig extends IFetchBaseConifg {
@@ -273,7 +281,7 @@ interface IFetchFileConfig extends IFetchBaseConifg {
 }
 ```
 
-#### IFetchPollingConfig
+### IFetchPollingConfig
 
 ```ts
 interface IFetchPollingConfig {
@@ -285,7 +293,7 @@ interface IFetchPollingConfig {
 }
 ```
 
-#### IFetchCommon
+### IFetchCommon
 
 ```ts
 type IFetchCommon<T> = {
@@ -296,7 +304,7 @@ type IFetchCommon<T> = {
 }[]
 ```
 
-#### IFileInfo
+### IFileInfo
 
 ```ts
 interface IFileInfo {
@@ -307,14 +315,14 @@ interface IFileInfo {
 }
 ```
 
-#### IFetchHTML
+### IFetchHTML
 
 ```ts
 interface IFetchHTML {
   statusCode: number | undefined
   headers: IncomingHttpHeaders
   data: {
-    raw: string // HTML String
+    html: string // HTML String
     jsdom: JSDOM // HTML parsing using the jsdom library
   }
 }

document/cn.md

@@ -2,14 +2,20 @@
 
 [English](https://github.com/coder-hxl/x-crawl#x-crawl) | 简体中文
 
-XCrawl 是 Nodejs 多功能爬虫库。只需简单的配置即可抓取 HTML 、JSON、文件资源等等。
+XCrawl 是 Nodejs 多功能爬虫库。
 
-## 亮点
+## 特点
 
-- 简单的配置即可抓取 HTML 、JSON 、文件资源等等
-- 批量请求可选择模式 异步 或 同步
+- 只需简单的配置即可抓取 HTML 、JSON、文件资源等等
+- 使用 JSDOM 库对 HTML 解析，也可自行解析 HTML
+- 批量请求时可选择模式 异步/同步
 - 轮询功能
 - 拟人化的请求间隔时间
+- 使用 TypeScript 编写
+
+## 目录
+
+[TOC]
 
 ## 安装
 
@@ -206,21 +212,21 @@ myXCrawl.fetchPolling({ h: 1, m: 30 }, () => {
 
 ## 类型
 
-#### IAnyObject
+### IAnyObject
 
 ```ts
 interface IAnyObject extends Object {
   [key: string | number | symbol]: any
 }
 ```
 
-#### IMethod
+### IMethod
 
 ```ts
 type IMethod = 'get' | 'GET' | 'delete' | 'DELETE' | 'head' | 'HEAD' | 'options' | 'OPTIONS' | 'post' | 'POST' | 'put' | 'PUT' | 'patch' | 'PATCH' | 'purge' | 'PURGE' | 'link' | 'LINK' | 'unlink' | 'UNLINK'
 ```
 
-#### IRequestConfig
+### IRequestConfig
 
 ```ts 
 interface IRequestConfig {
@@ -230,10 +236,11 @@ interface IRequestConfig {
   params?: IAnyObject
   data?: any
   timeout?: number
+  proxy?: string
 }
 ```
 
-#### IIntervalTime
+### IIntervalTime
 
 ```ts
 type IIntervalTime = number | {
@@ -242,7 +249,7 @@ type IIntervalTime = number | {
 }
 ```
 
-#### IFetchBaseConifg
+### IFetchBaseConifg
 
 ```ts
 interface IFetchBaseConifg {
@@ -251,31 +258,32 @@ interface IFetchBaseConifg {
 }
 ```
 
-#### IXCrawlBaseConifg
+### IXCrawlBaseConifg
 
 ```ts
 interface IXCrawlBaseConifg {
   baseUrl?: string
   timeout?: number
   intervalTime?: IIntervalTime
   mode?: 'async' | 'sync'
+  proxy?: string
 }
 ```
 
-#### IFetchHTMLConfig
+### IFetchHTMLConfig
 
 ```ts
 type IFetchHTMLConfig = string | IRequestConfig
 ```
 
-#### IFetchDataConfig
+### IFetchDataConfig
 
 ```ts
 interface IFetchDataConfig extends IFetchBaseConifg {
 }
 ```
 
-#### IFetchFileConfig
+### IFetchFileConfig
 
 ```ts
 interface IFetchFileConfig extends IFetchBaseConifg {
@@ -285,7 +293,7 @@ interface IFetchFileConfig extends IFetchBaseConifg {
 }
 ```
 
-#### IFetchPollingConfig
+### IFetchPollingConfig
 
 ```ts
 interface IFetchPollingConfig {
@@ -297,7 +305,7 @@ interface IFetchPollingConfig {
 }
 ```
 
-#### IFetchCommon
+### IFetchCommon
 
 ```ts
 type IFetchCommon<T> = {
@@ -308,7 +316,7 @@ type IFetchCommon<T> = {
 }[]
 ```
 
-#### IFileInfo
+### IFileInfo
 
 ```ts
 interface IFileInfo {
@@ -319,14 +327,14 @@ interface IFileInfo {
 }
 ```
 
-#### IFetchHTML
+### IFetchHTML
 
 ```ts
 interface IFetchHTML {
   statusCode: number | undefined
   headers: IncomingHttpHeaders
   data: {
-    raw: string // HTML String
+    html: string // HTML String
     jsdom: JSDOM // 使用了 jsdom 库对 HTML 解析
   }
 }

package.json

@@ -1,9 +1,9 @@
 {
   "private": true,
   "name": "x-crawl",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "author": "CoderHxl",
-  "description": "XCrawl is a Nodejs multifunctional crawler library. Crawl HTML, JSON, file resources, etc. through simple configuration.",
+  "description": "XCrawl is a Nodejs multifunctional crawler library.",
   "license": "MIT",
   "main": "src/index.ts",
   "scripts": {
@@ -16,6 +16,7 @@
   },
   "dependencies": {
     "chalk": "4.1.2",
+    "https-proxy-agent": "^5.0.1",
     "jsdom": "^21.1.0",
     "x-crawl": "link:"
   },