Sorting of error messages and fetchData/fetchFile API results

Author: coder-hxl

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "x-crawl",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library.",
   "license": "MIT",

publish/package.json

@@ -1,6 +1,6 @@
 {
   "name": "x-crawl",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "author": "CoderHxl",
   "description": "XCrawl is a Nodejs multifunctional crawler library.",
   "license": "MIT",

src/index.ts

@@ -3,6 +3,7 @@ import path from 'node:path'
 import { JSDOM } from 'jsdom'
 
 import { batchRequest, syncBatchRequest, request } from './request'
+import { quickSort } from './sort'
 import {
   isArray,
   isString,
@@ -140,7 +141,10 @@ export default class XCrawl {
 
     await this.useBatchRequestByMode(requestConifg, intervalTime, handleResItem)
 
-    return container
+    const res = quickSort(
+      container.map((item) => ({ ...item, valueOf: () => item.id }))
+    )
+    return res
   }
 
   async fetchFile(
@@ -189,8 +193,11 @@ export default class XCrawl {
         success
       )}, error: ${logError(error)}`
     )
+    const res = quickSort(
+      container.map((item) => ({ ...item, valueOf: () => item.id }))
+    )
 
-    return container
+    return res
   }
 
   fetchPolling(config: IFetchPollingConfig, callback: (count: number) => void) {

src/request.ts

@@ -3,6 +3,7 @@ import https from 'node:https'
 import Url, { URL } from 'node:url'
 import HttpsProxyAgent from 'https-proxy-agent'
 
+import { quickSort } from './sort'
 import {
   isNumber,
   isUndefined,
@@ -176,7 +177,9 @@ export async function batchRequest(
   let index = 0
   let successTotal = 0
   let errorTotal = 0
-  const requestQueue: Promise<undefined | string>[] = []
+  const requestQueue: Promise<void>[] = []
+  const errorMessage: { id: number; message: string; valueOf: () => number }[] =
+    []
   for (const requestConifg of requestConifgs) {
     const id = ++index
 
@@ -190,10 +193,15 @@ export async function batchRequest(
     const requestItem = request(requestConifg)
       .catch((error: any) => {
         errorTotal++
-        return `Request ${id} is an error: ${error.message}`
+
+        const message = `Request ${id} is an error: ${error.message}`
+        // valueOf 为排序做准备
+        const valueOf = () => id
+
+        errorMessage.push({ id, message, valueOf })
       })
       .then((requestRes) => {
-        if (typeof requestRes === 'string') return requestRes
+        if (!requestRes) return
 
         successTotal++
         callback({ id, ...requestRes })
@@ -204,10 +212,11 @@ export async function batchRequest(
 
   log(logSuccess('All requests have been sent!'))
 
-  const res = await Promise.all(requestQueue)
+  // 等待所有请求结束
+  await Promise.all(requestQueue)
 
-  // 打印错误消息
-  res.forEach((item) => (item ? log(logError(item)) : ''))
+  // 排序后打印错误消息
+  quickSort(errorMessage).forEach((item) => log(logError(item.message)))
 
   log(
     `requestsTotal: ${logNumber(requestConifgs.length)}, success: ${logSuccess(

src/sort.ts

@@ -0,0 +1,48 @@
+function swap(arr: any[], i: number, j: number) {
+  const temp = arr[i]
+  arr[i] = arr[j]
+  arr[j] = temp
+}
+
+export function quickSort<T extends any[]>(arr: T): T {
+  const n = arr.length
+
+  partition(0, n - 1)
+
+  function partition(left: number, right: number) {
+    if (left >= right) return
+
+    // 1.找基准元素
+    const pivot = arr[right]
+
+    // 2.定义双指针进行交换(左小右大)
+    let i = left
+    let j = right - 1
+    while (i <= j) {
+      while (arr[i] < pivot) {
+        i++
+      }
+
+      while (arr[j] > pivot) {
+        j--
+      }
+
+      if (i <= j) {
+        swap(arr, i, j)
+        i++
+        j--
+      }
+    }
+
+    // 3.将 pivot 放到正确位置
+    swap(arr, i, right)
+
+    // 4.左右划分区域
+    partition(left, i - 1)
+    partition(i + 1, right)
+  }
+
+  return arr
+}
+
+// console.log(quickSort([7, 3, 6, 4, 9, 2, 1, 5]))

test/start/index.js

@@ -3,7 +3,7 @@ import XCrawl from '../../src'
 
 const testXCrawl = new XCrawl({
   timeout: 10000,
-  intervalTime: { max: 2000, min: 1000 },
+  intervalTime: { max: 0, min: 0 },
   mode: 'async'
 })
 
@@ -17,23 +17,23 @@ const testXCrawl = new XCrawl({
 //   ]
 // })
 
-testXCrawl.fetchPolling({ m: 3 }, () => {
-  testXCrawl
-    .fetchHTML('https://www.bilibili.com/guochuang/', (res) => {
-      console.log('fetchHTML Callback: ', res.statusCode)
-    })
-    .then((res) => {
-      const { jsdom } = res.data
+// testXCrawl.fetchPolling({ m: 3 }, () => {
+testXCrawl
+  .fetchHTML('https://www.bilibili.com/guochuang/', (res) => {
+    console.log('fetchHTML Callback: ', res.statusCode)
+  })
+  .then((res) => {
+    const { jsdom } = res.data
 
-      const imgSrc: string[] = []
-      const recomEls =
-        jsdom.window.document.querySelectorAll('.chief-recom-item')
-      recomEls.forEach((item) => imgSrc.push(item.querySelector('img')!.src))
+    const imgSrc: string[] = []
+    const recomEls = jsdom.window.document.querySelectorAll('.chief-recom-item')
+    recomEls.forEach((item) => imgSrc.push(item.querySelector('img')!.src))
 
-      const requestConifg = imgSrc.map((src) => ({ url: `https:${src}` }))
-      requestConifg.pop()
+    const requestConifg = imgSrc.map((src) => ({ url: `https:${src}` }))
+    requestConifg.pop()
 
-      testXCrawl.fetchFile(
+    testXCrawl
+      .fetchFile(
         {
           requestConifg,
           fileConfig: { storeDir: path.resolve(__dirname, './upload') }
@@ -42,5 +42,6 @@ testXCrawl.fetchPolling({ m: 3 }, () => {
           console.log(res.id, res.statusCode, res.data.fileName)
         }
       )
-    })
-})
+      .then((res) => console.log(res))
+  })
+// })