feat: better timeout handling

Soontao · May 29, 2024 · bf66073 · bf66073
1 parent 836b6e3
commit bf66073
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 12 deletions.
diff --git a/package.json b/package.json
@@ -20,6 +20,7 @@
   },
   "dependencies": {
     "@newdash/newdash": "^5.22.1",
+    "axios": "^1.7.2",
     "express": "^4.19.2",
     "puppeteer-core": "^22.9.0",
     "turndown": "^7.1.3",

diff --git a/src/utils.mjs b/src/utils.mjs
@@ -1,7 +1,7 @@
+import axios from "axios";
 import console from "console";
 import process from "process";
 import puppeteer from "puppeteer-core";
-import { fetch } from "undici";
 
 /**
  *
@@ -68,18 +68,25 @@ export function createCommonSearchAPI(options) {
           console.warn("TF_URL is not set, skipping text extraction");
           return item;
         }
-        const res = await fetch(process.env.TF_URL + "/extract", {
-          method: "POST",
-          headers: {
-            "Content-Type": "application/json",
-            "User-Agent": defaultUserAgent(),
-          },
-          body: JSON.stringify({ url: item.link }),
-        });
-        const data = await res.json();
-        if (res.ok && data.text) {
-          return { ...item, text: data.text };
+        try {
+          const res = await axios.post(
+            process.env.TF_URL + "/extract",
+            { url: item.link },
+            {
+              headers: {
+                "Content-Type": "application/json",
+                "User-Agent": defaultUserAgent(),
+              },
+              timeout: 5_000,
+            },
+          );
+          if (res.status < 300 && res.data?.text) {
+            return { ...item, text: res.data };
+          }
+        } catch (error) {
+          console.error("Failed to extract text", error.message, item.link);
         }
+
         return item;
       }),
     );