diff --git a/package.json b/package.json index 232c03b..da04fd1 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ }, "dependencies": { "@newdash/newdash": "^5.22.1", + "axios": "^1.7.2", "express": "^4.19.2", "puppeteer-core": "^22.9.0", "turndown": "^7.1.3", diff --git a/src/utils.mjs b/src/utils.mjs index c67f894..2fb6318 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -1,7 +1,7 @@ +import axios from "axios"; import console from "console"; import process from "process"; import puppeteer from "puppeteer-core"; -import { fetch } from "undici"; /** * @@ -68,18 +68,25 @@ export function createCommonSearchAPI(options) { console.warn("TF_URL is not set, skipping text extraction"); return item; } - const res = await fetch(process.env.TF_URL + "/extract", { - method: "POST", - headers: { - "Content-Type": "application/json", - "User-Agent": defaultUserAgent(), - }, - body: JSON.stringify({ url: item.link }), - }); - const data = await res.json(); - if (res.ok && data.text) { - return { ...item, text: data.text }; + try { + const res = await axios.post( + process.env.TF_URL + "/extract", + { url: item.link }, + { + headers: { + "Content-Type": "application/json", + "User-Agent": defaultUserAgent(), + }, + timeout: 5_000, + }, + ); + if (res.status < 300 && res.data?.text) { + return { ...item, text: res.data }; + } + } catch (error) { + console.error("Failed to extract text", error.message, item.link); } + return item; }), );