feat: use myFetch with default options

fixed #20

auto-imports.app.d.ts

@@ -39,6 +39,7 @@ declare global {
   const loadable: typeof import('jotai/utils')['loadable']
   const memo: typeof import('react')['memo']
   const metadata: typeof import('./shared/metadata')['metadata']
+  const myFetch: typeof import('./src/utils/index')['myFetch']
   const originSources: typeof import('./shared/sources')['originSources']
   const preprocessMetadata: typeof import('./src/atoms/primitiveMetadataAtom')['preprocessMetadata']
   const primitiveMetadataAtom: typeof import('./src/atoms/primitiveMetadataAtom')['primitiveMetadataAtom']

server/api/oauth/github.ts

@@ -12,7 +12,7 @@ export default defineEventHandler(async (event) => {
     access_token: string
     token_type: string
     scope: string
-  } = await $fetch(
+  } = await myFetch(
     `https://github.com/login/oauth/access_token`,
     {
       method: "POST",
@@ -33,7 +33,7 @@ export default defineEventHandler(async (event) => {
     avatar_url: string
     email: string
     notification_email: string
-  } = await $fetch(`https://api.github.com/user`, {
+  } = await myFetch(`https://api.github.com/user`, {
     headers: {
       "Accept": "application/vnd.github+json",
       "Authorization": `token ${response.access_token}`,

server/api/s/[id].ts

@@ -1,6 +1,7 @@
 import type { SourceID, SourceResponse } from "@shared/types"
 import { getters } from "#/getters"
 import { getCacheTable } from "#/database/cache"
+import type { CacheInfo } from "#/types"
 
 export default defineEventHandler(async (event): Promise<SourceResponse> => {
   try {
@@ -17,8 +18,9 @@ export default defineEventHandler(async (event): Promise<SourceResponse> => {
 
     const cacheTable = await getCacheTable()
     const now = Date.now()
+    let cache: CacheInfo
     if (cacheTable) {
-      const cache = await cacheTable.get(id)
+      cache = await cacheTable.get(id)
       if (cache) {
         // interval 刷新间隔，对于缓存失效也要执行的。本质上表示本来内容更新就很慢，这个间隔内可能内容压根不会更新。
         // 默认 10 分钟，是低于 TTL 的，但部分 Source 的更新间隔会超过 TTL，甚至有的一天更新一次。
@@ -52,17 +54,30 @@ export default defineEventHandler(async (event): Promise<SourceResponse> => {
       }
     }
 
-    const data = (await getters[id]()).slice(0, 30)
-    logger.success(`fetch ${id} latest`)
-    if (cacheTable) {
-      if (event.context.waitUntil) event.context.waitUntil(cacheTable.set(id, data))
-      else await cacheTable.set(id, data)
-    }
-    return {
-      status: "success",
-      id,
-      updatedTime: now,
-      items: data,
+    try {
+      const newData = (await getters[id]()).slice(0, 30)
+      if (cacheTable && newData) {
+        if (event.context.waitUntil) event.context.waitUntil(cacheTable.set(id, newData))
+        else await cacheTable.set(id, newData)
+      }
+      logger.success(`fetch ${id} latest`)
+      return {
+        status: "success",
+        id,
+        updatedTime: now,
+        items: newData,
+      }
+    } catch (e) {
+      if (cache!) {
+        return {
+          status: "cache",
+          id,
+          updatedTime: cache.updated,
+          items: cache.data,
+        }
+      } else {
+        throw e
+      }
     }
   } catch (e: any) {
     logger.error(e)

server/sources/_36kr.ts

@@ -4,7 +4,7 @@ import { load } from "cheerio"
 const quick = defineSource(async () => {
   const baseURL = "https://www.36kr.com"
   const url = `${baseURL}/newsflashes`
-  const response = await $fetch(url) as any
+  const response = await myFetch(url) as any
   const $ = load(response)
   const news: NewsItem[] = []
   const $items = $(".newsflash-item")

server/sources/baidu.ts

@@ -12,7 +12,7 @@ interface Res {
 }
 
 export default defineSource(async () => {
-  const rawData: string = await $fetch(`https://top.baidu.com/board?tab=realtime`)
+  const rawData: string = await myFetch(`https://top.baidu.com/board?tab=realtime`)
   const jsonStr = (rawData as string).match(/<!--s-data:(.*?)-->/s)
   const data: Res = JSON.parse(jsonStr![1])
 

server/sources/bilibili.ts

@@ -36,7 +36,7 @@ interface WapRes {
 
 const hotSearch = defineSource(async () => {
   const url = "https://s.search.bilibili.com/main/hotword?limit=30"
-  const res: WapRes = await $fetch(url)
+  const res: WapRes = await myFetch(url)
 
   return res.list.map(k => ({
     id: k.keyword,

server/sources/cankaoxiaoxi.ts

@@ -11,7 +11,7 @@ interface Res {
 }
 
 export default defineSource(async () => {
-  const res = await Promise.all(["zhongguo", "guandian", "gj"].map(k => $fetch(`https://china.cankaoxiaoxi.com/json/channel/${k}/list.json`) as Promise<Res>))
+  const res = await Promise.all(["zhongguo", "guandian", "gj"].map(k => myFetch(`https://china.cankaoxiaoxi.com/json/channel/${k}/list.json`) as Promise<Res>))
   return res.map(k => k.list).flat().map(k => ({
     id: k.data.id,
     title: k.data.title,

server/sources/cls/index.ts

@@ -23,9 +23,10 @@ interface Depthes {
   }
 }
 
+// 失效
 const depth = defineSource(async () => {
   const apiUrl = `https://www.cls.cn/v3/depth/home/assembled/1000`
-  const res: Depthes = await $fetch(apiUrl, {
+  const res: Depthes = await myFetch(apiUrl, {
     query: await getSearchParams(),
   })
   return res.data.depth_list.sort((m, n) => n.ctime - m.ctime).map((k) => {
@@ -40,13 +41,13 @@ const depth = defineSource(async () => {
     }
   })
 })
-
 // hot 失效
 
 const telegraph = defineSource(async () => {
   const apiUrl = `https://www.cls.cn/nodeapi/updateTelegraphList`
-  const res: TelegraphRes = await $fetch(apiUrl, {
+  const res: TelegraphRes = await myFetch(apiUrl, {
     query: await getSearchParams({ }),
+    timeout: 10000,
   })
   return res.data.roll_data.filter(k => !k.is_ad).map((k) => {
     return {

server/sources/coolapk/index.ts

@@ -23,7 +23,7 @@ interface Res {
 export default defineSource({
   coolapk: async () => {
     const url = "https://api.coolapk.com/v6/page/dataList?url=%2Ffeed%2FstatList%3FcacheExpires%3D300%26statType%3Dday%26sortField%3Ddetailnum%26title%3D%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&title=%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&subTitle=&page=1"
-    const r: Res = await $fetch(url, {
+    const r: Res = await myFetch(url, {
       headers: await genHeaders(),
     })
     if (!r.data.length) throw new Error("Failed to fetch")

server/sources/douyin.ts

@@ -12,7 +12,7 @@ interface Res {
 export default defineSource(async () => {
   const url = "https://www.douyin.com/aweme/v1/web/hot/search/list/?device_platform=webapp&aid=6383&channel=channel_pc_web&detail_list=1"
   const cookie = (await $fetch.raw("https://www.douyin.com/passport/general/login_guiding_strategy/?aid=6383")).headers.getSetCookie()
-  const res: Res = await $fetch(url, {
+  const res: Res = await myFetch(url, {
     headers: {
       cookie: cookie.join("; "),
     },

server/sources/fastbull.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 const express = defineSource(async () => {
   const baseURL = "https://www.fastbull.cn"
-  const html: any = await $fetch(`${baseURL}/express-news`)
+  const html: any = await myFetch(`${baseURL}/express-news`)
   const $ = cheerio.load(html)
   const $main = $(".news-list")
   const news: NewsItem[] = []
@@ -27,7 +27,7 @@ const express = defineSource(async () => {
 
 const news = defineSource(async () => {
   const baseURL = "https://www.fastbull.cn"
-  const html: any = await $fetch(`${baseURL}/news`)
+  const html: any = await myFetch(`${baseURL}/news`)
   const $ = cheerio.load(html)
   const $main = $(".trending_type")
   const news: NewsItem[] = []

server/sources/gelonghui.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
   const baseURL = "https://www.gelonghui.com"
-  const html: any = await $fetch("https://www.gelonghui.com/news/")
+  const html: any = await myFetch("https://www.gelonghui.com/news/")
   const $ = cheerio.load(html)
   const $main = $(".article-content")
   const news: NewsItem[] = []

server/sources/github.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 const trending = defineSource(async () => {
   const baseURL = "https://github.com"
-  const html: any = await $fetch("https://github.com/trending?spoken_language_code=")
+  const html: any = await myFetch("https://github.com/trending?spoken_language_code=")
   const $ = cheerio.load(html)
   const $main = $("main .Box div[data-hpc] > article")
   const news: NewsItem[] = []

server/sources/hackernews.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
   const baseURL = "https://news.ycombinator.com"
-  const html: any = await $fetch(baseURL)
+  const html: any = await myFetch(baseURL)
   const $ = cheerio.load(html)
   const $main = $(".athing")
   const news: NewsItem[] = []

server/sources/ithome.ts

@@ -2,7 +2,7 @@ import * as cheerio from "cheerio"
 import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
-  const response: any = await $fetch("https://www.ithome.com/list/")
+  const response: any = await myFetch("https://www.ithome.com/list/")
   const $ = cheerio.load(response)
   const $main = $("#list > div.fl > ul > li")
   const news: NewsItem[] = []

server/sources/jin10.ts

@@ -23,7 +23,7 @@ export default defineSource(async () => {
   const timestamp = Date.now()
   const url = `https://www.jin10.com/flash_newest.js?t=${timestamp}`
 
-  const rawData: string = await $fetch(url)
+  const rawData: string = await myFetch(url)
 
   const jsonStr = (rawData as string)
     .replace(/^var\s+newest\s*=\s*/, "") // 移除开头的变量声明

server/sources/kaopu.ts

@@ -7,7 +7,7 @@ type Res = {
   title: string
 }[]
 export default defineSource(async () => {
-  const res = await Promise.all(["https://kaopucdn.azureedge.net/jsondata/news_list_beta_hans_0.json", "https://kaopucdn.azureedge.net/jsondata/news_list_beta_hans_1.json"].map(url => $fetch(url) as Promise<Res>))
+  const res = await Promise.all(["https://kaopucdn.azureedge.net/jsondata/news_list_beta_hans_0.json", "https://kaopucdn.azureedge.net/jsondata/news_list_beta_hans_1.json"].map(url => myFetch(url) as Promise<Res>))
   return res.flat().filter(k => ["财新", "公视"].every(h => k.publisher !== h)).map((k) => {
     return {
       id: k.link,

server/sources/kuaishou.ts

@@ -26,7 +26,7 @@ interface HotRankData {
 
 export default defineSource(async () => {
   // 获取快手首页HTML
-  const html = await $fetch("https://www.kuaishou.com/?isHome=1")
+  const html = await myFetch("https://www.kuaishou.com/?isHome=1")
   // 提取window.__APOLLO_STATE__中的数据
   const matches = (html as string).match(/window\.__APOLLO_STATE__\s*=\s*(\{.+?\});/)
   if (!matches) {

server/sources/producthunt.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
   const baseURL = "https://www.producthunt.com/"
-  const html: any = await $fetch(baseURL)
+  const html: any = await myFetch(baseURL)
   const $ = cheerio.load(html)
   const $main = $("[data-test^=post-item]")
   const news: NewsItem[] = []

server/sources/solidot.ts

@@ -3,7 +3,7 @@ import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
   const baseURL = "https://www.solidot.org"
-  const html: any = await $fetch(baseURL)
+  const html: any = await myFetch(baseURL)
   const $ = cheerio.load(html)
   const $main = $(".block_m")
   const news: NewsItem[] = []

server/sources/sputniknewscn.ts

@@ -2,7 +2,7 @@ import * as cheerio from "cheerio"
 import type { NewsItem } from "@shared/types"
 
 export default defineSource(async () => {
-  const response: any = await $fetch("https://sputniknews.cn/services/widget/lenta/")
+  const response: any = await myFetch("https://sputniknews.cn/services/widget/lenta/")
   const $ = cheerio.load(response)
   const $items = $(".lenta__item")
   const news: NewsItem[] = []

server/sources/thepaper.ts

@@ -10,7 +10,7 @@ interface Res {
 
 export default defineSource(async () => {
   const url = "https://cache.thepaper.cn/contentapi/wwwIndex/rightSidebar"
-  const res: Res = await $fetch(url)
+  const res: Res = await myFetch(url)
   return res.data.hotNews
     .map((k) => {
       return {

server/sources/tieba.ts

@@ -14,7 +14,7 @@ interface Res {
 
 export default defineSource(async () => {
   const url = "https://tieba.baidu.com/hottopic/browse/topicList"
-  const res: Res = await $fetch(url)
+  const res: Res = await myFetch(url)
   return res.data.bang_topic.topic_list
     .map((k) => {
       return {

server/sources/toutiao.ts

@@ -11,7 +11,7 @@ interface Res {
 
 export default defineSource(async () => {
   const url = "https://www.toutiao.com/hot-event/hot-board/?origin=toutiao_pc"
-  const res: Res = await $fetch(url)
+  const res: Res = await myFetch(url)
   return res.data
     .map((k) => {
       return {

server/sources/v2ex.ts

@@ -17,7 +17,8 @@ interface Res {
 }
 
 const share = defineSource(async () => {
-  const res = await Promise.all(["create", "ideas", "programmer", "share"].map(k => $fetch(`https://www.v2ex.com/feed/${k}.json`) as Promise< Res>))
+  const res = await Promise.all(["create", "ideas", "programmer", "share"]
+    .map(k => myFetch(`https://www.v2ex.com/feed/${k}.json`) as Promise<Res>))
   return res.map(k => k.items).flat().map(k => ({
     id: k.id,
     title: k.title,

server/sources/wallstreetcn.ts

@@ -33,7 +33,7 @@ interface HotRes {
 const live = defineSource(async () => {
   const apiUrl = `https://api-one.wallstcn.com/apiv1/content/lives?channel=global-channel&limit=30`
 
-  const res: LiveRes = await $fetch(apiUrl)
+  const res: LiveRes = await myFetch(apiUrl)
   return res.data.items
     .map((k) => {
       return {
@@ -50,7 +50,7 @@ const live = defineSource(async () => {
 const news = defineSource(async () => {
   const apiUrl = `https://api-one.wallstcn.com/apiv1/content/information-flow?channel=global-channel&accept=article&limit=30`
 
-  const res: NewsRes = await $fetch(apiUrl)
+  const res: NewsRes = await myFetch(apiUrl)
   return res.data.items
     .filter(k => k.resource_type !== "ad" && k.resource.type !== "live" && k.resource.uri)
     .map(({ resource: h }) => {
@@ -68,7 +68,7 @@ const news = defineSource(async () => {
 const hot = defineSource(async () => {
   const apiUrl = `https://api-one.wallstcn.com/apiv1/content/articles/hot?period=all`
 
-  const res: HotRes = await $fetch(apiUrl)
+  const res: HotRes = await myFetch(apiUrl)
   return res.data.day_items
     .map((h) => {
       return {

server/sources/weibo.ts

@@ -27,7 +27,7 @@ interface Res {
 
 export default defineSource(async () => {
   const url = "https://weibo.com/ajax/side/hotSearch"
-  const res: Res = await $fetch(url)
+  const res: Res = await myFetch(url)
   return res.data.realtime
     .filter(k => !k.is_ad)
     .map((k) => {

server/sources/xueqiu.ts

@@ -16,7 +16,7 @@ interface StockRes {
 const hotstock = defineSource(async () => {
   const url = "https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size=30&_type=10&type=10"
   const cookie = (await $fetch.raw("https://xueqiu.com")).headers.getSetCookie()
-  const res: StockRes = await $fetch(url, {
+  const res: StockRes = await myFetch(url, {
     headers: {
       cookie: cookie.join("; "),
     },

server/sources/zaobao.ts

@@ -2,10 +2,9 @@ import { Buffer } from "node:buffer"
 import * as cheerio from "cheerio"
 import iconv from "iconv-lite"
 import type { NewsItem } from "@shared/types"
-import { $fetch } from "ofetch"
 
 export default defineSource(async () => {
-  const response: ArrayBuffer = await $fetch("https://www.kzaobao.com/top.html", {
+  const response: ArrayBuffer = await myFetch("https://www.kzaobao.com/top.html", {
     responseType: "arrayBuffer",
   })
   const base = "https://www.kzaobao.com"

server/sources/zhihu.ts

@@ -22,7 +22,7 @@ interface Res {
 export default defineSource({
   zhihu: async () => {
     const url = "https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=20&desktop=true"
-    const res: Res = await $fetch(url)
+    const res: Res = await myFetch(url)
     return res.data
       .map((k) => {
         return {

server/utils/fetch.ts

@@ -0,0 +1,9 @@
+import { $fetch } from "ofetch"
+
+export const myFetch = $fetch.create({
+  headers: {
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
+  },
+  timeout: 5000,
+  retry: 3,
+})