diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 58b584d0..00000000 --- a/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM golang:1.20.7-alpine AS builder - -WORKDIR /app/fulltclash-origin -RUN apk add --no-cache git && \ - git clone https://github.com/AirportR/FullTCore.git /app/fulltclash-origin && \ - go build -ldflags="-s -w" fulltclash.go - -WORKDIR /app/fulltclash-meta -RUN git clone -b meta https://github.com/AirportR/FullTCore.git /app/fulltclash-meta && \ - go build -tags with_gvisor -ldflags="-s -w" fulltclash.go && \ - mkdir /app/FullTCore-file && \ - cp /app/fulltclash-origin/fulltclash /app/FullTCore-file/fulltclash-origin && \ - cp /app/fulltclash-meta/fulltclash /app/FullTCore-file/fulltclash-meta - - -FROM python:alpine3.18 - -WORKDIR /app - -RUN apk add --no-cache \ - git gcc g++ make libffi-dev tzdata && \ - git clone -b dev https://github.com/AirportR/FullTclash.git /app && \ - pip3 install --no-cache-dir -r requirements.txt && \ - cp resources/config.yaml.example resources/config.yaml && \ - cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ - echo "Asia/Shanghai" > /etc/timezone && \ - apk del gcc g++ make libffi-dev tzdata && \ - rm -f bin/* - -COPY --from=builder /app/FullTCore-file/* ./bin/ - -CMD ["main.py"] -ENTRYPOINT ["python3"] diff --git a/README.md b/README.md index 6ac3d221..f0e10ff9 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,11 @@ FullTClash bot 是承载其测试任务的Telegram 机器人(以下简称bot 2. 链路拓扑测试(节点出入口分析)。 3. 下行速度测试 -## 具备特性 +## 分支说明 +* [master](https://github.com/AirportR/FullTclash/tree/master) 主分支,主打稳定。 +* [backend](https://github.com/AirportR/FullTclash/tree/backend) 纯后端代码,无前端BOT,意味着需要额外的bot作主端。 +* [dev](https://github.com/AirportR/FullTclash/tree/dev) 开发进度最前沿。 +* [old](https://github.com/AirportR/FullTclash/tree/dev) 依靠调用原版Clash Restful API进行测试。可随意更换内核,但已停止新功能开发。 ## 支持协议 @@ -180,8 +184,6 @@ bot: 等待初始化操作,出现“程序已启动!”字样就说明在运行了。 运行之后和bot私聊指令: ->/clash start 用于启动clash,否则测试结果会全部是N/A。 - >/testurl <订阅地址>(clash配置格式)即可开始测试 >/help 可查看所有命令说明 diff --git a/addons/unlockTest/ip_risk.py b/addons/builtin/ip_risk.py similarity index 100% rename from addons/unlockTest/ip_risk.py rename to addons/builtin/ip_risk.py diff --git a/addons/unlockTest/netflix.py b/addons/builtin/netflix.py similarity index 100% rename from addons/unlockTest/netflix.py rename to addons/builtin/netflix.py diff --git a/addons/unlockTest/openai.py b/addons/builtin/openai.py similarity index 100% rename from addons/unlockTest/openai.py rename to addons/builtin/openai.py diff --git a/addons/unlockTest/primevideo.py b/addons/builtin/primevideo.py similarity index 100% rename from addons/unlockTest/primevideo.py rename to addons/builtin/primevideo.py diff --git a/addons/unlockTest/steam.py b/addons/builtin/steam.py similarity index 100% rename from addons/unlockTest/steam.py rename to addons/builtin/steam.py diff --git a/addons/unlockTest/tvb.py b/addons/builtin/tvb.py similarity index 100% rename from addons/unlockTest/tvb.py rename to addons/builtin/tvb.py diff --git a/addons/unlockTest/viu.py b/addons/builtin/viu.py similarity index 100% rename from addons/unlockTest/viu.py rename to addons/builtin/viu.py diff --git a/addons/unlockTest/wikipedia.py b/addons/builtin/wikipedia.py similarity index 100% rename from addons/unlockTest/wikipedia.py rename to addons/builtin/wikipedia.py diff --git a/addons/unlockTest/youtube.py b/addons/builtin/youtube.py similarity index 100% rename from addons/unlockTest/youtube.py rename to addons/builtin/youtube.py diff --git a/botmodule/command/authority.py b/botmodule/command/authority.py index e9945c6f..88e06554 100644 --- a/botmodule/command/authority.py +++ b/botmodule/command/authority.py @@ -121,7 +121,7 @@ async def get_url_from_invite(_, message2): include_text = texts_li[1] if len(texts_li) > 2: exclude_text = texts_li[2] - url_li = geturl(text_li) + url_li = geturl(text_li, True) if url_li: await temp_queue.put((url_li, include_text, exclude_text)) else: diff --git a/botmodule/command/setting.py b/botmodule/command/setting.py index e9811332..efb386a9 100644 --- a/botmodule/command/setting.py +++ b/botmodule/command/setting.py @@ -620,7 +620,7 @@ async def select_sort_only(_: "Client", call: Union["CallbackQuery", "Message"], IKB("⬇️平均速度降序", f"{api_route}arspeed")]) content_keyboard.append([IKB("⬆️最大速度升序", f"{api_route}mspeed"), IKB("⬇️最大速度降序", f"{api_route}mrspeed")]) - content_keyboard.append([dbtn['b_cancel']]) + content_keyboard.append([dbtn['b_close']]) botmsg = await call.reply(f"请选择排序方式(你有{timeout}s的时间选择): ", reply_markup=InlineKeyboardMarkup(content_keyboard), quote=True) recvkey = gen_msg_key(botmsg) diff --git a/botmodule/command/test.py b/botmodule/command/test.py index e0d48435..617e2b1f 100644 --- a/botmodule/command/test.py +++ b/botmodule/command/test.py @@ -168,7 +168,7 @@ async def process(app: Client, message: Message, **kwargs): back_message = await message.reply("⏳任务接收成功,测试进行中...", quote=True) tgtext = str(message.text) tgargs = cleaner.ArgCleaner().getall(tgtext) - suburl = cleaner.geturl(tgtext) if kwargs.get('url', None) is None else kwargs.get('url', None) + suburl = cleaner.geturl(tgtext, True) if kwargs.get('url', None) is None else kwargs.get('url', None) put_type = kwargs.pop('put_type', '') if kwargs.get('put_type', '') else tgargs[0].split("@")[0] logger.info("测试指令: " + str(put_type)) if not put_type: diff --git a/botmodule/register.py b/botmodule/register.py index 97d7ebfd..efbf1bf3 100644 --- a/botmodule/register.py +++ b/botmodule/register.py @@ -1,6 +1,8 @@ import random import urllib.parse import aiohttp +from pyrogram.enums import ParseMode +from pyrogram.types import Message from loguru import logger from aiohttp.client_exceptions import ClientConnectorError from utils.cleaner import geturl @@ -60,16 +62,16 @@ async def getsub_async(url: str, username: str, pwd: str, proxy=None): return str(e) -async def baipiao(_, message): +async def baipiao(_, message: "Message"): back_message = await message.reply("正在尝试注册...") # 发送提示 regisurl = geturl(str(message.text)) if regisurl: suburl = await getsub_async(regisurl, random_value(10), random_value(8), proxy=proxies) else: - await back_message.edit_text("❌发生错误,请检查注册地址是否正确") + await back_message.edit_text("❌发生错误,请检查注册地址是否正确", parse_mode=ParseMode.DISABLED) return if suburl: - await back_message.edit_text(suburl) + await back_message.edit_text(suburl, parse_mode=ParseMode.DISABLED) return else: await back_message.edit_text("❌发生错误,请检查注册地址是否正确") diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..51dfc4ac --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,43 @@ +FROM python:3.9.18-slim-bookworm AS compile-image + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + gcc g++ make ca-certificates + +RUN python -m venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" +ADD https://raw.githubusercontent.com/AirportR/FullTclash/dev/requirements.txt . +RUN pip3 install --no-cache-dir -r requirements.txt && \ + pip3 install --no-cache-dir supervisor + +FROM python:3.9.18-slim-bookworm + +WORKDIR /app + +RUN apt-get update && \ + apt-get install --no-install-recommends -y \ + git tzdata curl wget jq bash nano cron && \ + git clone -b dev --single-branch --depth=1 https://github.com/AirportR/FullTclash.git /app && \ + cp resources/config.yaml.example resources/config.yaml && \ + rm -f /etc/localtime && \ + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \ + echo "Asia/Shanghai" > /etc/timezone && \ + mkdir /etc/supervisord.d && \ + mv /app/docker/supervisord.conf /etc/supervisord.conf && \ + mv /app/docker/fulltclash.conf /etc/supervisord.d/fulltclash.conf && \ + chmod +x /app/docker/fulltcore.sh && \ + /app/docker/fulltcore.sh && \ + cp /app/docker/crontab /etc/cron.d/crontab && \ + chmod 0644 /etc/cron.d/crontab && \ + /usr/bin/crontab /etc/cron.d/crontab && \ + chmod +x /app/docker/update.sh && \ + chmod +x /app/docker/docker-entrypoint.sh && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=compile-image /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=compile-image /opt/venv /opt/venv + +ENV PATH="/opt/venv/bin:$PATH" + +ENTRYPOINT ["/app/docker/docker-entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..060ede96 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,51 @@ +# 使用Docker安装 + +> 这能让你在Windows、Mac、Linux、openwrt、Nas几乎任何支持Docker(目前仅Amd64和Arm64)的环境下使用此项目! + +## 1.创建配置文件 +新建配置文件保存目录`mkdir /etc/FullTclash` +下载并编辑配置文件 +``` +wget -O /etc/FullTclash/config.yaml https://raw.githubusercontent.com/AirportR/FullTclash/dev/resources/config.yaml.example +``` +修改 config.yaml (path是必须修改的配置,不能使用默认的) +``` +clash: + path: './bin/fulltclash-origin' + branch: origin +``` +或者 [Meta内核](https://github.com/AirportR/FullTCore/tree/meta) +``` +clash: + path: './bin/fulltclash-meta' + branch: meta +``` + +## 构建Docker镜像 + +### 下载Dockerfile +``` +wget -N https://raw.githubusercontent.com/AirportR/FullTclash/dev/docker/Dockerfile +``` + +### 构建镜像 +``` +docker build -t fulltclash:dev . +``` + +启动 +``` +docker run -itd --name=fulltclash --restart=always -v /etc/FullTclash/config.yaml:/app/resources/config.yaml fulltclash:dev +``` +查看日志 +``` +docker exec -it fulltclash tail -f /var/log/fulltclash.log +``` +更新版本 +``` +docker exec -it fulltclash bash /app/docker/update.sh +``` +重启程序 +``` +docker exec -it fulltclash supervisorctl restart fulltclash +``` diff --git a/docker/crontab b/docker/crontab new file mode 100644 index 00000000..b545befd --- /dev/null +++ b/docker/crontab @@ -0,0 +1 @@ +00 6 * * * bash /app/docker/update.sh diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100644 index 00000000..32c3cfce --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +supervisord -c /etc/supervisord.conf + +cron -f > /dev/null 2>&1 \ No newline at end of file diff --git a/docker/fulltclash.conf b/docker/fulltclash.conf new file mode 100644 index 00000000..c5a86a47 --- /dev/null +++ b/docker/fulltclash.conf @@ -0,0 +1,4 @@ +[program:fulltclash] +command=python3 /app/main.py +directory=/app +stdout_logfile=/var/log/fulltclash.log \ No newline at end of file diff --git a/docker/fulltcore.sh b/docker/fulltcore.sh new file mode 100644 index 00000000..91700815 --- /dev/null +++ b/docker/fulltcore.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +rm -f /app/bin/* + +ORIGIN_AMD64_URL=https://github.com/AirportR/FullTCore/releases/download/v1.0/FullTCore_1.0_linux_amd64.tar.gz +META_AMD64_URL=https://github.com/AirportR/FullTCore/releases/download/v1.1-meta/FullTCore_1.1-meta_linux_amd64.tar.gz +ORIGIN_ARM64_URL=https://github.com/AirportR/FullTCore/releases/download/v1.0/FullTCore_1.0_linux_arm64.tar.gz +META_ARM64_URL=https://github.com/AirportR/FullTCore/releases/download/v1.1-meta/FullTCore_1.1-meta_linux_arm64.tar.gz + +arch=$(arch) + +if [[ $arch == "x86_64" || $arch == "x64" || $arch == "amd64" ]]; then + arch="amd64" + wget -O /app/bin/FullTCore_origin.tar.gz ${ORIGIN_AMD64_URL} > /dev/null 2>&1 + wget -O /app/bin/FullTCore_meta.tar.gz ${META_AMD64_URL} > /dev/null 2>&1 +elif [[ $arch == "aarch64" || $arch == "arm64" ]]; then + arch="arm64" + wget -O /app/bin/FullTCore_origin.tar.gz ${ORIGIN_ARM64_URL} > /dev/null 2>&1 + wget -O /app/bin/FullTCore_meta.tar.gz ${META_ARM64_URL} > /dev/null 2>&1 +fi + +tar -C /app/bin/ -xvzf /app/bin/FullTCore_origin.tar.gz +mv /app/bin/FullTCore /app/bin/fulltclash-origin + +tar -C /app/bin/ -xvzf /app/bin/FullTCore_meta.tar.gz +mv /app/bin/FullTCore /app/bin/fulltclash-meta + +find /app/bin/* | egrep -v "fulltclash-origin|fulltclash-meta" | xargs rm -f + +chmod +x /app/bin/fulltclash-origin +chmod +x /app/bin/fulltclash-meta + +echo "架构: ${arch}下载FullTCore完成" \ No newline at end of file diff --git a/docker/supervisord.conf b/docker/supervisord.conf new file mode 100644 index 00000000..b636e352 --- /dev/null +++ b/docker/supervisord.conf @@ -0,0 +1,22 @@ +[unix_http_server] +file=/tmp/supervisor.sock + +[supervisord] +logfile=/tmp/supervisord.log +logfile_maxbytes=50MB +logfile_backups=10 +loglevel=info +pidfile=/tmp/supervisord.pid +nodaemon=false +silent=false +minfds=1024 +minprocs=200 + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = supervisord.d/*.conf \ No newline at end of file diff --git a/docker/update.sh b/docker/update.sh new file mode 100644 index 00000000..fa265d3c --- /dev/null +++ b/docker/update.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +git_version=$(git --git-dir='/app/.git' --work-tree='/app' rev-parse HEAD) +last_version=$(curl -Ls "https://api.github.com/repos/AirportR/FullTclash/commits/dev" | jq .sha | sed -E 's/.*"([^"]+)".*/\1/') + +update() { + git --git-dir='/app/.git' --work-tree='/app' fetch --all + git --git-dir='/app/.git' --work-tree='/app' reset --hard origin/dev + git --git-dir='/app/.git' --work-tree='/app' pull +} + +if [[ $last_version == "$git_version" ]]; then + echo -e "已是最新版本,无需更新" +else + echo -e "检查到新版本,正在更新" + update +fi \ No newline at end of file diff --git a/glovar.py b/glovar.py index 49fc1087..94adb7a9 100644 --- a/glovar.py +++ b/glovar.py @@ -16,7 +16,7 @@ BUILD_TOKEN = init_bot.config.getBuildToken() userbot_config = bot_config.config.get('userbot', {}) # 项目版本号 -__version__ = '3.6.2' +__version__ = '3.6.4' # 客户端 app = Client("my_bot", api_id=init_bot.api_id, diff --git a/resources/config.yaml.example b/resources/config.yaml.example index 67720df4..e6f6173b 100644 --- a/resources/config.yaml.example +++ b/resources/config.yaml.example @@ -29,7 +29,7 @@ bot: #如果要http代理要验证,配置格式为: proxy: "用户名:密码@host:端口" 比如: user1:112233@127.0.0.1:7890 #geoip-api: "ip-api.com" # GEOIP 测试api,取二级域名,目前支持 ip-api.com ip.sb ipleak.net ipdata.co ipapi.co 五种,其中 ipdata 需要配置下面的geoip-key #geoip-key: xxxxxxx #ipdata 的 apikey,如果使用其他api则无需填写 -#subconvertor: #订阅转换(此配置主要开发者已无心维护,能用但不稳定) +#subconverter: #订阅转换(此配置主要开发者已无心维护,能用但不稳定) # enable: true #是否启用 # host: '127.0.0.1:25500' #域名或者ip加端口 # remoteconfig: "https://raw.githubusercontent.com/ACL4SSR/ACL4SSR/master/Clash/config/ACL4SSR_Online.ini" #远程配置 diff --git a/utils/cleaner.py b/utils/cleaner.py index 7f918443..271f5cea 100644 --- a/utils/cleaner.py +++ b/utils/cleaner.py @@ -26,7 +26,7 @@ def __init__(self, data): def get(self, key, _default=None): try: if self._data is None: - return {} + return "" return self._data[key] except KeyError: return _default @@ -406,16 +406,27 @@ def __init__(self, _config, _config2: Union[str, bytes] = None): if not isinstance(self.yaml, dict): self.yaml = {} - def load(self, _config, _config2: Union[str, bytes]): - if type(_config).__name__ == 'str': + def notag(self, _config: Union[bytes, str]): + """ + 去除制表符,yaml反序列化不允许制表符出现在标量以外的地方 + """ + return _config.replace(b'\t', b' ') + + def load(self, _config, _config2: Union[str, bytes] = None): + if isinstance(_config, str): if _config == ':memory:': try: if _config2 is None: self.yaml = yaml.safe_load(preTemplate()) else: - self.yaml = yaml.safe_load(_config2) + try: + self.yaml = yaml.safe_load(_config2) + except yaml.MarkedYAMLError: + _config2 = self.notag(_config2) + self.yaml = yaml.safe_load(_config2) self.check_type() return + except Exception as e: logger.error(str(e)) self.yaml = {} @@ -644,17 +655,9 @@ def node_filter(self, include: str = '', exclude: str = '', issave=False): try: if include: - if len(include) < 32: - pattern1 = remodule.compile(include) - else: - pattern1 = None - logger.warning(f"包含过滤器的文本: {include} 大于32个长度,无法生效!") + pattern1 = remodule.compile(include) if exclude: - if len(exclude) < 32: - pattern2 = remodule.compile(exclude) - else: - pattern2 = None - logger.warning(f"排除过滤器的文本: {exclude} 大于32个长度,无法生效!") + pattern2 = remodule.compile(exclude) except remodule.error: logger.error("正则错误!请检查正则表达式!") return self.nodesName() @@ -703,7 +706,7 @@ def node_filter(self, include: str = '', exclude: str = '', issave=False): @logger.catch def save(self, savePath: str = "./sub.yaml"): with open(savePath, "w", encoding="UTF-8") as fp: - yaml.dump(self.yaml, fp) + yaml.safe_dump(self.yaml, fp, encoding='utf-8') class ConfigManager: @@ -1093,7 +1096,7 @@ def del_user(self, user: list or str or int): def save(self, savePath: str = "./resources/config.yaml"): with open(savePath, "w+", encoding="UTF-8") as fp: try: - yaml.dump(self.yaml, fp) + yaml.safe_dump(self.yaml, fp, encoding='utf-8') return True except Exception as e: logger.error(e) @@ -1220,7 +1223,7 @@ def get_all(self): info[i] = task(self) continue if i == "Youtube": - from addons.unlockTest import youtube + from addons.builtin import youtube you = youtube.get_youtube_info(self) info['Youtube'] = you elif i == "Disney": @@ -1233,25 +1236,25 @@ def get_all(self): dazn = self.get_dazn_info() info['Dazn'] = dazn elif i == "Netflix": - from addons.unlockTest import netflix + from addons.builtin import netflix info['Netflix'] = netflix.get_netflix_info_new(self) elif i == "TVB": - from addons.unlockTest import tvb + from addons.builtin import tvb info['TVB'] = tvb.get_TVBAnywhere_info(self) elif i == "Viu": - from addons.unlockTest import viu + from addons.builtin import viu info['Viu'] = viu.get_viu_info(self) elif i == "iprisk" or i == "落地IP风险": - from addons.unlockTest import ip_risk + from addons.builtin import ip_risk info['落地IP风险'] = ip_risk.get_iprisk_info(self) elif i == "steam货币": - from addons.unlockTest import steam + from addons.builtin import steam info['steam货币'] = steam.get_steam_info(self) elif i == "维基百科": - from addons.unlockTest import wikipedia + from addons.builtin import wikipedia info['维基百科'] = wikipedia.get_wikipedia_info(self) elif item == "OpenAI": - from addons.unlockTest import openai + from addons.builtin import openai info['OpenAI'] = openai.get_openai_info(self) else: pass @@ -1582,7 +1585,43 @@ def getall(self, string: str = None): return arg -def geturl(string: str): +def protocol_join(protocol_link: str): + if not protocol_link: + return '' + protocol_prefix = ['vmess', 'vless', 'ss', 'ssr', 'trojan', 'hysteria2', 'hysteria', + 'socks5', 'snell', 'tuic', 'juicity'] + p = protocol_link.split('://') + if len(p) < 2: + return '' + if p[0] not in protocol_prefix: + return '' + + from urllib.parse import quote + subcvtconf = config.config.get('subconverter', {}) + enable = subcvtconf.get('enable', False) + if not isinstance(enable, bool): # 如果没有解析成bool值,强制禁用subconverter + enable = False + if not enable: + return '' + subcvtaddr = subcvtconf.get('host', '') + remoteconfig = subcvtconf.get('remoteconfig', '') + if not remoteconfig: + remoteconfig = "https%3A%2F%2Fraw.githubusercontent.com%2FACL4SSR%2FACL4SSR%2Fmaster%2FClash%2F" \ + "config%2FACL4SSR_Online.ini" + else: + remoteconfig = quote(remoteconfig) + + new_link = f"https://{subcvtaddr}/sub?target=clash&new_name=true&url=" + quote(protocol_link) + \ + f"&insert=false&config={remoteconfig}" + return new_link + + +def geturl(string: str, protocol_match: bool = False): + """ + 获取URL + + :param: protocol_match: 是否匹配协议URI,并拼接成ubconverter形式 + """ text = string pattern = re.compile( r"https?://(?:[a-zA-Z]|\d|[$-_@.&+]|[!*,]|[\w\u4e00-\u9fa5])+") # 匹配订阅地址 @@ -1591,7 +1630,13 @@ def geturl(string: str): url = pattern.findall(text)[0] # 列表中第一个项为订阅地址 return url except IndexError: - return None + if protocol_match: + args = ArgCleaner.getarg(string) + protocol_link = args[1] if len(args) > 1 else text.strip() if not text.startswith("/") else '' + new_link = protocol_join(protocol_link) + return new_link if new_link else None + else: + return None @logger.catch diff --git a/utils/collector.py b/utils/collector.py index 8d9ca00e..783588b2 100644 --- a/utils/collector.py +++ b/utils/collector.py @@ -9,6 +9,7 @@ from aiohttp.client_exceptions import ClientConnectorError, ContentTypeError from aiohttp_socks import ProxyConnector, ProxyConnectionError from loguru import logger + from utils import cleaner """ @@ -21,7 +22,7 @@ 需要注意的是,这些类/函数仅作采集工作,并不负责清洗。我们需要将拿到的数据给cleaner类清洗。 ** 开发建议 ** -如果你想自己添加一个流媒体测试项,建议继承Collector类,重写类中的create_tasks方法,以及自定义自己的流媒体测试函数 fetch_XXX() +如果你想自己添加一个流媒体测试项,建议查看 ./resources/dos/新增流媒体测试项指南.md """ config = cleaner.ConfigManager() @@ -191,10 +192,13 @@ class SubCollector(BaseCollector): """ @logger.catch() - def __init__(self, suburl: str, include: str = '', exclude: str = ''): + def __init__(self, suburl: str, include: str = '', exclude: str = '', force_convert: bool = False): """ 这里在初始化中读取了subconverter的相关配置,但是由于sunconverter无人维护,容易出问题,因此之后我不会再维护此功能。也就是在下载订阅时 订阅转换 + + :param: force_convert: 是否强制转换,如果传进来的url本身就已经是subconverter拼接过的,那么套娃转换会拖慢拉去订阅的速度。 + 设置为False会检查是否为subconverter拼接过的 """ super().__init__() self.text = None @@ -208,18 +212,31 @@ def __init__(self, suburl: str, include: str = '', exclude: str = ''): self.code_include = quote(include, encoding='utf-8') self.code_exclude = quote(exclude, encoding='utf-8') self.cvt_host = str(self.subconverter.get('host', '127.0.0.1:25500')) - self.cvt_url = f"http://{self.cvt_host}/sub?target=clash&new_name=true&url={self.codeurl}" \ + self.cvt_scheme = self.parse_cvt_scheme() + self.cvt_url = f"{self.cvt_scheme}://{self.cvt_host}/sub?target=clash&new_name=true&url={self.codeurl}" \ + f"&include={self.code_include}&exclude={self.code_exclude}" self.sub_remote_config = self.subconverter.get('remoteconfig', '') self.config_include = quote(self.subconverter.get('include', ''), encoding='utf-8') # 这两个 self.config_exclude = quote(self.subconverter.get('exclude', ''), encoding='utf-8') # print(f"配置文件过滤,包含:{self.config_include} 排除:{self.config_exclude}") if self.config_include or self.config_exclude: - self.cvt_url = f"http://{self.cvt_host}/sub?target=clash&new_name=true&url={self.cvt_url}" \ + self.cvt_url = f"{self.cvt_scheme}://{self.cvt_host}/sub?target=clash&new_name=true&url={self.cvt_url}" \ + f"&include={self.code_include}&exclude={self.code_exclude}" if self.sub_remote_config: self.sub_remote_config = quote(self.sub_remote_config, encoding='utf-8') self.cvt_url = self.cvt_url + "&config=" + self.sub_remote_config + if not force_convert: + if "/sub?target=" in self.url: + self.cvt_url = self.url + + def parse_cvt_scheme(self) -> str: + temp_cvt = self.cvt_host.split(":") + cvt_scheme = 'http' + if len(temp_cvt) == 2: + hostname = temp_cvt[0] + if hostname != "127.0.0.1": + cvt_scheme = 'https' + return cvt_scheme async def start(self, proxy=None): try: @@ -279,28 +296,43 @@ async def getSubConfig(self, save_path: str = "./", proxy=proxies, inmemory: boo suburl = self.cvt_url if self.cvt_enable else self.url cvt_text = "subconverter状态: {}".format("已启用" if self.cvt_enable else "未启用") logger.info(cvt_text) + + async def safe_read(_response: aiohttp.ClientResponse, limit: int = 52428800): + if _response.content_length and _response.content_length > limit: + logger.warning(f"订阅文件大小超过了{limit / 1024 / 1024}MB的阈值,已取消获取。") + return False + _data = b'' + if inmemory: + while True: + _chunk = await _response.content.read(1024) + if not _chunk: + logger.info("获取订阅成功") + break + _data += _chunk + if len(_data) > limit: + logger.warning(f"订阅文件大小超过了{limit / 1024 / 1024}MB的阈值,已取消获取。") + return False + return _data + else: + with open(save_path, 'wb+') as fd: + while True: + _chunk = await _response.content.read(1024) + if not _chunk: + logger.info("获取订阅成功") + break + fd.write(_chunk) + return True + try: async with aiohttp.ClientSession(headers=_headers) as session: async with session.get(suburl, proxy=proxy, timeout=20) as response: if response.status == 200: - data = b'' - if inmemory: - while True: - chunk = await response.content.read() - if not chunk: - logger.info("获取订阅成功") - break - data += chunk - return data - with open(save_path, 'wb+') as fd: - while True: - chunk = await response.content.read() - if not chunk: - logger.info("获取订阅成功") - break - fd.write(chunk) - return True - return False + return await safe_read(response) + else: + if self.url == self.cvt_url: + return False + self.cvt_url = self.url + return await self.getSubConfig(inmemory=True) except asyncio.exceptions.TimeoutError: logger.info("获取订阅超时") return False @@ -390,7 +422,6 @@ def __init__(self, proxyconfig: list, host: str = '127.0.0.1', port: int = 1112, class Collector: def __init__(self, script: List[str] = None): - self.session = None self.tasks = [] self._script = script self._headers = { @@ -399,25 +430,9 @@ def __init__(self, script: List[str] = None): self._headers_json = { 'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/106.0.0.0 Safari/537.36", "Content-Type": 'application/json'} - self.ipurl = "https://api.ip.sb/geoip" - self.youtubeurl = "https://www.youtube.com/premium" - self.youtubeHeaders = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + - 'Chrome/80.0.3987.87 Safari/537.36', - 'Accept-Language': 'en' - } - self.youtubeCookie = { - 'YSC': 'BiCUU3-5Gdk', - 'CONSENT': 'YES+cb.20220301-11-p0.en+FX+700', - 'GPS': '1', - 'VISITOR_INFO1_LIVE': '4VwPMkB7W5A', - '_gcl_au': '1.1.1809531354.1646633279', - 'PREF': 'tz=Asia.Shanghai' - } self.info = {} self.disneyurl1 = "https://www.disneyplus.com/" self.disneyurl2 = "https://global.edge.bamgrid.com/token" - self.daznurl = "https://startup.core.indazn.com/misl/v5/Startup" @logger.catch def create_tasks(self, session: aiohttp.ClientSession, proxy=None): @@ -437,31 +452,31 @@ def create_tasks(self, session: aiohttp.ClientSession, proxy=None): self.tasks.append(task(self, session, proxy=proxy)) continue if i == "Youtube": - from addons.unlockTest import youtube + from addons.builtin import youtube self.tasks.append(youtube.task(self, session, proxy=proxy)) elif i == "Disney" or i == "Disney+": task5 = asyncio.create_task(self.fetch_dis(session, proxy=proxy)) self.tasks.append(task5) elif i == "Netflix": - from addons.unlockTest import netflix + from addons.builtin import netflix self.tasks.append(netflix.task(self, session, proxy=proxy, netflixurl=netflix_url)) elif i == "TVB": - from addons.unlockTest import tvb + from addons.builtin import tvb self.tasks.append(tvb.task(self, session, proxy=proxy)) elif i == "Viu": - from addons.unlockTest import viu + from addons.builtin import viu self.tasks.append(viu.task(self, session, proxy=proxy)) elif i == "Iprisk" or i == "落地IP风险": - from addons.unlockTest import ip_risk + from addons.builtin import ip_risk self.tasks.append(ip_risk.task(self, session, proxy=proxy)) elif i == "steam货币": - from addons.unlockTest import steam + from addons.builtin import steam self.tasks.append(steam.task(self, session, proxy=proxy)) elif i == "维基百科": - from addons.unlockTest import wikipedia + from addons.builtin import wikipedia self.tasks.append(wikipedia.task(self, session, proxy=proxy)) elif item == "OpenAI": - from addons.unlockTest import openai + from addons.builtin import openai self.tasks.append(openai.task(self, session, proxy=proxy)) else: pass @@ -470,34 +485,6 @@ def create_tasks(self, session: aiohttp.ClientSession, proxy=None): logger.error(e) return [] - async def fetch_ip(self, session: aiohttp.ClientSession, proxy=None): - """ - ip查询 - :param session: - :param proxy: - :return: - """ - try: - res = await session.get(self.ipurl, proxy=proxy, timeout=5) - logger.info("ip查询状态:" + str(res.status)) - if res.status != 200: - self.info['ip'] = None - self.info['netflix1'] = None - self.info['netflix2'] = None - self.info['youtube'] = None - self.info['ne_status_code1'] = None - self.info['ne_status_code2'] = None - logger.warning("无法查询到代理ip") - return self.info - else: - self.info['ip'] = await res.json() - except ClientConnectorError as c: - logger.warning(c) - self.info['ip'] = None - return self.info - except Exception as e: - logger.error(str(e)) - async def fetch_dis(self, session: aiohttp.ClientSession, proxy=None, reconnection=2): """ Disney+ 解锁检测 @@ -582,8 +569,6 @@ async def start(self, host: str, port: int, proxy=None): try: conn = ProxyConnector(host=host, port=port, limit=0) session = aiohttp.ClientSession(connector=conn, headers=self._headers) - # if proxy is None: - # proxy = f"http://{host}:{port}" tasks = self.create_tasks(session, proxy=proxy) if tasks: try: @@ -591,8 +576,7 @@ async def start(self, host: str, port: int, proxy=None): except (ConnectionRefusedError, ProxyConnectionError, ssl.SSLError) as e: logger.error(str(e)) return self.info - finally: - await session.close() + await session.close() return self.info except Exception as e: logger.error(str(e)) diff --git a/utils/export.py b/utils/export.py index 7d9792a9..75ad6180 100644 --- a/utils/export.py +++ b/utils/export.py @@ -14,7 +14,7 @@ from utils.cleaner import ConfigManager import utils.emoji_custom as emoji_source -__version__ = '3.6.2' +__version__ = '3.6.4' # 这是将测试的结果输出为图片的模块。 # 设计思路: