-
Notifications
You must be signed in to change notification settings - Fork 2
/
1_Crawler.py
65 lines (59 loc) · 3.44 KB
/
1_Crawler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import urllib.request
import urllib.parse
import urllib.error
from bs4 import BeautifulSoup
import os
import re
def downloader(link):
"""
创建以该字幕命名的文件夹,将元信息存储到meta-info.txt中,并将字幕文件保存至该文件夹中
"""
directory = os.getcwd() #保存原始目录位置
try:
page = BeautifulSoup(urllib.request.urlopen(link),'html5lib')
if page.title.text == '字幕发布详情说明页': #确定该页面是字幕页面
title = namemodifier(page.find(name='h2',attrs={'class':'subtitle-tit'}).string) #获取本字幕标题
info = page.find(name='ul',attrs={'class':'subtitle-info'}) #获取本字幕元信息
items = [i for i in info.children]
os.mkdir(title) #以字幕名为名创建新文件夹
os.chdir(title) #定位到此目录
metainfo = open('meta-info.txt','a+',encoding='utf-8') #创建meta-info.txt存储元信息
for i in range(1,15,2):
metainfo.write(items[i].text + '\n') #写入元信息
metainfo.write('【原始链接】'+link) #写入原始链接地址
metainfo.close()
reallink = page.find(name='a',attrs={'class':'f3'})['href'] #定位下载页面
dlpage = BeautifulSoup(urllib.request.urlopen(reallink),'html5lib') #打开下载页面
dllink = dlpage.find(name='a',attrs={'class':'btn-click'})['href'] #定位下载地址
filetitle = namemodifier(dlpage.find(name='a',attrs={'class':'btn-click'}).text) #获取本字幕文件的名称
if filetitle.find('.') == -1: #没有后缀名时,根据链接地址添加后缀名
suffix = re.search('\....',os.path.basename(dllink)).group()
filetitle = filetitle + suffix
urllib.request.urlretrieve(urllib.parse.quote(dllink,safe=':/'),filetitle) #保存本字幕文件至本地
except(FileExistsError,urllib.error.HTTPError):
pass
finally:
os.chdir(directory) #回到原始目录
def namemodifier(string):
"""
将文件夹及文件名中的非法字符删除
"""
string = string.replace('\\','')
string = string.replace('/','')
string = string.replace(':','')
string = string.replace('*','')
string = string.replace('"','')
string = string.replace('<','')
string = string.replace('>','')
string = string.replace('|','')
string = string.replace('?','')
return string
log = open('log.txt','a+',encoding='utf-8')
os.mkdir('Download')
os.chdir('Download')
for i in range(1,56300):
link = 'http://www.zimuzu.tv/subtitle/' + str(i)
print('downloading ' + link)
downloader(link)