-
Notifications
You must be signed in to change notification settings - Fork 0
/
games.py
125 lines (94 loc) · 5.18 KB
/
games.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.loader import ItemLoader
import json
class OfficialSpider(CrawlSpider):
name = 'official'
start_urls = [
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-01#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-03#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-05#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-08#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-10#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-11#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-12#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-13#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-14#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-15#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-16#jt_list/',
'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-17#jt_list/',
# 'http://www.dofus.com/fr/mmorpg/communaute/tournois/goultarminator/calendrier?date=2016-08-18#jt_list/',
]
def parse(self, response):
with open('allgames.json') as data_file:
games = json.load(data_file)
done_id = self.extract_done_id(games)
for href in response.css('table.ak-ladder tr td:last-child a::attr(href)'):
done = False
if self.make_url(href.extract())[-4:] in done_id:
done = True
if not done:
url_object = response.urljoin(self.make_url(href.extract()))
yield scrapy.Request(url_object, callback=self.parse_combat_result_page)
def extract_done_id(self, games):
done_id = [game['id'] for game in games]
return done_id
def parse_combat_result_page(self, response):
teams = self.get_teams(response)
result = self.get_result(response)
winner_result = result['teams'][teams['winner']['name']]
teams['winner']['points'] = winner_result['points']
teams['winner']['deaths'] = winner_result['deaths']
loser_result = result['teams'][teams['loser']['name']]
teams['loser']['points'] = loser_result['points']
teams['loser']['deaths'] = loser_result['deaths']
yield {
'id': self.get_id(response),
'winner': teams['winner'],
'loser': teams['loser'],
'turns': result['turns'],
}
def get_result(self, response):
container = response.css('.row.ak-container')
result = {}
result['turns'] = container.css('div:first-child div:first-child strong::text').extract_first().strip()
teams = {}
for team_container in container.css('.ak-column.ak-container.col-md-6'):
team_name = team_container.css('.ak-team-match-result strong::text').extract_first().lower().strip()
teams[team_name] = {}
teams[team_name]['points'] = team_container.css('.ak-team-match-result-points::text').extract_first()[+28:].strip()
team_death = []
for player in team_container.css('.ak-team-match-result-players .ak-panel-content .ak-list-element'):
name = player.css('.ak-content .ak-title a::text').extract_first()
dead = player.css('.ak-character-dead').extract_first()
if dead:
team_death.append(name)
teams[team_name]['deaths'] = team_death
result['teams'] = teams
return result
def get_teams(self, response):
teams = {}
winner_name = self.get_winner_name(response).lower()
for td in response.css('table.ak-ladder tbody tr td:first-child'):
names = td.css('a::text').extract()
urls = td.css('a::attr(href)').extract()
winner = {}
loser = {}
for i in [0,1]:
if names[i].lower() == winner_name:
winner['name'] = names[i].lower()
winner['url'] = self.make_url(urls[i])
else:
loser['name'] = names[i].lower()
loser['url'] = self.make_url(urls[i])
teams['winner'] = winner
teams['loser'] = loser
return teams
def get_winner_name(self, response):
return response.css('table.ak-ladder tbody tr td:last-child a::text').extract_first()
def get_id(self, response):
return response.url.split("/")[-1]
def make_url(self, url):
return 'http://www.dofus.com{}'.format(url)