-
Notifications
You must be signed in to change notification settings - Fork 3
/
gen_xml.py
209 lines (172 loc) · 5.61 KB
/
gen_xml.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
# gen_xml.py
import getopt
import os
import re
import string
import sys
import xml.etree.ElementTree as ET
from BeautifulSoup import BeautifulSoup as BS
#
from cardinfo import CardInfoGatherer
import grab_html
import sanity
import sets
import special
import symbols
import tools
import xmltools
XML_VERSION = "1.4.0"
# this should be bumped up every time a change is made to the XML output
# (directly or indirectly), or if sanity checks were added.
# these tags can be added to a set
SET_TAGS= [
"deck", # preconstructed deck or special package
"online", # MTGO only
]
# ElementTree does not write the XML header, so we need to provide this
XML_HEADER = '<?xml version="1.0" ?>'
DEBUG = False
def gen_xml(short_set):
ids = grab_html.read_ids(short_set)
root = generate_base_xml(short_set)
cards = root.find('.//cards')
for idx, id in enumerate(ids):
if short_set == 'UGL' and id in special.UNGLUED_TOKENS:
print "Skipping token:", id
continue
print id
sp = open_with_bs(short_set, id, 'p')
so = open_with_bs(short_set, id, 'o')
try:
d = gather_data(so, id)
except:
print "Multiverse id:", id
raise
d['type_oracle'] = d['type']
d['rules_oracle'] = d['rules']
del d['type'], d['rules']
d['type_printed'], d['rules_printed'] = extract_printed_data(sp, id)
d['multiverseid'] = str(id)
try:
back = special.double_faced_cards[id]
except KeyError:
pass
else:
d['doublefaced'] = ('front', str(back))
try:
front = special.doublefaced_reverse[id]
except KeyError:
pass
else:
d['doublefaced'] = ('back', str(front))
print d
sanity.check_card_dict(d)
add_xml_element(cards, d)
if DEBUG and idx > 1: break # FIXME
write_xml(short_set, root)
print "Done"
ATTRS = ['name', 'manacost', 'type', 'rules',
'rarity', 'number', 'artist', 'power', 'toughness',
'loyalty', 'flavor_text']
SPECIAL = ['manacost', 'doublefaced']
# these attribute names should correspond to CardInfoGatherer methods and the
# eventual XML tags that we're going to use
def generate_base_xml(short_set):
s = sets.set_info[short_set]
root = ET.Element('root')
meta = ET.SubElement(root, 'meta')
version = ET.SubElement(meta, 'version')
version.text = XML_VERSION
set = ET.SubElement(root, 'set')
set_name = ET.SubElement(set, 'name')
set_name.text = s.name
set_shortname = ET.SubElement(set, 'shortname')
set_shortname.text = short_set
noc = ET.SubElement(set, 'number_of_cards')
noc.text = str(s.cards)
date = ET.SubElement(set, 'release_date')
tags = ET.SubElement(set, 'tags')
# add tags, if any
for tag in SET_TAGS:
if getattr(s, tag, None):
_ = ET.SubElement(tags, tag) # value does not matter
date.text = s.date
cards = ET.SubElement(set, 'cards')
return root
def gather_data(soup, id):
d = {}
cig = CardInfoGatherer(soup, id)
for attr in ATTRS:
value = getattr(cig, attr)()
if value is not None:
d[attr] = value
return d
def extract_printed_data(soup, id):
cig = CardInfoGatherer(soup, id)
return cig.type(), cig.rules()
def open_with_bs(short_set, id, suffix):
path = os.path.join('html', short_set, "%s-%s.html" % (id, suffix))
with open(path, 'rb') as f:
data = f.read()
return BS(data)
def add_xml_element(elem, d):
card = ET.SubElement(elem, 'card')
for key, value in d.items():
if key in SPECIAL: continue
x = ET.SubElement(card, key)
x.text = value
manacost = ET.SubElement(card, 'manacost')
for symbol in d['manacost']:
s = ET.SubElement(manacost, 'symbol')
s.text = symbol
# add double-faced indicator
try:
side, counterpart = d['doublefaced']
except KeyError:
pass
else:
dblfaced = ET.SubElement(card, 'doublefaced')
sidenode = ET.SubElement(dblfaced, 'side')
sidenode.text = side
cntrnode = ET.SubElement(dblfaced, 'other')
cntrnode.text = counterpart
return card
def write_xml(short_set, root):
if not os.path.exists('xml'): os.makedirs('xml')
print "Writing XML..."
path = os.path.join('xml', short_set + ".xml")
with open(path, 'wb') as f:
f.write(XML_HEADER + "\n")
xmltools.pprint_xml(root, f=f)
sanity.validate_xml(path)
re_version = re.compile("<version>(.*?)</version>")
def find_updates():
""" Return the sets whose XML files needs updated, i.e. they have a
version < the latest version. """
behind = []
for short_set in sets.set_info.keys():
path = os.path.join("xml", short_set + ".xml")
with open(path, 'rb') as f:
chunk = f.read(2048) # read first 2K
m = re_version.search(chunk)
if m:
file_version = m.group(1)
if file_version < XML_VERSION:
behind.append(short_set)
else:
# no version?! must be really old
behind.append(short_set)
return behind
if __name__ == "__main__":
opts, args = getopt.getopt(sys.argv[1:], "du", ["update"])
if args == ["all"]:
args = sorted(sets.set_info.keys())
for o, a in opts:
if o == '-d':
print "Running in debug mode"
DEBUG = True
elif o in ("--update", "-u"):
args = find_updates()
print "Updating:", args
for short_set in args:
gen_xml(short_set)