-
Notifications
You must be signed in to change notification settings - Fork 0
/
scrape.py
executable file
·86 lines (71 loc) · 3.41 KB
/
scrape.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
#!/usr/bin/env python3
import csv
import requests
import sys
from bs4 import BeautifulSoup
def main():
if len(sys.argv) != 1+1:
print("Unexpected arg count. Please specify output file.")
sys.exit()
url_base = "https://www.haasjr.org/grants/search?page="
page = 0
with open(sys.argv[1], "w", newline="") as f:
fieldnames = ["grantee", "grantee_url", "year", "amount", "issue_area",
"sub_issue_area", "grant_info"]
writer = csv.DictWriter(f, fieldnames=fieldnames)
writer.writeheader()
while True:
print("On page " + str(page), file=sys.stderr)
url = url_base + str(page)
r = requests.get(url)
soup = BeautifulSoup(r.content, "lxml")
if soup.body.find_all(text="Your search didn’t match any grants. Please try again."):
break
tables = soup.find_all("table", {"class": "grants-search-results-table"})
assert len(tables) == 1, "Error: there must be exactly one grants table on the page"
table = tables[0]
headers_found = list(map(lambda x: x.text.strip(), table.find_all("th")))
headers_expected = ['Grantee', 'Grant Info', 'Year', 'Amount', 'Issue Area',
'Sub-Issue Area']
assert headers_found == headers_expected
# Build header access map so that we can say cells[h[key]] rather than
# cells[idx]
h = {key: idx for idx, key in enumerate(headers_expected)}
rows = table.find("tbody").find_all("tr")
for index, row in enumerate(rows):
# This is a "grant info row", which means it was processed
# already as part of the previous row (see comment below). So
# we can safely skip it.
if row.get("class") == ["grants-search-grant-info-row"]:
continue
cells = row.find_all("td")
grantee = cells[h["Grantee"]].text.strip()
grantee_url = cells[h["Grantee"]].a.get("href").strip()
year = cells[h["Year"]].text.strip()
amount = cells[h["Amount"]].text.strip()
issue_area = cells[h["Issue Area"]].text.strip()
sub_issue_area = cells[h["Sub-Issue Area"]].text.strip()
# The HTML table structure is pretty weird: if a grant has
# extra "grant info", it is shown only as a [+] (to be clicked)
# in the row for the grant, and the info itself is shown as a
# separate row below the grant. Therefore to get the grant
# info, we have to peek at the next row to see if it's a "grant
# info row".
grant_info = ""
try:
if rows[index+1].get("class") == ["grants-search-grant-info-row"]:
grant_info = rows[index+1].text.strip()
except IndexError:
pass
writer.writerow({
"grantee": grantee,
"grantee_url": grantee_url,
"year": year,
"amount": amount,
"issue_area": issue_area,
"sub_issue_area": sub_issue_area,
"grant_info": grant_info,
})
page += 1
if __name__ == "__main__":
main()