-
Notifications
You must be signed in to change notification settings - Fork 1
/
2.5. Download RIN.R
39 lines (28 loc) · 835 Bytes
/
2.5. Download RIN.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
require(data.table)
require(dplyr)
require(RCurl)
require(parallel)
require(httr)
require(rvest)
dwnld.files <- function(i) {
url <- year.masterfile$html_url[i] %>%
read_html() %>%
html_nodes(xpath = '//*[@class="metadata_list"]') %>%
html_text() %>%
try()
return(url[1])
}
for(yr in 1998:2004){
year.masterfile <- yr %>%
paste0("/Users/evolkova/Dropbox/Projects/Govt Agenda/Data/Master and Texts/", ., ".rds") %>%
readRDS(.)
print(yr)
print(dim(year.masterfile))
print(Sys.time())
year.masterfile$Info <- ""
for (i in 1:length(year.masterfile$pdf_url)) {
if (i %% 1000 == 0) print(Sys.time())
try(year.masterfile$Info[i] <- dwnld.files(i))
}
saveRDS(year.masterfile, paste0("/Users/evolkova/Dropbox/Projects/Govt Agenda/Data/Master and Texts/",yr, ".rds"))
}