-
Notifications
You must be signed in to change notification settings - Fork 6
/
iem_scraper_example.r
78 lines (65 loc) · 2.75 KB
/
iem_scraper_example.r
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# DEPRECATED
# Recommend using the 'riem' package instead
#example script that scrapes data from the IEM ASOS download service
#adapted from python code here: https://github.com/akrherz/iem/blob/master/scripts/asos/iem_scraper_example.py
#modified to download data for a single ASOS station specified below
#
###USER INPUTS###
iem.wd <- "C:/Data" #download location
date1 <- ISOdate(2005,1,1) #start date in year, month, day format
date2 <- ISOdate(2017,6,26) #end date in year, month, day format
user.network <- c("ASOS")
user.state <- c("IA") #state
user.faaid <- c("SUX") #site FAA identifier - leave empty and a list will print for your reference
#################
library(jsonlite)
library(RCurl)
library(lubridate)
library(stringr)
#create subdirectories
download.wd <- str_c(iem.wd, user.network, user.faaid, sep="/")
if(user.faaid != "") {dir.create(download.wd, recursive=T)}
setwd(download.wd)
service <- "https://mesonet.agron.iastate.edu/cgi-bin/request/asos.py?"
service <- str_c(service, "data=all&tz=Etc/UTC&format=comma&latlon=yes&", sep="")
service <- str_c(service, "year1=", year(date1), "&month1=", month(date1), "&day1=", mday(date1), "&", sep="")
service <- str_c(service, "year2=", year(date2), "&month2=", month(date2), "&day2=", mday(date2), "&", sep="")
states <- c("AK AL AR AZ CA CO CT DE FL GA ")
states <- str_c(states,"HI IA ID IL IN KS KY LA MA MD ")
states <- str_c(states,"ME MI MN MO MS MT NC ND NE NH ")
states <- str_c(states,"NJ NM NV NY OH OK OR PA RI SC ")
states <- str_c(states,"SD TN TX UT VA VT WA WI WV WY")
states <- unlist(strsplit(states, " "))
networks <- "AWOS"
for (i in 1:length(states)) {
networks[i+1] <- str_c(states[i], "_ASOS", sep="")
}
if (user.network == "ASOS"){
networks <- networks[which(networks %in% str_c(user.state, "_", user.network))]
} else {
networks <- subset(networks %in% str_c(user.network))
}
for (network in networks){
#get metadata
uri <- str_c("https://mesonet.agron.iastate.edu/geojson/network/", network, ".geojson", sep="")
data <- url(uri)
jdict <- fromJSON(data)
for (i in 1:nrow(jdict$features)){
site <- jdict$features[i,]
faaid <- site$properties$sid
if (faaid == user.faaid) {
sitename <- site$properties$sname
uri <- str_c(service, "station=", faaid)
print(str_c("Network:", network, "Downloading:", sitename, faaid, sep=" "))
data <- url(uri)
#print(data) #uncomment to print metadata
datestring1 <- format(date1, "%Y%m%d")
datestring2 <- format(date2, "%Y%m%d")
outfn <- str_c(network, "_", faaid, "_", datestring1, "_to_", datestring2, sep="")
download.file(uri, str_c(outfn, ".txt"), "auto")
}
if (user.faaid == "" & i == 1) {
print(data.frame(jdict$features$properties[c("sname", "sid")]))
}
}
}