-
Notifications
You must be signed in to change notification settings - Fork 4
/
metadata_csv_download.py
208 lines (174 loc) · 6.92 KB
/
metadata_csv_download.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
import argparse
import gzip
import logging
import os
import requests
def main(csv_folder, years=None, overwrite_flag=False):
"""Download Landsat Collection 1 bulk metadata CSV GZ files and extract
Parameters
----------
csv_folder : str
Folder path where the Landsat bulk metadata CSV files will be saved.
years : list, optional
Comma separated values or ranges of years to include.
CSV files will only be downloaded if they include data for target years.
The default is None which will download the Landsat 5, 7, & 8 CSV files.
Example: ['1984', '2000-2015']
overwrite_flag : bool, optional
If True, overwrite existing CSV files (the default is False).
Notes
-----
Main Landsat Bulk Metadata Site:
https://landsat.usgs.gov/download-entire-collection-metadata
Example CSV download URL:
https://landsat.usgs.gov/landsat/metadata_service/bulk_metadata_files/
LANDSAT_TM-1980-1989.csv
"""
logging.info('\nDownloading Landsat Bulk Metadata CSV GZ files')
download_url = 'https://landsat.usgs.gov/landsat/metadata_service/bulk_metadata_files'
# The CSV GZ files are 15-20% of the full file size
gz_file_list = [
'LANDSAT_8_C1.csv.gz',
'LANDSAT_ETM_C1.csv.gz',
'LANDSAT_TM_C1.csv.gz',
]
gz_years = {
'LANDSAT_8_C1.csv.gz': set(range(2013, 2099)),
'LANDSAT_ETM_C1.csv.gz': set(range(1999, 2099)),
'LANDSAT_TM_C1.csv.gz': set(range(1984, 2012)),
}
if years is not None:
user_years = set([x for y in years for x in parse_int_set(y)])
else:
user_years = set()
for gz_name in gz_file_list:
logging.info('{}'.format(gz_name))
csv_name = gz_name.replace('.gz', '')
gz_path = os.path.join(csv_folder, gz_name)
csv_path = os.path.join(csv_folder, csv_name)
file_url = '{}/{}'.format(download_url, gz_name)
if user_years and not gz_years[gz_name].intersection(user_years):
logging.info(' No data for target year(s), skipping file')
continue
# Don't redownload unless overwrite or both files don't exist
if os.path.isfile(csv_path):
if overwrite_flag:
logging.debug(' CSV already exists, removing')
os.remove(csv_path)
else:
logging.debug(' CSV already exists, skipping')
continue
elif os.path.isfile(gz_path):
if overwrite_flag:
logging.debug(' GZ already exists, removing')
os.remove(gz_path)
else:
logging.debug(' GZ already exists')
decompress_gz(gz_path, csv_path)
continue
# Trying to catch errors when the bulk metadata site is down
download_file(file_url, gz_path)
# Unpack the CSV gz file
decompress_gz(gz_path, csv_path)
def download_file(file_url, file_path):
""""""
logging.debug(' Downloading file')
logging.debug(' {}'.format(file_url))
try:
r = requests.get(file_url)
with open(file_path, 'wb') as output_f:
for chunk in r.iter_content(chunk_size=128):
output_f.write(chunk)
except Exception as e:
logging.info(' {}\n Try manually checking the bulk metadata '
'website\n'.format(e))
def decompress_gz(input_path, output_path, blocksize=1 << 14):
""""""
logging.debug(' Extracting CSV file')
try:
with gzip.open(input_path, 'rb') as input_f:
with open(output_path, 'wb') as output_f:
while True:
block = input_f.read(blocksize)
if block == '' or block == b'':
break
output_f.write(block)
# with open(output_path, 'wb') as output_f:
# output_f.write(input_f.read())
except Exception as e:
logging.error(' Unhandled Exception: {}'.format(e))
try:
os.remove(output_path)
except:
pass
# def get_csv_path(workspace):
# import Tkinter, tkFileDialog
# root = Tkinter.Tk()
# ini_path = tkFileDialog.askopenfilename(
# initialdir=workspace, parent=root, filetypes=[('XML files', '.xml')],
# title='Select the target XML file')
# root.destroy()
# return ini_path
def is_valid_folder(parser, arg):
if not os.path.isdir(os.path.abspath(arg)):
parser.error('The folder {} does not exist!'.format(arg))
else:
return arg
def parse_int_set(nputstr=""):
"""Return list of numbers given a string of ranges
http://thoughtsbyclayg.blogspot.com/2008/10/parsing-list-of-numbers-in-python.html
"""
selection = set()
invalid = set()
# tokens are comma separated values
tokens = [x.strip() for x in nputstr.split(',')]
for i in tokens:
try:
# typically tokens are plain old integers
selection.add(int(i))
except:
# if not, then it might be a range
try:
token = [int(k.strip()) for k in i.split('-')]
if len(token) > 1:
token.sort()
# we have items separated by a dash
# try to build a valid range
first = token[0]
last = token[len(token) - 1]
for x in range(first, last + 1):
selection.add(x)
except:
# not an int and not a range...
invalid.add(i)
# Report invalid tokens before returning valid selection
# print "Invalid set: " + str(invalid)
return selection
def arg_parse():
""""""
parser = argparse.ArgumentParser(
description='Download Landsat Collection 1 bulk metadata CSV files',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--csv', default=os.getcwd(), metavar='FOLDER',
type=lambda x: is_valid_folder(parser, x),
help='Landsat bulk metadata CSV folder')
parser.add_argument(
'-y', '--years', default=None, nargs='+',
help='Space/comma separated list of years or year ranges to keep '
'(i.e. "--years 1984 2000-2015")')
parser.add_argument(
'-o', '--overwrite', default=False, action='store_true',
help='Force overwrite of existing files')
parser.add_argument(
'-d', '--debug', default=logging.INFO, const=logging.DEBUG,
help='Debug level logging', action='store_const', dest='loglevel')
args = parser.parse_args()
# Convert relative paths to absolute paths
if args.csv and os.path.isdir(os.path.abspath(args.csv)):
args.csv = os.path.abspath(args.csv)
return args
if __name__ == '__main__':
args = arg_parse()
logging.basicConfig(level=args.loglevel, format='%(message)s')
main(csv_folder=args.csv, years=args.years, overwrite_flag=args.overwrite)