-
Notifications
You must be signed in to change notification settings - Fork 1
/
azure_task.py
88 lines (62 loc) · 3.16 KB
/
azure_task.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
import os.path
import datetime
import numpy
import argparse
import logging
import timeit
from astropy.utils.iers import iers
from common import helpers
from db import azure
from lib import calc_pos, crutils, crstats
if __name__ == '__main__':
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
parser = argparse.ArgumentParser()
parser.add_argument('--filepath', required=True,
help='The path to the text file to process. The path'
'may include a compute node\'s environment'
'variables, such as'
'$AZ_BATCH_NODE_SHARED_DIR/filename.txt')
parser.add_argument('--storageaccount', required=True,
help='The name the Azure Storage account that owns the'
'blob storage container to which to upload'
'results.')
parser.add_argument('--storagecontainer', required=True,
help='The Azure Blob storage container to which to'
'upload results.')
parser.add_argument('--sastoken', required=True,
help='The SAS token providing write access to the'
'Storage container.')
args = parser.parse_args()
input_file = os.path.realpath(args.filepath)
output_file = '{}_OUTPUT{}.txt'.format(
os.path.splitext(args.filepath)[0],
os.path.splitext(args.filepath)[1])
try:
logging.basicConfig(filename=output_file, filemode='w+', level=logging.INFO)
logging.info('Started processing at {} '.format(datetime.datetime.now().replace(microsecond=0)))
tic = timeit.default_timer()
data_ext = helpers.extension_from_filename(input_file)
pos_ext = helpers.pos_ext_from_data_ext(data_ext)
pos_input_file = input_file.replace(data_ext, pos_ext)
img = crutils.load(input_file, pos_input_file)
logging.info('Loaded image {} for instrument {}'.format(img.file_name, img.instrument.NAME))
_, cr_pixels = crutils.clean_cr(img.data, mask=None, iterations=2, gain=img.gain)
logging.info('Got {} cr pixels'.format(numpy.sum(cr_pixels)))
crs, normalized_img = crutils.reduce_cr(cr_pixels, img.exposition_duration)
logging.info('Got {} cosmic rays'.format(len(crs)))
long, lat, height = calc_pos.calc_pos(img)
logging.info('Output: CR {}, Lat {}, Long {}, Height {}'.format(len(crs), long, lat, height))
stats = crstats.calculate(crs, normalized_img)
logging.info('Stats: {}'.format(stats))
toc = timeit.default_timer()
logging.info('Finished processing at {} '.format(datetime.datetime.now().replace(microsecond=0)))
azure.save_image(img, crs, lat, long, height, stats, toc - tic)
logging.info('Done with everything at {} '.format(datetime.datetime.now().replace(microsecond=0)))
azure.save_output(args, output_file)
except Exception as e:
logging.exception('Unexpected error')
azure.save_output(args, output_file)
raise