-
Notifications
You must be signed in to change notification settings - Fork 1
/
Makefile
364 lines (305 loc) · 15 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
.ONE_SHELL:
SHELL := /bin/bash
.EXPORT_ALL_VARIABLES:
.SILENT:
.PHONY:
ifndef SUNDIAL_BASE_PATH
SUNDIAL_BASE_PATH := $(shell pwd -P)
endif
ifndef SUNDIAL_ENV_NAME
SUNDIAL_ENV_NAME := sundial
endif
ifndef SUNDIAL_PROCESSING
SUNDIAL_PROCESSING := hpc
endif
ifndef SUNDIAL_NODES
SUNDIAL_NODES :=
endif
ifndef SUNDIAL_PACKAGE_FORMAT
SUNDIAL_PACKAGE_FORMAT := tar
endif
SUNDIAL_EXPERIMENT_BASE_NAME := $(SUNDIAL_EXPERIMENT_PREFIX)_$(SUNDIAL_SAMPLE_NAME)
ifndef SUNDIAL_EXPERIMENT_SUFFIX
SUNDIAL_EXPERIMENT_FULL_NAME := $(SUNDIAL_EXPERIMENT_BASE_NAME)
else
SUNDIAL_EXPERIMENT_FULL_NAME := $(SUNDIAL_EXPERIMENT_BASE_NAME)_$(SUNDIAL_EXPERIMENT_SUFFIX)
endif
default:
echo "Welcome to Sundial!"
echo
echo " Methods:"
echo
echo " setup_env: Sets up Sundial directories and environment. The existing environment will be overwritten."
echo " setup: Sets up Sundial experiment directories and config files using defaults found in src/settings.py."
echo " clink: Copies Sundial experiment configs from source experiment to new experiment and links sample configs and data. Requires SSOURCE variable."
echo
echo " sample: Generates chip sample polygons using Google Earth Engine and provided shapefile."
echo " annotate: Collects image annotations for experiment using sample polygons."
echo " download: Downloads image chips for experiment using sample polygons."
echo " index: Creates indicies for training from chip and anno data."
echo " calculate: Calculate means and stds for experiment sample and verify simple sums"
echo
echo " fit: Train model using subset of data from sample."
echo " validate: Validate model using subset of data from sample."
echo " test: Test model using subset of data from sample."
echo " predict: Predict an image using subset of data from sample."
echo " package: Convert predictions to GeoTIFF if not already in format and compress for download."
echo
echo " status: Check status of all jobs for user."
echo " vars: Print all Sundial variables."
echo " (method)_err: Print ERRORs and CRITICALs in log file and print stderr from file on HPC."
echo
echo " clean: Removes all logs, checkpoints, and predictions for experiment."
echo " clean_outs: Removes all stdouts for experiment."
echo " clean_logs: Removes all logs for experiment."
echo " clean_samp: Removes all sample data for experiment."
echo " clean_dnld: Removes all image chip data for experiment."
echo " clean_anno: Removes all annotation data for experiment."
echo " clean_ckpt: Removes all checkpoints for experiment."
echo " clean_pred: Removes all predictions for experiment."
echo " clean_expt: Removes all logs, sample data, checkpoints, and predictions for experiment."
echo " clean_nuke: Removes all data for experiment including configs."
echo
echo " Variables:"
echo " These may be set at submake or as environment variables."
echo
echo " SUNDIAL_BASE_PATH: Base path for Sundial scripts. Default: 'shell pwd' of makefile"
echo " SUNDIAL_SAMPLE_NAME: Sample name. REQUIRED"
echo " SUNDIAL_EXPERIMENT_PREFIX: Sundial experiment name prefix used for naming. Default: ''"
echo " SUNDIAL_EXPERIMENT_SUFFIX: Experiment suffix used only for logging."
echo " SUNDIAL_ENV_NAME: Sundial environment name. Default: 'sundial'"
echo " SUNDIAL_PROCESSING: Sundial processing method. Default: 'hpc'"
echo " SUNDIAL_NODES: Node within hpc to run job. Default: any node"
echo " SUNDIAL_GPU_PARTITION: Partition within hpc to run gpu based jobs."
echo " SUNDIAL_CPU_PARTITION: Partition within hpc to run cpu based jobs."
echo " SUNDIAL_PACKAGE_FORMAT: Format to package predictions. Default: 'tar'"
echo
setup_env:
echo "Setting up directories for $(SUNDIAL_BASE_PATH) and conda environment "$(SUNDIAL_ENV_NAME)". The existing environment will be overwritten...";
mkdir -p $(SUNDIAL_BASE_PATH)/logs;
mkdir -p $(SUNDIAL_BASE_PATH)/samples;
mkdir -p $(SUNDIAL_BASE_PATH)/checkpoints;
mkdir -p $(SUNDIAL_BASE_PATH)/predictions;
mkdir -p $(SUNDIAL_BASE_PATH)/configs;
mkdir -p $(SUNDIAL_BASE_PATH)/models/backbones;
mkdir -p $(SUNDIAL_BASE_PATH)/models/heads;
conda env create -f $(SUNDIAL_BASE_PATH)/environment.yml -n $(SUNDIAL_ENV_NAME) -p $(CONDA_PREFIX) -y --force;
conda activate $(SUNDIAL_ENV_NAME);
echo "Sundial setup complete!";
setup: _experiment_name_check
echo "Setting up Sundial experiment for $(SUNDIAL_EXPERIMENT_BASE_NAME)...";
mkdir -p $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME);
mkdir -p $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME);
mkdir -p $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME);
mkdir -p $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME);
if [[ -d $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME) ]]; then \
echo "WARNING: Configs directory found. To restart experiment from scratch, use 'make clean_nuke' then 'make setup'..."; \
else \
echo "Generateing Sundial config files for experiment with values from settings.py..."; \
python $(SUNDIAL_BASE_PATH)/src/settings.py; \
fi;
clink: _experiment_name_check
if [[ -z "$(SSOURCE)" ]]; then \
echo "Please provide a source experiment to copy from in variable "SSOURCE" (eg 'make clink SSOURCE=14k')..."; \
exit 1; \
fi; \
source_exp=$(SSOURCE)_$(SUNDIAL_SAMPLE_NAME); \
echo "Copying Sundial experiment $$source_exp configs to $(SUNDIAL_EXPERIMENT_BASE_NAME)..."; \
cp -r $(SUNDIAL_BASE_PATH)/configs/$$source_exp $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME); \
rm -rfv $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/sample.yaml; \
echo "WARNING: This will create soft links to the source sammple data and sample config file."; \
ln -s $(SUNDIAL_BASE_PATH)/samples/$$source_exp $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME); \
ln -s $(SUNDIAL_BASE_PATH)/configs/$$source_exp/sample.yaml $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/sample.yaml;
vars: _experiment_name_check
echo "SUNDIAL_BASE_PATH: $(SUNDIAL_BASE_PATH)";
echo "SUNDIAL_SAMPLE_NAME: $(SUNDIAL_SAMPLE_NAME)";
echo "SUNDIAL_EXPERIMENT_PREFIX: $(SUNDIAL_EXPERIMENT_PREFIX)";
echo "SUNDIAL_EXPERIMENT_SUFFIX: $(SUNDIAL_EXPERIMENT_SUFFIX)";
echo "SUNDIAL_EXPERIMENT_BASE_NAME: $(SUNDIAL_EXPERIMENT_BASE_NAME)";
echo "SUNDIAL_EXPERIMENT_FULL_NAME: $(SUNDIAL_EXPERIMENT_FULL_NAME)";
echo "SUNDIAL_ENV_NAME: $(SUNDIAL_ENV_NAME)";
echo "SUNDIAL_PROCESSING: $(SUNDIAL_PROCESSING)";
echo "SUNDIAL_NODES: $(SUNDIAL_NODES)";
echo "SUNDIAL_GPU_PARTITION: $(SUNDIAL_GPU_PARTITION)";
echo "SUNDIAL_CPU_PARTITION: $(SUNDIAL_CPU_PARTITION)";
echo "SUNDIAL_PACKAGE_FORMAT: $(SUNDIAL_PACKAGE_FORMAT)";
rename: _experiment_name_check
if [[ -z "$(SNEW)" ]]; then \
echo "Please provide a new experiment name in variable "SNEW" (eg 'make rename SNEW=15k')..."; \
exit 1; \
fi; \
new_exp=$(SNEW)_$(SUNDIAL_SAMPLE_NAME); \
echo "Renaming Sundial experiment $(SUNDIAL_EXPERIMENT_BASE_NAME) to $$new_exp..."; \
mv $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME) $(SUNDIAL_BASE_PATH)/configs/$$new_exp; \
mv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME) $(SUNDIAL_BASE_PATH)/samples/$$new_exp; \
mv $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME) $(SUNDIAL_BASE_PATH)/checkpoints/$$new_exp; \
mv $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME) $(SUNDIAL_BASE_PATH)/predictions/$$new_exp; \
mv $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME) $(SUNDIAL_BASE_PATH)/logs/$$new_exp;
sample: _sample
echo "Processing polygon sample from $(SUNDIAL_SAMPLE_NAME) shapefile for $(SUNDIAL_EXPERIMENT_PREFIX). Uno momento...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
annotate: _annotate
echo "Collecting image annotations for $(SUNDIAL_EXPERIMENT_FULL_NAME). This might take a sec...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
download: _download
echo "Downloading image chips for $(SUNDIAL_EXPERIMENT_FULL_NAME). Sit tight...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
calculate: _calculate
echo "Calculating metrics for $(SUNDIAL_EXPERIMENT_FULL_NAME)...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
index: _index
echo "Creating indicies for $(SUNDIAL_EXPERIMENT_FULL_NAME)...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
fit: _fit
echo "Training model for $(SUNDIAL_EXPERIMENT_FULL_NAME)... This may take a while...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_GPU_PARTITION))
$(MAKE) -s _run
validate: _validate
echo "Validating model for $(SUNDIAL_EXPERIMENT_FULL_NAME)... Go ahead and hold your breath...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_GPU_PARTITION))
$(MAKE) -s _run
test: _test
echo "Testing model for $(SUNDIAL_EXPERIMENT_FULL_NAME)... Use 'make test_err' to check for critical errors...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_GPU_PARTITION))
$(MAKE) -s _run
predict: _predict
echo "Making predictions for $(SUNDIAL_EXPERIMENT_FULL_NAME)... Lets see if this works!";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_GPU_PARTITION))
$(MAKE) -s _run
package: _package
echo "Packaging predictions for $(SUNDIAL_EXPERIMENT_FULL_NAME)...";
$(eval export SUNDIAL_PARTITION=$(SUNDIAL_CPU_PARTITION))
$(MAKE) -s _run
status: _hpc_check
squeue -u $(USER) --format="%.18i %.9P %.40j %.8u %.8T %.10M %.9l %.6D %R";
sample_err: _sample
$(MAKE) -s _read_err;
annotate_err: _annotate
$(MAKE) -s _read_err;
download_err: _download
$(MAKE) -s _read_err;
calculate_err: _calculate
$(MAKE) -s _read_err;
index_err: _index
$(MAKE) -s _read_err;
fit_err: _fit
$(MAKE) -s _read_err;
validate_err: _validate
$(MAKE) -s _read_err;
test_err: _test
$(MAKE) -s _read_err;
predict_err: _predict
$(MAKE) -s _read_err;
package_err: _package
$(MAKE) -s _read_err;
clean: _experiment_name_check
echo "Cleaning up logs, checkpoints, and predictions for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
rm -rfv $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
rm -rfv $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_logs: _experiment_name_check
echo "Cleaning up logs for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_samp: _experiment_name_check
echo "Cleaning up all sample data for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_dnld: _experiment_name_check
echo "Cleaning up chip data for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME)/chip_data*;
clean_anno: _experiment_name_check
echo "Cleaning up annotation data for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME)/anno_data*;
clean_ckpt: _experiment_name_check
echo "Cleaning up checkpoints generated for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_pred: _experiment_name_check
echo "Cleaning up predictions generated for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_expt: _experiment_name_check
echo "Cleaning up logs, sample data, checkpoints, and predictions for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
rm -rfv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
rm -rfv $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
rm -rfv $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME)/*;
clean_nuke: _experiment_name_check
echo "Cleaning up all data for $(SUNDIAL_EXPERIMENT_BASE_NAME).";
rm -rfv $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME);
rm -rfv $(SUNDIAL_BASE_PATH)/samples/$(SUNDIAL_EXPERIMENT_BASE_NAME);
rm -rfv $(SUNDIAL_BASE_PATH)/checkpoints/$(SUNDIAL_EXPERIMENT_BASE_NAME);
rm -rfv $(SUNDIAL_BASE_PATH)/predictions/$(SUNDIAL_EXPERIMENT_BASE_NAME);
rm -rfv $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME);
_config_dir_check:
if [[ ! -d $(SUNDIAL_BASE_PATH)/configs/$(SUNDIAL_EXPERIMENT_BASE_NAME) ]]; then \
echo "No configs found. Please run make setup first..."; \
exit 1; \
fi;
_experiment_name_check:
if [[ -z "$(SUNDIAL_SAMPLE_NAME)" ]]; then \
echo "Please provide sample name. (eg 'make setup SUNDIAL_SAMPLE_NAME=shapefile_name') "; \
exit 1; \
fi; \
if [[ -z "$(SUNDIAL_EXPERIMENT_PREFIX)" ]]; then \
echo "Please provide a Sundial experiment prefix. (eg 'make setup SUNDIAL_EXPERIMENT_PREFIX=14k')..."; \
exit 1; \
fi;
_hpc_check:
if [[ "$(SUNDIAL_PROCESSING)" != hpc ]]; then \
echo "This method is only available on an HPC with slurm..."; \
exit 1; \
fi;
_sample:
$(eval export SUNDIAL_METHOD=sample)
_annotate:
$(eval export SUNDIAL_METHOD=annotate)
_download:
$(eval export SUNDIAL_METHOD=download)
_calculate:
$(eval export SUNDIAL_METHOD=calculate)
_index:
$(eval export SUNDIAL_METHOD=index)
_fit:
$(eval export SUNDIAL_METHOD=fit)
_validate:
$(eval export SUNDIAL_METHOD=validate)
_test:
$(eval export SUNDIAL_METHOD=test)
_predict:
$(eval export SUNDIAL_METHOD=predict)
_package:
$(eval export SUNDIAL_METHOD=package)
_run: _experiment_name_check
if [[ "$(SUNDIAL_PROCESSING)" == hpc ]]; then \
job_name=$(SUNDIAL_METHOD)_$(SUNDIAL_EXPERIMENT_FULL_NAME); \
if [[ -z "$(SUNDIAL_NODES)" ]]; then \
nodes=$(shell sinfo -h -p $(SUNDIAL_PARTITION) -o "%N"); \
else \
nodes=$(SUNDIAL_NODES); \
fi; \
echo "Running on HPC..."; \
sbatch \
--job-name=$$job_name \
--output=$(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).o \
--error=$(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).e \
--chdir=$(SUNDIAL_BASE_PATH) \
--partition=$(SUNDIAL_PARTITION) \
--nodelist=$$nodes \
--export=ALL \
$(SUNDIAL_BASE_PATH)/slurm/$(SUNDIAL_METHOD).slurm; \
else \
echo "Running on local machine..."; \
python $(SUNDIAL_BASE_PATH)/src/runner.py; \
fi;
_read_err: _experiment_name_check
if [[ -f $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).e ]]; then \
cat $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).e; \
fi; \
if [[ -f $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).log ]]; then \
cat $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).log | grep ERROR; \
cat $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).log | grep CRITICAL; \
tail $(SUNDIAL_BASE_PATH)/logs/$(SUNDIAL_EXPERIMENT_BASE_NAME)/$(SUNDIAL_METHOD).log; \
fi;