forked from ocaml/ocaml
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
329 lines (290 loc) · 11.2 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
#**************************************************************************
#* *
#* OCaml *
#* *
#* Xavier Clerc, SED, INRIA Rocquencourt *
#* *
#* Copyright 2010 Institut National de Recherche en Informatique et *
#* en Automatique. *
#* *
#* All rights reserved. This file is distributed under the terms of *
#* the GNU Lesser General Public License version 2.1, with the *
#* special exception on linking described in the file LICENSE. *
#* *
#**************************************************************************
.NOTPARALLEL:
BASEDIR := $(shell pwd)
ifneq "$(words |$(BASEDIR)|)" "1"
$(error The Testsuite does not handle spaces\
in the ocaml working directory path: $(BASEDIR))
endif
ROOTDIR = ..
include $(ROOTDIR)/Makefile.common
ifeq "$(UNIX_OR_WIN32)" "win32"
CYGPATH=cygpath -m
# Ensure that the test runners definitely use Cygwin's sort and not the
# Windows sort command
SORT=/usr/bin/sort
else
CYGPATH=echo
SORT=sort
endif
BASEDIR_HOST := $(shell $(CYGPATH) "$(BASEDIR)")
ROOTDIR_HOST := $(BASEDIR_HOST)/$(ROOTDIR)
OCAMLTESTDIR = $(BASEDIR_HOST)/$(DIR)/_ocamltest
failstamp := failure.stamp
TESTLOG ?= _log
ocamltest_directory := ../ocamltest
ocamltest_program := $(or \
$(wildcard $(ocamltest_directory)/ocamltest.opt$(EXE)),\
$(wildcard $(ocamltest_directory)/ocamltest$(EXE)))
ifeq "$(UNIX_OR_WIN32)" "unix"
ifeq "$(SYSTEM)" "cygwin"
find := /usr/bin/find
else # Non-cygwin Unix
find := find
endif
# $(IFS_LINE) for Unix simply clears IFS, so entire lines are read
IFS_LINE = IFS=''
else # Windows
find := /usr/bin/find
# On Windows, ocamltest will produce Windows line-endings (\r\n) and the final
# \r is kept by the shell. This can either be stripped with tr -d '\r' but we
# can avoid the additional process by instead setting IFS in the while loop to
# be the CR character (i.e. treat \r as a field delimiter).
# The dance with $(CR_CHAR) is because POSIX doesn't provide a way to write \r
# in a string.
export CR_CHAR := $(shell printf "\r")
IFS_LINE = IFS="$$CR_CHAR"
endif
ifeq "$(ocamltest_program)" ""
ocamltest = $(error ocamltest not found in $(ocamltest_directory))
else
ocamltest := SORT=$(SORT) MAKE=$(MAKE) $(ocamltest_program)
endif
# PROMOTE is only meant to be used internally in recursive calls;
# users should call the 'promote' target explicitly.
PROMOTE =
ifeq "$(PROMOTE)" ""
OCAMLTEST_PROMOTE_FLAG =
else
OCAMLTEST_PROMOTE_FLAG = -promote
endif
# KEEP_TEST_DIR_ON_SUCCESS should be set by the user (to a non-empty value)
# if they want to pass the -keep-test-dir-on-success option to ocamltest,
# to preserve test data of successful tests.
# KEEP is provided as a rather easier to remmber alias of
# KEEP_TEST_DIR_ON_SUCCESS, but to prevent the risk of naming conflict it's only
# recognised when used as make -C testsuite KEEP=1 ...
ifeq "$(origin KEEP)" "command line"
KEEP_TEST_DIR_ON_SUCCESS ?= $(KEEP)
else
KEEP_TEST_DIR_ON_SUCCESS ?=
endif
ifeq "$(KEEP_TEST_DIR_ON_SUCCESS)" ""
OCAMLTEST_KEEP_TEST_DIR_ON_SUCCESS_FLAG =
else
OCAMLTEST_KEEP_TEST_DIR_ON_SUCCESS_FLAG = -keep-test-dir-on-success
endif
TIMEOUT ?= 600 # 10 minutes
# SHOW_TIMINGS should be set by the user (to a non-empty value) if they want
# the timings for each test file to be included in the log
SHOW_TIMINGS ?=
OCAMLTEST_SHOW_TIMINGS_FLAG =
ifneq "$(SHOW_TIMINGS)" ""
ifeq "$(lib_unix)" "true"
OCAMLTEST_SHOW_TIMINGS_FLAG = -show-timings
endif
endif
OCAMLTESTFLAGS = \
-timeout $(TIMEOUT) \
$(OCAMLTEST_PROMOTE_FLAG) \
$(OCAMLTEST_KEEP_TEST_DIR_ON_SUCCESS_FLAG) \
$(OCAMLTEST_SHOW_TIMINGS_FLAG)
# Make sure USE_RUNTIME is defined
USE_RUNTIME ?=
ifneq ($(USE_RUNTIME),)
# Check USE_RUNTIME value
ifeq ($(findstring $(USE_RUNTIME),d i),)
$(error If set, USE_RUNTIME must be equal to "d" (debug runtime) \
or "i" (instrumented runtime))
endif
# When using the debug or instrumented runtime,
# set the runtime's verbosity to 0 by default
export OCAMLRUNPARAM?=v=0
endif
.PHONY: default
default:
@echo "Available targets:"
@echo " all launch all tests"
@echo " all-foo launch all tests beginning with foo"
@echo " parallel launch all tests using GNU parallel"
@echo " parallel-foo launch all tests beginning with foo using \
GNU parallel"
@echo " one (TEST|DIR|LIST)=x launch the specified tests ..."
@echo " promote (TEST|DIR|LIST)=x promote the output for the specified \
tests ..."
@echo " ... TEST=f ... the single test in the file f"
@echo " ... DIR=d ... the tests located in the directory d"
@echo " ... LIST=f ... the tests listed in the file f (one \
per line)"
@echo " clean delete generated files"
@echo " report print the report for the last execution"
@echo
@echo "By default, artefacts from tests which pass are not kept, but this can"
@echo "be changed by adding KEEP=1 to the make command line or by setting the"
@echo "KEEP_TEST_DIR_ON_SUCCESS environment variable to a non-empty value."
@echo
@echo "Set the environment variable USE_RUNTIME to \"d\" or \"i\" to run"
@echo "the tests with the debug or the instrumented runtime."
.PHONY: all
all:
@$(MAKE) --no-print-directory new-without-report
@$(MAKE) --no-print-directory report
.PHONY: new-without-report
new-without-report:
@rm -f $(failstamp)
@($(ocamltest) -find-test-dirs tests | while $(IFS_LINE) read -r dir; do \
echo Running tests from \'$$dir\' ... ; \
$(MAKE) exec-ocamltest DIR="$$dir" \
OCAMLTESTENV=""; \
done || echo outer loop >> $(failstamp)) 2>&1 | tee $(TESTLOG)
@$(MAKE) check-failstamp
.PHONY: check-failstamp
check-failstamp:
@if [ -f $(failstamp) ]; then \
echo 'Unexpected error in the test infrastructure:'; \
cat $(failstamp); \
rm $(failstamp); \
exit 1; \
fi
.PHONY: all-%
all-%:
@for dir in tests/$**; do \
$(MAKE) --no-print-directory exec-one DIR=$$dir; \
done 2>&1 | tee $(TESTLOG)
@$(MAKE) report
# The targets below use GNU parallel to parallelize tests
# 'make all' and 'make parallel' should be equivalent
#
# parallel uses specific logic to make sure the output of the commands
# run in parallel are not mangled. By default, it will reproduce
# the output of each completed command atomically, in order of completion.
#
# With the --keep-order option, we ask it to save the completed output
# and replay them in invocation order instead. In theory this costs
# a tiny bit of performance, but I could not measure any difference.
# In theory again, the reporting logic works fine with test outputs
# coming in in arbitrary order (so we should not need --keep-order),
# but keeping the output deterministic is guaranteed to make
# someone's life easier at least once in the future.
#
# Finally, note that the command we run has a 2>&1 redirection, as
# in the other make targets. If we removed the quoting around
# "$(MAKE) ... 2>&1", the rediction would apply to the complete output
# of parallel, and have a slightly different behavior: by default parallel
# cleanly separates the stdout and stderr output of each completed command,
# printing stderr first then stdout second (for each command).
# I chose to keep the previous behavior exactly unchanged,
# but the demangling separation is arguably nicer behavior that we might
# want to implement at the exec-one level to also have it in the 'all' target.
# If make has been invoked with "-j n", pass this on to GNU parallel. parallel
# does not support -j without an argument, hence the double-filter. Note that
# GNU make normalises -j in $(MAKEFLAGS) so it will either be -j alone or -jn
# (i.e. with no space).
J_ARGUMENT = $(filter-out -j,$(filter -j%,$(MAKEFLAGS)))
.PHONY: parallel-%
parallel-%:
@echo | parallel >/dev/null 2>/dev/null \
|| (echo "Unable to run the GNU parallel tool;";\
echo "You should install it before using the parallel* targets.";\
exit 1)
@echo | parallel --gnu --no-notice >/dev/null 2>/dev/null \
|| (echo "Your 'parallel' tool seems incompatible with GNU parallel.";\
echo "This target requires GNU parallel.";\
exit 1)
@for dir in tests/$**; do echo $$dir; done \
| parallel --gnu --no-notice --keep-order $(J_ARGUMENT) \
"$(MAKE) --no-print-directory exec-one DIR={} 2>&1" \
| tee $(TESTLOG)
@$(MAKE) report
.PHONY: parallel
parallel: parallel-*
.PHONY: list
list:
@if [ -z "$(FILE)" ]; \
then echo "No value set for variable 'FILE'."; \
exit 1; \
fi
@$(MAKE) --no-print-directory one LIST="$(FILE)"
.PHONY: one
one:
@case "$(words $(DIR) $(LIST) $(TEST))" in \
0) echo 'No value set for variable DIR, LIST or TEST'>&2; exit 1;; \
1) exit 0;; \
*) echo 'Please specify exactly one of DIR, LIST or TEST'>&2; exit 1;; \
esac
@if [ -n '$(DIR)' ] && [ ! -d '$(DIR)' ]; then \
echo "Directory '$(DIR)' does not exist."; exit 1; \
fi
@if [ -n '$(TEST)' ] && [ ! -e '$(TEST)' ]; then \
echo "Test '$(TEST)' does not exist."; exit 1; \
fi
@if [ -n '$(LIST)' ] && [ ! -e '$(LIST)' ]; then \
echo "File '$(LIST)' does not exist."; exit 1; \
fi
@if [ -n '$(DIR)' ] ; then \
$(MAKE) --no-print-directory exec-one DIR=$(DIR) \
2>&1 | tee $(TESTLOG).one ; \
fi
@if [ -n '$(TEST)' ] ; then \
TERM=dumb $(OCAMLTESTENV) $(ocamltest) $(OCAMLTESTFLAGS) $(TEST) \
2>&1 | tee $(TESTLOG).one; fi
@if [ -n '$(LIST)' ] ; then \
while IFS='' read -r LINE; do \
$(MAKE) --no-print-directory exec-one DIR=$$LINE ; \
done < $$LIST 2>&1 | tee $(TESTLOG).one ; \
fi
@$(MAKE) check-failstamp
@$(MAKE) TESTLOG=$(TESTLOG).one report
.PHONY: exec-one
exec-one:
@$(ocamltest) -find-test-dirs $(DIR) | while $(IFS_LINE) read -r dir; do \
echo "Running tests from '$$dir' ..."; \
$(MAKE) exec-ocamltest DIR="$$dir" \
OCAMLTESTENV="OCAMLTESTDIR=$(OCAMLTESTDIR)"; \
done
.PHONY: exec-ocamltest
exec-ocamltest:
@if [ -z "$(DIR)" ]; then exit 1; fi
@if [ ! -d "$(DIR)" ]; then exit 1; fi
@($(ocamltest) -list-tests $(DIR) | while $(IFS_LINE) read -r testfile; do \
TERM=dumb $(OCAMLTESTENV) \
$(ocamltest) $(OCAMLTESTFLAGS) "$(DIR)/$$testfile" || \
echo " ... testing '$$testfile' => unexpected error"; \
done) || echo directory "$(DIR)" >>$(failstamp)
.PHONY: clean-one
clean-one:
@if [ ! -f $(DIR)/Makefile ]; then \
for dir in $(DIR)/*; do \
if [ -d $$dir ]; then \
$(MAKE) clean-one DIR=$$dir; \
fi; \
done; \
else \
cd $(DIR) && $(MAKE) TERM=dumb BASEDIR=$(BASEDIR) clean; \
fi
.PHONY: promote
promote:
@$(MAKE) one PROMOTE=true
.PHONY: clean
clean:
find . -name '*_ocamltest*' | xargs rm -rf
rm -f $(failstamp)
.PHONY: distclean
distclean: clean
rm -f _log*
.PHONY: report
report:
@if [ ! -f $(TESTLOG) ]; then echo "No $(TESTLOG) file."; exit 1; fi
@$(AWK) -f ./summarize.awk < $(TESTLOG)