-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile.toolkit
229 lines (190 loc) · 8.69 KB
/
Makefile.toolkit
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
# vim:noet:ts=3:nowrap
# @file Makefile.toolkit
# @brief Master toolkit with common targets and recipes for all modules.
#
# @author Samuel Larkin
#
# Technologies langagieres interactives / Interactive Language Technologies
# Inst. de technologie de l'information / Institute for Information Technology
# Conseil national de recherches Canada / National Research Council Canada
# Copyright 2010, Sa Majeste la Reine du Chef du Canada /
# Copyright 2010, Her Majesty in Right of Canada
# Make sure plugins in the plugins directory will be used by the framework.
ifeq (${MAKELEVEL},0)
export PATH := $(abspath $(dir $(lastword ${MAKEFILE_LIST}))plugins):${PATH}
endif
################################################################################
# HELPERS
########################################
# Helper function to create reference files based on a stem.
# Usage: $(call CREATE_REFERENCE_NAMES, ${TUNE_RESCORE}[, EXTENSION])
CREATE_REFERENCE_NAMES = $(strip $(foreach c,$1,$(addprefix $c_${TGT_LANG},$(or $(addsuffix $(or $(strip $2),${LANGX}),$(strip ${REFERENCE_INDICES})),$(or $(strip $2),${LANGX})))))
# Helper function to adjust relative paths. A trailing "/" is added to the prefix if missing.
# Usage: $(call FIX_RELATIVE_PATH, PREFIX_TO_ADD, PATH_TO_FIX)
# e.g. pretrained_lm_paths = $(foreach f, ${LM_PRETRAINED_LMS}, $(call FIX_RELATIVE_PATH, ../.., $f))
FIX_RELATIVE_PATH = $(strip $(or $(filter /% ~%, $(2)), $(addprefix $(1:/=)$(if $(strip $(1)),/), $(2))))
# Helper recipe to list all corpora in use.
define HELP_LIST_ALL_CORPORA
@echo "Your corpora are:"
$(if $(strip ${TRAIN_LM}),@echo " train LMs (non-mix): ${TRAIN_LM}")
$(if $(strip ${MIXLM}),@echo " train LMs for MixLM: ${MIXLM}")
$(if $(strip ${TUNE_LM}),@echo " tune LM parameters: ${TUNE_LM}")
$(if $(strip ${TRAIN_COARSELM}),@echo " train coarse LMs: ${TRAIN_COARSELM}")
$(if $(strip ${TUNE_COARSELM}),@echo " tune coarse LM parameters: ${TUNE_COARSELM}")
$(if $(strip ${TRAIN_BILM}),@echo " train BiLMs: ${TRAIN_BILM}")
$(if $(strip ${TUNE_BILM}),@echo " tune BiLM parameters: ${TUNE_BILM}")
$(if $(strip ${TRAIN_WCL}),@echo " train word classes: ${TRAIN_WCL}")
$(if $(strip ${TRAIN_TC}),@echo " train TC: ${TRAIN_TC}")
@echo " train TM: ${TRAIN_TM}"
$(if $(strip ${MIXTM}),@echo " train TMs for MixTM: ${MIXTM}")
$(if $(strip ${MIXTM_TRAIN_MIX}),@echo " tune MixTM mix: ${MIXTM_TRAIN_MIX}")
$(if $(strip ${TRAIN_LDM}),@echo " train LDM: ${TRAIN_LDM}")
$(if $(strip ${TRAIN_HLDM}),@echo " train HLDM: ${TRAIN_HLDM}")
$(if $(strip ${TRAIN_SPARSE}),@echo " train sparse: ${TRAIN_SPARSE}")
@echo " tune decode: ${TUNE_DECODE}"
$(if $(strip ${TUNE_DECODE_VARIANTS}),@echo " tune decode variants: $(addprefix ${TUNE_DECODE}, ${TUNE_DECODE_VARIANTS})")
$(if $(strip ${TUNE_RESCORE}),@echo " tune rescore: ${TUNE_RESCORE}")
$(if $(strip ${TUNE_CE}),@echo " tune CE: ${TUNE_CE}")
endef
# Helper to list other important parameters in the help
define HELP_OTHER_PARAMS
@echo "Other important parameters:"
$(if $(strip ${LM_PRETRAINED_TGT_LMS}),@echo " pretrained LMs (non-mix): ${LM_PRETRAINED_TGT_LMS}")
$(if $(strip ${MIXLM_PRETRAINED_TGT_LMS}),@echo " pretrained LMs for MixLM: ${MIXLM_PRETRAINED_TGT_LMS}")
$(if $(strip ${COARSELM_NCLS_LIST}),@echo " coarse LM num classes: ${COARSELM_NCLS_LIST}")
$(if $(strip ${BILM_SPEC}),@echo " BiLM specification: ${BILM_SPEC}")
$(if $(strip ${TM_PRETRAINED_TMS}),@echo " pretrained TMs (non-mix): ${TM_PRETRAINED_TMS}")
$(if $(strip ${MIXTM_PRETRAINED_TMS}),@echo " pretrained TMs for MixTM: ${MIXTM_PRETRAINED_TMS}")
@echo " src locale: ${SRC_LOCALE}"
@echo " tgt locale: ${TGT_LOCALE}"
endef
# Helper recipe to list the main targets for a Makefile:
define HELP_LIST_MAIN_TARGETS
@echo
@echo "The main targets in this Makefile are:"
@echo " " $(sort ${MAIN_TARGETS})
endef
# Helper recipe to list the expected output files for a Makefile:
define HELP_LIST_EXPECTED_FILES
@echo
@echo "Expected output files are:"
@tr ' ' '\n' <<< "$(sort ${EXPECTED_FILES})" | sed -e 's/^/ /'
endef
# Helper recipes for LM_TOOLKIT help for a *lm Makefile:
define HELP_LM_TOOLKIT
@echo "In Makefile.params, define LM_TOOLKIT=SRI, LM_TOOLKIT=MIT or LM_TOOLKIT=IRST"
@echo "to specify the LM toolkit to use."
@echo "LM_TOOLKIT=${LM_TOOLKIT}"
@echo
endef
define HELP_LM_TOOLKIT_2
@echo "In Makefile.params, define LM_TOOLKIT=SRI or LM_TOOLKIT=MIT to specify the"
@echo "LM toolkit to use."
@echo "LM_TOOLKIT=${LM_TOOLKIT}"
@echo
endef
# Helper recipes for IRSTLM help for a Makefile:
ifeq (${LM_TOOLKIT},IRST)
define HELP_IRSTLM
@echo "Please run the following in order for this framework to run properly:"
@echo " export PATH=${IRSTLM_BIN_DIR}:\$$PATH"
@echo " export IRSTLM=${IRSTLM}"
@echo
endef
else
define HELP_IRSTLM
endef
endif
########################################
# List of files that should read-only for safety. Mainly this list
# should be composed of final targets from a Makefile.
FILES_TO_BE_LOCKED ?=
# Change to read-only aka "Locks" precious files.
.PHONY: lock
lock: SHELL=${LOCAL_SHELL}
lock:
-chmod u-w ${FILES_TO_BE_LOCKED}
# Change to read-write aka "Unlocks" precious files.
.PHONY: unlock
unlock: SHELL=${LOCAL_SHELL}
unlock:
-chmod u+w ${FILES_TO_BE_LOCKED}
########################################
# Hide logs.
.PHONY: hide_logs_sub
hide_logs_sub: SHELL=${LOCAL_SHELL}
hide_logs_sub:
mkdir --parents .logs
-mv log.* run-parallel-logs-* .logs
-ls * | egrep '\.[joe][0-9]{1,6}' | xargs -i mv {} .logs
########################################
# Summarize resources usage for this system.
.PHONY: resource_summary_sub
resource_summary_sub: SHELL=${LOCAL_SHELL}
resource_summary_sub:
@find -type f -name log.\* -o -name \*.log \
| sort \
| sed 's/^\.\///' \
| xargs time-mem-tally.pl -no-dir -m ${MODULE_NAME} \
| second-to-hms.pl \
| expand-auto.pl
################################################################################
# UNITTESTS
.PHONY: toolkit_unittests
toolkit_unittests: unittests_create_references
toolkit_unittests: unittests_fix_relative_path
toolkit_unittests:
@echo "ALL Makefile.toolkit unittests passed."
.PHONY: unittests_create_references
unittests_create_references: unittest_create_references_multiple_1
unittests_create_references: unittest_create_references_multiple_2
unittests_create_references: unittest_create_references_single_1
unittests_create_references: unittest_create_references_single_2
unittests_create_references: unittest_create_references_empty
unittest_create_references%: TGT_LANG=l
unittest_create_references%: LANGX=.lc
unittest_create_references_multiple%: REFERENCE_INDICES=1 2
.PHONY: unittest_create_references_multiple_1
# Create multiple references with the default extension.
unittest_create_references_multiple_1:
@[[ "$(call CREATE_REFERENCE_NAMES, a)" == "a_l1.lc a_l2.lc" ]]
.PHONY: unittest_create_references_multiple_2
# Create multiple references with a different extehsion.
unittest_create_references_multiple_2:
@[[ "$(call CREATE_REFERENCE_NAMES, a, .al)" == "a_l1.al a_l2.al" ]]
unittest_create_references_single%: REFERENCE_INDICES=
.PHONY: unittest_create_references_single_1
# Create a single reference with the default extension.
unittest_create_references_single_1:
@[[ "$(call CREATE_REFERENCE_NAMES, a)" == "a_l.lc" ]]
.PHONY: unittest_create_references_single_2
# Create a single reference with a different extehsion.
unittest_create_references_single_2:
@[[ "$(call CREATE_REFERENCE_NAMES, a, .al)" == "a_l.al" ]]
.PHONY: unittest_create_references_empty
unittest_create_references_empty:
@[[ "$(call CREATE_REFERENCE_NAMES,)" == "" ]]
.PHONY: unittests_fix_relative_path
unittests_fix_relative_path: unittest_fix_relative_path_1
unittests_fix_relative_path: unittest_fix_relative_path_2
unittests_fix_relative_path: unittest_fix_relative_path_3
unittests_fix_relative_path: unittest_fix_relative_path_4
.PHONY: unittest_fix_relative_path_1
unittest_fix_relative_path_1:
@[[ "$(call FIX_RELATIVE_PATH, ../.., /home/a/b)" == "/home/a/b" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../.., ~/a/b)" == "~/a/b" ]]
.PHONY: unittest_fix_relative_path_2
unittest_fix_relative_path_2:
@[[ "$(call FIX_RELATIVE_PATH, ../.., a)" == "../../a" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../.., a/b)" == "../../a/b" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../.., ./b)" == "../.././b" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../.., ../b)" == "../../../b" ]]
.PHONY: unittest_fix_relative_path_3
unittest_fix_relative_path_3:
@[[ "$(call FIX_RELATIVE_PATH, ../../, a)" == "../../a" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../../, a/b)" == "../../a/b" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../../, ./b)" == "../.././b" ]]
@[[ "$(call FIX_RELATIVE_PATH, ../../, ../b)" == "../../../b" ]]
.PHONY: unittest_fix_relative_path_4
unittest_fix_relative_path_4:
@[[ "$(call FIX_RELATIVE_PATH, , a)" == "a" ]]