-
Notifications
You must be signed in to change notification settings - Fork 3
/
Makefile.CLEF2022
315 lines (223 loc) · 16.2 KB
/
Makefile.CLEF2022
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
CLEF_DATA_PATH=/home/kai.labusch/qurator/HIPE-2022-data/data/v2.1
RESULT_PATH=data/HIPE2022
SUBMISSION_PATH=/home/kai.labusch/qurator/HIPE-2022-data/data/submission
MAX_DIST?=0.05
MAX_CANDIDATES?=25
MAX_FREQUENCY?=1000
NED_THRESHOLD?=0.2
NED_PARAMS ?=md$(MAX_DIST)_mc$(MAX_CANDIDATES)_mf$(MAX_FREQUENCY)_nthr$(NED_THRESHOLD)
NED_JSON_PARAMS ?=md$(MAX_DIST)_mc$(MAX_CANDIDATES)_mf$(MAX_FREQUENCY)
FIND_ENTITY_PARAMS ?=--ned-threshold=$(NED_THRESHOLD) --max-candidates=$(MAX_CANDIDATES) --max-dist=$(MAX_DIST)
NED_ENDPOINTS?='{"de": "http://localhost:5015", "fr": "http://localhost:5016", "en": "http://localhost:5017"}'
RUNNUMBER?= 1
TEAM?=SBB
TASKBUNDLEID=bundle5
HISTO_NORM_FILE=/home/labusch/qurator/HIPE-2022-eval/HIPE-scorer/historico-fuzzy-QID-mapping-ID-mapping.tsv
SCORING_DIR=scoring
SCORING_LOG=scoring.log
define CREATE_SUBMISSION_NAME
import sys,re,os
infile = sys.argv[1]
taskbundleid = sys.argv[2]
lang = sys.argv[3]
team = sys.argv[4]
runnumber = sys.argv[5]
infile = os.path.basename(infile)
#print("{}: {}, {}".format(infile, taskbundleid, lang, team, runnumber))
m = re.match(r'neat-HIPE-2022-v2.1-(\w+)-test.*', infile)
if not m:
raise RuntimeError("File name does not match!")
datasetalias = m.group(1)
print("{}_{}_{}_{}_{}.tsv".format(team, taskbundleid, datasetalias, lang, runnumber))
endef
export CREATE_SUBMISSION_NAME
define GS_FILENAME
import sys,re,os
infile = sys.argv[1]
ned_params = sys.argv[2]
lang = sys.argv[3]
taskbundleid = sys.argv[4]
dirname = os.path.dirname(infile)
infile = os.path.basename(infile)
# print(infile)
m = re.match('prediction-neat-HIPE-2022-v2.1-(\w+)-([\w-]+)-{}-.*.tsv'.format(lang), infile)
if not m:
raise RuntimeError("File name does not match!")
datasetalias = m.group(1)
kind_of_dataset = m.group(2)
if kind_of_dataset.startswith('test'):
gs_dataset = 'test'
else:
gs_dataset = kind_of_dataset
print("{}/HIPE-2022-v2.1-{}-{}-{}.tsv ".format(dirname, datasetalias, gs_dataset , lang))
endef
export GS_FILENAME
define JOIN_SCORING
import sys,re,os
import pandas as pd
from glob import glob
scoring_dir = sys.argv[1]
target_file = sys.argv[2]
df_all=[]
for f in glob('{}/*.tsv'.format(scoring_dir)):
m = re.match('prediction-neat-HIPE-2022-v2.1-(\w+)-([\w-]+)-(de|fr|en)-md([0-9.]+)_mc([0-9.]+)_.*nthr([0-9.]+).*.tsv', os.path.basename(f))
if not m:
raise RuntimeError("File does not match!")
dataset = m.group(1)
kind = m.group(2)
lang = m.group(3)
max_dist = m.group(4)
max_candidates = m.group(5)
threshold = m.group(6)
df = pd.read_csv(f, sep='\t')
df['dataset'] = dataset
df['kind'] = kind
df['lang'] = lang
df['max_dist'] = max_dist
df['max_candidates'] = max_candidates
df['threshold'] = threshold
df_all.append(df)
df_all=pd.concat(df_all).drop(columns=['System'])
df_all.to_pickle(target_file)
endef
export JOIN_SCORING
# Convert the CLEF-TSVs into the TSV-format that is understood by neat and tools like find-entities
%-neat-tsvs:
find $(CLEF_DATA_PATH) -wholename "*/$*/HIPE-2022*.tsv" | parallel 'clef2tsv-2022 {} `dirname {}`/neat-`basename {}`'
neat-tsvs: de-neat-tsvs fr-neat-tsvs en-neat-tsvs
# Convert the neat - prediction TSVs (starting with ned-full) back into CLEF-format files (starting with prediction-) that is understood by the hipe-scorer
%-test-prediction-tsvs:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*test?ELmasked*.tsv"`;do tsv2clef-2022 `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv $$i `dirname $$i`/prediction-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv;done
test-prediction-tsvs: de-test-prediction-tsvs fr-test-prediction-tsvs en-test-prediction-tsvs
%-dev-prediction-tsvs:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*dev*.tsv"`;do tsv2clef-2022 `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv $$i `dirname $$i`/prediction-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv;done
dev-prediction-tsvs: de-dev-prediction-tsvs fr-dev-prediction-tsvs en-dev-prediction-tsvs
# Create properly named submission files
%-clef-submission:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*test?ELmasked*.tsv"`;do tsv2clef-2022 `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv $$i $(SUBMISSION_PATH)/`python -c "$$CREATE_SUBMISSION_NAME" "$$i" $(TASKBUNDLEID) "$*" $(TEAM) $(RUNNUMBER)`;done
submission: de-clef-submission fr-clef-submission en-clef-submission
# remove all the neat TSVs
clean-%-tsvs:
find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*.tsv" -exec rm -f {} \;
clean-tsvs: clean-de-tsvs clean-fr-tsvs clean-en-tsvs
# The following 3 targets create EL-only prediction TSV files in neat format that start with ned-full--
%-dev-ned:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*dev*.tsv"`;do find-entities $(FIND_ENTITY_PARAMS) --noproxy --ned-json-file=`dirname $$i`/`basename -s .tsv $$i`-$(NED_JSON_PARAMS).json --ned-rest-endpoints=$(NED_ENDPOINTS) --lang $* --context-split --not-after-context-field "hipe2022:date" $$i `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv ;done
%-train-ned:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*train*.tsv"`;do find-entities $(FIND_ENTITY_PARAMS) --noproxy --ned-json-file=`dirname $$i`/`basename -s .tsv $$i`-$(NED_JSON_PARAMS).json --ned-rest-endpoints=$(NED_ENDPOINTS) --lang $* --context-split --not-after-context-field "hipe2022:date" $$i `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv ;done
%-test-ELmasked-ned:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*test?ELmasked*.tsv"`;do find-entities $(FIND_ENTITY_PARAMS) --noproxy --ned-json-file=`dirname $$i`/`basename -s .tsv $$i`-$(NED_JSON_PARAMS).json --ned-rest-endpoints=$(NED_ENDPOINTS) --lang $* --context-split --not-after-context-field "hipe2022:date" $$i `dirname $$i`/ned-full-`basename -s .tsv $$i`-$(NED_PARAMS)-$(TASKBUNDLEID).tsv ;done
dev-ned: de-dev-ned fr-dev-ned en-dev-ned
train-ned: de-train-ned fr-train-ned en-train-ned
test-ELmasked-ned: de-test-ELmasked-ned fr-test-ELmasked-ned en-test-ELmasked-ned
# Create pickled dataframes with decider training data for each dataset
%-train-stat:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/neat-*train*.tsv"`;do sentence-stat-2022 --context-split $$i `dirname $$i`/`basename -s .tsv $$i`-$(NED_JSON_PARAMS).json $$i `dirname $$i`/train-data-`basename -s .tsv $$i`-$(NED_PARAMS).pkl; done
# Join all the pickled dataframes into one big file for decider training over all datasets.
%-train-data:
mkdir -p $(RESULT_PATH)
df-concatenate --reset-index $(RESULT_PATH)/$*-train-$(NED_JSON_PARAMS).pkl `find $(CLEF_DATA_PATH) -wholename "*/$*/train-data*-$(NED_PARAMS).pkl" -printf "%p "`
# Perform the decider training per language
%-decider:
train-decider $(RESULT_PATH)/$*-train-$(NED_JSON_PARAMS).pkl $(RESULT_PATH)/$*-decider-$(NED_JSON_PARAMS).pkl > $(RESULT_PATH)/$*-decider-$(NED_JSON_PARAMS).log 2>&1
train-stat: de-train-stat fr-train-stat en-train-stat
train-data: de-train-data fr-train-data en-train-data
decider: de-decider fr-decider en-decider
# Joined target that does the complete required chain in order to train the deciders
%-train-all: %-train-ned %-train-stat %-train-data %-decider ;
train-all: de-train-all fr-train-all en-train-all
# Starts EL-webservice with a decider that has been trained for MAX_DIST=0.05 and MAX_CANDIDATES=25
de-service:
env CONFIG=de-HIPE2022.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5015
fr-service:
env CONFIG=fr-HIPE2022.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5016
en-service:
env CONFIG=en-HIPE2022.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5017
# Starts EL-webservice with a decider that has been trained for MAX_DIST=0.13 and MAX_CANDIDATES=50
de-service-2:
env CONFIG=de-HIPE2022-2.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5015
fr-service-2:
env CONFIG=fr-HIPE2022-2.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5016
en-service-2:
env CONFIG=en-HIPE2022-2.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5017
# Starts EL-webservice in no-decider mode. Only suitable in order to generate decider training data.
de-service-no-decider:
env CONFIG=de-config-no-decider.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5015
fr-service-no-decider:
env CONFIG=fr-config-no-decider.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5016
en-service-no-decider:
env CONFIG=en-config-no-decider.json env FLASK_APP=qurator/sbb_ned/webapp/app.py env FLASK_ENV=development env USE_CUDA=True flask run --host=0.0.0.0 --port=5017
%-normalize-linking:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/HIPE-2022-v2.1-*-test-$**.tsv"`;do normalize-linking -i $$i -o `dirname $$i`/`basename -s .tsv $$i`-normalized.tsv --hipe_edition=hipe-2022 --norm-time --norm-histo -m $(HISTO_NORM_FILE);done
%-test-scoring-test:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/prediction-neat-HIPE-2022-v2.1-*-test?ELmasked-$**.tsv"`;do ls -lh `python -c "$$GS_FILENAME" "$$i" "$(NED_PARAMS)" "$*" "$(TASKBUNDLEID)"`;done
%-test-scoring:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/prediction-neat-HIPE-2022-v2.1-*-test?ELmasked-$**.tsv"`;do hipe-scorer --pred=$$i --ref=`python -c "$$GS_FILENAME" "$$i" "$(NED_PARAMS)" "$*" "$(TASKBUNDLEID)"` --task=nel --n_best=1,3,5 --suffix=eval --skip-check --hipe_edition=hipe-2022 --outdir=$(SCORING_DIR) --log=$(SCORING_LOG);done
test-scoring: de-test-scoring fr-test-scoring en-test-scoring
%-dev-scoring-test:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/prediction-neat-HIPE-2022-v2.1-*-dev*-$**.tsv"`;do ls -lh `python -c "$$GS_FILENAME" "$$i" "$(NED_PARAMS)" "$*" "$(TASKBUNDLEID)"`;done
%-dev-scoring:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/prediction-neat-HIPE-2022-v2.1-*-dev*-$**.tsv"`;do hipe-scorer --pred=$$i --ref=`python -c "$$GS_FILENAME" "$$i" "$(NED_PARAMS)" "$*" "$(TASKBUNDLEID)"` --task=nel --n_best=1,3,5 --suffix=eval --skip-check --hipe_edition=hipe-2022 --outdir=$(SCORING_DIR) --log=$(SCORING_LOG);done
dev-scoring: de-dev-scoring fr-dev-scoring en-dev-scoring
join-scoring:
python -c "$$JOIN_SCORING" "$(SCORING_DIR)" scoring.pkl
#########
test-iterate-threshold:
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.05
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.1
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.15
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.2
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.25
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.3
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.35
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.4
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.45
make -f Makefile.CLEF2022 test-ELmasked-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.5
test-iterate:
make -f Makefile.CLEF2022 test-iterate-threshold MAX_CANDIDATES=25 MAX_DIST=0.05
make -f Makefile.CLEF2022 test-iterate-threshold MAX_CANDIDATES=50 MAX_DIST=0.13
dev-iterate-threshold:
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.05
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.1
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.15
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.2
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.25
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.3
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.35
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.4
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.45
make -f Makefile.CLEF2022 dev-ned MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.5
dev-iterate:
make -f Makefile.CLEF2022 dev-iterate-threshold MAX_CANDIDATES=25 MAX_DIST=0.05
make -f Makefile.CLEF2022 dev-iterate-threshold MAX_CANDIDATES=50 MAX_DIST=0.13
##########
test-predictions-iterate-threshold:
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.05
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.1
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.15
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.2
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.25
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.3
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.35
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.4
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.45
make -f Makefile.CLEF2022 test-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.5
test-predictions-iterate:
make -f Makefile.CLEF2022 test-predictions-iterate-threshold MAX_CANDIDATES=25 MAX_DIST=0.05
make -f Makefile.CLEF2022 test-predictions-iterate-threshold MAX_CANDIDATES=50 MAX_DIST=0.13
dev-predictions-iterate-threshold:
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.05
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.1
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.15
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.2
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.25
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.3
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.35
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.4
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.45
make -f Makefile.CLEF2022 dev-prediction-tsvs MAX_CANDIDATES=$(MAX_CANDIDATES) MAX_DIST=$(MAX_DIST) NED_THRESHOLD=0.5
dev-predictions-iterate:
make -f Makefile.CLEF2022 dev-predictions-iterate-threshold MAX_CANDIDATES=25 MAX_DIST=0.05
make -f Makefile.CLEF2022 dev-predictions-iterate-threshold MAX_CANDIDATES=50 MAX_DIST=0.13
%-coverage:
for i in `find $(CLEF_DATA_PATH) -wholename "*/$*/HIPE-2022*test-$*.tsv"`;do compute-knb-coverage-2022 data/wikidata/$*-wikipedia-ner-entities.pkl $$i ;done