This repository has been archived by the owner on Oct 31, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 61
/
evaluate_retrieved_passages.py
66 lines (53 loc) · 1.96 KB
/
evaluate_retrieved_passages.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.
import argparse
import json
import logging
import glob
import numpy as np
import torch
import src.utils
from src.evaluation import calculate_matches
logger = logging.getLogger(__name__)
def validate(data, workers_num):
match_stats = calculate_matches(data, workers_num)
top_k_hits = match_stats.top_k_hits
#logger.info('Validation results: top k documents hits %s', top_k_hits)
top_k_hits = [v / len(data) for v in top_k_hits]
#logger.info('Validation results: top k documents hits accuracy %s', top_k_hits)
return top_k_hits
def main(opt):
logger = src.utils.init_logger(opt, stdout_only=True)
datapaths = glob.glob(args.data)
r20, r100 = [], []
for path in datapaths:
data = []
with open(path, 'r') as fin:
for line in fin:
data.append(json.loads(line))
#data = json.load(fin)
answers = [ex['answers'] for ex in data]
top_k_hits = validate(data, args.validation_workers)
message = f"Evaluate results from {path}:"
for k in [5, 10, 20, 100]:
if k <= len(top_k_hits):
recall = 100 * top_k_hits[k-1]
if k == 20:
r20.append(f"{recall:.1f}")
if k == 100:
r100.append(f"{recall:.1f}")
message += f' R@{k}: {recall:.1f}'
logger.info(message)
print(datapaths)
print('\t'.join(r20))
print('\t'.join(r100))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--data', required=True, type=str, default=None)
parser.add_argument('--validation_workers', type=int, default=16,
help="Number of parallel processes to validate results")
args = parser.parse_args()
main(args)