Skip to content

Commit

Permalink
fix: add missing benchmark to benchmarks.py (#1641)
Browse files Browse the repository at this point in the history
add missing benchmark
  • Loading branch information
gowitheflow-1998 authored Dec 30, 2024
1 parent 815a1b4 commit 27eb549
Showing 1 changed file with 33 additions and 0 deletions.
33 changes: 33 additions & 0 deletions mteb/benchmarks/benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,39 @@ def load_results(
}""",
)

RAR_b = Benchmark(
name="RAR-b",
tasks=get_tasks(
tasks=[
"ARCChallenge",
"AlphaNLI",
"HellaSwag",
"WinoGrande",
"PIQA",
"SIQA",
"Quail",
"SpartQA",
"TempReasonL1",
"TempReasonL2Pure",
"TempReasonL2Fact",
"TempReasonL2Context",
"TempReasonL3Pure",
"TempReasonL3Fact",
"TempReasonL3Context",
"RARbCode",
"RARbMath",
]
),
description="A benchmark to evaluate reasoning capabilities of retrievers.",
reference="https://arxiv.org/abs/2404.06347",
citation="""@article{xiao2024rar,
title={RAR-b: Reasoning as Retrieval Benchmark},
author={Xiao, Chenghao and Hudson, G Thomas and Al Moubayed, Noura},
journal={arXiv preprint arXiv:2404.06347},
year={2024}
}""",
)

MTEB_FRA = Benchmark(
name="MTEB(fra)",
tasks=get_tasks(
Expand Down

0 comments on commit 27eb549

Please sign in to comment.