CLEVA Scenarios, Perturbations and Metrics (#1824)

Co-authored-by: Jianqiao-Zhao <jz2778@columbia.edu>
Co-authored-by: zd11024 <zzzd3434@gmail.com>
Co-authored-by: HenryHZY <168133331@qq.com>

setup.cfg

@@ -117,6 +117,13 @@ models =
     crfm-helm[tsinghua]
     crfm-helm[yandex]
 
+cleva = 
+    unidecode==1.3.6
+    pypinyin==0.49.0
+    jieba==0.42.1
+    opencc==1.1.6
+    langdetect==1.0.9
+
 # Install everything
 all =
     crfm-helm[server]
@@ -126,6 +133,7 @@ all =
     crfm-helm[metrics]
     crfm-helm[plots]
     crfm-helm[slurm]
+    crfm-helm[cleva]
 
 # Development only
 # Do not include in all

src/helm/benchmark/adaptation/adapters/language_modeling_adapter.py

@@ -28,7 +28,7 @@ class LanguageModelingAdapter(Adapter):
     @htrack(None)
     def adapt(self, instances: List[Instance], parallelism: int) -> ScenarioState:
         """
-        Takes a a list of `Instance`s and builds a list of corresponding `RequestState`s.
+        Takes a list of `Instance`s and builds a list of corresponding `RequestState`s.
         Only requires eval instances.
         """
         # Pick out evaluation instances. This includes both valid and test splits.
@@ -73,7 +73,7 @@ def generate_requests(self, eval_instance: Instance) -> List[RequestState]:
         # Why is this limit needed ?
         # Because some vendors like Anthropic have this limit in their backend set to something lower than
         # max_sequence_length + max_generated_tokens_length.
-        # Note that max_generated_tokens_length is not explicitely set and checked in our codebase.
+        # Note that max_generated_tokens_length is not explicitly set and checked in our codebase.
         # This will be handled by the vendor backend.
 
         max_sequence_length: int = self.window_service.max_sequence_length