Fixes

stanford-crfm · Dec 19, 2024 · ff86da0 · ff86da0
1 parent 830c2da
commit ff86da0
Show file tree

Hide file tree

Showing 3 changed files with 2 additions and 24 deletions.
diff --git a/src/helm/benchmark/metrics/czech_bank_qa_metrics.py b/src/helm/benchmark/metrics/czech_bank_qa_metrics.py
@@ -2,34 +2,12 @@
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.basic_metrics import compute_request_state_metrics
-from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 from helm.benchmark.metrics.metric import Metric
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 
 
-# class AIRBench2024BasicGenerationMetric(Metric):
-#     """Replacement for BasicGenerationMetric for AIRBench 2024.
-
-#     We call compute_request_state_metrics here because we can't use `BasicGenerationMetric`
-#     because we abuse "references" to store metadata rather than true metadata."""
-
-#     def __init__(self):
-#         super().__init__()
-#         self.efficiency_metric = EfficiencyMetric()
-
-#     def evaluate_generation(
-#         self,
-#         adapter_spec: AdapterSpec,
-#         request_state: RequestState,
-#         metric_service: MetricService,
-#         eval_cache_path: str,
-#     ) -> List[Stat]:
-#         return compute_request_state_metrics(self.efficiency_metric, adapter_spec, request_state, metric_service)
-
-
 class CzechBankQAMetrics(Metric):
     """Score metrics for AIRBench 2024."""
 

diff --git a/src/helm/benchmark/run_specs/experimental_run_specs.py b/src/helm/benchmark/run_specs/experimental_run_specs.py
@@ -8,7 +8,7 @@
 from helm.benchmark.annotation.annotator import AnnotatorSpec
 from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs, get_exact_match_metric_specs
 from helm.benchmark.metrics.metric import MetricSpec
-from helm.benchmark.run_spec import AnnotatorSpec, RunSpec, run_spec_function
+from helm.benchmark.run_spec import RunSpec, run_spec_function
 from helm.benchmark.scenarios.scenario import ScenarioSpec
 
 

diff --git a/src/helm/benchmark/scenarios/czech_bank_qa_scenario.py b/src/helm/benchmark/scenarios/czech_bank_qa_scenario.py
@@ -104,7 +104,7 @@ class CzechBankQAScenario(Scenario):
 ,  "account" integer  DEFAULT NULL
 ,  PRIMARY KEY ("trans_id")
 ,  CONSTRAINT "trans_ibfk_1" FOREIGN KEY ("account_id") REFERENCES "account" ("account_id")
-);"""
+);"""  # noqa: E501
 
     """CzechBankQA"""
     name = "czech_bank_qa"