upgrade to mypy 1.9 and fix errors

Most significant change is the removing of Optional types from ai eval spreadsheet

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
     hooks:
       - id: pyupgrade
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.910
+    rev: v1.9.0
     hooks:
       - id: mypy
         exclude: .+/snapshots/.+

automation-api/lib/ai_eval_spreadsheet/schemas.py

@@ -16,13 +16,13 @@
 class Question(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True)
 
-    include_in_next_evaluation: Optional[bool] = Field(
-        None, title="Include in next evaluation", validate_default=True
+    include_in_next_evaluation: bool = Field(
+        False, title="Include in next evaluation", validate_default=True
     )
-    question_id: Optional[str] = Field(None, title="Question ID")
-    language: Optional[str] = Field(None, title="Language")
-    published_version_of_question: Optional[str] = Field(
-        None, title="Published version of question"
+    question_id: str = Field("", title="Question ID")
+    language: str = Field("", title="Language")
+    published_version_of_question: str = Field(
+        "", title="Published version of question"
     )
 
     @field_validator("include_in_next_evaluation", mode="before")
@@ -40,14 +40,12 @@ class Config:
 class QuestionOption(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True)
 
-    question_option_id: Optional[str] = Field(None, title="Question Option ID")
-    question_id: Optional[str] = Field(None, title="Question ID")
-    language: Optional[str] = Field(None, title="Language")
-    letter: Optional[str] = Field(None, title="Letter")
-    question_option: Optional[str] = Field(None, title="Question option")
-    correctness_of_answer_option: Optional[int] = Field(
-        None, title="Correctness of answer option"
-    )
+    question_option_id: str = Field("", title="Question Option ID")
+    question_id: str = Field("", title="Question ID")
+    language: str = Field("", title="Language")
+    letter: str = Field("", title="Letter")
+    question_option: str = Field("", title="Question option")
+    correctness_of_answer_option: int = Field(-1, title="Correctness of answer option")
 
 
 class QuestionOptionsDf(pa.DataFrameModel):
@@ -59,22 +57,18 @@ class Config:
 class PromptVariation(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True)
 
-    include_in_next_evaluation: Optional[bool] = Field(
-        None, title="Include in next evaluation"
-    )
-    variation_id: Optional[str] = Field(None, title="Variation ID")
-    language: Optional[str] = Field(None, title="Language")
-    question_template: Optional[str] = Field(None, title="Question template")
-    question_prefix: Optional[str] = Field(None, title="Question prefix")
-    ai_prefix: Optional[str] = Field(None, title="AI prefix")
-    question_prompt_template: Optional[str] = Field(
-        None, title="Question prompt template"
+    include_in_next_evaluation: bool = Field(False, title="Include in next evaluation")
+    variation_id: str = Field("", title="Variation ID")
+    language: str = Field("", title="Language")
+    question_template: str = Field("", title="Question template")
+    question_prefix: str = Field("", title="Question prefix")
+    ai_prefix: str = Field("", title="AI prefix")
+    question_prompt_template: str = Field("", title="Question prompt template")
+    question_prompt_template_example: str = Field(
+        "", title="Question prompt template example"
     )
-    question_prompt_template_example: Optional[str] = Field(
-        None, title="Question prompt template example"
-    )
-    follow_up_answer_correctness_evaluation_prompt_template: Optional[str] = Field(
-        None, title="Follow-up answer correctness evaluation prompt template"
+    follow_up_answer_correctness_evaluation_prompt_template: str = Field(
+        "", title="Follow-up answer correctness evaluation prompt template"
     )
 
 
@@ -87,9 +81,9 @@ class Config:
 class GenAiModel(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True, protected_namespaces=())
 
-    model_id: Optional[str] = Field(None, title="Model ID")
-    vendor: Optional[str] = Field(None, title="Vendor")
-    model_name: Optional[str] = Field(None, title="Model name")
+    model_id: str = Field("", title="Model ID")
+    vendor: str = Field("", title="Vendor")
+    model_name: str = Field("", title="Model name")
 
 
 class GenAiModelsDf(pa.DataFrameModel):
@@ -101,15 +95,13 @@ class Config:
 class GenAiModelConfig(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True, protected_namespaces=())
 
-    include_in_next_evaluation: Optional[bool] = Field(
-        None, title="Include in next evaluation"
-    )
-    model_config_id: Optional[str] = Field(None, title="Model configuration ID")
-    model_id: Optional[str] = Field(None, title="Model ID")
-    model_parameters: Optional[str] = Field(None, title="Model Parameters")
-    repeat_times: Optional[int] = Field(None, title="Repeat Times")
-    memory: Optional[bool] = Field(None, title="Memory")
-    memory_size: Optional[int] = Field(None, title="Memory Size")
+    include_in_next_evaluation: bool = Field(False, title="Include in next evaluation")
+    model_config_id: str = Field("", title="Model configuration ID")
+    model_id: str = Field("", title="Model ID")
+    model_parameters: str = Field("", title="Model Parameters")
+    repeat_times: int = Field(-1, title="Repeat Times")
+    memory: bool = Field(False, title="Memory")
+    memory_size: int = Field(-1, title="Memory Size")
 
 
 class GenAiModelConfigsDf(pa.DataFrameModel):
@@ -119,11 +111,11 @@ class Config:
 
 
 class Metric(BaseModel):
-    name: Optional[str] = Field(None, title="Name")
-    description: Optional[str] = Field(None, title="Description")
-    prompt: Optional[str] = Field(None, title="Prompt")
-    choices: Optional[str] = Field(None, title="Choices")
-    choice_scores: Optional[str] = Field(None, title="Choice Scores")
+    name: str = Field("", title="Name")
+    description: str = Field("", title="Description")
+    prompt: str = Field("", title="Prompt")
+    choices: str = Field("", title="Choices")
+    choice_scores: str = Field("", title="Choice Scores")
 
 
 class MetricsDf(pa.DataFrameModel):
@@ -135,17 +127,17 @@ class Config:
 class EvalResult(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True, protected_namespaces=())
 
-    question_id: Optional[str] = Field(None, title="Question ID")
-    language: Optional[str] = Field(None, title="Language")
-    prompt_variation_id: Optional[str] = Field(None, title="Prompt variation ID")
-    model_configuration_id: Optional[str] = Field(None, title="Model Configuration ID")
-    last_evaluation_datetime: Optional[str] = Field(None, title="Last Evaluation")
-    percent_correct: Optional[float] = Field(None, title="Percent Correct")
-    percent_wrong: Optional[float] = Field(None, title="Percent Wrong")
-    percent_very_wrong: Optional[float] = Field(None, title="Percent Very Wrong")
-    percent_eval_failed: Optional[float] = Field(None, title="Percent Eval Failed")
-    rounds: Optional[int] = Field(None, title="Rounds")
-    result: Optional[str] = Field(None, title="Result")
+    question_id: str = Field("", title="Question ID")
+    language: str = Field("", title="Language")
+    prompt_variation_id: str = Field("", title="Prompt variation ID")
+    model_configuration_id: str = Field("", title="Model Configuration ID")
+    last_evaluation_datetime: str = Field("", title="Last Evaluation")
+    percent_correct: Optional[float] = Field("", title="Percent Correct")
+    percent_wrong: Optional[float] = Field("", title="Percent Wrong")
+    percent_very_wrong: Optional[float] = Field("", title="Percent Very Wrong")
+    percent_eval_failed: Optional[float] = Field("", title="Percent Eval Failed")
+    rounds: int = Field(-1, title="Rounds")
+    result: str = Field("", title="Result")
 
 
 class EvalResultsDf(pa.DataFrameModel):
@@ -157,16 +149,16 @@ class Config:
 class SessionResult(BaseModel):
     model_config = ConfigDict(coerce_numbers_to_str=True, protected_namespaces=())
 
-    session_id: Optional[str] = Field(None, title="Session ID")
-    session_time: Optional[str] = Field(None, title="Session Time")
-    prompt_variation_id: Optional[str] = Field(None, title="Prompt Variation ID")
-    model_configuration_id: Optional[str] = Field(None, title="Model Configuration ID")
-    survey_id: Optional[str] = Field(None, title="Survey ID")
-    question_id: Optional[str] = Field(None, title="Question ID")
-    language: Optional[str] = Field(None, title="Language")
-    question_number: Optional[int] = Field(None, title="Question No.")
-    output: Optional[str] = Field(None, title="Response Text")
-    grade: Optional[str] = Field(None, title="Grade")
+    session_id: str = Field("", title="Session ID")
+    session_time: str = Field("", title="Session Time")
+    prompt_variation_id: str = Field("", title="Prompt Variation ID")
+    model_configuration_id: str = Field("", title="Model Configuration ID")
+    survey_id: str = Field("", title="Survey ID")
+    question_id: str = Field("", title="Question ID")
+    language: str = Field("", title="Language")
+    question_number: int = Field(-1, title="Question No.")
+    output: str = Field("", title="Response Text")
+    grade: str = Field("", title="Grade")
 
 
 class SessionResultsDf(pa.DataFrameModel):

automation-api/lib/ai_eval_spreadsheet/wrapper.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import Optional
+from typing import Type
 
 from gspread import Spreadsheet
 
@@ -27,24 +27,20 @@
 
 @dataclass
 class AiEvalData:
-    questions: Optional[GsheetsWorksheetEditor[QuestionsDf, Question]] = None
-    question_options: Optional[
-        GsheetsWorksheetEditor[QuestionOptionsDf, QuestionOption]
-    ] = None
-    prompt_variations: Optional[
-        GsheetsWorksheetEditor[PromptVariationsDf, PromptVariation]
-    ] = None
-    gen_ai_models: Optional[GsheetsWorksheetEditor[GenAiModelsDf, GenAiModel]] = None
-    gen_ai_model_configs: Optional[
-        GsheetsWorksheetEditor[GenAiModelConfigsDf, GenAiModelConfig]
-    ] = None
-    metrics: Optional[GsheetsWorksheetEditor[MetricsDf, Metric]] = None
-    evaluation_results: Optional[
-        GsheetsWorksheetEditor[EvalResult, EvalResultsDf]
-    ] = None
-    session_results: Optional[
-        GsheetsWorksheetEditor[SessionResult, SessionResultsDf]
-    ] = None
+    prompt_variations: GsheetsWorksheetEditor[
+        Type[PromptVariationsDf], Type[PromptVariation]
+    ]
+    questions: GsheetsWorksheetEditor[Type[QuestionsDf], Type[Question]]
+    question_options: GsheetsWorksheetEditor[
+        Type[QuestionOptionsDf], Type[QuestionOption]
+    ]
+    gen_ai_models: GsheetsWorksheetEditor[Type[GenAiModelsDf], Type[GenAiModel]]
+    gen_ai_model_configs: GsheetsWorksheetEditor[
+        Type[GenAiModelConfigsDf], Type[GenAiModelConfig]
+    ]
+    metrics: GsheetsWorksheetEditor[Type[MetricsDf], Type[Metric]]
+    evaluation_results: GsheetsWorksheetEditor[Type[EvalResult], Type[EvalResultsDf]]
+    session_results: GsheetsWorksheetEditor[Type[SessionResult], Type[SessionResultsDf]]
 
 
 sheet_names = {

automation-api/lib/pilot/helpers.py

@@ -1,7 +1,7 @@
 import json
 import uuid
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 import pandas as pd
 from langchain.chains import LLMChain
@@ -178,9 +178,13 @@ def create_question_dataset_for_test(
     return [create_question_data_for_test(question_tmpl, q) for q in question_list]
 
 
-def create_question_data_for_eval(question: QuestionAndOptions) -> Dict[str, str]:
+def create_question_data_for_eval(
+    question: QuestionAndOptions,
+) -> Dict[str, Union[str, int]]:
     q, options = question
-    question_dict = {"question": q.published_version_of_question}
+    question_dict: Dict[str, Union[str, int]] = {
+        "question": q.published_version_of_question
+    }
     for opt in options:
         if opt.letter == "A":
             question_dict["option_a"] = opt.question_option
@@ -196,7 +200,7 @@ def create_question_data_for_eval(question: QuestionAndOptions) -> Dict[str, str
 
 def create_question_dataset_for_eval(
     question_list: List[QuestionAndOptions],
-) -> List[Dict[str, str]]:
+) -> List[Dict[str, Union[str, int]]]:
     return [create_question_data_for_eval(q) for q in question_list]
 
 

automation-api/mypy.ini

@@ -13,7 +13,3 @@ warn_unreachable = True
 warn_unused_ignores = True
 
 plugins = pandera.mypy
-
-; # Disable mypy for tests
-; [mypy-tests.*]
-; ignore_errors = True