diff --git a/data_extractor/code/kpi_inference_data_pipeline/kpi_inference_data_pipeline/components/text_kpi_inference_curator.py b/data_extractor/code/kpi_inference_data_pipeline/kpi_inference_data_pipeline/components/text_kpi_inference_curator.py index 564a941..a444149 100644 --- a/data_extractor/code/kpi_inference_data_pipeline/kpi_inference_data_pipeline/components/text_kpi_inference_curator.py +++ b/data_extractor/code/kpi_inference_data_pipeline/kpi_inference_data_pipeline/components/text_kpi_inference_curator.py @@ -11,7 +11,8 @@ import numpy as np import pandas as pd from fuzzywuzzy import fuzz -from kpi_inference_data_pipeline.utils.utils import aggregate_csvs, clean_annotation +from kpi_inference_data_pipeline.utils.utils import (aggregate_csvs, + clean_annotation) from .base_kpi_inference_curator import BaseKPIInferenceCurator diff --git a/data_extractor/code/model_pipeline/metrics_per_kpi.py b/data_extractor/code/model_pipeline/metrics_per_kpi.py index 26b56cf..7a25276 100644 --- a/data_extractor/code/model_pipeline/metrics_per_kpi.py +++ b/data_extractor/code/model_pipeline/metrics_per_kpi.py @@ -6,14 +6,11 @@ from farm.evaluation.metrics import squad_EM, squad_f1 from farm.infer import QAInferencer from farm.modeling.predictions import QACandidate -from model_pipeline.config_qa_farm_train import ( - QAFileConfig, - QAInferConfig, - QAModelConfig, - QAProcessorConfig, - QATokenizerConfig, - QATrainingConfig, -) +from model_pipeline.config_qa_farm_train import (QAFileConfig, QAInferConfig, + QAModelConfig, + QAProcessorConfig, + QATokenizerConfig, + QATrainingConfig) from model_pipeline.utils.qa_metrics import relaxed_squad_f1 diff --git a/data_extractor/code/model_pipeline/model_pipeline/__init__.py b/data_extractor/code/model_pipeline/model_pipeline/__init__.py index 27ede02..c082f21 100644 --- a/data_extractor/code/model_pipeline/model_pipeline/__init__.py +++ b/data_extractor/code/model_pipeline/model_pipeline/__init__.py @@ -1,21 +1,9 @@ -from .config_farm_train import ( - FileConfig, - InferConfig, - MLFlowConfig, - ModelConfig, - ProcessorConfig, - TokenizerConfig, - TrainingConfig, -) -from .config_qa_farm_train import ( - QAFileConfig, - QAInferConfig, - QAMLFlowConfig, - QAModelConfig, - QAProcessorConfig, - QATokenizerConfig, - QATrainingConfig, -) +from .config_farm_train import (FileConfig, InferConfig, MLFlowConfig, + ModelConfig, ProcessorConfig, TokenizerConfig, + TrainingConfig) +from .config_qa_farm_train import (QAFileConfig, QAInferConfig, QAMLFlowConfig, + QAModelConfig, QAProcessorConfig, + QATokenizerConfig, QATrainingConfig) from .farm_trainer import FARMTrainer from .qa_farm_trainer import QAFARMTrainer from .trainer_optuna import TrainerOptuna diff --git a/data_extractor/code/model_pipeline/model_pipeline/inference_server.py b/data_extractor/code/model_pipeline/model_pipeline/inference_server.py index 2d651b1..826b9e6 100644 --- a/data_extractor/code/model_pipeline/model_pipeline/inference_server.py +++ b/data_extractor/code/model_pipeline/model_pipeline/inference_server.py @@ -13,24 +13,15 @@ import torch from flask import Flask, Response, request from kpi_inference_data_pipeline import TextKPIInferenceCurator, config -from model_pipeline.config_farm_train import ( - FileConfig, - InferConfig, - MLFlowConfig, - ModelConfig, - ProcessorConfig, - TokenizerConfig, - TrainingConfig, -) -from model_pipeline.config_qa_farm_train import ( - QAFileConfig, - QAInferConfig, - QAMLFlowConfig, - QAModelConfig, - QAProcessorConfig, - QATokenizerConfig, - QATrainingConfig, -) +from model_pipeline.config_farm_train import (FileConfig, InferConfig, + MLFlowConfig, ModelConfig, + ProcessorConfig, TokenizerConfig, + TrainingConfig) +from model_pipeline.config_qa_farm_train import (QAFileConfig, QAInferConfig, + QAMLFlowConfig, QAModelConfig, + QAProcessorConfig, + QATokenizerConfig, + QATrainingConfig) from model_pipeline.farm_trainer import FARMTrainer from model_pipeline.qa_farm_trainer import QAFARMTrainer from model_pipeline.relevance_infer import TextRelevanceInfer diff --git a/data_extractor/code/model_pipeline/model_pipeline/optuna_hyp.py b/data_extractor/code/model_pipeline/model_pipeline/optuna_hyp.py index 320b7dd..3cf6d87 100644 --- a/data_extractor/code/model_pipeline/model_pipeline/optuna_hyp.py +++ b/data_extractor/code/model_pipeline/model_pipeline/optuna_hyp.py @@ -1,14 +1,7 @@ import model_pipeline import optuna -from model_pipeline import ( - FARMTrainer, - FileConfig, - MLFlowConfig, - ModelConfig, - ProcessorConfig, - TokenizerConfig, - TrainingConfig, -) +from model_pipeline import (FARMTrainer, FileConfig, MLFlowConfig, ModelConfig, + ProcessorConfig, TokenizerConfig, TrainingConfig) def objective(trial): diff --git a/data_extractor/code/rule_based_pipeline/rule_based_pipeline/main_find_xy.py b/data_extractor/code/rule_based_pipeline/rule_based_pipeline/main_find_xy.py index ff30965..e5cbef2 100644 --- a/data_extractor/code/rule_based_pipeline/rule_based_pipeline/main_find_xy.py +++ b/data_extractor/code/rule_based_pipeline/rule_based_pipeline/main_find_xy.py @@ -11,7 +11,6 @@ import config import pandas as pd - # TODO: from globals import * from HTMLDirectory import * diff --git a/data_extractor/code/tests/test_train_on_pdf.py b/data_extractor/code/tests/test_train_on_pdf.py index 1f8cf05..a03dcc6 100644 --- a/data_extractor/code/tests/test_train_on_pdf.py +++ b/data_extractor/code/tests/test_train_on_pdf.py @@ -1,7 +1,6 @@ import shutil import sys import traceback - # types import typing from pathlib import Path diff --git a/data_extractor/code/tests/test_utils/test_convert_xls_to_csv.py b/data_extractor/code/tests/test_utils/test_convert_xls_to_csv.py index 8240b85..d990737 100644 --- a/data_extractor/code/tests/test_utils/test_convert_xls_to_csv.py +++ b/data_extractor/code/tests/test_utils/test_convert_xls_to_csv.py @@ -6,11 +6,8 @@ import pytest import s3_communication import train_on_pdf -from tests.utils_test import ( - create_multiple_xlsx_files, - create_single_xlsx_file, - write_to_file, -) +from tests.utils_test import (create_multiple_xlsx_files, + create_single_xlsx_file, write_to_file) from train_on_pdf import convert_xls_to_csv diff --git a/data_extractor/code/tests/test_utils/test_generate_text.py b/data_extractor/code/tests/test_utils/test_generate_text.py index 8ed35d2..2a32dd4 100644 --- a/data_extractor/code/tests/test_utils/test_generate_text.py +++ b/data_extractor/code/tests/test_utils/test_generate_text.py @@ -1,5 +1,4 @@ import shutil - # types import typing from pathlib import Path diff --git a/data_extractor/code/tests/test_utils/test_run_router.py b/data_extractor/code/tests/test_utils/test_run_router.py index 49d53ad..263c741 100644 --- a/data_extractor/code/tests/test_utils/test_run_router.py +++ b/data_extractor/code/tests/test_utils/test_run_router.py @@ -1,5 +1,4 @@ import shutil - # types import typing from pathlib import Path @@ -10,7 +9,8 @@ import requests_mock import train_on_pdf from _pytest.capture import CaptureFixture -from tests.test_utils.test_convert_xls_to_csv import prerequisites_convert_xls_to_csv +from tests.test_utils.test_convert_xls_to_csv import \ + prerequisites_convert_xls_to_csv from tests.test_utils.test_generate_text import prerequisites_generate_text from train_on_pdf import run_router