From 9c922c1848fada47b1af8d1416bb86031a82cf04 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 24 Sep 2024 03:51:50 +0000 Subject: [PATCH 1/2] Revert "feat: Factor Implement Search Enhancement (#294)" This reverts commit 4ecf25f0acf2389a172b14d3dab20895daf2ab89. --- .../coder/factor_coder/CoSTEER/__init__.py | 7 -- .../factor_coder/CoSTEER/evolving_strategy.py | 50 ------------ .../CoSTEER/knowledge_management.py | 76 ++++--------------- .../components/coder/factor_coder/config.py | 3 - .../coder/factor_coder/prompts.yaml | 31 +------- 5 files changed, 17 insertions(+), 150 deletions(-) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py index 0ff1cbc62..58bb8c397 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/__init__.py @@ -1,4 +1,3 @@ -import json import pickle from pathlib import Path @@ -50,11 +49,6 @@ def __init__( if FACTOR_IMPLEMENT_SETTINGS.new_knowledge_base_path is not None else None ) - self.data_tables_knowledge_path = ( - Path(FACTOR_IMPLEMENT_SETTINGS.data_tables_knowledge_path) - if FACTOR_IMPLEMENT_SETTINGS.data_tables_knowledge_path is not None - else None - ) self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen @@ -78,7 +72,6 @@ def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, c factor_knowledge_base = ( FactorGraphKnowledgeBase( init_component_list=component_init_list, - data_set_knowledge_path=self.data_tables_knowledge_path, ) if self.evolving_version == 2 else FactorKnowledgeBaseV1() diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 66d852f26..2dbfa9c56 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -183,19 +183,6 @@ def __init__(self, *args, **kwargs) -> None: self.num_loop = 0 self.haveSelected = False - def _query_data_tables(self, user_prompt, session): - for _ in range(10): # max attempt to reduce the length of user_prompt - response = session.build_chat_completion( - user_prompt=user_prompt, - json_mode=True, - ) - try: - result = json.loads(response) - return result - except json.JSONDecodeError: - continue - return None - def implement_one_factor( self, target_task: FactorTask, @@ -231,42 +218,6 @@ def implement_one_factor( queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else [] ) - queried_data_tables = ( - queried_knowledge.data_set_knowledge_dict[target_factor_task_information] - if queried_knowledge is not None - else [] - ) - queried_data_tables_str = json.dumps(queried_data_tables, indent=2) - - system_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - implement_prompts["evolving_strategy_search_data_table_system_prompt"], - ) - .render() - ) - - user_prompt = ( - Environment(undefined=StrictUndefined) - .from_string( - implement_prompts["evolving_strategy_search_data_table"], - ) - .render( - scenario=self.scen.get_scenario_all_desc(), - factor_information_str=target_factor_task_information, - data_tables=queried_data_tables_str, - ) - ) - session = APIBackend(use_chat_cache=FACTOR_IMPLEMENT_SETTINGS.coder_use_cache).build_chat_session( - session_system_prompt=system_prompt, - ) - - useful_data_table = self._query_data_tables(user_prompt, session) - selected_knowledge_dict = {} - for key in useful_data_table: - if key in queried_knowledge.data_set_knowledge_dict: - selected_knowledge_dict[key] = queried_knowledge.data_set_knowledge_dict[key] - queried_former_failed_knowledge_to_render = queried_former_failed_knowledge system_prompt = ( @@ -277,7 +228,6 @@ def implement_one_factor( .render( scenario=self.scen.get_scenario_all_desc(), queried_former_failed_knowledge=queried_former_failed_knowledge_to_render, - selected_knowledge_dict=selected_knowledge_dict, ) ) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py index 65cc84bac..d5f76286c 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py @@ -1,7 +1,6 @@ from __future__ import annotations import copy -import heapq import json import random import re @@ -151,9 +150,9 @@ def query( for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_task_information() if target_factor_task_information in self.knowledgebase.success_task_info_set: - queried_knowledge.success_task_to_knowledge_dict[ - target_factor_task_information - ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1] + queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = ( + self.knowledgebase.implementation_trace[target_factor_task_information][-1] + ) elif ( len( self.knowledgebase.implementation_trace.setdefault( @@ -165,14 +164,12 @@ def query( ): queried_knowledge.failed_task_info_set.add(target_factor_task_information) else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[ - target_factor_task_information - ] = self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - )[ - -v1_query_former_trace_limit: - ] + queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( + self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + )[-v1_query_former_trace_limit:] + ) knowledge_base_success_task_list = list( self.knowledgebase.success_task_info_set, @@ -193,9 +190,9 @@ def query( )[-1] for index in similar_indexes ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[ - target_factor_task_information - ] = similar_successful_knowledge + queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = ( + similar_successful_knowledge + ) return queried_knowledge @@ -206,13 +203,11 @@ def __init__( former_traces: dict = {}, component_with_success_task: dict = {}, error_with_success_task: dict = {}, - data_set_knowledge_dict: dict = {}, **kwargs, ) -> None: self.former_traces = former_traces self.component_with_success_task = component_with_success_task self.error_with_success_task = error_with_success_task - self.data_set_knowledge_dict = data_set_knowledge_dict super().__init__(**kwargs) @@ -312,10 +307,6 @@ def query(self, evo: EvolvableSubjects, evolving_trace: list[EvoStep]) -> Querie FACTOR_IMPLEMENT_SETTINGS.v2_query_error_limit, knowledge_sampler=conf_knowledge_sampler, ) - factor_implementation_queried_graph_knowledge = self.dataset_query( - evo, - factor_implementation_queried_graph_knowledge, - ) return factor_implementation_queried_graph_knowledge def analyze_component( @@ -436,9 +427,9 @@ def former_trace_query( else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[ - target_factor_task_information - ] = former_trace_knowledge[-v2_query_former_trace_limit:] + factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( + former_trace_knowledge[-v2_query_former_trace_limit:] + ) else: factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = [] @@ -718,37 +709,9 @@ def error_query( return factor_implementation_queried_graph_knowledge - def dataset_query( - self, - evo: EvolvableSubjects, - factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge, - ) -> QueriedKnowledge | None: - for task_index, target_factor_task in enumerate(evo.sub_tasks): - target_factor_task_information = target_factor_task.get_task_information() - related_info = {} - - knowledge_dict = self.knowledgebase.data_set_knowledge_dict - table_explanations = [f"{key}: {json.dumps(value)}" for key, value in knowledge_dict.items()] - - similarity = calculate_embedding_distance_between_str_list( - [target_factor_task_information], table_explanations - )[0] - - top_related_indexes = heapq.nlargest(10, range(len(similarity)), key=lambda i: similarity[i]) - - for index in top_related_indexes: - key = list(knowledge_dict.keys())[index] - related_info[key] = knowledge_dict[key] - - factor_implementation_queried_graph_knowledge.data_set_knowledge_dict[ - target_factor_task_information - ] = related_info - - return factor_implementation_queried_graph_knowledge - class FactorGraphKnowledgeBase(EvolvingKnowledgeBase): - def __init__(self, init_component_list=None, path: str | Path = None, data_set_knowledge_path=None) -> None: + def __init__(self, init_component_list=None, path: str | Path = None) -> None: """ Load knowledge, offer brief information of knowledge and common handle interfaces """ @@ -776,13 +739,6 @@ def __init__(self, init_component_list=None, path: str | Path = None, data_set_k # store the task description to component nodes self.task_to_component_nodes = {} - # data set: data set information - self.data_set_knowledge_dict = {} - if data_set_knowledge_path: - with open(data_set_knowledge_path, "r") as f: - self.data_set_knowledge_dict = json.load(f) - super().__init__(path) - def get_all_nodes_by_label(self, label: str) -> list[UndirectedNode]: return self.graph.get_all_nodes_by_label(label) diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index 2120319e1..5b573a78a 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -55,9 +55,6 @@ class Config: knowledge_base_path: Union[str, None] = None """Path to the knowledge base""" - data_tables_knowledge_path: Union[str, None] = None - """Path to the data tables knowledge""" - new_knowledge_base_path: Union[str, None] = None """Path to the new knowledge base""" diff --git a/rdagent/components/coder/factor_coder/prompts.yaml b/rdagent/components/coder/factor_coder/prompts.yaml index 977b9393b..a3423ace4 100644 --- a/rdagent/components/coder/factor_coder/prompts.yaml +++ b/rdagent/components/coder/factor_coder/prompts.yaml @@ -46,8 +46,7 @@ evolving_strategy_factor_implementation_v1_system: |- 1. The user might provide you the correct code to similar factors. Your should learn from these code to write the correct code. 2. The user might provide you the failed former code and the corresponding feedback to the code. The feedback contains to the execution, the code and the factor value. You should analyze the feedback and try to correct the latest code. 3. The user might provide you the suggestion to the latest fail code and some similar fail to correct pairs. Each pair contains the fail code with similar error and the corresponding corrected version code. You should learn from these suggestion to write the correct code. - 4. The user might provide you some data table names and their explanations, as well as their corresponding data fields. You should find the data fields that contains the data needed for the factor implementation. - + Your must write your code based on your former latest attempt below which consists of your former code and code feedback, you should read the former attempt carefully and must not modify the right part of your former code. {% if queried_former_failed_knowledge|length != 0 %} @@ -58,13 +57,6 @@ evolving_strategy_factor_implementation_v1_system: |- {{ queried_former_failed_knowledge[-1].feedback }} {% endif %} - {% if selected_knowledge_dict|length != 0 %} - --------------Some datasets and their fields you may need:--------------- - {% for selected_knowledge in selected_knowledge_dict.keys() %} - {{ selected_knowledge }}: {{ selected_knowledge_dict[selected_knowledge] }} - {% endfor %} - {% endif %} - Please response the code in the following json format. Here is an example structure for the JSON output: { "code": "The Python code as a string." @@ -127,27 +119,6 @@ evolving_strategy_factor_implementation_v2_user: |- {% endfor %} {% endif %} -evolving_strategy_search_data_table_system_prompt: |- - The user will provide you with a task and some data. You need to determine which data tables are needed by the user. - - Respond with your analysis in JSON format. The JSON schema should include: - { - "name of data table 1": ["name of data field 1", "name of data field 2"], - "name of data table 2": ["name of data field 1"], - } - -evolving_strategy_search_data_table: |- - User is trying to implement some factors in the following scenario: - {{ scenario }} - User is doing the following task: - {{factor_information_str}} - - Here are some data table names and their explanations. - You should find the data table and data columns that contains the data needed for the factor implementation. - Please response the data table name in the json format. - -------------------Data Tables------------------- - {{ data_tables }} - evolving_strategy_error_summary_v2_system: |- User is trying to implement some factors in the following scenario: {{ scenario }} From 1d87a9e03e7aeae60827cbec04ec57b822404e3f Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Tue, 24 Sep 2024 03:57:53 +0000 Subject: [PATCH 2/2] fix CI --- .../CoSTEER/knowledge_management.py | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py index d5f76286c..c1cc66fa3 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py @@ -150,9 +150,9 @@ def query( for target_factor_task in evo.sub_tasks: target_factor_task_information = target_factor_task.get_task_information() if target_factor_task_information in self.knowledgebase.success_task_info_set: - queried_knowledge.success_task_to_knowledge_dict[target_factor_task_information] = ( - self.knowledgebase.implementation_trace[target_factor_task_information][-1] - ) + queried_knowledge.success_task_to_knowledge_dict[ + target_factor_task_information + ] = self.knowledgebase.implementation_trace[target_factor_task_information][-1] elif ( len( self.knowledgebase.implementation_trace.setdefault( @@ -164,12 +164,14 @@ def query( ): queried_knowledge.failed_task_info_set.add(target_factor_task_information) else: - queried_knowledge.working_task_to_former_failed_knowledge_dict[target_factor_task_information] = ( - self.knowledgebase.implementation_trace.setdefault( - target_factor_task_information, - [], - )[-v1_query_former_trace_limit:] - ) + queried_knowledge.working_task_to_former_failed_knowledge_dict[ + target_factor_task_information + ] = self.knowledgebase.implementation_trace.setdefault( + target_factor_task_information, + [], + )[ + -v1_query_former_trace_limit: + ] knowledge_base_success_task_list = list( self.knowledgebase.success_task_info_set, @@ -190,9 +192,9 @@ def query( )[-1] for index in similar_indexes ] - queried_knowledge.working_task_to_similar_successful_knowledge_dict[target_factor_task_information] = ( - similar_successful_knowledge - ) + queried_knowledge.working_task_to_similar_successful_knowledge_dict[ + target_factor_task_information + ] = similar_successful_knowledge return queried_knowledge @@ -427,9 +429,9 @@ def former_trace_query( else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( - former_trace_knowledge[-v2_query_former_trace_limit:] - ) + factor_implementation_queried_graph_knowledge.former_traces[ + target_factor_task_information + ] = former_trace_knowledge[-v2_query_former_trace_limit:] else: factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = []