diff --git a/examples/benchmarks/dspy_entity.py b/examples/benchmarks/dspy_entity.py index bc3dd54..955dd78 100644 --- a/examples/benchmarks/dspy_entity.py +++ b/examples/benchmarks/dspy_entity.py @@ -114,19 +114,20 @@ async def run_benchmark(text: str): system_prompt_dspy = f"{system_prompt} Time: {time.time()}." lm = dspy.OpenAI( model="deepseek-chat", - model_type="chat", + model_type="chat", + api_provider="openai", api_key=os.environ["DEEPSEEK_API_KEY"], base_url=os.environ["DEEPSEEK_BASE_URL"], - system_prompt=system_prompt_dspy, + system_prompt=system_prompt, temperature=1.0, - top_p=1, - max_tokens=4096 + max_tokens=8192 ) - dspy.settings.configure(lm=lm) + dspy.settings.configure(lm=lm, experimental=True) graph_storage_with_dspy, time_with_dspy = await benchmark_entity_extraction(text, system_prompt_dspy, use_dspy=True) print(f"Execution time with DSPy-AI: {time_with_dspy:.2f} seconds") print_extraction_results(graph_storage_with_dspy) + import pdb; pdb.set_trace() print("Running benchmark without DSPy-AI:") system_prompt_no_dspy = f"{system_prompt} Time: {time.time()}." graph_storage_without_dspy, time_without_dspy = await benchmark_entity_extraction(text, system_prompt_no_dspy, use_dspy=False) diff --git a/examples/finetune_entity_relationship_dspy.ipynb b/examples/finetune_entity_relationship_dspy.ipynb index f637125..7503ae1 100644 --- a/examples/finetune_entity_relationship_dspy.ipynb +++ b/examples/finetune_entity_relationship_dspy.ipynb @@ -1,8 +1,24 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Evaluating Entity Relationship Extraction with DSPy & Fine-Tune Prompt Instructions\n", + "\n", + "## Steps\n", + "- Load DSPy examples (separated into train, val, dev) that are saved locally.\n", + "- Evaluate the extraction module with the dev examples to determine the baseline scores, i.e: the fine-tuned extraction module should score higher.\n", + "- Run bootstrapping with random search with train examples, evaluate its compiled extraction module on the same dev examples to compare against baseline scores.\n", + "- Run MIPROv2 with train and dev examples, evaluate its compiled extraction module on the same dev examples to compare against baseline and bootstrapping with random search scores.\n", + "\n", + "## Why Use MIPROv2?\n", + "MIPROv2 is an optimizer that generates candidate few-shot examples and instructions for each prompt in the extraction module, and then optimizes over the fewshot examples and instructions as hyperparameters for a number of batches. For each batch, the optimizer evaluates different combinations of prompts on a subset of training inputs and learns the combinations that maximizes performance. This optimizer is quite useful when dealing with a large number of training examples. " + ] + }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,36 +28,41 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniconda/base/envs/nano-graphrag/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], "source": [ "import dspy\n", - "from dspy.teleprompt.random_search import BootstrapFewShotWithRandomSearch\n", - "from dspy.teleprompt.mipro_optimizer_v2 import MIPROv2\n", + "from dspy.teleprompt import BootstrapFewShotWithRandomSearch, MIPROv2\n", "from dspy.evaluate import Evaluate\n", - "import asyncio\n", "import os\n", "import numpy as np\n", "from dotenv import load_dotenv\n", - "from datasets import load_dataset\n", "import logging\n", "import pickle\n", + "import matplotlib.pyplot as plt\n", "\n", - "from nano_graphrag._utils import compute_mdhash_id\n", - "from nano_graphrag.entity_extraction.extract import generate_dataset\n", - "from nano_graphrag.entity_extraction.module import EntityRelationshipExtractor\n", - "from nano_graphrag.entity_extraction.metric import relationship_similarity_metric, entity_recall_metric" + "from nano_graphrag.entity_extraction.module import TypedEntityRelationshipExtractor\n", + "from nano_graphrag.entity_extraction.metric import relationships_similarity_metric, entity_recall_metric" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "WORKING_DIR = \"./nano_graphrag_cache_finetune_entity_relationship_dspy\"\n", - "\n", + "EXAMPLES_DIR = \"./nano_graphrag_cache_generate_dspy_examples\"\n", "load_dotenv()\n", "\n", "logging.basicConfig(level=logging.WARNING)\n", @@ -52,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -62,161 +83,43 @@ " If you detect that you made a mistake in your reasoning at any point, correct yourself.\n", " Think carefully.\n", "\"\"\"\n", - "lm = dspy.OpenAI(\n", + "deepseek = dspy.OpenAI(\n", " model=\"deepseek-chat\", \n", - " model_type=\"chat\", \n", + " model_type=\"chat\",\n", " api_key=os.environ[\"DEEPSEEK_API_KEY\"], \n", " base_url=os.environ[\"DEEPSEEK_BASE_URL\"], \n", " system_prompt=system_prompt, \n", " temperature=1.0,\n", - " top_p=1.0,\n", - " max_tokens=4096\n", + " max_tokens=8192\n", ")\n", - "llama_lm = dspy.OllamaLocal(\n", - " model=\"llama3.1\", \n", - " model_type=\"chat\",\n", + "qwen2 = dspy.OllamaLocal(\n", + " model=\"qwen2\", \n", " system=system_prompt,\n", - " max_tokens=4096\n", + " temperature=1.0,\n", + " max_tokens=4096,\n", + " num_ctx=32000,\n", + " format=\"json\",\n", + " timeout_s=240,\n", ")\n", - "dspy.settings.configure(lm=lm)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/homebrew/Caskroom/miniconda/base/envs/nano-graphrag/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by promote_options='default'.\n", - " table = cls._concat_blocks(blocks, axis=0)\n" - ] - } - ], - "source": [ - "os.makedirs(WORKING_DIR, exist_ok=True)\n", - "train_len = 20\n", - "val_len = 2\n", - "dev_len = 3\n", - "entity_relationship_trainset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_trainset.pkl\")\n", - "entity_relationship_valset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_valset.pkl\")\n", - "entity_relationship_devset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_devset.pkl\")\n", - "entity_relationship_module_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news.json\")\n", - "fin_news = load_dataset(\"ashraq/financial-news-articles\")\n", - "cnn_news = load_dataset(\"AyoubChLin/CNN_News_Articles_2011-2022\")\n", - "fin_shuffled_indices = np.random.permutation(len(fin_news['train']))\n", - "cnn_train_shuffled_indices = np.random.permutation(len(cnn_news['train']))\n", - "cnn_test_shuffled_indices = np.random.permutation(len(cnn_news['test']))\n", - "train_data = cnn_news['train'].select(cnn_train_shuffled_indices[:train_len])\n", - "val_data = cnn_news['test'].select(cnn_test_shuffled_indices[:val_len])\n", - "dev_data = fin_news['train'].select(fin_shuffled_indices[:dev_len])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "train_data['text'][:2]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "val_data['text']" + "dspy.settings.configure(lm=deepseek, experimental=True)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ - "dev_data['text'][:2]" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:nano-graphrag:Entities: 17 | Missed Entities: 15 | Total Entities: 32\n", - "DEBUG:nano-graphrag:Entities: 9 | Missed Entities: 7 | Total Entities: 16\n", - "DEBUG:nano-graphrag:Entities: 27 | Missed Entities: 21 | Total Entities: 48\n", - "DEBUG:nano-graphrag:Entities: 18 | Missed Entities: 10 | Total Entities: 28\n", - "DEBUG:nano-graphrag:Entities: 9 | Missed Entities: 9 | Total Entities: 18\n", - "DEBUG:nano-graphrag:Entities: 13 | Missed Entities: 6 | Total Entities: 19\n", - "DEBUG:nano-graphrag:Entities: 14 | Missed Entities: 7 | Total Entities: 21\n", - "DEBUG:nano-graphrag:Entities: 8 | Missed Entities: 10 | Total Entities: 18\n", - "DEBUG:nano-graphrag:Entities: 28 | Missed Entities: 6 | Total Entities: 34\n", - "DEBUG:nano-graphrag:Entities: 13 | Missed Entities: 5 | Total Entities: 18\n", - "DEBUG:nano-graphrag:Entities: 15 | Missed Entities: 8 | Total Entities: 23\n", - "DEBUG:nano-graphrag:Entities: 14 | Missed Entities: 5 | Total Entities: 19\n", - "DEBUG:nano-graphrag:Entities: 21 | Missed Entities: 5 | Total Entities: 26\n", - "DEBUG:nano-graphrag:Entities: 11 | Missed Entities: 6 | Total Entities: 17\n", - "DEBUG:nano-graphrag:Entities: 16 | Missed Entities: 9 | Total Entities: 25\n", - "DEBUG:nano-graphrag:Entities: 25 | Missed Entities: 10 | Total Entities: 35\n", - "DEBUG:nano-graphrag:Relationships: 27 | Missed Relationships: 22 | Total Relationships: 49\n", - "DEBUG:nano-graphrag:Relationships: 11 | Missed Relationships: 9 | Total Relationships: 20\n", - "DEBUG:nano-graphrag:Relationships: 18 | Missed Relationships: 20 | Total Relationships: 38\n", - "DEBUG:nano-graphrag:Relationships: 15 | Missed Relationships: 7 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Relationships: 12 | Missed Relationships: 10 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Relationships: 15 | Missed Relationships: 9 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Relationships: 12 | Missed Relationships: 9 | Total Relationships: 21\n", - "DEBUG:nano-graphrag:Relationships: 7 | Missed Relationships: 8 | Total Relationships: 15\n", - "DEBUG:nano-graphrag:Relationships: 17 | Missed Relationships: 6 | Total Relationships: 23\n", - "DEBUG:nano-graphrag:Relationships: 10 | Missed Relationships: 6 | Total Relationships: 16\n", - "DEBUG:nano-graphrag:Relationships: 16 | Missed Relationships: 8 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Relationships: 15 | Missed Relationships: 5 | Total Relationships: 20\n", - "DEBUG:nano-graphrag:Relationships: 19 | Missed Relationships: 5 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Relationships: 10 | Missed Relationships: 8 | Total Relationships: 18\n", - "DEBUG:nano-graphrag:Relationships: 13 | Missed Relationships: 9 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Relationships: 22 | Missed Relationships: 10 | Total Relationships: 32\n", - "DEBUG:nano-graphrag:Direct Relationships: 44 | Second-order: 5 | Third-order: 0 | Total Relationships: 49\n", - "DEBUG:nano-graphrag:Direct Relationships: 16 | Second-order: 4 | Third-order: 0 | Total Relationships: 20\n", - "DEBUG:nano-graphrag:Direct Relationships: 38 | Second-order: 0 | Third-order: 0 | Total Relationships: 38\n", - "DEBUG:nano-graphrag:Direct Relationships: 22 | Second-order: 0 | Third-order: 0 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Direct Relationships: 22 | Second-order: 0 | Third-order: 0 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Direct Relationships: 24 | Second-order: 0 | Third-order: 0 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Direct Relationships: 21 | Second-order: 0 | Third-order: 0 | Total Relationships: 21\n", - "DEBUG:nano-graphrag:Direct Relationships: 15 | Second-order: 0 | Third-order: 0 | Total Relationships: 15\n", - "DEBUG:nano-graphrag:Direct Relationships: 23 | Second-order: 0 | Third-order: 0 | Total Relationships: 23\n", - "DEBUG:nano-graphrag:Direct Relationships: 16 | Second-order: 0 | Third-order: 0 | Total Relationships: 16\n", - "DEBUG:nano-graphrag:Direct Relationships: 24 | Second-order: 0 | Third-order: 0 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Direct Relationships: 17 | Second-order: 3 | Third-order: 0 | Total Relationships: 20\n", - "DEBUG:nano-graphrag:Direct Relationships: 24 | Second-order: 0 | Third-order: 0 | Total Relationships: 24\n", - "DEBUG:nano-graphrag:Direct Relationships: 13 | Second-order: 5 | Third-order: 0 | Total Relationships: 18\n", - "DEBUG:nano-graphrag:Direct Relationships: 22 | Second-order: 0 | Third-order: 0 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Direct Relationships: 32 | Second-order: 0 | Third-order: 0 | Total Relationships: 32\n", - "DEBUG:nano-graphrag:Entities: 10 | Missed Entities: 5 | Total Entities: 15\n", - "DEBUG:nano-graphrag:Entities: 6 | Missed Entities: 5 | Total Entities: 11\n", - "DEBUG:nano-graphrag:Entities: 18 | Missed Entities: 15 | Total Entities: 33\n", - "DEBUG:nano-graphrag:Entities: 15 | Missed Entities: 10 | Total Entities: 25\n", - "DEBUG:nano-graphrag:Relationships: 11 | Missed Relationships: 5 | Total Relationships: 16\n", - "DEBUG:nano-graphrag:Relationships: 5 | Missed Relationships: 5 | Total Relationships: 10\n", - "DEBUG:nano-graphrag:Relationships: 13 | Missed Relationships: 15 | Total Relationships: 28\n", - "DEBUG:nano-graphrag:Relationships: 16 | Missed Relationships: 10 | Total Relationships: 26\n", - "DEBUG:nano-graphrag:Direct Relationships: 11 | Second-order: 5 | Third-order: 0 | Total Relationships: 16\n", - "DEBUG:nano-graphrag:Direct Relationships: 9 | Second-order: 1 | Third-order: 0 | Total Relationships: 10\n", - "DEBUG:nano-graphrag:Direct Relationships: 28 | Second-order: 0 | Third-order: 0 | Total Relationships: 28\n", - "DEBUG:nano-graphrag:Direct Relationships: 26 | Second-order: 0 | Third-order: 0 | Total Relationships: 26\n", - "INFO:nano-graphrag:Saved 20 examples with keys: ['input_text', 'entities', 'relationships']\n" - ] - } - ], - "source": [ - "train_chunks = {compute_mdhash_id(text, prefix=f\"chunk-\"): {\"content\": text} for text in train_data[\"text\"]}\n", - "trainset = asyncio.run(generate_dataset(chunks=train_chunks, filepath=entity_relationship_trainset_path))" + "os.makedirs(WORKING_DIR, exist_ok=True)\n", + "entity_relationship_rs_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_rs.json\")\n", + "entity_relationship_miprov2_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_miprov2.json\")\n", + "entity_relationship_trainset_path = os.path.join(EXAMPLES_DIR, \"entity_relationship_extraction_news_trainset.pkl\")\n", + "entity_relationship_valset_path = os.path.join(EXAMPLES_DIR, \"entity_relationship_extraction_news_valset.pkl\")\n", + "entity_relationship_devset_path = os.path.join(EXAMPLES_DIR, \"entity_relationship_extraction_news_devset.pkl\")\n", + "\n", + "trainset = pickle.load(open(entity_relationship_trainset_path, \"rb\"))\n", + "valset = pickle.load(open(entity_relationship_valset_path, \"rb\"))\n", + "devset = pickle.load(open(entity_relationship_devset_path, \"rb\"))" ] }, { @@ -226,8 +129,8 @@ "outputs": [], "source": [ "for example in trainset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 2:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 2:\n", " print(relationship)" ] }, @@ -238,8 +141,8 @@ "outputs": [], "source": [ "for example in trainset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 3:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 3:\n", " print(relationship)" ] }, @@ -249,52 +152,16 @@ "metadata": {}, "outputs": [], "source": [ - "trainset[0].relationships.context[:2]" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:nano-graphrag:Entities: 21 | Missed Entities: 14 | Total Entities: 35\n", - "DEBUG:nano-graphrag:Entities: 10 | Missed Entities: 5 | Total Entities: 15\n", - "DEBUG:nano-graphrag:Relationships: 22 | Missed Relationships: 14 | Total Relationships: 36\n", - "DEBUG:nano-graphrag:Relationships: 10 | Missed Relationships: 5 | Total Relationships: 15\n", - "DEBUG:nano-graphrag:Direct Relationships: 36 | Second-order: 0 | Third-order: 0 | Total Relationships: 36\n", - "DEBUG:nano-graphrag:Direct Relationships: 12 | Second-order: 3 | Third-order: 0 | Total Relationships: 15\n", - "INFO:nano-graphrag:Saved 2 examples with keys: ['input_text', 'entities', 'relationships']\n" - ] - } - ], - "source": [ - "val_chunks = {compute_mdhash_id(text, prefix=f\"chunk-\"): {\"content\": text} for text in val_data[\"text\"]}\n", - "valset = asyncio.run(generate_dataset(chunks=val_chunks, filepath=entity_relationship_valset_path))" + "trainset[0].relationships[:2]" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Relationship(src_id='PORTUGAL', tgt_id='EURO 2016', description='Portugal qualified for the final of Euro 2016.', weight=0.9, order=1),\n", - " Relationship(src_id='PORTUGAL', tgt_id='WALES', description='Portugal defeated Wales in the semifinal of Euro 2016.', weight=0.9, order=1)]" - ] - }, - "execution_count": 30, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "valset[0].relationships.context[:2]" + "valset[0].relationships[:2]" ] }, { @@ -304,8 +171,8 @@ "outputs": [], "source": [ "for example in valset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 2:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 2:\n", " print(relationship)" ] }, @@ -316,45 +183,18 @@ "outputs": [], "source": [ "for example in valset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 3:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 3:\n", " print(relationship)" ] }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "DEBUG:nano-graphrag:Entities: 27 | Missed Entities: 9 | Total Entities: 36\n", - "DEBUG:nano-graphrag:Entities: 14 | Missed Entities: 7 | Total Entities: 21\n", - "DEBUG:nano-graphrag:Entities: 7 | Missed Entities: 4 | Total Entities: 11\n", - "DEBUG:nano-graphrag:Relationships: 19 | Missed Relationships: 8 | Total Relationships: 27\n", - "DEBUG:nano-graphrag:Relationships: 14 | Missed Relationships: 8 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Relationships: 8 | Missed Relationships: 8 | Total Relationships: 16\n", - "DEBUG:nano-graphrag:Direct Relationships: 27 | Second-order: 0 | Third-order: 0 | Total Relationships: 27\n", - "DEBUG:nano-graphrag:Direct Relationships: 18 | Second-order: 4 | Third-order: 0 | Total Relationships: 22\n", - "DEBUG:nano-graphrag:Direct Relationships: 12 | Second-order: 4 | Third-order: 0 | Total Relationships: 16\n", - "INFO:nano-graphrag:Saved 3 examples with keys: ['input_text', 'entities', 'relationships']\n" - ] - } - ], - "source": [ - "dev_chunks = {compute_mdhash_id(text, prefix=f\"chunk-\"): {\"content\": text} for text in dev_data[\"text\"]}\n", - "devset = asyncio.run(generate_dataset(chunks=dev_chunks, filepath=entity_relationship_devset_path))" - ] - }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "devset[0].relationships.context[:2]" + "devset[0].relationships[:2]" ] }, { @@ -364,8 +204,8 @@ "outputs": [], "source": [ "for example in devset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 2:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 2:\n", " print(relationship)" ] }, @@ -376,131 +216,172 @@ "outputs": [], "source": [ "for example in devset:\n", - " for relationship in example.relationships.context:\n", - " if relationship.order == 3:\n", + " for relationship in example.relationships:\n", + " if relationship['order'] == 3:\n", " print(relationship)" ] }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "extractor.predictor = Predict(CombinedExtraction(input_text, entity_types -> entities, relationships\n", - " instructions='Signature for extracting both entities and relationships from input text.'\n", + "extractor.predictor.predictor = Predict(StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given a text document that is potentially relevant to this activity and a list of entity types, \\nidentify all entities of those types from the text and all relationships among the identified entities.\\n\\nEntity Guidelines:\\n1. Each entity name should be an actual atomic word from the input text. \\n2. Avoid duplicates and generic terms.\\n3. Make sure descriptions are detailed and comprehensive. Use multiple complete sentences for each point below:\\n a). The entity\\'s role or significance in the context\\n b). Key attributes or characteristics\\n c). Relationships to other entities (if applicable)\\n d). Historical or cultural relevance (if applicable)\\n e). Any notable actions or events associated with the entity\\n4. All entity types from the text must be included. \\n5. IMPORTANT: Only use entity types from the provided \\'entity_types\\' list. Do not introduce new entity types.\\n\\nRelationship Guidelines:\\n1. Make sure relationship descriptions are detailed and comprehensive. Use multiple complete sentences for each point below:\\n a). The nature of the relationship (e.g., familial, professional, causal)\\n b). The impact or significance of the relationship on both entities\\n c). Any historical or contextual information relevant to the relationship\\n d). How the relationship evolved over time (if applicable)\\n e). Any notable events or actions that resulted from this relationship\\n2. Include direct relationships (order 1) as well as higher-order relationships (order 2 and 3):\\n a). Direct relationships: Immediate connections between entities.\\n b). Second-order relationships: Indirect effects or connections that result from direct relationships.\\n c). Third-order relationships: Further indirect effects that result from second-order relationships.\\n3. The \"src_id\" and \"tgt_id\" fields must exactly match entity names from the extracted entities list.'\n", " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", - " entity_types = Field(annotation=EntityTypes required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Entity Types:', 'desc': '${entity_types}'})\n", - " entities = Field(annotation=Entities required=True json_schema_extra={'desc': '\\n Format:\\n {\\n \"context\": [\\n {\\n \"entity_name\": \"ENTITY NAME\",\\n \"entity_type\": \"ENTITY TYPE\",\\n \"description\": \"Detailed description\",\\n \"importance_score\": 0.8\\n },\\n ...\\n ]\\n }\\n Each entity name should be an actual atomic word from the input text. Avoid duplicates and generic terms.\\n Make sure descriptions are concise and specific, and all entity types are included from the text. \\n Entities must have an importance score greater than 0.5.\\n IMPORTANT: Only use entity types from the provided \\'entity_types\\' list. Do not introduce new entity types.\\n Ensure the output is strictly JSON formatted without any trailing text or comments.\\n ', '__dspy_field_type': 'output', 'prefix': 'Entities:'})\n", - " relationships = Field(annotation=Relationships required=True json_schema_extra={'desc': '\\n Format:\\n {\\n \"context\": [\\n {\\n \"src_id\": \"SOURCE ENTITY\",\\n \"tgt_id\": \"TARGET ENTITY\",\\n \"description\": \"Detailed description of the relationship\",\\n \"weight\": 0.7,\\n \"order\": 1 # 1 for direct relationships, 2 for second-order, 3 for third-order, etc.\\n },\\n ...\\n ]\\n }\\n Make sure relationships are detailed and specific.\\n Include direct relationships (order 1) as well as higher-order relationships (order 2 and 3):\\n - Direct relationships: Immediate connections between entities.\\n - Second-order relationships: Indirect effects or connections that result from direct relationships.\\n - Third-order relationships: Further indirect effects that result from second-order relationships.\\n IMPORTANT: Only include relationships between existing entities from the extracted entities. Do not introduce new entities here.\\n The \"src_id\" and \"tgt_id\" fields must exactly match entity names from the extracted entities list.\\n Ensure the output is strictly JSON formatted without any trailing text or comments.\\n ', '__dspy_field_type': 'output', 'prefix': 'Relationships:'})\n", - "))\n", - "self_reflection.predictor = Predict(CombinedSelfReflection(input_text, entity_types, entities, relationships -> missing_entities, missing_relationships\n", - " instructions='Signature for combined self-reflection on extracted entities and relationships.\\nSelf-reflection is on the completeness and quality of both the extracted entities and relationships.'\n", - " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The original input text.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", - " entity_types = Field(annotation=EntityTypes required=True json_schema_extra={'__dspy_field_type': 'input', 'prefix': 'Entity Types:', 'desc': '${entity_types}'})\n", - " entities = Field(annotation=Entities required=True json_schema_extra={'desc': 'List of extracted entities.', '__dspy_field_type': 'input', 'prefix': 'Entities:'})\n", - " relationships = Field(annotation=Relationships required=True json_schema_extra={'desc': 'List of extracted relationships.', '__dspy_field_type': 'input', 'prefix': 'Relationships:'})\n", - " missing_entities = Field(annotation=Entities required=True json_schema_extra={'desc': '\\n Format:\\n {\\n \"context\": [\\n {\\n \"entity_name\": \"ENTITY NAME\",\\n \"entity_type\": \"ENTITY TYPE\",\\n \"description\": \"Detailed description\",\\n \"importance_score\": 0.8\\n },\\n ...\\n ]\\n }\\n More specifically:\\n 1. Entities mentioned in the text but not captured in the initial extraction.\\n 2. Implicit entities that are crucial to the context but not explicitly mentioned.\\n 3. Entities that belong to the identified entity types but were overlooked.\\n 4. Subtypes or more specific instances of the already extracted entities.\\n Ensure the output is strictly JSON formatted without any trailing text or comments.\\n ', '__dspy_field_type': 'output', 'prefix': 'Missing Entities:'})\n", - " missing_relationships = Field(annotation=Relationships required=True json_schema_extra={'desc': '\\n Format:\\n {\\n \"context\": [\\n {\\n \"src_id\": \"SOURCE ENTITY\",\\n \"tgt_id\": \"TARGET ENTITY\",\\n \"description\": \"Detailed description of the relationship\",\\n \"weight\": 0.7,\\n \"order\": 1 # 1 for direct, 2 for second-order, 3 for third-order\\n },\\n ...\\n ]\\n }\\n More specifically:\\n 1. Direct relationships (order 1) between entities that were not captured initially.\\n 2. Second-order relationships (order 2): Indirect effects or connections resulting from direct relationships.\\n 3. Third-order relationships (order 3): Further indirect effects resulting from second-order relationships.\\n 4. Implicit relationships that can be inferred from the context.\\n 5. Hierarchical, causal, or temporal relationships that may have been overlooked.\\n 6. Relationships involving the newly identified missing entities.\\n Only include relationships between entities in the combined entities list (extracted + missing).\\n Ensure the output is strictly JSON formatted without any trailing text or comments.\\n ', '__dspy_field_type': 'output', 'prefix': 'Missing Relationships:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", "))" ] }, - "execution_count": 33, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "model = EntityRelationshipExtractor()\n", + "model = TypedEntityRelationshipExtractor()\n", "model" ] }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:nano-graphrag:Entities: 13 | Missed Entities: 10 | Total Entities: 23\n", - "DEBUG:nano-graphrag:Entities: 22 | Missed Entities: 14 | Total Entities: 36\n", - " 0%| | 0/3 [00:00\n", - "#T_34e56 th {\n", + "#T_18824 th {\n", " text-align: left;\n", "}\n", - "#T_34e56 td {\n", + "#T_18824 td {\n", " text-align: left;\n", "}\n", - "#T_34e56_row0_col0, #T_34e56_row0_col1, #T_34e56_row0_col2, #T_34e56_row0_col3, #T_34e56_row0_col4, #T_34e56_row0_col5, #T_34e56_row1_col0, #T_34e56_row1_col1, #T_34e56_row1_col2, #T_34e56_row1_col3, #T_34e56_row1_col4, #T_34e56_row1_col5, #T_34e56_row2_col0, #T_34e56_row2_col1, #T_34e56_row2_col2, #T_34e56_row2_col3, #T_34e56_row2_col4, #T_34e56_row2_col5 {\n", + "#T_18824_row0_col0, #T_18824_row0_col1, #T_18824_row0_col2, #T_18824_row0_col3, #T_18824_row0_col4, #T_18824_row0_col5, #T_18824_row1_col0, #T_18824_row1_col1, #T_18824_row1_col2, #T_18824_row1_col3, #T_18824_row1_col4, #T_18824_row1_col5, #T_18824_row2_col0, #T_18824_row2_col1, #T_18824_row2_col2, #T_18824_row2_col3, #T_18824_row2_col4, #T_18824_row2_col5, #T_18824_row3_col0, #T_18824_row3_col1, #T_18824_row3_col2, #T_18824_row3_col3, #T_18824_row3_col4, #T_18824_row3_col5, #T_18824_row4_col0, #T_18824_row4_col1, #T_18824_row4_col2, #T_18824_row4_col3, #T_18824_row4_col4, #T_18824_row4_col5 {\n", " text-align: left;\n", " white-space: pre-wrap;\n", " word-wrap: break-word;\n", " max-width: 400px;\n", "}\n", "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 input_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsentity_recall_metricinput_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsentity_recall_metric
0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...context=[Entity(entity_name='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', entity_type='ORGANIZATION', description='A high school in Florida where a mass shooting occurred.', importance_score=0.9), Entity(entity_name='NIKOLAS CRUZ', entity_type='PERSON', description='The gunman who carried out...context=[Relationship(src_id='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', tgt_id='NIKOLAS CRUZ', description='Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', weight=0.9, order=1), Relationship(src_id='NIKOLAS CRUZ', tgt_id='FLORIDA',...context=[Entity(entity_name='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', entity_type='ORGANIZATION', description='A high school in Florida where a mass shooting occurred.', importance_score=0.9), Entity(entity_name='NIKOLAS CRUZ', entity_type='PERSON', description='The gunman who carried out...context=[Relationship(src_id='NIKOLAS CRUZ', tgt_id='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', description='Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', weight=0.9, order=1), Relationship(src_id='LAURENZO PRADO', tgt_id='MARJORY...✔️ [0.8055555555555556]0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about restricting sales of assault rifles.', 'importance_score':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Students from Marjory Stoneman Douglas High School are located in the state of Florida.', 'weight': 0.9,...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about assault rifle sales.', 'importance_score': 0.9}, {'entity_name':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Marjory Stoneman Douglas High School is located in Florida.', 'weight': 0.9, 'order': 1}, {'src_id': 'LAURENZO PRADO',...✔️ [0.6666666666666666]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.8, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...✔️ [0.8888888888888888]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...[{'entity_name': 'YONGHUI SUPERSTORES CO LTD', 'entity_type': 'ORGANIZATION', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'importance_score': 1.0}, {'entity_name':...[{'src_id': 'YONGHUI SUPERSTORES CO LTD', 'tgt_id': '166.3 MILLION SHARES', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'weight':...[{'entity_name': 'SHANGHAI STOCK EXCHANGE', 'entity_type': 'ORGANIZATION', 'description': 'The Shanghai Stock Exchange is a stock exchange located in Shanghai, China.', 'importance_score': 0.9}, {'entity_name': 'YONGHUI SUPERSTORES CO...[{'src_id': 'SHANGHAI STOCK EXCHANGE', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': 'The Shanghai Stock Exchange filed a block trade involving YONGHUI SUPERSTORES Co Ltd.', 'weight': 0.9,...✔️ [0.8]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...context=[Entity(entity_name='RAILWAYS', entity_type='VEHICLE', description='Transportation system used for ferrying people and transporting nuclear waste and coal.', importance_score=0.9), Entity(entity_name='BELGIUM', entity_type='LOCATION', description='Country where a business is looking to innovate...context=[Relationship(src_id='RAILNOVA', tgt_id='BRUSSELS', description='Railnova is based in Brussels.', weight=0.9, order=1), Relationship(src_id='RAILNOVA', tgt_id='DEUTSCHE BAHN', description='Railnova serves Deutsche Bahn as a client.', weight=0.8, order=1), Relationship(src_id='RAILNOVA', tgt_id='SNCF', description='Railnova serves...context=[Entity(entity_name='RAILWAYS', entity_type='VEHICLE', description='A mode of transportation that involves trains running on tracks, used for various purposes including passenger and cargo transport.', importance_score=0.9), Entity(entity_name='BELGIUM', entity_type='LOCATION', description='A...context=[Relationship(src_id='RAILNOVA', tgt_id='DEUTSCHE BAHN', description='Railnova provides innovative technology solutions to Deutsche Bahn, a German railway company.', weight=0.8, order=1), Relationship(src_id='RAILNOVA', tgt_id='SNCF', description='Railnova offers its technology services to...✔️ [0.8095238095238095]3LONDON (Reuters) - Britain’s economy was weaker than previously thought in 2017, official data showed on Thursday, leaving the country lagging further behind the global...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BANK OF ENGLAND', 'tgt_id': 'INTEREST RATES',...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy is discussed in the text.', 'importance_score': 1.0}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The political and...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BRITAIN', 'tgt_id': 'BANK OF ENGLAND', 'description':...✔️ [1.0]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...context=[Entity(entity_name='YONGHUI SUPERSTORES', entity_type='ORGANIZATION', description='A company involved in a block trade of its shares.', importance_score=0.9), Entity(entity_name='SHANGHAI STOCK EXCHANGE', entity_type='ORGANIZATION', description='The stock exchange where the block trade...context=[Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='SHANGHAI STOCK EXCHANGE', description=\"YONGHUI SUPERSTORES' shares were traded on the SHANGHAI STOCK EXCHANGE.\", weight=0.9, order=1), Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='166.3 MILLION SHARES', description='YONGHUI SUPERSTORES was...context=[Entity(entity_name='YONGHUI SUPERSTORES', entity_type='ORGANIZATION', description='A company involved in a block trade of its shares.', importance_score=0.9), Entity(entity_name='SHANGHAI STOCK EXCHANGE', entity_type='ORGANIZATION', description='The stock exchange where the block trade...context=[Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='166.3 MILLION SHARES', description='YONGHUI SUPERSTORES was involved in a block trade of 166.3 million shares.', weight=0.9, order=1), Relationship(src_id='166.3 MILLION SHARES', tgt_id='1.63 BILLION YUAN',...✔️ [0.7272727272727273]4Trump taps White House doctor as new VA secretary 2 Hours Ago CNBC's Kayla Tausche reports President Trump has tapped White House physician Rear Admiral...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson as new VA secretary.', 'importance_score': 1.0}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'President Trump taps Ronny Jackson as new VA secretary.', 'weight': 1.0, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'VA', 'description':...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson.', 'importance_score': 0.9}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White House where Ronny Jackson...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'Trump taps Ronny Jackson.', 'weight': 0.9, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'DEPARTMENT OF VETERANS AFFAIRS', 'description': 'Ronny Jackson...✔️ [0.8571428571428571]
\n" ], "text/plain": [ - "" + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " ... 15 more rows not displayed ...\n", + "
\n", + " " + ], + "text/plain": [ + "" ] }, "metadata": {}, @@ -510,93 +391,132 @@ "name": "stderr", "output_type": "stream", "text": [ - "DEBUG:nano-graphrag:Entities: 22 | Missed Entities: 14 | Total Entities: 36\n", - "DEBUG:nano-graphrag:Entities: 13 | Missed Entities: 10 | Total Entities: 23\n", - " 0%| | 0/3 [00:00\n", - "#T_465ae th {\n", + "#T_7104e th {\n", " text-align: left;\n", "}\n", - "#T_465ae td {\n", + "#T_7104e td {\n", " text-align: left;\n", "}\n", - "#T_465ae_row0_col0, #T_465ae_row0_col1, #T_465ae_row0_col2, #T_465ae_row0_col3, #T_465ae_row0_col4, #T_465ae_row0_col5, #T_465ae_row1_col0, #T_465ae_row1_col1, #T_465ae_row1_col2, #T_465ae_row1_col3, #T_465ae_row1_col4, #T_465ae_row1_col5, #T_465ae_row2_col0, #T_465ae_row2_col1, #T_465ae_row2_col2, #T_465ae_row2_col3, #T_465ae_row2_col4, #T_465ae_row2_col5 {\n", + "#T_7104e_row0_col0, #T_7104e_row0_col1, #T_7104e_row0_col2, #T_7104e_row0_col3, #T_7104e_row0_col4, #T_7104e_row0_col5, #T_7104e_row1_col0, #T_7104e_row1_col1, #T_7104e_row1_col2, #T_7104e_row1_col3, #T_7104e_row1_col4, #T_7104e_row1_col5, #T_7104e_row2_col0, #T_7104e_row2_col1, #T_7104e_row2_col2, #T_7104e_row2_col3, #T_7104e_row2_col4, #T_7104e_row2_col5, #T_7104e_row3_col0, #T_7104e_row3_col1, #T_7104e_row3_col2, #T_7104e_row3_col3, #T_7104e_row3_col4, #T_7104e_row3_col5, #T_7104e_row4_col0, #T_7104e_row4_col1, #T_7104e_row4_col2, #T_7104e_row4_col3, #T_7104e_row4_col4, #T_7104e_row4_col5 {\n", " text-align: left;\n", " white-space: pre-wrap;\n", " word-wrap: break-word;\n", " max-width: 400px;\n", "}\n", "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
 input_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsrelationship_similarity_metricinput_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsrelationships_similarity_metric
0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...context=[Entity(entity_name='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', entity_type='ORGANIZATION', description='A high school in Florida where a mass shooting occurred.', importance_score=0.9), Entity(entity_name='NIKOLAS CRUZ', entity_type='PERSON', description='The gunman who carried out...context=[Relationship(src_id='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', tgt_id='NIKOLAS CRUZ', description='Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', weight=0.9, order=1), Relationship(src_id='NIKOLAS CRUZ', tgt_id='FLORIDA',...context=[Entity(entity_name='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', entity_type='ORGANIZATION', description='A high school in Florida where a mass shooting occurred.', importance_score=0.9), Entity(entity_name='NIKOLAS CRUZ', entity_type='PERSON', description='The gunman who carried out...context=[Relationship(src_id='NIKOLAS CRUZ', tgt_id='MARJORY STONEMAN DOUGLAS HIGH SCHOOL', description='Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', weight=0.9, order=1), Relationship(src_id='LAURENZO PRADO', tgt_id='MARJORY...✔️ [0.946203351020813]0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about restricting sales of assault rifles.', 'importance_score':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Students from Marjory Stoneman Douglas High School are located in the state of Florida.', 'weight': 0.9,...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about assault rifle sales.', 'importance_score': 0.9}, {'entity_name':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Marjory Stoneman Douglas High School is located in Florida.', 'weight': 0.9, 'order': 1}, {'src_id': 'LAURENZO PRADO',...✔️ [0.45]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.8, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...✔️ [0.75]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...[{'entity_name': 'YONGHUI SUPERSTORES CO LTD', 'entity_type': 'ORGANIZATION', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'importance_score': 1.0}, {'entity_name':...[{'src_id': 'YONGHUI SUPERSTORES CO LTD', 'tgt_id': '166.3 MILLION SHARES', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'weight':...[{'entity_name': 'SHANGHAI STOCK EXCHANGE', 'entity_type': 'ORGANIZATION', 'description': 'The Shanghai Stock Exchange is a stock exchange located in Shanghai, China.', 'importance_score': 0.9}, {'entity_name': 'YONGHUI SUPERSTORES CO...[{'src_id': 'SHANGHAI STOCK EXCHANGE', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': 'The Shanghai Stock Exchange filed a block trade involving YONGHUI SUPERSTORES Co Ltd.', 'weight': 0.9,...✔️ [0.8]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...context=[Entity(entity_name='RAILWAYS', entity_type='VEHICLE', description='Transportation system used for ferrying people and transporting nuclear waste and coal.', importance_score=0.9), Entity(entity_name='BELGIUM', entity_type='LOCATION', description='Country where a business is looking to innovate...context=[Relationship(src_id='RAILNOVA', tgt_id='BRUSSELS', description='Railnova is based in Brussels.', weight=0.9, order=1), Relationship(src_id='RAILNOVA', tgt_id='DEUTSCHE BAHN', description='Railnova serves Deutsche Bahn as a client.', weight=0.8, order=1), Relationship(src_id='RAILNOVA', tgt_id='SNCF', description='Railnova serves...context=[Entity(entity_name='RAILWAYS', entity_type='VEHICLE', description='A mode of transportation that involves trains running on tracks, used for various purposes including passenger and cargo transport.', importance_score=0.9), Entity(entity_name='BELGIUM', entity_type='LOCATION', description='A...context=[Relationship(src_id='RAILNOVA', tgt_id='DEUTSCHE BAHN', description='Railnova provides innovative technology solutions to Deutsche Bahn, a German railway company.', weight=0.8, order=1), Relationship(src_id='RAILNOVA', tgt_id='SNCF', description='Railnova offers its technology services to...✔️ [0.9310485124588013]3LONDON (Reuters) - Britain’s economy was weaker than previously thought in 2017, official data showed on Thursday, leaving the country lagging further behind the global...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BANK OF ENGLAND', 'tgt_id': 'INTEREST RATES',...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy is discussed in the text.', 'importance_score': 1.0}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The political and...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BRITAIN', 'tgt_id': 'BANK OF ENGLAND', 'description':...✔️ [0.2]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...context=[Entity(entity_name='YONGHUI SUPERSTORES', entity_type='ORGANIZATION', description='A company involved in a block trade of its shares.', importance_score=0.9), Entity(entity_name='SHANGHAI STOCK EXCHANGE', entity_type='ORGANIZATION', description='The stock exchange where the block trade...context=[Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='SHANGHAI STOCK EXCHANGE', description=\"YONGHUI SUPERSTORES' shares were traded on the SHANGHAI STOCK EXCHANGE.\", weight=0.9, order=1), Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='166.3 MILLION SHARES', description='YONGHUI SUPERSTORES was...context=[Entity(entity_name='YONGHUI SUPERSTORES', entity_type='ORGANIZATION', description='A company involved in a block trade of its shares.', importance_score=0.9), Entity(entity_name='SHANGHAI STOCK EXCHANGE', entity_type='ORGANIZATION', description='The stock exchange where the block trade...context=[Relationship(src_id='YONGHUI SUPERSTORES', tgt_id='166.3 MILLION SHARES', description='YONGHUI SUPERSTORES was involved in a block trade of 166.3 million shares.', weight=0.9, order=1), Relationship(src_id='166.3 MILLION SHARES', tgt_id='1.63 BILLION YUAN',...✔️ [0.9334976673126221]4Trump taps White House doctor as new VA secretary 2 Hours Ago CNBC's Kayla Tausche reports President Trump has tapped White House physician Rear Admiral...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson as new VA secretary.', 'importance_score': 1.0}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'President Trump taps Ronny Jackson as new VA secretary.', 'weight': 1.0, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'VA', 'description':...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson.', 'importance_score': 0.9}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White House where Ronny Jackson...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'Trump taps Ronny Jackson.', 'weight': 0.9, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'DEPARTMENT OF VETERANS AFFAIRS', 'description': 'Ronny Jackson...✔️ [0.65]
\n" ], "text/plain": [ - "" + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " ... 15 more rows not displayed ...\n", + "
\n", + " " + ], + "text/plain": [ + "" ] }, "metadata": {}, @@ -604,16 +524,142 @@ } ], "source": [ - "metrics = [entity_recall_metric, relationship_similarity_metric]\n", + "metrics = [entity_recall_metric, relationships_similarity_metric]\n", + "baseline_scores = []\n", "for metric in metrics:\n", " evaluate = Evaluate(\n", - " devset=devset, \n", + " devset=devset[:20], \n", " metric=metric, \n", " num_threads=os.cpu_count(), \n", " display_progress=True,\n", " display_table=5,\n", " )\n", - " evaluate(model)" + " baseline_scores.append(evaluate(model))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Assess the similarity between gold and predicted relationships:\n", + "1. Match relationships based on src_id and tgt_id pairs, allowing for slight variations in entity names.\n", + "2. For matched pairs, compare:\n", + " a) Description similarity (semantic meaning)\n", + " b) Weight similarity\n", + " c) Order similarity\n", + "3. Consider unmatched relationships as penalties.\n", + "4. Aggregate scores, accounting for precision and recall.\n", + "5. Return a final similarity score between 0 (no similarity) and 1 (perfect match).\n", + "\n", + "Key considerations:\n", + "- Prioritize matching based on entity pairs over exact string matches.\n", + "- Use semantic similarity for descriptions rather than exact matches.\n", + "- Weight the importance of different aspects (e.g., entity matching, description, weight, order).\n", + "- Balance the impact of matched and unmatched relationships in the final score.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Gold Relationships: The gold-standard relationships to compare against.\n", + "\n", + "Predicted Relationships: The predicted relationships to compare against the gold-standard relationships.\n", + "\n", + "Reasoning: Let's think step by step in order to ${produce the similarity_score}. We ...\n", + "\n", + "Similarity Score: Similarity score between 0 and 1, with 1 being the highest similarity. (Respond with a single float value)\n", + "\n", + "---\n", + "\n", + "Gold Relationships: [{\"src_id\":\"INSTITUTE FOR FISCAL STUDIES\",\"tgt_id\":\"EUROPEAN UNION\",\"description\":\"The Institute for Fiscal Studies said cutting trade tariffs completely when leaving the European Union would reduce prices in British shops by only 1.2 percent.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"KLÉPIERRE SA\",\"tgt_id\":\"HAMMERSON PLC\",\"description\":\"Klépierre SA made a 4.9 billion pound takeover approach to Hammerson Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"SNAP INC\",\"tgt_id\":\"TWITTER\",\"description\":\"Snap Inc's Snapchat is so popular in Britain that its advertising revenue will overtake Twitter's UK revenue in 2019.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"PERSIMMON PLC\",\"tgt_id\":\"JEFF FAIRBURN\",\"description\":\"Persimmon Plc paid its Chief Executive, Jeff Fairburn, and two other executives a combined 104 million pounds last year.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"SHERBORNE INVESTORS\",\"tgt_id\":\"BARCLAYS PLC\",\"description\":\"Sherborne Investors, led by British-born Edward Bramson, has taken a 5 percent stake in Barclays Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"MONZO\",\"description\":\"The Royal Bank of Scotland is working on secret plans to create a standalone digital bank to compete with emerging British fintech champions including Monzo.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"REVOLUT\",\"description\":\"The Royal Bank of Scotland is working on secret plans to create a standalone digital bank to compete with emerging British fintech champions including Revolut.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"HERSHEY CO\",\"tgt_id\":\"TYRRELLS\",\"description\":\"Hershey Co, the U.S. confectionery giant, is at the early stages of exploring options for the British crisps brand Tyrrells.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"FINANCIAL REPORTING COUNCIL\",\"tgt_id\":\"CARILLION PLC\",\"description\":\"The Financial Reporting Council, UK's accountancy watchdog has launched an investigation into two former finance directors of collapsed construction firm Carillion Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"RAC FOUNDATION\",\"tgt_id\":\"CAR-OWNING HOUSEHOLDS\",\"description\":\"The RAC Foundation reported that running a car has become more than a third more expensive in one year for the poorest families.\",\"weight\":0.7,\"order\":1}]\n", + "\n", + "Predicted Relationships: [{\"src_id\":\"HAMMERSON PLC\",\"tgt_id\":\"KLÉPIERRE SA\",\"description\":\"Hammerson Plc received a takeover approach from Klépierre SA.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"SNAP INC\",\"tgt_id\":\"TWITTER\",\"description\":\"Snap Inc's Snapchat is expected to overtake Twitter's UK revenue.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"PERSIMMON PLC\",\"tgt_id\":\"JEFF FAIRBURN\",\"description\":\"Persimmon Plc paid its Chief Executive, Jeff Fairburn, a large sum.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"SHERBORNE INVESTORS\",\"tgt_id\":\"BARCLAYS PLC\",\"description\":\"Sherborne Investors took a 5 percent stake in Barclays Plc.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"MONZO\",\"description\":\"Royal Bank of Scotland is competing with Monzo in the digital banking space.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"REVOLUT\",\"description\":\"Royal Bank of Scotland is competing with Revolut in the digital banking space.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"HERSHEY CO\",\"tgt_id\":\"TYRRELLS\",\"description\":\"Hershey Co is exploring options for the British crisps brand Tyrrells.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"FINANCIAL REPORTING COUNCIL\",\"tgt_id\":\"CARILLION PLC\",\"description\":\"The Financial Reporting Council is investigating former finance directors of Carillion Plc.\",\"weight\":0.6,\"order\":1}]\n", + "\n", + "Please provide the output fields Reasoning then Similarity Score. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n", + "\n", + "\u001b[32mReasoning: Let's think step by step in order to produce the similarity_score. We start by matching relationships based on src_id and tgt_id pairs, allowing for slight variations in entity names. We then compare the matched pairs on description similarity (semantic meaning), weight similarity, and order similarity. Unmatched relationships are considered penalties. Finally, we aggregate scores, accounting for precision and recall, to return a final similarity score between 0 (no similarity) and 1 (perfect match).\n", + "\n", + "1. **Entity Pair Matching:**\n", + " - Matched pairs: \n", + " - (HAMMERSON PLC, KLÉPIERRE SA)\n", + " - (SNAP INC, TWITTER)\n", + " - (PERSIMMON PLC, JEFF FAIRBURN)\n", + " - (SHERBORNE INVESTORS, BARCLAYS PLC)\n", + " - (ROYAL BANK OF SCOTLAND, MONZO)\n", + " - (ROYAL BANK OF SCOTLAND, REVOLUT)\n", + " - (HERSHEY CO, TYRRELLS)\n", + " - (FINANCIAL REPORTING COUNCIL, CARILLION PLC)\n", + " - Unmatched pairs in gold: \n", + " - (INSTITUTE FOR FISCAL STUDIES, EUROPEAN UNION)\n", + " - (RAC FOUNDATION, CAR-OWNING HOUSEHOLDS)\n", + "\n", + "2. **Comparison of Matched Pairs:**\n", + " - **Description Similarity:**\n", + " - (HAMMERSON PLC, KLÉPIERRE SA): High similarity.\n", + " - (SNAP INC, TWITTER): High similarity.\n", + " - (PERSIMMON PLC, JEFF FAIRBURN): Moderate similarity.\n", + " - (SHERBORNE INVESTORS, BARCLAYS PLC): High similarity.\n", + " - (ROYAL BANK OF SCOTLAND, MONZO): Moderate similarity.\n", + " - (ROYAL BANK OF SCOTLAND, REVOLUT): Moderate similarity.\n", + " - (HERSHEY CO, TYRRELLS): High similarity.\n", + " - (FINANCIAL REPORTING COUNCIL, CARILLION PLC): High similarity.\n", + " - **Weight Similarity:**\n", + " - (HAMMERSON PLC, KLÉPIERRE SA): 0.9 (gold) vs 0.8 (predicted)\n", + " - (SNAP INC, TWITTER): 0.8 (gold) vs 0.7 (predicted)\n", + " - (PERSIMMON PLC, JEFF FAIRBURN): 0.9 (gold) vs 0.7 (predicted)\n", + " - (SHERBORNE INVESTORS, BARCLAYS PLC): 0.9 (gold) vs 0.7 (predicted)\n", + " - (ROYAL BANK OF SCOTLAND, MONZO): 0.8 (gold) vs 0.6 (predicted)\n", + " - (ROYAL BANK OF SCOTLAND, REVOLUT): 0.8 (gold) vs 0.6 (predicted)\n", + " - (HERSHEY CO, TYRRELLS): 0.7 (gold) vs 0.6 (predicted)\n", + " - (FINANCIAL REPORTING COUNCIL, CARILLION PLC): 0.9 (gold) vs 0.6 (predicted)\n", + " - **Order Similarity:**\n", + " - All matched pairs have the same order (1).\n", + "\n", + "3. **Penalties for Unmatched Relationships:**\n", + " - Two unmatched pairs in gold: (INSTITUTE FOR FISCAL STUDIES, EUROPEAN UNION) and (RAC FOUNDATION, CAR-OWNING HOUSEHOLDS).\n", + "\n", + "4. **Aggregation of Scores:**\n", + " - **Precision and Recall:**\n", + " - Precision: 8/8 (all predicted relationships are matched).\n", + " - Recall: 8/10 (8 out of 10 gold relationships are matched).\n", + " - **Weighted Score:**\n", + " - Description similarity: High for most pairs, moderate for some.\n", + " - Weight similarity: Varied, with some discrepancies.\n", + " - Order similarity: Perfect match.\n", + "\n", + "5. **Final Similarity Score:**\n", + " - Considering the high description similarity, moderate weight similarity, perfect order similarity, and the penalties for unmatched relationships, the final similarity score is calculated as follows:\n", + " - Precision: 1.0\n", + " - Recall: 0.8\n", + " - Weighted average considering the importance of each aspect: 0.85\n", + "\n", + "Similarity Score: 0.85\u001b[0m\n", + "\n", + "\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n\\n\\nAssess the similarity between gold and predicted relationships:\\n1. Match relationships based on src_id and tgt_id pairs, allowing for slight variations in entity names.\\n2. For matched pairs, compare:\\n a) Description similarity (semantic meaning)\\n b) Weight similarity\\n c) Order similarity\\n3. Consider unmatched relationships as penalties.\\n4. Aggregate scores, accounting for precision and recall.\\n5. Return a final similarity score between 0 (no similarity) and 1 (perfect match).\\n\\nKey considerations:\\n- Prioritize matching based on entity pairs over exact string matches.\\n- Use semantic similarity for descriptions rather than exact matches.\\n- Weight the importance of different aspects (e.g., entity matching, description, weight, order).\\n- Balance the impact of matched and unmatched relationships in the final score.\\n\\n---\\n\\nFollow the following format.\\n\\nGold Relationships: The gold-standard relationships to compare against.\\n\\nPredicted Relationships: The predicted relationships to compare against the gold-standard relationships.\\n\\nReasoning: Let\\'s think step by step in order to ${produce the similarity_score}. We ...\\n\\nSimilarity Score: Similarity score between 0 and 1, with 1 being the highest similarity. (Respond with a single float value)\\n\\n---\\n\\nGold Relationships: [{\"src_id\":\"INSTITUTE FOR FISCAL STUDIES\",\"tgt_id\":\"EUROPEAN UNION\",\"description\":\"The Institute for Fiscal Studies said cutting trade tariffs completely when leaving the European Union would reduce prices in British shops by only 1.2 percent.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"KLÉPIERRE SA\",\"tgt_id\":\"HAMMERSON PLC\",\"description\":\"Klépierre SA made a 4.9 billion pound takeover approach to Hammerson Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"SNAP INC\",\"tgt_id\":\"TWITTER\",\"description\":\"Snap Inc\\'s Snapchat is so popular in Britain that its advertising revenue will overtake Twitter\\'s UK revenue in 2019.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"PERSIMMON PLC\",\"tgt_id\":\"JEFF FAIRBURN\",\"description\":\"Persimmon Plc paid its Chief Executive, Jeff Fairburn, and two other executives a combined 104 million pounds last year.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"SHERBORNE INVESTORS\",\"tgt_id\":\"BARCLAYS PLC\",\"description\":\"Sherborne Investors, led by British-born Edward Bramson, has taken a 5 percent stake in Barclays Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"MONZO\",\"description\":\"The Royal Bank of Scotland is working on secret plans to create a standalone digital bank to compete with emerging British fintech champions including Monzo.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"REVOLUT\",\"description\":\"The Royal Bank of Scotland is working on secret plans to create a standalone digital bank to compete with emerging British fintech champions including Revolut.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"HERSHEY CO\",\"tgt_id\":\"TYRRELLS\",\"description\":\"Hershey Co, the U.S. confectionery giant, is at the early stages of exploring options for the British crisps brand Tyrrells.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"FINANCIAL REPORTING COUNCIL\",\"tgt_id\":\"CARILLION PLC\",\"description\":\"The Financial Reporting Council, UK\\'s accountancy watchdog has launched an investigation into two former finance directors of collapsed construction firm Carillion Plc.\",\"weight\":0.9,\"order\":1},{\"src_id\":\"RAC FOUNDATION\",\"tgt_id\":\"CAR-OWNING HOUSEHOLDS\",\"description\":\"The RAC Foundation reported that running a car has become more than a third more expensive in one year for the poorest families.\",\"weight\":0.7,\"order\":1}]\\n\\nPredicted Relationships: [{\"src_id\":\"HAMMERSON PLC\",\"tgt_id\":\"KLÉPIERRE SA\",\"description\":\"Hammerson Plc received a takeover approach from Klépierre SA.\",\"weight\":0.8,\"order\":1},{\"src_id\":\"SNAP INC\",\"tgt_id\":\"TWITTER\",\"description\":\"Snap Inc\\'s Snapchat is expected to overtake Twitter\\'s UK revenue.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"PERSIMMON PLC\",\"tgt_id\":\"JEFF FAIRBURN\",\"description\":\"Persimmon Plc paid its Chief Executive, Jeff Fairburn, a large sum.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"SHERBORNE INVESTORS\",\"tgt_id\":\"BARCLAYS PLC\",\"description\":\"Sherborne Investors took a 5 percent stake in Barclays Plc.\",\"weight\":0.7,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"MONZO\",\"description\":\"Royal Bank of Scotland is competing with Monzo in the digital banking space.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"ROYAL BANK OF SCOTLAND\",\"tgt_id\":\"REVOLUT\",\"description\":\"Royal Bank of Scotland is competing with Revolut in the digital banking space.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"HERSHEY CO\",\"tgt_id\":\"TYRRELLS\",\"description\":\"Hershey Co is exploring options for the British crisps brand Tyrrells.\",\"weight\":0.6,\"order\":1},{\"src_id\":\"FINANCIAL REPORTING COUNCIL\",\"tgt_id\":\"CARILLION PLC\",\"description\":\"The Financial Reporting Council is investigating former finance directors of Carillion Plc.\",\"weight\":0.6,\"order\":1}]\\n\\nPlease provide the output fields Reasoning then Similarity Score. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\\n\\n\\x1b[32mReasoning: Let\\'s think step by step in order to produce the similarity_score. We start by matching relationships based on src_id and tgt_id pairs, allowing for slight variations in entity names. We then compare the matched pairs on description similarity (semantic meaning), weight similarity, and order similarity. Unmatched relationships are considered penalties. Finally, we aggregate scores, accounting for precision and recall, to return a final similarity score between 0 (no similarity) and 1 (perfect match).\\n\\n1. **Entity Pair Matching:**\\n - Matched pairs: \\n - (HAMMERSON PLC, KLÉPIERRE SA)\\n - (SNAP INC, TWITTER)\\n - (PERSIMMON PLC, JEFF FAIRBURN)\\n - (SHERBORNE INVESTORS, BARCLAYS PLC)\\n - (ROYAL BANK OF SCOTLAND, MONZO)\\n - (ROYAL BANK OF SCOTLAND, REVOLUT)\\n - (HERSHEY CO, TYRRELLS)\\n - (FINANCIAL REPORTING COUNCIL, CARILLION PLC)\\n - Unmatched pairs in gold: \\n - (INSTITUTE FOR FISCAL STUDIES, EUROPEAN UNION)\\n - (RAC FOUNDATION, CAR-OWNING HOUSEHOLDS)\\n\\n2. **Comparison of Matched Pairs:**\\n - **Description Similarity:**\\n - (HAMMERSON PLC, KLÉPIERRE SA): High similarity.\\n - (SNAP INC, TWITTER): High similarity.\\n - (PERSIMMON PLC, JEFF FAIRBURN): Moderate similarity.\\n - (SHERBORNE INVESTORS, BARCLAYS PLC): High similarity.\\n - (ROYAL BANK OF SCOTLAND, MONZO): Moderate similarity.\\n - (ROYAL BANK OF SCOTLAND, REVOLUT): Moderate similarity.\\n - (HERSHEY CO, TYRRELLS): High similarity.\\n - (FINANCIAL REPORTING COUNCIL, CARILLION PLC): High similarity.\\n - **Weight Similarity:**\\n - (HAMMERSON PLC, KLÉPIERRE SA): 0.9 (gold) vs 0.8 (predicted)\\n - (SNAP INC, TWITTER): 0.8 (gold) vs 0.7 (predicted)\\n - (PERSIMMON PLC, JEFF FAIRBURN): 0.9 (gold) vs 0.7 (predicted)\\n - (SHERBORNE INVESTORS, BARCLAYS PLC): 0.9 (gold) vs 0.7 (predicted)\\n - (ROYAL BANK OF SCOTLAND, MONZO): 0.8 (gold) vs 0.6 (predicted)\\n - (ROYAL BANK OF SCOTLAND, REVOLUT): 0.8 (gold) vs 0.6 (predicted)\\n - (HERSHEY CO, TYRRELLS): 0.7 (gold) vs 0.6 (predicted)\\n - (FINANCIAL REPORTING COUNCIL, CARILLION PLC): 0.9 (gold) vs 0.6 (predicted)\\n - **Order Similarity:**\\n - All matched pairs have the same order (1).\\n\\n3. **Penalties for Unmatched Relationships:**\\n - Two unmatched pairs in gold: (INSTITUTE FOR FISCAL STUDIES, EUROPEAN UNION) and (RAC FOUNDATION, CAR-OWNING HOUSEHOLDS).\\n\\n4. **Aggregation of Scores:**\\n - **Precision and Recall:**\\n - Precision: 8/8 (all predicted relationships are matched).\\n - Recall: 8/10 (8 out of 10 gold relationships are matched).\\n - **Weighted Score:**\\n - Description similarity: High for most pairs, moderate for some.\\n - Weight similarity: Varied, with some discrepancies.\\n - Order similarity: Perfect match.\\n\\n5. **Final Similarity Score:**\\n - Considering the high description similarity, moderate weight similarity, perfect order similarity, and the penalties for unmatched relationships, the final similarity score is calculated as follows:\\n - Precision: 1.0\\n - Recall: 0.8\\n - Weighted average considering the importance of each aspect: 0.85\\n\\nSimilarity Score: 0.85\\x1b[0m\\n\\n\\n'" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "deepseek.inspect_history(n=1)" ] }, { @@ -623,11 +669,11 @@ "outputs": [], "source": [ "optimizer = BootstrapFewShotWithRandomSearch(\n", - " metric=relationship_similarity_metric, \n", + " metric=relationships_similarity_metric, \n", " num_threads=os.cpu_count(),\n", - " num_candidate_programs=4,\n", + " num_candidate_programs=10,\n", " max_labeled_demos=5,\n", - " max_bootstrapped_demos=3,\n", + " max_bootstrapped_demos=2,\n", ")\n", "rs_model = optimizer.compile(model, trainset=trainset, valset=valset)\n", "rs_model" @@ -639,7 +685,7 @@ "metadata": {}, "outputs": [], "source": [ - "metrics = [entity_recall_metric, relationship_similarity_metric]\n", + "metrics = [entity_recall_metric, relationships_similarity_metric]\n", "for metric in metrics:\n", " evaluate = Evaluate(\n", " devset=devset, \n", @@ -657,33 +703,13754 @@ "metadata": {}, "outputs": [], "source": [ - "optimizer = MIPROv2(\n", - " prompt_model=lm,\n", - " task_model=llama_lm,\n", - " metric=relationship_similarity_metric,\n", - " init_temperature=1.0,\n", - " num_candidates=4\n", - ")\n", - "miprov2_model = optimizer.compile(model, trainset=trainset, valset=valset, num_batches=5, max_labeled_demos=5, max_bootstrapped_demos=3)\n", - "miprov2_model" + "rs_model.save(entity_relationship_rs_path)" ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "metrics = [entity_recall_metric, relationship_similarity_metric]\n", - "for metric in metrics:\n", - " evaluate = Evaluate(\n", - " devset=devset, \n", - " metric=metric, \n", - " num_threads=os.cpu_count(), \n", - " display_progress=True,\n", - " display_table=5,\n", - " )\n", - " evaluate(miprov2_model)" + "## MIPROv2\n", + "- Let's take a small LLM to complete the entity relationship extraction, while making a larger LLM create the prompt instructions and few-shot examples.\n", + "- Feel free to generate sufficient amount of train dataset to avoid overfitting." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[93m\u001b[1mWARNING: Projected Language Model (LM) Calls\u001b[0m\n", + "\n", + "Please be advised that based on the parameters you have set, the maximum number of LM calls is projected as follows:\n", + "\n", + "\n", + "\u001b[93m- Prompt Model: \u001b[94m\u001b[1m10\u001b[0m\u001b[93m data summarizer calls + \u001b[94m\u001b[1m10\u001b[0m\u001b[93m * \u001b[94m\u001b[1m1\u001b[0m\u001b[93m lm calls in program + (\u001b[94m\u001b[1m2\u001b[0m\u001b[93m) lm calls in program aware proposer = \u001b[94m\u001b[1m22\u001b[0m\u001b[93m prompt model calls\u001b[0m\n", + "\u001b[93m- Task Model: \u001b[94m\u001b[1m25\u001b[0m\u001b[93m examples in minibatch * \u001b[94m\u001b[1m20\u001b[0m\u001b[93m batches + \u001b[94m\u001b[1m50\u001b[0m\u001b[93m examples in train set * \u001b[94m\u001b[1m2\u001b[0m\u001b[93m full evals = \u001b[94m\u001b[1m600\u001b[0m\u001b[93m task model calls\u001b[0m\n", + "\n", + "\u001b[93m\u001b[1mEstimated Cost Calculation:\u001b[0m\n", + "\n", + "\u001b[93mTotal Cost = (Number of calls to task model * (Avg Input Token Length per Call * Task Model Price per Input Token + Avg Output Token Length per Call * Task Model Price per Output Token) \n", + " + (Number of calls to prompt model * (Avg Input Token Length per Call * Task Prompt Price per Input Token + Avg Output Token Length per Call * Prompt Model Price per Output Token).\u001b[0m\n", + "\n", + "For a preliminary estimate of potential costs, we recommend you perform your own calculations based on the task\n", + "and prompt models you intend to use. If the projected costs exceed your budget or expectations, you may consider:\n", + "\n", + "\u001b[93m- Reducing the number of trials (`num_batches`), the size of the trainset, or the number of LM calls in your program.\u001b[0m\n", + "\u001b[93m- Using a cheaper task model to optimize the prompt.\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b: 10\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b: 20\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b: 30\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "b: 40\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "summary: Prediction(\n", + " summary='The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.'\n", + ")\n", + "DATA SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/50 [00:00 reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from a given text input. It uses a pipeline that involves a language model to perform this extraction. The program works by defining a signature that specifies the input fields (input_text and entity_types) and the output fields (reasoning and entities_relationships). The TypedPredictor class ensures that the type annotations in the signature are enforced, and it handles retries and error explanations if the output is invalid. The TypedEntityRelationshipExtractor class integrates this predictor to extract entities and relationships from the input text, converting the results into a structured format. The program uses a combination of type checking, JSON schema validation, and language model predictions to achieve its task.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S): \n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from a given text input. It uses a pipeline that involves a language model to perform this extraction. The program works by defining a signature that specifies the input fields (input_text and entity_types) and the output fields (reasoning and entities_relationships). The TypedPredictor class ensures that the type annotations in the signature are enforced, and it handles retries and error explanations if the output is invalid. The TypedEntityRelationshipExtractor class integrates this predictor to extract entities and relationships from the input text, converting the results into a structured format. The program uses a combination of type checking, JSON schema validation, and language model predictions to achieve its task.\n", + "task_demos \n", + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from a given text input. It uses a pipeline that involves a language model to perform this extraction. The program works by defining a signature that specifies the input fields (input_text and entity_types) and the output fields (reasoning and entities_relationships). The TypedPredictor class ensures that the type annotations in the signature are enforced, and it handles retries and error explanations if the output is invalid. The TypedEntityRelationshipExtractor class integrates this predictor to extract entities and relationships from the input text, converting the results into a structured format. The program uses a combination of type checking, JSON schema validation, and language model predictions to achieve its task.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S): \n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from input text. It works by first identifying entities in the text based on provided entity types, then determining the relationships between these entities considering their interactions and dependencies within the context of the text. Finally, it formats the entities and relationships according to a specified JSON schema. The program uses a pipeline that includes a language model to perform these tasks, ensuring type annotations and handling retries and errors in the extraction process.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: Melbourne, Australia (CNN)After spending part of the off-season training with Mike Tyson, Serena Williams is hoping to deliver a knockout punch at the Australian Open. Follow @cnnsport\n", + "\n", + "For Williams that would mean winning a record-tying 24th grand slam title, which has so far proved elusive despite getting close four times since returning to the tour after giving birth to daughter Alexis Olympia. Her preparation for the year's first major couldn't have gone much better, suggesting the mini grand slam drought for arguably tennis' greatest ever player is about to cease. Williams let rip into a punching bag in December -- drawing a compliment from former heavyweight boxing champion Tyson, whose daughter happens to be a budding tennis star -- and then won a buildup tournament in Auckland last week to incredibly land a title in a fourth straight decade. She also mingled with one of tennis' youngest sensations, Coco Gauff, as part of the training camp in Florida organized by her coach, Patrick Mouratoglou. JUST WATCHEDMouratoglou on Serena comeback and McEnroeReplayMore Videos ...MUST WATCHMouratoglou on Serena comeback and McEnroe 02:19Read MoreWilliams overcame singles opponents with differing styles, reached the doubles final with one of her best friends -- the soon-to-be retired Caroline Wozniacki -- and most crucially, ended a five-match losing streak in finals with her daughter and husband Alexis Ohanian looking on. No wonder the 38-year-old said following her straight-set victory over fellow American Jessica Pegula: \"It's pretty satisfying just to get a win in the final. That was really important for me. And I just want to build on it,\" added Williams, who donated her prize money check of $43,000 to bush fire relief efforts in Australia. \"It's just a step towards the next goal.\"Indeed. READ: Can Rafael Nadal match Roger Federer's all-time grand slam record?READ: Player brands Australian Open email a 'slap in the face'Eyes on bigger prizeYes, as nice as it was to be holding the winners' trophy in Auckland -- where Williams once hit 88 unforced errors in a loss to Madison Brengle -- she didn't make the long trip to simply prosper in New Zealand. The much bigger prize is the Australian Open, where Williams triumphed while in the early stages of pregnancy in 2017. If Williams makes the final in Melbourne -- and she might have to defeat the likes of twice grand slam winner Naomi Osaka and current world No. 1 Ashleigh Barty along the way -- she will probably have to then defeat someone with a heftier reputation than the 66th-ranked Pegula. Helping Williams, however, is that one of the main contenders, Bianca Andreescu, isn't in Melbourne because of another injury, this time to a knee. But winning any final -- after losses in grand slam finals to Osaka, Andreescu, Angelique Kerber and Simona Halep and retiring against Andreescu in the Rogers Cup finale last August -- could potentially be turning point as Williams attempts to draw level with the grand slam haul of Australia's Margaret Court. JUST WATCHEDSerena Williams falls short in major title chaseReplayMore Videos ...MUST WATCHSerena Williams falls short in major title chase 01:00\"Serena, she certainly looks hungry, and I think she's got a little momentum going into the Australian Open,\" Chris Evert, the 18-time grand slam champion, told reporters in an ESPN conference call last week. \"And it would probably be the least pressure, this grand slam, to win for her. \"I think every other tournament, the French Open, the clay isn't her best surface. Wimbledon is a lot of pressure, US Open is a lot of pressure. \"This one, the first one of the year, it's a 'Happy Slam,'\" referring to the Australian Open's nickname. \"I think if she just takes a little bit of pressure off herself and she can just play her brand of tennis, I think she's got a good shot at winning it.\"She's better at grand slams than any other player when you look at the last two years.\"The way Wozniacki put it, Williams has a \"big chance\" to match Court. pic.twitter.com/skoZilynH8— Serena Williams (@serenawilliams) January 12, 2020 Other high-profile players, such as Halep and former world No. 1 Karolina Pliskova -- who saved four match points last year in Melbourne against Williams and rallied from 5-1 down in the third set after the American rolled her ankle -- aren't discounting her chances, either, despite just falling short recently at grand slams. \"I'm very impressed about her, that she keeps playing at this level, with being a mother and also being a little bit older than us,\" said Halep, who made a mere three unforced errors in downing Williams in the 2019 Wimbledon final. \"It's impressive what she does.\"Challenges aheadBut Evert also underscored the stiffer challenges facing Williams. \"There are some darned good players out there that I have a lot of respect for, and the way that Osaka, Barty, Pliskova, Halep ... there's not one or two threats to Serena, there's probably about eight threats, eight players that can probably do some damage and that can compete against her.\"And one more thing is -- I always felt this way -- the older you get, I think the more bad days you may have, days when you feel burned out, days when you don't want to get out of bed, days when you don't have incentive. You don't want to have that day during a grand slam, but sometimes you can't help it.\"JUST WATCHEDSimona Halep on winning her first Wimbledon titleReplayMore Videos ...MUST WATCHSimona Halep on winning her first Wimbledon title 01:56Visit our tennis page for more news and videosWilliams figures to still be around towards the end of the Australian Open, but whether it is as the last person standing continues to be the question. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Melbourne\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A city in Australia where the Australian Open is held.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australia\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The country where Melbourne is located and where the Australian Open takes place.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Serena Williams\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player aiming to win the Australian Open.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Mike Tyson\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former heavyweight boxing champion who trained with Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australian Open\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held annually in Melbourne, Australia.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Alexis Olympia\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The daughter of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Caroline Wozniacki\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and friend of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Coco Gauff\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A young tennis sensation who trained with Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Patrick Mouratoglou\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The coach of Serena Williams who organized the training camp.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Naomi Osaka\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and grand slam winner who could face Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ashleigh Barty\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The current world No. 1 tennis player who could face Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Bianca Andreescu\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player who is a main contender but is not in Melbourne due to injury.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Simona Halep\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player who defeated Serena Williams in the 2019 Wimbledon final.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Chris Evert\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An 18-time grand slam champion who commented on Serena Williams' chances at the Australian Open.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Margaret Court\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An Australian tennis player with a record-tying 24 grand slam titles.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jessica Pegula\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A fellow American tennis player who Serena Williams defeated in a match.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Alexis Ohanian\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The husband of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Karolina Pliskova\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former world No. 1 tennis player who saved match points against Serena Williams in Melbourne.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Mike Tyson\",\n", + "\n", + "\"description\": \"Serena Williams trained with Mike Tyson during the off-season.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Australian Open\",\n", + "\n", + "\"description\": \"Serena Williams is aiming to win the Australian Open.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Alexis Olympia\",\n", + "\n", + "\"description\": \"Serena Williams is the mother of Alexis Olympia.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Caroline Wozniacki\",\n", + "\n", + "\"description\": \"Serena Williams reached the doubles final with Caroline Wozniacki.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Coco Gauff\",\n", + "\n", + "\"description\": \"Serena Williams trained with Coco Gauff.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Patrick Mouratoglou\",\n", + "\n", + "\"description\": \"Patrick Mouratoglou is the coach of Serena Williams.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Naomi Osaka\",\n", + "\n", + "\"description\": \"Serena Williams might have to defeat Naomi Osaka to reach the final of the Australian Open.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Ashleigh Barty\",\n", + "\n", + "\"description\": \"Serena Williams might have to defeat Ashleigh Barty to reach the final of the Australian Open.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Bianca Andreescu\",\n", + "\n", + "\"description\": \"Bianca Andreescu is a main contender but is not in Melbourne due to injury.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Simona Halep\",\n", + "\n", + "\"description\": \"Simona Halep defeated Serena Williams in the 2019 Wimbledon final.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Chris Evert\",\n", + "\n", + "\"description\": \"Chris Evert commented on Serena Williams' chances at the Australian Open.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Margaret Court\",\n", + "\n", + "\"description\": \"Serena Williams is attempting to match Margaret Court's record of 24 grand slam titles.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Jessica Pegula\",\n", + "\n", + "\"description\": \"Serena Williams defeated Jessica Pegula in a match.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Alexis Ohanian\",\n", + "\n", + "\"description\": \"Alexis Ohanian is the husband of Serena Williams.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Karolina Pliskova\",\n", + "\n", + "\"description\": \"Karolina Pliskova saved match points against Serena Williams in Melbourne.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Neymar surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.Two penalties and a last-minute tap-in took the Paris Saint-Germain forward's tally to 64 for his country, two ahead of Ronaldo and 13 behind Pele's all-time record of 77.Everton's Richarlison scored Brazil's other goal as the five-time world champion twice had to come from behind to secure the win.Brazil head coach Tite told reporters after the match that it was \"unfair\" to compare Neymar and Ronaldo.\"What I can say is Neymar has this unpredictability,\" he said. \"He is the bow and the arrow, he's a player who both makes and takes chances. And he gets better and better, and more mature.\"Read MoreNeymar celebrates after completing his hat-trick against Peru.The match had a number of controversial VAR moments, particularly the decision to award Neymar a second penalty late in the game with the score tied at 2-2, a decision which surprised even the Brazilian players.Peru's anger was compounded in stoppage time when Carlos Zambrano was shown a red card for an elbow on Richarlison. The Everton forward had escaped punishment for a similar incident earlier in the match that left Peru's Miguel Trauco with a bloody wound above his eye.The latest incidents added to a growing list of decisions that have left South American fans scratching their heads during this international break, the first time VAR has been used in CONMEBOL World Cup qualifiersBrazil's victory ensures it begins the grueling 18-month campaign with a 100% record to top the standings on six points along with Argentina, which secured an impressive 2-1 win in the altitude of La Paz, Bolivia earlier in the day.The top four teams will qualify automatically for Qatar 2022, with the fifth-placed team competing in a two-legged playoff against a country from another continent.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neymar\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Brazilian footballer who surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ronaldo\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former Brazilian footballer who was previously second on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Pele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A legendary Brazilian footballer who holds the all-time record for Brazil with 77 goals.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brazil\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country where the football match took place and where Neymar and Ronaldo are from.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peru\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The opposing country in the football match against Brazil.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"World Cup\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"An international football competition where the qualifying match between Brazil and Peru took place.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Ronaldo\",\n", + "\n", + "\"description\": \"Neymar surpassed Ronaldo on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Pele\",\n", + "\n", + "\"description\": \"Neymar is 13 goals behind Pele's all-time record for Brazil.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brazil\",\n", + "\n", + "\"tgt_id\": \"Peru\",\n", + "\n", + "\"description\": \"Brazil won a football match against Peru in World Cup qualifying.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)The New York City Marathon returns on Sunday following a two-year hiatus, and at the front of the field, legendary distance runner Kenenisa Bekele is primed to \"make more history\" in his storied career. Bekele, a three-time Olympic gold medalist over 5,000 and 10,000 meters, is the headline name competing in the men's elite field in New York. With his only previous marathon in the United States a fourth-place finish in Chicago seven years ago, the Ethiopian is hungry for success as he prepares to take to the streets of the Big Apple.\"Really, I wanted to have a good result in the USA, that's why I chose the New York marathon,\" Bekele told reporters this week.\"The New York marathon is big publicity and a really big marathon race. To achieve a good result in this marathon would be perfect, and for me, it's also really good to make more history in sports.\"Read MoreBekele has a point to prove after a disappointing outing six weeks ago at the Berlin Marathon -- the race at which he came within two seconds of Eliud Kipchoge's world record time two years ago. Bekele catches his breath after this year's Berlin Marathon. Despite expectations that he could challenge Kipchoge's record of two hours, one minute and 39 seconds in Berlin this year, Bekele says he fell foul to a poor night's sleep as he finished third, more than a minute behind winner Guye Adola.\"I was not ready for that race,\" Bekele explained. \"A few weeks before the race, I was not really confident. It's a little bit tough for me because the day before also I couldn't sleep well, I really had bad luck the day before in the night. That also made me very tired.\"I've recovered well ... At this moment, I'm really strong. I hope I'll achieve a good result on Sunday.\"For race organizers, who called off last year's New York City Marathon amid the pandemic, it is a boon to have a figure like Bekele on the start line on Sunday.On top of his three Olympic gold medals -- including a double in the 5,000 and 10,000 meters at Beijing 2008 -- and one silver medal, he also claimed five world athletics championship titles between 2003 to 2009. Bekele's 5,000m world record stood for 16 years before it was broken by Uganda's Joshua Cheptegei last year, while his 10,000m record stood for 15 before also being broken last year by Cheptegei. Eliud Kipchoge: Marathon world record holder has 'the qualities of an ascetic monk'Many consider Bekele to be the greatest male distance runner of all time, such has been his supremacy across multiple distances over the past two decades; others point towards Kenya's Kipchoge, who has been so dominant over the 26.2 miles of a marathon and is the first man to break the distance's two-hour barrier, albeit in unofficial conditions. \"I still feel that I am the best and better than anyone,\" Bekele told Sports Illustrated in August after it was announced that he would make his debut in New York. \"I think every athlete and others should think like that.\"Having struggled with injuries and form in recent years, Bekele now has a second opportunity to win a major marathon in the US. There is little chance that he will be able to come close to his best marathon time in New York; unlike Berlin, the course, which undulates through the city's five boroughs, rarely produces fast performances.\"I know the course is tough and there are also no pacemakers in the race. It's more about concentration and a tactical race,\" said Bekele. He will be up against the Netherlands' Adbi Nageeye, who won the silver medal at the Olympics in Sapporo three months ago, and the half marathon world record holder Kibiwott Kandie of Kenya, who is making his marathon debut. The women's race is headlined by Kenya's Peres Jepchirchir, the gold medalist at the Olympics. She will be joined by the USA's Olympic bronze medalist Molly Seidel. Seidel poses with her bronze medal from the Tokyo Olympics. Visit CNN.com/sport for more news, videos and featuresAnother athlete to keep an eye on is US star Shalane Flanagan as she attempts to run six marathons in 42 days. Having already completed marathons in Berlin, London, Chicago, Boston and Portland -- all between times of 2:35:14 and 2:46:39 -- Flanagan will aim to complete her challenge in New York. And as for Bekele, like so many other runners who will take to the start line on Sunday, he's grateful to be able to race in New York following the pandemic and last year's cancellation.\"To see the people of New York participate in this race again -- to see this for me is fantastic,\" he said. \"I want to wish all participants good luck.\"\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenenisa Bekele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Legendary distance runner and three-time Olympic gold medalist over 5,000 and 10,000 meters.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New York City Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in New York City, returning after a two-year hiatus due to the pandemic.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eliud Kipchoge\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Marathon world record holder and dominant runner over the 26.2 miles of a marathon.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Berlin Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in Berlin, where Kenenisa Bekele had a disappointing outing six weeks ago.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guye Adola\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Winner of the Berlin Marathon, finishing more than a minute ahead of Kenenisa Bekele.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Adbi Nageeye\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Dutch runner who won the silver medal at the Olympics in Sapporo and will compete in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kibiwott Kandie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Half marathon world record holder from Kenya, making his marathon debut in New York.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peres Jepchirchir\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Kenyan runner and gold medalist at the Olympics, headlining the women's race in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Molly Seidel\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"American runner and Olympic bronze medalist, competing in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Shalane Flanagan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"US star attempting to run six marathons in 42 days, including the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"New York City Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele is primed to compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Eliud Kipchoge\",\n", + "\n", + "\"description\": \"Kenenisa Bekele aims to make more history in his storied career, comparing himself to Eliud Kipchoge.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Berlin Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele had a disappointing outing at the Berlin Marathon six weeks ago.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Guye Adola\",\n", + "\n", + "\"description\": \"Kenenisa Bekele finished third in the Berlin Marathon, more than a minute behind winner Guye Adola.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Adbi Nageeye\",\n", + "\n", + "\"description\": \"Adbi Nageeye will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Kibiwott Kandie\",\n", + "\n", + "\"description\": \"Kibiwott Kandie will make his marathon debut in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Peres Jepchirchir\",\n", + "\n", + "\"description\": \"Peres Jepchirchir will headline the women's race in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Molly Seidel\",\n", + "\n", + "\"description\": \"Molly Seidel will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Shalane Flanagan\",\n", + "\n", + "\"description\": \"Shalane Flanagan will aim to complete her challenge of running six marathons in 42 days in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from input text. It works by first identifying entities in the text based on provided entity types, then determining the relationships between these entities considering their interactions and dependencies within the context of the text. Finally, it formats the entities and relationships according to a specified JSON schema. The program uses a pipeline that includes a language model to perform these tasks, ensuring type annotations and handling retries and errors in the extraction process.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: Melbourne, Australia (CNN)After spending part of the off-season training with Mike Tyson, Serena Williams is hoping to deliver a knockout punch at the Australian Open. Follow @cnnsport\n", + "\n", + "For Williams that would mean winning a record-tying 24th grand slam title, which has so far proved elusive despite getting close four times since returning to the tour after giving birth to daughter Alexis Olympia. Her preparation for the year's first major couldn't have gone much better, suggesting the mini grand slam drought for arguably tennis' greatest ever player is about to cease. Williams let rip into a punching bag in December -- drawing a compliment from former heavyweight boxing champion Tyson, whose daughter happens to be a budding tennis star -- and then won a buildup tournament in Auckland last week to incredibly land a title in a fourth straight decade. She also mingled with one of tennis' youngest sensations, Coco Gauff, as part of the training camp in Florida organized by her coach, Patrick Mouratoglou. JUST WATCHEDMouratoglou on Serena comeback and McEnroeReplayMore Videos ...MUST WATCHMouratoglou on Serena comeback and McEnroe 02:19Read MoreWilliams overcame singles opponents with differing styles, reached the doubles final with one of her best friends -- the soon-to-be retired Caroline Wozniacki -- and most crucially, ended a five-match losing streak in finals with her daughter and husband Alexis Ohanian looking on. No wonder the 38-year-old said following her straight-set victory over fellow American Jessica Pegula: \"It's pretty satisfying just to get a win in the final. That was really important for me. And I just want to build on it,\" added Williams, who donated her prize money check of $43,000 to bush fire relief efforts in Australia. \"It's just a step towards the next goal.\"Indeed. READ: Can Rafael Nadal match Roger Federer's all-time grand slam record?READ: Player brands Australian Open email a 'slap in the face'Eyes on bigger prizeYes, as nice as it was to be holding the winners' trophy in Auckland -- where Williams once hit 88 unforced errors in a loss to Madison Brengle -- she didn't make the long trip to simply prosper in New Zealand. The much bigger prize is the Australian Open, where Williams triumphed while in the early stages of pregnancy in 2017. If Williams makes the final in Melbourne -- and she might have to defeat the likes of twice grand slam winner Naomi Osaka and current world No. 1 Ashleigh Barty along the way -- she will probably have to then defeat someone with a heftier reputation than the 66th-ranked Pegula. Helping Williams, however, is that one of the main contenders, Bianca Andreescu, isn't in Melbourne because of another injury, this time to a knee. But winning any final -- after losses in grand slam finals to Osaka, Andreescu, Angelique Kerber and Simona Halep and retiring against Andreescu in the Rogers Cup finale last August -- could potentially be turning point as Williams attempts to draw level with the grand slam haul of Australia's Margaret Court. JUST WATCHEDSerena Williams falls short in major title chaseReplayMore Videos ...MUST WATCHSerena Williams falls short in major title chase 01:00\"Serena, she certainly looks hungry, and I think she's got a little momentum going into the Australian Open,\" Chris Evert, the 18-time grand slam champion, told reporters in an ESPN conference call last week. \"And it would probably be the least pressure, this grand slam, to win for her. \"I think every other tournament, the French Open, the clay isn't her best surface. Wimbledon is a lot of pressure, US Open is a lot of pressure. \"This one, the first one of the year, it's a 'Happy Slam,'\" referring to the Australian Open's nickname. \"I think if she just takes a little bit of pressure off herself and she can just play her brand of tennis, I think she's got a good shot at winning it.\"She's better at grand slams than any other player when you look at the last two years.\"The way Wozniacki put it, Williams has a \"big chance\" to match Court. pic.twitter.com/skoZilynH8— Serena Williams (@serenawilliams) January 12, 2020 Other high-profile players, such as Halep and former world No. 1 Karolina Pliskova -- who saved four match points last year in Melbourne against Williams and rallied from 5-1 down in the third set after the American rolled her ankle -- aren't discounting her chances, either, despite just falling short recently at grand slams. \"I'm very impressed about her, that she keeps playing at this level, with being a mother and also being a little bit older than us,\" said Halep, who made a mere three unforced errors in downing Williams in the 2019 Wimbledon final. \"It's impressive what she does.\"Challenges aheadBut Evert also underscored the stiffer challenges facing Williams. \"There are some darned good players out there that I have a lot of respect for, and the way that Osaka, Barty, Pliskova, Halep ... there's not one or two threats to Serena, there's probably about eight threats, eight players that can probably do some damage and that can compete against her.\"And one more thing is -- I always felt this way -- the older you get, I think the more bad days you may have, days when you feel burned out, days when you don't want to get out of bed, days when you don't have incentive. You don't want to have that day during a grand slam, but sometimes you can't help it.\"JUST WATCHEDSimona Halep on winning her first Wimbledon titleReplayMore Videos ...MUST WATCHSimona Halep on winning her first Wimbledon title 01:56Visit our tennis page for more news and videosWilliams figures to still be around towards the end of the Australian Open, but whether it is as the last person standing continues to be the question. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Melbourne\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A city in Australia where the Australian Open is held.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australia\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The country where Melbourne is located and where the Australian Open takes place.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Serena Williams\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player aiming to win the Australian Open.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Mike Tyson\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former heavyweight boxing champion who trained with Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australian Open\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held annually in Melbourne, Australia.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Alexis Olympia\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The daughter of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Caroline Wozniacki\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and friend of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Coco Gauff\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A young tennis sensation who trained with Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Patrick Mouratoglou\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The coach of Serena Williams who organized the training camp.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Naomi Osaka\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and grand slam winner who could face Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ashleigh Barty\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The current world No. 1 tennis player who could face Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Bianca Andreescu\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player who is a main contender but is not in Melbourne due to injury.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Simona Halep\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player who defeated Serena Williams in the 2019 Wimbledon final.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Chris Evert\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An 18-time grand slam champion who commented on Serena Williams' chances at the Australian Open.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Margaret Court\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An Australian tennis player with a record-tying 24 grand slam titles.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jessica Pegula\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A fellow American tennis player who Serena Williams defeated in a match.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Alexis Ohanian\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The husband of Serena Williams.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Karolina Pliskova\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former world No. 1 tennis player who saved match points against Serena Williams in Melbourne.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Mike Tyson\",\n", + "\n", + "\"description\": \"Serena Williams trained with Mike Tyson during the off-season.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Australian Open\",\n", + "\n", + "\"description\": \"Serena Williams is aiming to win the Australian Open.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Alexis Olympia\",\n", + "\n", + "\"description\": \"Serena Williams is the mother of Alexis Olympia.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Caroline Wozniacki\",\n", + "\n", + "\"description\": \"Serena Williams reached the doubles final with Caroline Wozniacki.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Coco Gauff\",\n", + "\n", + "\"description\": \"Serena Williams trained with Coco Gauff.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Patrick Mouratoglou\",\n", + "\n", + "\"description\": \"Patrick Mouratoglou is the coach of Serena Williams.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Naomi Osaka\",\n", + "\n", + "\"description\": \"Serena Williams might have to defeat Naomi Osaka to reach the final of the Australian Open.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Ashleigh Barty\",\n", + "\n", + "\"description\": \"Serena Williams might have to defeat Ashleigh Barty to reach the final of the Australian Open.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Bianca Andreescu\",\n", + "\n", + "\"description\": \"Bianca Andreescu is a main contender but is not in Melbourne due to injury.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Simona Halep\",\n", + "\n", + "\"description\": \"Simona Halep defeated Serena Williams in the 2019 Wimbledon final.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Chris Evert\",\n", + "\n", + "\"description\": \"Chris Evert commented on Serena Williams' chances at the Australian Open.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Margaret Court\",\n", + "\n", + "\"description\": \"Serena Williams is attempting to match Margaret Court's record of 24 grand slam titles.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Jessica Pegula\",\n", + "\n", + "\"description\": \"Serena Williams defeated Jessica Pegula in a match.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Alexis Ohanian\",\n", + "\n", + "\"description\": \"Alexis Ohanian is the husband of Serena Williams.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Serena Williams\",\n", + "\n", + "\"tgt_id\": \"Karolina Pliskova\",\n", + "\n", + "\"description\": \"Karolina Pliskova saved match points against Serena Williams in Melbourne.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Neymar surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.Two penalties and a last-minute tap-in took the Paris Saint-Germain forward's tally to 64 for his country, two ahead of Ronaldo and 13 behind Pele's all-time record of 77.Everton's Richarlison scored Brazil's other goal as the five-time world champion twice had to come from behind to secure the win.Brazil head coach Tite told reporters after the match that it was \"unfair\" to compare Neymar and Ronaldo.\"What I can say is Neymar has this unpredictability,\" he said. \"He is the bow and the arrow, he's a player who both makes and takes chances. And he gets better and better, and more mature.\"Read MoreNeymar celebrates after completing his hat-trick against Peru.The match had a number of controversial VAR moments, particularly the decision to award Neymar a second penalty late in the game with the score tied at 2-2, a decision which surprised even the Brazilian players.Peru's anger was compounded in stoppage time when Carlos Zambrano was shown a red card for an elbow on Richarlison. The Everton forward had escaped punishment for a similar incident earlier in the match that left Peru's Miguel Trauco with a bloody wound above his eye.The latest incidents added to a growing list of decisions that have left South American fans scratching their heads during this international break, the first time VAR has been used in CONMEBOL World Cup qualifiersBrazil's victory ensures it begins the grueling 18-month campaign with a 100% record to top the standings on six points along with Argentina, which secured an impressive 2-1 win in the altitude of La Paz, Bolivia earlier in the day.The top four teams will qualify automatically for Qatar 2022, with the fifth-placed team competing in a two-legged playoff against a country from another continent.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neymar\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Brazilian footballer who surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ronaldo\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former Brazilian footballer who was previously second on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Pele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A legendary Brazilian footballer who holds the all-time record for Brazil with 77 goals.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brazil\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country where the football match took place and where Neymar and Ronaldo are from.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peru\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The opposing country in the football match against Brazil.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"World Cup\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"An international football competition where the qualifying match between Brazil and Peru took place.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Ronaldo\",\n", + "\n", + "\"description\": \"Neymar surpassed Ronaldo on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Pele\",\n", + "\n", + "\"description\": \"Neymar is 13 goals behind Pele's all-time record for Brazil.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brazil\",\n", + "\n", + "\"tgt_id\": \"Peru\",\n", + "\n", + "\"description\": \"Brazil won a football match against Peru in World Cup qualifying.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)The New York City Marathon returns on Sunday following a two-year hiatus, and at the front of the field, legendary distance runner Kenenisa Bekele is primed to \"make more history\" in his storied career. Bekele, a three-time Olympic gold medalist over 5,000 and 10,000 meters, is the headline name competing in the men's elite field in New York. With his only previous marathon in the United States a fourth-place finish in Chicago seven years ago, the Ethiopian is hungry for success as he prepares to take to the streets of the Big Apple.\"Really, I wanted to have a good result in the USA, that's why I chose the New York marathon,\" Bekele told reporters this week.\"The New York marathon is big publicity and a really big marathon race. To achieve a good result in this marathon would be perfect, and for me, it's also really good to make more history in sports.\"Read MoreBekele has a point to prove after a disappointing outing six weeks ago at the Berlin Marathon -- the race at which he came within two seconds of Eliud Kipchoge's world record time two years ago. Bekele catches his breath after this year's Berlin Marathon. Despite expectations that he could challenge Kipchoge's record of two hours, one minute and 39 seconds in Berlin this year, Bekele says he fell foul to a poor night's sleep as he finished third, more than a minute behind winner Guye Adola.\"I was not ready for that race,\" Bekele explained. \"A few weeks before the race, I was not really confident. It's a little bit tough for me because the day before also I couldn't sleep well, I really had bad luck the day before in the night. That also made me very tired.\"I've recovered well ... At this moment, I'm really strong. I hope I'll achieve a good result on Sunday.\"For race organizers, who called off last year's New York City Marathon amid the pandemic, it is a boon to have a figure like Bekele on the start line on Sunday.On top of his three Olympic gold medals -- including a double in the 5,000 and 10,000 meters at Beijing 2008 -- and one silver medal, he also claimed five world athletics championship titles between 2003 to 2009. Bekele's 5,000m world record stood for 16 years before it was broken by Uganda's Joshua Cheptegei last year, while his 10,000m record stood for 15 before also being broken last year by Cheptegei. Eliud Kipchoge: Marathon world record holder has 'the qualities of an ascetic monk'Many consider Bekele to be the greatest male distance runner of all time, such has been his supremacy across multiple distances over the past two decades; others point towards Kenya's Kipchoge, who has been so dominant over the 26.2 miles of a marathon and is the first man to break the distance's two-hour barrier, albeit in unofficial conditions. \"I still feel that I am the best and better than anyone,\" Bekele told Sports Illustrated in August after it was announced that he would make his debut in New York. \"I think every athlete and others should think like that.\"Having struggled with injuries and form in recent years, Bekele now has a second opportunity to win a major marathon in the US. There is little chance that he will be able to come close to his best marathon time in New York; unlike Berlin, the course, which undulates through the city's five boroughs, rarely produces fast performances.\"I know the course is tough and there are also no pacemakers in the race. It's more about concentration and a tactical race,\" said Bekele. He will be up against the Netherlands' Adbi Nageeye, who won the silver medal at the Olympics in Sapporo three months ago, and the half marathon world record holder Kibiwott Kandie of Kenya, who is making his marathon debut. The women's race is headlined by Kenya's Peres Jepchirchir, the gold medalist at the Olympics. She will be joined by the USA's Olympic bronze medalist Molly Seidel. Seidel poses with her bronze medal from the Tokyo Olympics. Visit CNN.com/sport for more news, videos and featuresAnother athlete to keep an eye on is US star Shalane Flanagan as she attempts to run six marathons in 42 days. Having already completed marathons in Berlin, London, Chicago, Boston and Portland -- all between times of 2:35:14 and 2:46:39 -- Flanagan will aim to complete her challenge in New York. And as for Bekele, like so many other runners who will take to the start line on Sunday, he's grateful to be able to race in New York following the pandemic and last year's cancellation.\"To see the people of New York participate in this race again -- to see this for me is fantastic,\" he said. \"I want to wish all participants good luck.\"\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenenisa Bekele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Legendary distance runner and three-time Olympic gold medalist over 5,000 and 10,000 meters.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New York City Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in New York City, returning after a two-year hiatus due to the pandemic.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eliud Kipchoge\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Marathon world record holder and dominant runner over the 26.2 miles of a marathon.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Berlin Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in Berlin, where Kenenisa Bekele had a disappointing outing six weeks ago.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guye Adola\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Winner of the Berlin Marathon, finishing more than a minute ahead of Kenenisa Bekele.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Adbi Nageeye\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Dutch runner who won the silver medal at the Olympics in Sapporo and will compete in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kibiwott Kandie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Half marathon world record holder from Kenya, making his marathon debut in New York.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peres Jepchirchir\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Kenyan runner and gold medalist at the Olympics, headlining the women's race in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Molly Seidel\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"American runner and Olympic bronze medalist, competing in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Shalane Flanagan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"US star attempting to run six marathons in 42 days, including the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"New York City Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele is primed to compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Eliud Kipchoge\",\n", + "\n", + "\"description\": \"Kenenisa Bekele aims to make more history in his storied career, comparing himself to Eliud Kipchoge.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Berlin Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele had a disappointing outing at the Berlin Marathon six weeks ago.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Guye Adola\",\n", + "\n", + "\"description\": \"Kenenisa Bekele finished third in the Berlin Marathon, more than a minute behind winner Guye Adola.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Adbi Nageeye\",\n", + "\n", + "\"description\": \"Adbi Nageeye will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Kibiwott Kandie\",\n", + "\n", + "\"description\": \"Kibiwott Kandie will make his marathon debut in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Peres Jepchirchir\",\n", + "\n", + "\"description\": \"Peres Jepchirchir will headline the women's race in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Molly Seidel\",\n", + "\n", + "\"description\": \"Molly Seidel will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Shalane Flanagan\",\n", + "\n", + "\"description\": \"Shalane Flanagan will aim to complete her challenge of running six marathons in 42 days in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, identify and extract all entities within the text that match the specified entity types. After identifying the entities, determine the relationships between these entities based on their interactions and dependencies within the context of the text. Ensure that the extracted entities and relationships are formatted according to the specified JSON schema, including the fields `entity_name`, `entity_type`, `description`, `importance_score` for entities, and `src_id`, `tgt_id`, `description`, `weight`, and `order` for relationships. Provide a detailed step-by-step reasoning process to justify the extraction and relationship determination.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, identify and extract all entities within the text that match the specified entity types. After identifying the entities, determine the relationships between these entities based on their interactions and dependencies within the context of the text. Ensure that the extracted entities and relationships are formatted according to the specified JSON schema, including the fields `entity_name`, `entity_type`, `description`, `importance_score` for entities, and `src_id`, `tgt_id`, `description`, `weight`, and `order` for relationships. Provide a detailed step-by-step reasoning process to justify the extraction and relationship determination.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text. It works by first identifying entities in the text based on predefined entity types, then determining the relationships between these entities considering their interactions and the context provided in the text. The program ensures that each entity and relationship is described comprehensively and assigned appropriate importance scores and weights. The output is formatted according to a specified JSON schema, listing entities and their relationships in a structured manner.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: (CNN)Former Bosnian Serb army leader Ratko Mladic was sentenced to life in prison Wednesday after being found guilty of genocide for atrocities committed during the Bosnian war from 1992 to 1995.Verdict proceedings had been interrupted earlier when the 74-year-old's legal team claimed that his blood pressure was too high to continue.After outbursts from Mladic, Judge Alphons Orie, who was delivering a summation of the case, ordered the removal of the ex-general, telling him he could monitor proceedings by audio and video. \"I am very distraught,\" Mladic shouted inside the courtroom. \"Everything that you have said is pure lies. Shame on you. It's all lies.\"Mladic's legal team had asked for proceedings to be halted or for the summation of the case to be skipped, which the Judge refused. Read MoreMladic was charged with two counts of genocide and nine crimes against humanity and war crimes for his role in the conflict in the former Yugoslavia from 1992 to 1995, during which 100,000 people were killed and another 2.2 million displaced. He was found not guilty on one charge of genocide, but received a guilty verdict on each of the other 10 counts. Mladic's lawyer, Dragan Ivetic, said it was \"certain\" Mladic would appeal.\"Butcher of Bosnia\" Ratko Mladic has been found guilty of the highest crimes against international law, says CNN's @camanpour, who covered the Bosnian war https://t.co/4A7DtVjHn8 pic.twitter.com/HdPKU3tqRD— CNN International (@cnni) November 22, 2017The trial, which opened in 2012, took place at the International Criminal Tribunal for the former Yugoslavia in The Hague, Netherlands. The ad hoc court was established to prosecute crimes committed during the Balkans conflict. Mladic was accused of orchestrating a campaign of ethnic cleansing, including the slaughter of thousands of Muslim men and boys at Srebrenica in July 1995. It is the worst massacre to have taken place in Europe since the Second World War.Mladic judgment brings back stench of Bosnian genocideProsecutor Serge Brammertz told reporters that Mladic will be remembered by history \"for the many communities and lives he destroyed.\"\"Today's judgment is a milestone in the tribunal's history and for international justice,\" he added.The trial of Mladic, who was arrested in 2011, has lasted 530 days and included more than 500 witnesses and nearly 10,000 exhibits. Before the case was adjourned last December, prosecutors recommended a life sentence. Mladic had previously referred to the court as \"satanic\" and labeled the charges against him as \"obnoxious.\"ReactionAt a center for the association of women victims of war in Sarajevo, there was an outpouring of emotion during the judge's summation.There was particular frustration that Mladic was acquitted on one charge of genocide in Bosnian municipalities outside of Srebrenica.Amela Meduseljac (L) and Meliha Mrdzic were unhappy that Mladic was acquitted on one charge.Meliha Mrdzic, who said her father and brother were killed and thrown into the Drina River in Visegrad, told CNN she was left humiliated by the decision.\"The international community made me a victim a second time,\" she said. \"They make it seem like we killed ourselves, raped ourselves, slaughtered ourselves. I feel so hurt, I can't describe it.\"Amela Meduseljac, who said she was raped by Mladic's soldiers at Visegrad, said that victims will struggle to get over the judgment.\"Our mission as a rape survivor association was to stop victims from feeling like victims,\" she said. \"But it's getting worse from year to year and it will get especially worse after this verdict.\"People in Srebrenica celebrate as they watch a live TV broadcast of the trial Wednesday.UN High Commissioner for Human Rights Zeid Ra'ad Al Hussein called Mladic the \"epitome of evil\" and labeled his conviction a \"momentous victory for justice.\" \"Mladic presided over some of the darkest crimes to occur in Europe since World War II, bringing terror, death and destruction to thousands of victims, and sorrow, tragedy and trauma to countless more,\" Zeid said in a statement.\"His conviction is a testament to the courage and determination of those victims and witnesses who never gave up hope that they would see him brought to justice.\"A woman writes in a book inside a traveling monument called \"Prijedor 92\" outside the tribunal in The Hague on Wednesday.In Serbia, the country's president, Aleksandar Vučić, urged his people to look forward to the future.\"Today is not a day for joy, nor for sorrow, but to see what kind of future we want,\" he told reporters. \"We all knew that the judgment would be like that. There is no one who did not know it in advance. My call to all citizens of Serbia is to start looking at the future today. \"Let's think about where and how our children will live. How and in what way will we preserve peace and stability in the region\"In a separate development, Serbia's Minister for Justice Nela Kuburović urged that Mladic be released to undergo medical treatment.Who is Ratko Mladic?The ex-general -- accused of being \"the Butcher of Bosnia\" -- was in command of the Bosnian Serb army that entered the town of Srebrenica in July 1995. In the days that followed, 8,000 Muslim men and boys were systematically slaughtered by troops under his leadership.JUST WATCHEDAmanpour meets the 'Butcher of Bosnia'ReplayMore Videos ...MUST WATCHAmanpour meets the 'Butcher of Bosnia' 04:05The late Bosnia peace negotiator Richard Holbrooke once described Mladic as \"one of those lethal combinations that history thrusts up occasionally -- a charismatic murderer.\"Read: Bosnia's future is tied to justiceMladic faced charges over his actions during the siege of Sarajevo, where his heavily armed forces cut the city off from the outside world. Serb forces pounded the city from higher ground each day, trapping Sarajevo's residents in the valley below. More than 10,000 people, mostly civilians, perished.JUST WATCHEDMarking 20 years since Srebrenica: A survivor's storyReplayMore Videos ...MUST WATCHMarking 20 years since Srebrenica: A survivor's story 04:26After the war ended in 1995, Mladic went on the run before being found 16 years later when police burst into the garden of a small house in northern Serbia.Though he was carrying two handguns, he surrendered without a fight. He was extradited for trial in the Netherlands.Srebrenica massacre: Two decades on, wounds still raw, graves still openIn 2011, a tribunal judge entered pleas of not guilty for Mladic after he refused to cooperate and was forcibly removed from the courtroom at the judge's order.Mladic's judgment day comes more than a year after Bosnian Serb political leader Radovan Karadzic was sentenced to 40 years in prison for his role in the 1990s conflict. Former Serbian President Slobodan Milosevic was arrested in 2001 but died before his trial could be completed.At a news conference following the verdict, Mladic's son Darko said that his father felt sorry for every victim of the conflict. \"General Mladic cannot accept responsibility for things he did not do,\" Darko Mladic said. He argued that what had happened in Srebrenica was \"legitimate.\"Melina Borcak and Lindsay Isaac in Sarajevo contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. We ensure that each entity and relationship is described comprehensively and assigned appropriate importance scores and weights.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ratko Mladic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Bosnian Serb army leader found guilty of genocide and other crimes during the Bosnian war.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Bosnian war\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Conflict in the former Yugoslavia from 1992 to 1995 involving atrocities and genocide.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Srebrenica\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Site of the worst massacre in Europe since World War II, where thousands of Muslim men and boys were slaughtered.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"International Criminal Tribunal for the former Yugoslavia\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"Ad hoc court established to prosecute crimes committed during the Balkans conflict.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Sarajevo\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Capital of Bosnia and Herzegovina, heavily affected by the Bosnian war.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Drina River\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"River in Bosnia and Herzegovina where victims were thrown.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Aleksandar Vučić\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"President of Serbia during the trial of Ratko Mladic.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Radovan Karadzic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Bosnian Serb political leader sentenced for his role in the 1990s conflict.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Slobodan Milosevic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Serbian President who died before his trial could be completed.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Bosnian war\",\n", + "\n", + "\"description\": \"Ratko Mladic was a key figure in the Bosnian war, leading the Bosnian Serb army.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Bosnian war\",\n", + "\n", + "\"tgt_id\": \"Srebrenica\",\n", + "\n", + "\"description\": \"The Bosnian war included the massacre at Srebrenica.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"International Criminal Tribunal for the former Yugoslavia\",\n", + "\n", + "\"description\": \"Ratko Mladic's trial took place at the International Criminal Tribunal for the former Yugoslavia.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Sarajevo\",\n", + "\n", + "\"description\": \"Ratko Mladic's forces besieged Sarajevo during the Bosnian war.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Drina River\",\n", + "\n", + "\"description\": \"Victims of Ratko Mladic's forces were thrown into the Drina River.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Aleksandar Vučić\",\n", + "\n", + "\"tgt_id\": \"Ratko Mladic\",\n", + "\n", + "\"description\": \"Aleksandar Vučić, as the President of Serbia, commented on the trial of Ratko Mladic.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Radovan Karadzic\",\n", + "\n", + "\"description\": \"Ratko Mladic and Radovan Karadzic were both key figures in the Bosnian Serb leadership during the war.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Slobodan Milosevic\",\n", + "\n", + "\"description\": \"Ratko Mladic and Slobodan Milosevic were both involved in the conflicts in the former Yugoslavia.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Neymar surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.Two penalties and a last-minute tap-in took the Paris Saint-Germain forward's tally to 64 for his country, two ahead of Ronaldo and 13 behind Pele's all-time record of 77.Everton's Richarlison scored Brazil's other goal as the five-time world champion twice had to come from behind to secure the win.Brazil head coach Tite told reporters after the match that it was \"unfair\" to compare Neymar and Ronaldo.\"What I can say is Neymar has this unpredictability,\" he said. \"He is the bow and the arrow, he's a player who both makes and takes chances. And he gets better and better, and more mature.\"Read MoreNeymar celebrates after completing his hat-trick against Peru.The match had a number of controversial VAR moments, particularly the decision to award Neymar a second penalty late in the game with the score tied at 2-2, a decision which surprised even the Brazilian players.Peru's anger was compounded in stoppage time when Carlos Zambrano was shown a red card for an elbow on Richarlison. The Everton forward had escaped punishment for a similar incident earlier in the match that left Peru's Miguel Trauco with a bloody wound above his eye.The latest incidents added to a growing list of decisions that have left South American fans scratching their heads during this international break, the first time VAR has been used in CONMEBOL World Cup qualifiersBrazil's victory ensures it begins the grueling 18-month campaign with a 100% record to top the standings on six points along with Argentina, which secured an impressive 2-1 win in the altitude of La Paz, Bolivia earlier in the day.The top four teams will qualify automatically for Qatar 2022, with the fifth-placed team competing in a two-legged playoff against a country from another continent.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neymar\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Brazilian footballer who surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ronaldo\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former Brazilian footballer who was previously second on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Pele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A legendary Brazilian footballer who holds the all-time record for Brazil with 77 goals.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brazil\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country where the football match took place and where Neymar and Ronaldo are from.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peru\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The opposing country in the football match against Brazil.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"World Cup\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"An international football competition where the qualifying match between Brazil and Peru took place.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Ronaldo\",\n", + "\n", + "\"description\": \"Neymar surpassed Ronaldo on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Pele\",\n", + "\n", + "\"description\": \"Neymar is 13 goals behind Pele's all-time record for Brazil.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brazil\",\n", + "\n", + "\"tgt_id\": \"Peru\",\n", + "\n", + "\"description\": \"Brazil won a football match against Peru in World Cup qualifying.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text. It works by first identifying entities in the text based on predefined entity types, then determining the relationships between these entities considering their interactions and the context provided in the text. The program ensures that each entity and relationship is described comprehensively and assigned appropriate importance scores and weights. The output is formatted according to a specified JSON schema, listing entities and their relationships in a structured manner.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: (CNN)Former Bosnian Serb army leader Ratko Mladic was sentenced to life in prison Wednesday after being found guilty of genocide for atrocities committed during the Bosnian war from 1992 to 1995.Verdict proceedings had been interrupted earlier when the 74-year-old's legal team claimed that his blood pressure was too high to continue.After outbursts from Mladic, Judge Alphons Orie, who was delivering a summation of the case, ordered the removal of the ex-general, telling him he could monitor proceedings by audio and video. \"I am very distraught,\" Mladic shouted inside the courtroom. \"Everything that you have said is pure lies. Shame on you. It's all lies.\"Mladic's legal team had asked for proceedings to be halted or for the summation of the case to be skipped, which the Judge refused. Read MoreMladic was charged with two counts of genocide and nine crimes against humanity and war crimes for his role in the conflict in the former Yugoslavia from 1992 to 1995, during which 100,000 people were killed and another 2.2 million displaced. He was found not guilty on one charge of genocide, but received a guilty verdict on each of the other 10 counts. Mladic's lawyer, Dragan Ivetic, said it was \"certain\" Mladic would appeal.\"Butcher of Bosnia\" Ratko Mladic has been found guilty of the highest crimes against international law, says CNN's @camanpour, who covered the Bosnian war https://t.co/4A7DtVjHn8 pic.twitter.com/HdPKU3tqRD— CNN International (@cnni) November 22, 2017The trial, which opened in 2012, took place at the International Criminal Tribunal for the former Yugoslavia in The Hague, Netherlands. The ad hoc court was established to prosecute crimes committed during the Balkans conflict. Mladic was accused of orchestrating a campaign of ethnic cleansing, including the slaughter of thousands of Muslim men and boys at Srebrenica in July 1995. It is the worst massacre to have taken place in Europe since the Second World War.Mladic judgment brings back stench of Bosnian genocideProsecutor Serge Brammertz told reporters that Mladic will be remembered by history \"for the many communities and lives he destroyed.\"\"Today's judgment is a milestone in the tribunal's history and for international justice,\" he added.The trial of Mladic, who was arrested in 2011, has lasted 530 days and included more than 500 witnesses and nearly 10,000 exhibits. Before the case was adjourned last December, prosecutors recommended a life sentence. Mladic had previously referred to the court as \"satanic\" and labeled the charges against him as \"obnoxious.\"ReactionAt a center for the association of women victims of war in Sarajevo, there was an outpouring of emotion during the judge's summation.There was particular frustration that Mladic was acquitted on one charge of genocide in Bosnian municipalities outside of Srebrenica.Amela Meduseljac (L) and Meliha Mrdzic were unhappy that Mladic was acquitted on one charge.Meliha Mrdzic, who said her father and brother were killed and thrown into the Drina River in Visegrad, told CNN she was left humiliated by the decision.\"The international community made me a victim a second time,\" she said. \"They make it seem like we killed ourselves, raped ourselves, slaughtered ourselves. I feel so hurt, I can't describe it.\"Amela Meduseljac, who said she was raped by Mladic's soldiers at Visegrad, said that victims will struggle to get over the judgment.\"Our mission as a rape survivor association was to stop victims from feeling like victims,\" she said. \"But it's getting worse from year to year and it will get especially worse after this verdict.\"People in Srebrenica celebrate as they watch a live TV broadcast of the trial Wednesday.UN High Commissioner for Human Rights Zeid Ra'ad Al Hussein called Mladic the \"epitome of evil\" and labeled his conviction a \"momentous victory for justice.\" \"Mladic presided over some of the darkest crimes to occur in Europe since World War II, bringing terror, death and destruction to thousands of victims, and sorrow, tragedy and trauma to countless more,\" Zeid said in a statement.\"His conviction is a testament to the courage and determination of those victims and witnesses who never gave up hope that they would see him brought to justice.\"A woman writes in a book inside a traveling monument called \"Prijedor 92\" outside the tribunal in The Hague on Wednesday.In Serbia, the country's president, Aleksandar Vučić, urged his people to look forward to the future.\"Today is not a day for joy, nor for sorrow, but to see what kind of future we want,\" he told reporters. \"We all knew that the judgment would be like that. There is no one who did not know it in advance. My call to all citizens of Serbia is to start looking at the future today. \"Let's think about where and how our children will live. How and in what way will we preserve peace and stability in the region\"In a separate development, Serbia's Minister for Justice Nela Kuburović urged that Mladic be released to undergo medical treatment.Who is Ratko Mladic?The ex-general -- accused of being \"the Butcher of Bosnia\" -- was in command of the Bosnian Serb army that entered the town of Srebrenica in July 1995. In the days that followed, 8,000 Muslim men and boys were systematically slaughtered by troops under his leadership.JUST WATCHEDAmanpour meets the 'Butcher of Bosnia'ReplayMore Videos ...MUST WATCHAmanpour meets the 'Butcher of Bosnia' 04:05The late Bosnia peace negotiator Richard Holbrooke once described Mladic as \"one of those lethal combinations that history thrusts up occasionally -- a charismatic murderer.\"Read: Bosnia's future is tied to justiceMladic faced charges over his actions during the siege of Sarajevo, where his heavily armed forces cut the city off from the outside world. Serb forces pounded the city from higher ground each day, trapping Sarajevo's residents in the valley below. More than 10,000 people, mostly civilians, perished.JUST WATCHEDMarking 20 years since Srebrenica: A survivor's storyReplayMore Videos ...MUST WATCHMarking 20 years since Srebrenica: A survivor's story 04:26After the war ended in 1995, Mladic went on the run before being found 16 years later when police burst into the garden of a small house in northern Serbia.Though he was carrying two handguns, he surrendered without a fight. He was extradited for trial in the Netherlands.Srebrenica massacre: Two decades on, wounds still raw, graves still openIn 2011, a tribunal judge entered pleas of not guilty for Mladic after he refused to cooperate and was forcibly removed from the courtroom at the judge's order.Mladic's judgment day comes more than a year after Bosnian Serb political leader Radovan Karadzic was sentenced to 40 years in prison for his role in the 1990s conflict. Former Serbian President Slobodan Milosevic was arrested in 2001 but died before his trial could be completed.At a news conference following the verdict, Mladic's son Darko said that his father felt sorry for every victim of the conflict. \"General Mladic cannot accept responsibility for things he did not do,\" Darko Mladic said. He argued that what had happened in Srebrenica was \"legitimate.\"Melina Borcak and Lindsay Isaac in Sarajevo contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. We ensure that each entity and relationship is described comprehensively and assigned appropriate importance scores and weights.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ratko Mladic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Bosnian Serb army leader found guilty of genocide and other crimes during the Bosnian war.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Bosnian war\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Conflict in the former Yugoslavia from 1992 to 1995 involving atrocities and genocide.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Srebrenica\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Site of the worst massacre in Europe since World War II, where thousands of Muslim men and boys were slaughtered.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"International Criminal Tribunal for the former Yugoslavia\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"Ad hoc court established to prosecute crimes committed during the Balkans conflict.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Sarajevo\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Capital of Bosnia and Herzegovina, heavily affected by the Bosnian war.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Drina River\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"River in Bosnia and Herzegovina where victims were thrown.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Aleksandar Vučić\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"President of Serbia during the trial of Ratko Mladic.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Radovan Karadzic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Bosnian Serb political leader sentenced for his role in the 1990s conflict.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Slobodan Milosevic\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Serbian President who died before his trial could be completed.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Bosnian war\",\n", + "\n", + "\"description\": \"Ratko Mladic was a key figure in the Bosnian war, leading the Bosnian Serb army.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Bosnian war\",\n", + "\n", + "\"tgt_id\": \"Srebrenica\",\n", + "\n", + "\"description\": \"The Bosnian war included the massacre at Srebrenica.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"International Criminal Tribunal for the former Yugoslavia\",\n", + "\n", + "\"description\": \"Ratko Mladic's trial took place at the International Criminal Tribunal for the former Yugoslavia.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Sarajevo\",\n", + "\n", + "\"description\": \"Ratko Mladic's forces besieged Sarajevo during the Bosnian war.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Drina River\",\n", + "\n", + "\"description\": \"Victims of Ratko Mladic's forces were thrown into the Drina River.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Aleksandar Vučić\",\n", + "\n", + "\"tgt_id\": \"Ratko Mladic\",\n", + "\n", + "\"description\": \"Aleksandar Vučić, as the President of Serbia, commented on the trial of Ratko Mladic.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Radovan Karadzic\",\n", + "\n", + "\"description\": \"Ratko Mladic and Radovan Karadzic were both key figures in the Bosnian Serb leadership during the war.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Ratko Mladic\",\n", + "\n", + "\"tgt_id\": \"Slobodan Milosevic\",\n", + "\n", + "\"description\": \"Ratko Mladic and Slobodan Milosevic were both involved in the conflicts in the former Yugoslavia.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Neymar surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.Two penalties and a last-minute tap-in took the Paris Saint-Germain forward's tally to 64 for his country, two ahead of Ronaldo and 13 behind Pele's all-time record of 77.Everton's Richarlison scored Brazil's other goal as the five-time world champion twice had to come from behind to secure the win.Brazil head coach Tite told reporters after the match that it was \"unfair\" to compare Neymar and Ronaldo.\"What I can say is Neymar has this unpredictability,\" he said. \"He is the bow and the arrow, he's a player who both makes and takes chances. And he gets better and better, and more mature.\"Read MoreNeymar celebrates after completing his hat-trick against Peru.The match had a number of controversial VAR moments, particularly the decision to award Neymar a second penalty late in the game with the score tied at 2-2, a decision which surprised even the Brazilian players.Peru's anger was compounded in stoppage time when Carlos Zambrano was shown a red card for an elbow on Richarlison. The Everton forward had escaped punishment for a similar incident earlier in the match that left Peru's Miguel Trauco with a bloody wound above his eye.The latest incidents added to a growing list of decisions that have left South American fans scratching their heads during this international break, the first time VAR has been used in CONMEBOL World Cup qualifiersBrazil's victory ensures it begins the grueling 18-month campaign with a 100% record to top the standings on six points along with Argentina, which secured an impressive 2-1 win in the altitude of La Paz, Bolivia earlier in the day.The top four teams will qualify automatically for Qatar 2022, with the fifth-placed team competing in a two-legged playoff against a country from another continent.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neymar\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Brazilian footballer who surpassed Ronaldo and moved to second on Brazil's all-time goalscoring list with a hat-trick in a 4-2 win against Peru in World Cup qualifying.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ronaldo\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A former Brazilian footballer who was previously second on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Pele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A legendary Brazilian footballer who holds the all-time record for Brazil with 77 goals.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brazil\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country where the football match took place and where Neymar and Ronaldo are from.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peru\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The opposing country in the football match against Brazil.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"World Cup\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"An international football competition where the qualifying match between Brazil and Peru took place.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Ronaldo\",\n", + "\n", + "\"description\": \"Neymar surpassed Ronaldo on Brazil's all-time goalscoring list.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Neymar\",\n", + "\n", + "\"tgt_id\": \"Pele\",\n", + "\n", + "\"description\": \"Neymar is 13 goals behind Pele's all-time record for Brazil.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brazil\",\n", + "\n", + "\"tgt_id\": \"Peru\",\n", + "\n", + "\"description\": \"Brazil won a football match against Peru in World Cup qualifying.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide a comprehensive description and assign an importance score. Subsequently, determine the relationships between these entities, considering their interactions and the context provided in the text. Each relationship should include a description and appropriate weight. Ensure that the output is formatted according to the specified JSON schema, listing entities and their relationships in a structured manner.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide a comprehensive description and assign an importance score. Subsequently, determine the relationships between these entities, considering their interactions and the context provided in the text. Each relationship should include a description and appropriate weight. Ensure that the output is formatted according to the specified JSON schema, listing entities and their relationships in a structured manner.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text input. It works by first identifying entities in the text based on specified entity types. Then, it determines the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, it formats the entities and relationships according to a specified JSON schema. The program uses a language model to perform these tasks and includes mechanisms for error handling and retries to ensure accurate results.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: (CNN)The New York City Marathon returns on Sunday following a two-year hiatus, and at the front of the field, legendary distance runner Kenenisa Bekele is primed to \"make more history\" in his storied career. Bekele, a three-time Olympic gold medalist over 5,000 and 10,000 meters, is the headline name competing in the men's elite field in New York. With his only previous marathon in the United States a fourth-place finish in Chicago seven years ago, the Ethiopian is hungry for success as he prepares to take to the streets of the Big Apple.\"Really, I wanted to have a good result in the USA, that's why I chose the New York marathon,\" Bekele told reporters this week.\"The New York marathon is big publicity and a really big marathon race. To achieve a good result in this marathon would be perfect, and for me, it's also really good to make more history in sports.\"Read MoreBekele has a point to prove after a disappointing outing six weeks ago at the Berlin Marathon -- the race at which he came within two seconds of Eliud Kipchoge's world record time two years ago. Bekele catches his breath after this year's Berlin Marathon. Despite expectations that he could challenge Kipchoge's record of two hours, one minute and 39 seconds in Berlin this year, Bekele says he fell foul to a poor night's sleep as he finished third, more than a minute behind winner Guye Adola.\"I was not ready for that race,\" Bekele explained. \"A few weeks before the race, I was not really confident. It's a little bit tough for me because the day before also I couldn't sleep well, I really had bad luck the day before in the night. That also made me very tired.\"I've recovered well ... At this moment, I'm really strong. I hope I'll achieve a good result on Sunday.\"For race organizers, who called off last year's New York City Marathon amid the pandemic, it is a boon to have a figure like Bekele on the start line on Sunday.On top of his three Olympic gold medals -- including a double in the 5,000 and 10,000 meters at Beijing 2008 -- and one silver medal, he also claimed five world athletics championship titles between 2003 to 2009. Bekele's 5,000m world record stood for 16 years before it was broken by Uganda's Joshua Cheptegei last year, while his 10,000m record stood for 15 before also being broken last year by Cheptegei. Eliud Kipchoge: Marathon world record holder has 'the qualities of an ascetic monk'Many consider Bekele to be the greatest male distance runner of all time, such has been his supremacy across multiple distances over the past two decades; others point towards Kenya's Kipchoge, who has been so dominant over the 26.2 miles of a marathon and is the first man to break the distance's two-hour barrier, albeit in unofficial conditions. \"I still feel that I am the best and better than anyone,\" Bekele told Sports Illustrated in August after it was announced that he would make his debut in New York. \"I think every athlete and others should think like that.\"Having struggled with injuries and form in recent years, Bekele now has a second opportunity to win a major marathon in the US. There is little chance that he will be able to come close to his best marathon time in New York; unlike Berlin, the course, which undulates through the city's five boroughs, rarely produces fast performances.\"I know the course is tough and there are also no pacemakers in the race. It's more about concentration and a tactical race,\" said Bekele. He will be up against the Netherlands' Adbi Nageeye, who won the silver medal at the Olympics in Sapporo three months ago, and the half marathon world record holder Kibiwott Kandie of Kenya, who is making his marathon debut. The women's race is headlined by Kenya's Peres Jepchirchir, the gold medalist at the Olympics. She will be joined by the USA's Olympic bronze medalist Molly Seidel. Seidel poses with her bronze medal from the Tokyo Olympics. Visit CNN.com/sport for more news, videos and featuresAnother athlete to keep an eye on is US star Shalane Flanagan as she attempts to run six marathons in 42 days. Having already completed marathons in Berlin, London, Chicago, Boston and Portland -- all between times of 2:35:14 and 2:46:39 -- Flanagan will aim to complete her challenge in New York. And as for Bekele, like so many other runners who will take to the start line on Sunday, he's grateful to be able to race in New York following the pandemic and last year's cancellation.\"To see the people of New York participate in this race again -- to see this for me is fantastic,\" he said. \"I want to wish all participants good luck.\"\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenenisa Bekele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Legendary distance runner and three-time Olympic gold medalist over 5,000 and 10,000 meters.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New York City Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in New York City, returning after a two-year hiatus due to the pandemic.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eliud Kipchoge\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Marathon world record holder and dominant runner over the 26.2 miles of a marathon.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Berlin Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in Berlin, where Kenenisa Bekele had a disappointing outing six weeks ago.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guye Adola\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Winner of the Berlin Marathon, finishing more than a minute ahead of Kenenisa Bekele.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Adbi Nageeye\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Dutch runner who won the silver medal at the Olympics in Sapporo and will compete in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kibiwott Kandie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Half marathon world record holder from Kenya, making his marathon debut in New York.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peres Jepchirchir\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Kenyan runner and gold medalist at the Olympics, headlining the women's race in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Molly Seidel\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"American runner and Olympic bronze medalist, competing in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Shalane Flanagan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"US star attempting to run six marathons in 42 days, including the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"New York City Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele is primed to compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Eliud Kipchoge\",\n", + "\n", + "\"description\": \"Kenenisa Bekele aims to make more history in his storied career, comparing himself to Eliud Kipchoge.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Berlin Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele had a disappointing outing at the Berlin Marathon six weeks ago.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Guye Adola\",\n", + "\n", + "\"description\": \"Kenenisa Bekele finished third in the Berlin Marathon, more than a minute behind winner Guye Adola.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Adbi Nageeye\",\n", + "\n", + "\"description\": \"Adbi Nageeye will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Kibiwott Kandie\",\n", + "\n", + "\"description\": \"Kibiwott Kandie will make his marathon debut in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Peres Jepchirchir\",\n", + "\n", + "\"description\": \"Peres Jepchirchir will headline the women's race in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Molly Seidel\",\n", + "\n", + "\"description\": \"Molly Seidel will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Shalane Flanagan\",\n", + "\n", + "\"description\": \"Shalane Flanagan will aim to complete her challenge of running six marathons in 42 days in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text input. It works by first identifying entities in the text based on specified entity types. Then, it determines the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, it formats the entities and relationships according to a specified JSON schema. The program uses a language model to perform these tasks and includes mechanisms for error handling and retries to ensure accurate results.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: (CNN)The New York City Marathon returns on Sunday following a two-year hiatus, and at the front of the field, legendary distance runner Kenenisa Bekele is primed to \"make more history\" in his storied career. Bekele, a three-time Olympic gold medalist over 5,000 and 10,000 meters, is the headline name competing in the men's elite field in New York. With his only previous marathon in the United States a fourth-place finish in Chicago seven years ago, the Ethiopian is hungry for success as he prepares to take to the streets of the Big Apple.\"Really, I wanted to have a good result in the USA, that's why I chose the New York marathon,\" Bekele told reporters this week.\"The New York marathon is big publicity and a really big marathon race. To achieve a good result in this marathon would be perfect, and for me, it's also really good to make more history in sports.\"Read MoreBekele has a point to prove after a disappointing outing six weeks ago at the Berlin Marathon -- the race at which he came within two seconds of Eliud Kipchoge's world record time two years ago. Bekele catches his breath after this year's Berlin Marathon. Despite expectations that he could challenge Kipchoge's record of two hours, one minute and 39 seconds in Berlin this year, Bekele says he fell foul to a poor night's sleep as he finished third, more than a minute behind winner Guye Adola.\"I was not ready for that race,\" Bekele explained. \"A few weeks before the race, I was not really confident. It's a little bit tough for me because the day before also I couldn't sleep well, I really had bad luck the day before in the night. That also made me very tired.\"I've recovered well ... At this moment, I'm really strong. I hope I'll achieve a good result on Sunday.\"For race organizers, who called off last year's New York City Marathon amid the pandemic, it is a boon to have a figure like Bekele on the start line on Sunday.On top of his three Olympic gold medals -- including a double in the 5,000 and 10,000 meters at Beijing 2008 -- and one silver medal, he also claimed five world athletics championship titles between 2003 to 2009. Bekele's 5,000m world record stood for 16 years before it was broken by Uganda's Joshua Cheptegei last year, while his 10,000m record stood for 15 before also being broken last year by Cheptegei. Eliud Kipchoge: Marathon world record holder has 'the qualities of an ascetic monk'Many consider Bekele to be the greatest male distance runner of all time, such has been his supremacy across multiple distances over the past two decades; others point towards Kenya's Kipchoge, who has been so dominant over the 26.2 miles of a marathon and is the first man to break the distance's two-hour barrier, albeit in unofficial conditions. \"I still feel that I am the best and better than anyone,\" Bekele told Sports Illustrated in August after it was announced that he would make his debut in New York. \"I think every athlete and others should think like that.\"Having struggled with injuries and form in recent years, Bekele now has a second opportunity to win a major marathon in the US. There is little chance that he will be able to come close to his best marathon time in New York; unlike Berlin, the course, which undulates through the city's five boroughs, rarely produces fast performances.\"I know the course is tough and there are also no pacemakers in the race. It's more about concentration and a tactical race,\" said Bekele. He will be up against the Netherlands' Adbi Nageeye, who won the silver medal at the Olympics in Sapporo three months ago, and the half marathon world record holder Kibiwott Kandie of Kenya, who is making his marathon debut. The women's race is headlined by Kenya's Peres Jepchirchir, the gold medalist at the Olympics. She will be joined by the USA's Olympic bronze medalist Molly Seidel. Seidel poses with her bronze medal from the Tokyo Olympics. Visit CNN.com/sport for more news, videos and featuresAnother athlete to keep an eye on is US star Shalane Flanagan as she attempts to run six marathons in 42 days. Having already completed marathons in Berlin, London, Chicago, Boston and Portland -- all between times of 2:35:14 and 2:46:39 -- Flanagan will aim to complete her challenge in New York. And as for Bekele, like so many other runners who will take to the start line on Sunday, he's grateful to be able to race in New York following the pandemic and last year's cancellation.\"To see the people of New York participate in this race again -- to see this for me is fantastic,\" he said. \"I want to wish all participants good luck.\"\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenenisa Bekele\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Legendary distance runner and three-time Olympic gold medalist over 5,000 and 10,000 meters.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New York City Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in New York City, returning after a two-year hiatus due to the pandemic.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eliud Kipchoge\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Marathon world record holder and dominant runner over the 26.2 miles of a marathon.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Berlin Marathon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"Annual marathon in Berlin, where Kenenisa Bekele had a disappointing outing six weeks ago.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guye Adola\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Winner of the Berlin Marathon, finishing more than a minute ahead of Kenenisa Bekele.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Adbi Nageeye\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Dutch runner who won the silver medal at the Olympics in Sapporo and will compete in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kibiwott Kandie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Half marathon world record holder from Kenya, making his marathon debut in New York.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Peres Jepchirchir\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Kenyan runner and gold medalist at the Olympics, headlining the women's race in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Molly Seidel\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"American runner and Olympic bronze medalist, competing in the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Shalane Flanagan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"US star attempting to run six marathons in 42 days, including the New York City Marathon.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"New York City Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele is primed to compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Eliud Kipchoge\",\n", + "\n", + "\"description\": \"Kenenisa Bekele aims to make more history in his storied career, comparing himself to Eliud Kipchoge.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Berlin Marathon\",\n", + "\n", + "\"description\": \"Kenenisa Bekele had a disappointing outing at the Berlin Marathon six weeks ago.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Kenenisa Bekele\",\n", + "\n", + "\"tgt_id\": \"Guye Adola\",\n", + "\n", + "\"description\": \"Kenenisa Bekele finished third in the Berlin Marathon, more than a minute behind winner Guye Adola.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Adbi Nageeye\",\n", + "\n", + "\"description\": \"Adbi Nageeye will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Kibiwott Kandie\",\n", + "\n", + "\"description\": \"Kibiwott Kandie will make his marathon debut in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Peres Jepchirchir\",\n", + "\n", + "\"description\": \"Peres Jepchirchir will headline the women's race in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Molly Seidel\",\n", + "\n", + "\"description\": \"Molly Seidel will compete in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"New York City Marathon\",\n", + "\n", + "\"tgt_id\": \"Shalane Flanagan\",\n", + "\n", + "\"description\": \"Shalane Flanagan will aim to complete her challenge of running six marathons in 42 days in the New York City Marathon.\",\n", + "\n", + "\"weight\": 0.4,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract entities and their relationships from the text. First, identify all entities in the text that match the provided entity types, ensuring each entity includes its name, type, a brief description, and an importance score. Next, determine the relationships between these entities by analyzing their interactions and dependencies within the context of the text. Each relationship should include the source entity ID, target entity ID, a description of the relationship, a weight indicating the strength of the relationship, and an order indicating the sequence of the relationship. Finally, format the entities and relationships according to the specified JSON schema, ensuring all fields are correctly populated and validated.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously extract entities and their relationships from the text. First, identify all entities in the text that match the provided entity types, ensuring each entity includes its name, type, a brief description, and an importance score. Next, determine the relationships between these entities by analyzing their interactions and dependencies within the context of the text. Each relationship should include the source entity ID, target entity ID, a description of the relationship, a weight indicating the strength of the relationship, and an order indicating the sequence of the relationship. Finally, format the entities and relationships according to the specified JSON schema, ensuring all fields are correctly populated and validated.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text. It uses a language model to identify entities based on specified types and then determines the relationships between these entities. The program works by first defining a signature that includes input fields for the text and entity types, and output fields for reasoning and the extracted entities and relationships. It then uses a TypedPredictor to enforce type annotations and handle potential errors, ensuring that the output matches the expected format. The program iteratively attempts to extract and validate the entities and relationships, providing explanations for errors and suggesting corrections if necessary. Finally, it returns the extracted entities and their relationships in a structured format.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: (CNN)Roger Federer thinks the professional tennis circuit won't return for a while due to the coronavirus pandemic but, when the time does come, the Swiss superstar said he would find it difficult to play without fans. While European football's Bundesliga resurfaced last week behind closed doors and Spain's La Liga is set to resume the middle of next month, the last official word from tennis authorities essentially saw all action suspended through July. The hiatus began in March, with Federer already sidelined since he was recuperating from knee surgery. Sunday would have marked the first day of the French Open in its usual spot on the tennis calendar -- in March, though, it was rescheduled for September -- and another grand slam, Wimbledon in July, was called off. \"I'm not training at the moment because I don't see a reason for that to be honest,\" Federer told three-time French Open champion Gustavo Kuerten -- who is raising funds for coronavirus relief efforts in his native Brazil -- in a video interview reported by Tennis.com.Read More\"I am happy with my body now and I still believe that the return of the tour is a long way off,\" continued the 38-year-old. \"And I think it's important mentally to enjoy this break, having played so much tennis. \"When I'm getting towards returning and have a goal to train for, I think I will be super motivated.\"We should be sliding into @rolandgarros right now, thinking of our mates in Paris 👋 pic.twitter.com/0PLKryyIjj— #AusOpen (@AustralianOpen) May 24, 2020 Federer is arguably tennis' best supported player ever, and the prospect of competing without spectators doesn't appeal to him. \"Most of the time when we are training, there is no one,\" said the men's record 20-time grand slam champion. \"For us, of course, it is possible to play if there are no fans. But on the other hand, I really hope that the circuit can return as it normally is. \"And hold off till the time is appropriate, minimum a third of the stadium or half full. But, for me, completely empty when playing big tournaments is very difficult.\"Federer has been active on social media during the lockdown, sparking a public discussion on the merging of the men's and women's tours with a tweet last month and embarking on a funny Instagram Live with tennis rival Rafael Nadal.Nadal, unlike Federer, has started practicing, though only very recently. The Spaniard would have been favored to win a 20th major and tie Federer had the French Open been played as usual given he has collected a record 12 titles at Roland Garros. Here I am, the first pictures I am posting for you on court. This is my practice earlier today at @rnadalacademy #BackOnCourt #BabolatFamily 🎾👍🏻💪🏻😉 pic.twitter.com/x7tzgLj9pc— Rafa Nadal (@RafaelNadal) May 22, 2020 The next grand slam is scheduled to be late August's US Open in New York, with organizers expected to announce in June if it will go ahead. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We identify the entities in the text based on the provided entity types and then determine the relationships between these entities. We focus on key entities such as Roger Federer, Rafael Nadal, French Open, and Wimbledon, and their interactions and associations within the context of the text. We also consider the relationships between these entities and other relevant entities like the coronavirus pandemic and the tennis circuit.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Roger Federer\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Swiss professional tennis player, considered one of the greatest of all time.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rafael Nadal\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Spanish professional tennis player, known for his dominance on clay courts.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"French Open\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held over two weeks between late May and early June.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wimbledon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held at the All England Club in Wimbledon, London, every year.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"coronavirus pandemic\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A global outbreak of a novel coronavirus that has led to significant disruptions worldwide.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"Roger Federer discusses the impact of the coronavirus pandemic on the French Open's schedule.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"Wimbledon\",\n", + "\n", + "\"description\": \"Roger Federer mentions the cancellation of Wimbledon due to the coronavirus pandemic.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"Rafael Nadal\",\n", + "\n", + "\"description\": \"Roger Federer and Rafael Nadal engage in social media interactions during the lockdown.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Rafael Nadal\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"Rafael Nadal is expected to compete in the French Open, a tournament he has won multiple times.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"coronavirus pandemic\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"The coronavirus pandemic has caused the French Open to be rescheduled.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"coronavirus pandemic\",\n", + "\n", + "\"tgt_id\": \"Wimbledon\",\n", + "\n", + "\"description\": \"The coronavirus pandemic has led to the cancellation of Wimbledon.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and their relationships from a given text. It uses a language model to identify entities based on specified types and then determines the relationships between these entities. The program works by first defining a signature that includes input fields for the text and entity types, and output fields for reasoning and the extracted entities and relationships. It then uses a TypedPredictor to enforce type annotations and handle potential errors, ensuring that the output matches the expected format. The program iteratively attempts to extract and validate the entities and relationships, providing explanations for errors and suggesting corrections if necessary. Finally, it returns the extracted entities and their relationships in a structured format.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: (CNN)Roger Federer thinks the professional tennis circuit won't return for a while due to the coronavirus pandemic but, when the time does come, the Swiss superstar said he would find it difficult to play without fans. While European football's Bundesliga resurfaced last week behind closed doors and Spain's La Liga is set to resume the middle of next month, the last official word from tennis authorities essentially saw all action suspended through July. The hiatus began in March, with Federer already sidelined since he was recuperating from knee surgery. Sunday would have marked the first day of the French Open in its usual spot on the tennis calendar -- in March, though, it was rescheduled for September -- and another grand slam, Wimbledon in July, was called off. \"I'm not training at the moment because I don't see a reason for that to be honest,\" Federer told three-time French Open champion Gustavo Kuerten -- who is raising funds for coronavirus relief efforts in his native Brazil -- in a video interview reported by Tennis.com.Read More\"I am happy with my body now and I still believe that the return of the tour is a long way off,\" continued the 38-year-old. \"And I think it's important mentally to enjoy this break, having played so much tennis. \"When I'm getting towards returning and have a goal to train for, I think I will be super motivated.\"We should be sliding into @rolandgarros right now, thinking of our mates in Paris 👋 pic.twitter.com/0PLKryyIjj— #AusOpen (@AustralianOpen) May 24, 2020 Federer is arguably tennis' best supported player ever, and the prospect of competing without spectators doesn't appeal to him. \"Most of the time when we are training, there is no one,\" said the men's record 20-time grand slam champion. \"For us, of course, it is possible to play if there are no fans. But on the other hand, I really hope that the circuit can return as it normally is. \"And hold off till the time is appropriate, minimum a third of the stadium or half full. But, for me, completely empty when playing big tournaments is very difficult.\"Federer has been active on social media during the lockdown, sparking a public discussion on the merging of the men's and women's tours with a tweet last month and embarking on a funny Instagram Live with tennis rival Rafael Nadal.Nadal, unlike Federer, has started practicing, though only very recently. The Spaniard would have been favored to win a 20th major and tie Federer had the French Open been played as usual given he has collected a record 12 titles at Roland Garros. Here I am, the first pictures I am posting for you on court. This is my practice earlier today at @rnadalacademy #BackOnCourt #BabolatFamily 🎾👍🏻💪🏻😉 pic.twitter.com/x7tzgLj9pc— Rafa Nadal (@RafaelNadal) May 22, 2020 The next grand slam is scheduled to be late August's US Open in New York, with organizers expected to announce in June if it will go ahead. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We identify the entities in the text based on the provided entity types and then determine the relationships between these entities. We focus on key entities such as Roger Federer, Rafael Nadal, French Open, and Wimbledon, and their interactions and associations within the context of the text. We also consider the relationships between these entities and other relevant entities like the coronavirus pandemic and the tennis circuit.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Roger Federer\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Swiss professional tennis player, considered one of the greatest of all time.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rafael Nadal\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Spanish professional tennis player, known for his dominance on clay courts.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"French Open\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held over two weeks between late May and early June.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wimbledon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A major tennis tournament held at the All England Club in Wimbledon, London, every year.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"coronavirus pandemic\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A global outbreak of a novel coronavirus that has led to significant disruptions worldwide.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"Roger Federer discusses the impact of the coronavirus pandemic on the French Open's schedule.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"Wimbledon\",\n", + "\n", + "\"description\": \"Roger Federer mentions the cancellation of Wimbledon due to the coronavirus pandemic.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Roger Federer\",\n", + "\n", + "\"tgt_id\": \"Rafael Nadal\",\n", + "\n", + "\"description\": \"Roger Federer and Rafael Nadal engage in social media interactions during the lockdown.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Rafael Nadal\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"Rafael Nadal is expected to compete in the French Open, a tournament he has won multiple times.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"coronavirus pandemic\",\n", + "\n", + "\"tgt_id\": \"French Open\",\n", + "\n", + "\"description\": \"The coronavirus pandemic has caused the French Open to be rescheduled.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"coronavirus pandemic\",\n", + "\n", + "\"tgt_id\": \"Wimbledon\",\n", + "\n", + "\"description\": \"The coronavirus pandemic has led to the cancellation of Wimbledon.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide its `entity_name`, `entity_type`, a concise `description`, and an `importance_score`. Subsequently, determine and list the relationships between these entities, specifying the `src_id` (source entity), `tgt_id` (target entity), a `description` of the relationship, a `weight` indicating the strength of the relationship, and an `order` to indicate the sequence of relationships. Ensure that the relationships are coherent and directly derived from the context provided in the `input_text`.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide its `entity_name`, `entity_type`, a concise `description`, and an `importance_score`. Subsequently, determine and list the relationships between these entities, specifying the `src_id` (source entity), `tgt_id` (target entity), a `description` of the relationship, a `weight` indicating the strength of the relationship, and an `order` to indicate the sequence of relationships. Ensure that the relationships are coherent and directly derived from the context provided in the `input_text`.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of entity and relationship extraction from a given text. It works by taking an input text and a list of entity types, then using a language model to identify entities within the text that match the provided types. After identifying the entities, the program determines the relationships between these entities based on their interactions and dependencies within the context of the text. The process involves generating reasoning steps to justify the extraction, and the final output is a list of entities with their types, descriptions, and importance scores, along with relationships between these entities, including descriptions, weights, and order. The program ensures type safety and validation through the use of Pydantic models and retries the extraction process up to a specified number of times if errors are encountered.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: Story highlightsThe Man v Fat soccer league is exclusively for men with a body mass index of 30 or moreThe league helps players lose weight, but it also gives them a built-in support group (CNN)After reviewing restaurants for years, Andrew Shanahan became obese and was looking for a way to lose weight.But it wasn't easy to find a program tailored to men. Every time he tried joining a weight-loss group, he was the only man in the room. \"I wondered how many other men felt that there was something missing and who weren't getting the support that you desperately need when you're trying to lose weight,\" he said.So in 2014, with the help of a crowdfunding campaign, Shanahan launched his Man v Fat initiative to reach out to other men like him. The idea drew attention from celebrities such as chef Jamie Oliver, and Shanahan was encouraged to write a book on it the next year. Being based in England, a soccer league seemed like a natural outreach program. And Shanahan received nearly 1,000 applications for 80 spots in the inaugural league, which was created exclusively for men with a body mass index of 30 or more.Read MorePhotographer Simone PerolariMany of the men used to play soccer -- or as it's called in most of the world, football -- but their weight had become an obstacle in pursuing the sport they love.\"I saw people who want to have fun and slim down and believe in what they do,\" said Simone Perolari, who photographed the league and whose images can be seen in the gallery above. \"There is not a space for them in classic football, but some of them are really good players.\"Rob Bird, 40, read about the league in the Birmingham Mail. He played the sport until he was 25, and he calls the 15 years after that the \"fat years\" when he didn't feel comfortable playing. \"I've always loved football, but playing against 'normal-size' guys left me feeling like I couldn't compete as they were faster,\" Bird said. \"(This league) is a level playing field for bigger guys.\" Shanahan isn't sure what he was expecting to result from the league's first season, but the competitiveness of the players surprised him. \"For a long time they haven't had a level playing field to compete on, so they are all determined to give it everything they have,\" he said. \"Fortunately, even those teams who lose can see that the benefits of losing weight and getting healthy more than make up for not topping the league.\"Social mediaFollow @CNNPhotos on Twitter to join the conversation about photography.Shanahan watched bonds form between the men as they enjoyed their favorite sport and supported one another in their goals to get fit and healthy. After 14 weeks, the 80 team members collectively lost 1,727 pounds, with the average weight loss close to 22 pounds per player. Marcus Farnsworth lost more than 70 pounds and is enjoying the energy boost he's received in return. He hopes it will help with his knees and back as he continues to lose weight. But he also enjoyed the togetherness and support of the guys around him. \"It gives you good motivation if one person is feeling hungry or thinking about snacking,\" Farnsworth said. \"We can give advice and communicate through our WhatsApp group for support.\"Bird has dropped nearly 60 pounds and is now down to 210. His gout and leg ulcer have improved \"immeasurably.\"Being around other guys who struggled with weight fostered a great spirit amongst us of being in it together and sharing our knowledge and experience without any judging or snide comments.Rob Bird, a player in the Man v Fat soccer league\"I feel a million times better,\" Bird said. \"This has given me the confidence that if you are making the right choices, you will feel better for it. Being around other guys who struggled with weight fostered a great spirit amongst us of being in it together and sharing our knowledge and experience without any judging or snide comments.\"Shanahan is expanding the league across the rest of the United Kingdom beginning in July, and he hopes to start similar leagues in the United States and other countries. \"We saw a lot of health improvements, whether it was guys curing their sleep apnea, reversing their diabetes or lowering their blood pressure,\" Shanahan said. \"The other really notable thing was how much many of the players grew in confidence. \"I think it's a great thing to undertake a project with other people who desperately want the same thing. It gives you energy and impetus. Working with the guys has been one of the greatest privileges of my career. To share in their excitement and to witness their achievements has been amazing.\"Simone Perolari is an Italian photographer based in Paris. You can follow him on Facebook and Instagram.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and dependencies within the context of the text.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Man v Fat soccer league\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A soccer league exclusively for men with a body mass index of 30 or more, designed to help them lose weight and provide a support group.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Andrew Shanahan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A man who became obese after reviewing restaurants for years and later founded the Man v Fat initiative.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Simone Perolari\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An Italian photographer based in Paris who photographed the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rob Bird\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A 40-year-old man who played soccer until he was 25 and later joined the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Marcus Farnsworth\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A participant in the Man v Fat soccer league who lost more than 70 pounds.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jamie Oliver\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A celebrity chef who supported the Man v Fat initiative.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"England\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The country where the Man v Fat soccer league was initially launched.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"CNN\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A news organization that reported on the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andrew Shanahan\",\n", + "\n", + "\"tgt_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"description\": \"Andrew Shanahan founded the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Simone Perolari\",\n", + "\n", + "\"description\": \"Simone Perolari photographed the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Rob Bird\",\n", + "\n", + "\"description\": \"Rob Bird joined the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Marcus Farnsworth\",\n", + "\n", + "\"description\": \"Marcus Farnsworth participated in the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andrew Shanahan\",\n", + "\n", + "\"tgt_id\": \"Jamie Oliver\",\n", + "\n", + "\"description\": \"Jamie Oliver supported the Man v Fat initiative founded by Andrew Shanahan.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"England\",\n", + "\n", + "\"description\": \"The Man v Fat soccer league was initially launched in England.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"CNN\",\n", + "\n", + "\"description\": \"CNN reported on the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of entity and relationship extraction from a given text. It works by taking an input text and a list of entity types, then using a language model to identify entities within the text that match the provided types. After identifying the entities, the program determines the relationships between these entities based on their interactions and dependencies within the context of the text. The process involves generating reasoning steps to justify the extraction, and the final output is a list of entities with their types, descriptions, and importance scores, along with relationships between these entities, including descriptions, weights, and order. The program ensures type safety and validation through the use of Pydantic models and retries the extraction process up to a specified number of times if errors are encountered.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: Story highlightsThe Man v Fat soccer league is exclusively for men with a body mass index of 30 or moreThe league helps players lose weight, but it also gives them a built-in support group (CNN)After reviewing restaurants for years, Andrew Shanahan became obese and was looking for a way to lose weight.But it wasn't easy to find a program tailored to men. Every time he tried joining a weight-loss group, he was the only man in the room. \"I wondered how many other men felt that there was something missing and who weren't getting the support that you desperately need when you're trying to lose weight,\" he said.So in 2014, with the help of a crowdfunding campaign, Shanahan launched his Man v Fat initiative to reach out to other men like him. The idea drew attention from celebrities such as chef Jamie Oliver, and Shanahan was encouraged to write a book on it the next year. Being based in England, a soccer league seemed like a natural outreach program. And Shanahan received nearly 1,000 applications for 80 spots in the inaugural league, which was created exclusively for men with a body mass index of 30 or more.Read MorePhotographer Simone PerolariMany of the men used to play soccer -- or as it's called in most of the world, football -- but their weight had become an obstacle in pursuing the sport they love.\"I saw people who want to have fun and slim down and believe in what they do,\" said Simone Perolari, who photographed the league and whose images can be seen in the gallery above. \"There is not a space for them in classic football, but some of them are really good players.\"Rob Bird, 40, read about the league in the Birmingham Mail. He played the sport until he was 25, and he calls the 15 years after that the \"fat years\" when he didn't feel comfortable playing. \"I've always loved football, but playing against 'normal-size' guys left me feeling like I couldn't compete as they were faster,\" Bird said. \"(This league) is a level playing field for bigger guys.\" Shanahan isn't sure what he was expecting to result from the league's first season, but the competitiveness of the players surprised him. \"For a long time they haven't had a level playing field to compete on, so they are all determined to give it everything they have,\" he said. \"Fortunately, even those teams who lose can see that the benefits of losing weight and getting healthy more than make up for not topping the league.\"Social mediaFollow @CNNPhotos on Twitter to join the conversation about photography.Shanahan watched bonds form between the men as they enjoyed their favorite sport and supported one another in their goals to get fit and healthy. After 14 weeks, the 80 team members collectively lost 1,727 pounds, with the average weight loss close to 22 pounds per player. Marcus Farnsworth lost more than 70 pounds and is enjoying the energy boost he's received in return. He hopes it will help with his knees and back as he continues to lose weight. But he also enjoyed the togetherness and support of the guys around him. \"It gives you good motivation if one person is feeling hungry or thinking about snacking,\" Farnsworth said. \"We can give advice and communicate through our WhatsApp group for support.\"Bird has dropped nearly 60 pounds and is now down to 210. His gout and leg ulcer have improved \"immeasurably.\"Being around other guys who struggled with weight fostered a great spirit amongst us of being in it together and sharing our knowledge and experience without any judging or snide comments.Rob Bird, a player in the Man v Fat soccer league\"I feel a million times better,\" Bird said. \"This has given me the confidence that if you are making the right choices, you will feel better for it. Being around other guys who struggled with weight fostered a great spirit amongst us of being in it together and sharing our knowledge and experience without any judging or snide comments.\"Shanahan is expanding the league across the rest of the United Kingdom beginning in July, and he hopes to start similar leagues in the United States and other countries. \"We saw a lot of health improvements, whether it was guys curing their sleep apnea, reversing their diabetes or lowering their blood pressure,\" Shanahan said. \"The other really notable thing was how much many of the players grew in confidence. \"I think it's a great thing to undertake a project with other people who desperately want the same thing. It gives you energy and impetus. Working with the guys has been one of the greatest privileges of my career. To share in their excitement and to witness their achievements has been amazing.\"Simone Perolari is an Italian photographer based in Paris. You can follow him on Facebook and Instagram.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and dependencies within the context of the text.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Man v Fat soccer league\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A soccer league exclusively for men with a body mass index of 30 or more, designed to help them lose weight and provide a support group.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Andrew Shanahan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A man who became obese after reviewing restaurants for years and later founded the Man v Fat initiative.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Simone Perolari\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"An Italian photographer based in Paris who photographed the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rob Bird\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A 40-year-old man who played soccer until he was 25 and later joined the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Marcus Farnsworth\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A participant in the Man v Fat soccer league who lost more than 70 pounds.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jamie Oliver\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A celebrity chef who supported the Man v Fat initiative.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"England\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The country where the Man v Fat soccer league was initially launched.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"CNN\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A news organization that reported on the Man v Fat soccer league.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andrew Shanahan\",\n", + "\n", + "\"tgt_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"description\": \"Andrew Shanahan founded the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Simone Perolari\",\n", + "\n", + "\"description\": \"Simone Perolari photographed the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Rob Bird\",\n", + "\n", + "\"description\": \"Rob Bird joined the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"Marcus Farnsworth\",\n", + "\n", + "\"description\": \"Marcus Farnsworth participated in the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andrew Shanahan\",\n", + "\n", + "\"tgt_id\": \"Jamie Oliver\",\n", + "\n", + "\"description\": \"Jamie Oliver supported the Man v Fat initiative founded by Andrew Shanahan.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"England\",\n", + "\n", + "\"description\": \"The Man v Fat soccer league was initially launched in England.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Man v Fat soccer league\",\n", + "\n", + "\"tgt_id\": \"CNN\",\n", + "\n", + "\"description\": \"CNN reported on the Man v Fat soccer league.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text`, identify all entities that match the provided `entity_types`. For each identified entity, provide its `entity_name`, `entity_type`, `description`, and `importance_score`. Then, determine the relationships between these entities, including the `src_id` (source entity), `tgt_id` (target entity), `description` of the relationship, `weight` of the relationship, and `order` of the relationship. Ensure that the relationships are based on interactions and dependencies within the context of the text.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text`, identify all entities that match the provided `entity_types`. For each identified entity, provide its `entity_name`, `entity_type`, `description`, and `importance_score`. Then, determine the relationships between these entities, including the `src_id` (source entity), `tgt_id` (target entity), `description` of the relationship, `weight` of the relationship, and `order` of the relationship. Ensure that the relationships are based on interactions and dependencies within the context of the text.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to extract entities and relationships from a given text based on specified entity types. It uses a language model to perform this task, ensuring that the output adheres to strict type annotations and JSON schemas. The program follows a structured approach: it first identifies entities in the text according to the provided entity types, then determines the relationships between these entities based on their interactions and dependencies within the context of the text, and finally formats the entities and relationships according to a predefined JSON schema. The program includes error handling and retry mechanisms to ensure the output meets the required specifications.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: London (CNN)Prince Philip, the husband of Queen Elizabeth II, is being treated for an infection and is not expected to leave the hospital for several days, according to an update from Buckingham Palace on Tuesday.\"The Duke of Edinburgh remains at King Edward VII's Hospital where he is receiving medical attention for an infection. He is comfortable and responding to treatment but is not expected to leave hospital for several days,\" the palace said in a statement.Prince Philip was taken to the hospital last Tuesday after \"feeling unwell,\" Buckingham Palace said.On Monday, his grandson, Prince William, said the 99-year-old was doing \"OK\" and hospital staff were \"keeping an eye on him.\"Queen Elizabeth and Duke of Edinburgh receive Covid-19 vaccinePrince William made the remarks during a visit to a vaccination center in Norfolk, England, on Monday.Read MoreOn Tuesday, Prince Philip's youngest son, Prince Edward, told the UK's Sky News that he recently spoke to his father, who is doing \"a lot better\" and is looking forward to returning home.Prince Edward said the family was still keeping their \"fingers crossed\" for his father.\"We've had some brilliant and lovely messages from all sorts of people. And we really appreciate that and so does he. I've been passing them on,\" said Prince Edward.Prince Philip, who turns 100 in June, stepped back from public life in 2017 and has been taken to the hospital several times in recent years. In December 2019, he received hospital treatment for a pre-existing condition.His latest admission was not an emergency, according to a royal source, who said he traveled to King Edward VII Hospital by car and walked in unaided. The source added that the illness was not Covid-19 related.Prince Philip and Queen Elizabeth have both received at least one dose of a Covid-19 vaccine.The couple spent most of the past year at Windsor Castle, having moved away from Buckingham Palace during the first wave of the Covid-19 pandemic in spring 2020. CNN's Katharina Krebs contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince Philip\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The husband of Queen Elizabeth II, currently being treated for an infection at King Edward VII's Hospital.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Queen Elizabeth II\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The wife of Prince Philip, who has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"King Edward VII's Hospital\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The hospital where Prince Philip is receiving treatment for an infection.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Buckingham Palace\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The official residence of the British monarch, where updates on Prince Philip's condition are provided.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince William\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The grandson of Prince Philip, who visited a vaccination center in Norfolk and provided an update on his grandfather's condition.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince Edward\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The youngest son of Prince Philip, who provided an update on his father's condition to the UK's Sky News.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Covid-19\",\n", + "\n", + "\"entity_type\": \"DISEASE\",\n", + "\n", + "\"description\": \"The disease that both Prince Philip and Queen Elizabeth II have received vaccines for.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Windsor Castle\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The residence where Prince Philip and Queen Elizabeth II spent most of the past year during the Covid-19 pandemic.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"Queen Elizabeth II\",\n", + "\n", + "\"description\": \"Prince Philip is the husband of Queen Elizabeth II.\",\n", + "\n", + "\"weight\": 1.0,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"King Edward VII's Hospital\",\n", + "\n", + "\"description\": \"Prince Philip is being treated for an infection at King Edward VII's Hospital.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"Covid-19\",\n", + "\n", + "\"description\": \"Prince Philip has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Queen Elizabeth II\",\n", + "\n", + "\"tgt_id\": \"Covid-19\",\n", + "\n", + "\"description\": \"Queen Elizabeth II has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince William\",\n", + "\n", + "\"tgt_id\": \"Prince Philip\",\n", + "\n", + "\"description\": \"Prince William is the grandson of Prince Philip and provided an update on his condition.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 2\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Edward\",\n", + "\n", + "\"tgt_id\": \"Prince Philip\",\n", + "\n", + "\"description\": \"Prince Edward is the youngest son of Prince Philip and provided an update on his condition.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 2\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to extract entities and relationships from a given text based on specified entity types. It uses a language model to perform this task, ensuring that the output adheres to strict type annotations and JSON schemas. The program follows a structured approach: it first identifies entities in the text according to the provided entity types, then determines the relationships between these entities based on their interactions and dependencies within the context of the text, and finally formats the entities and relationships according to a predefined JSON schema. The program includes error handling and retry mechanisms to ensure the output meets the required specifications.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: London (CNN)Prince Philip, the husband of Queen Elizabeth II, is being treated for an infection and is not expected to leave the hospital for several days, according to an update from Buckingham Palace on Tuesday.\"The Duke of Edinburgh remains at King Edward VII's Hospital where he is receiving medical attention for an infection. He is comfortable and responding to treatment but is not expected to leave hospital for several days,\" the palace said in a statement.Prince Philip was taken to the hospital last Tuesday after \"feeling unwell,\" Buckingham Palace said.On Monday, his grandson, Prince William, said the 99-year-old was doing \"OK\" and hospital staff were \"keeping an eye on him.\"Queen Elizabeth and Duke of Edinburgh receive Covid-19 vaccinePrince William made the remarks during a visit to a vaccination center in Norfolk, England, on Monday.Read MoreOn Tuesday, Prince Philip's youngest son, Prince Edward, told the UK's Sky News that he recently spoke to his father, who is doing \"a lot better\" and is looking forward to returning home.Prince Edward said the family was still keeping their \"fingers crossed\" for his father.\"We've had some brilliant and lovely messages from all sorts of people. And we really appreciate that and so does he. I've been passing them on,\" said Prince Edward.Prince Philip, who turns 100 in June, stepped back from public life in 2017 and has been taken to the hospital several times in recent years. In December 2019, he received hospital treatment for a pre-existing condition.His latest admission was not an emergency, according to a royal source, who said he traveled to King Edward VII Hospital by car and walked in unaided. The source added that the illness was not Covid-19 related.Prince Philip and Queen Elizabeth have both received at least one dose of a Covid-19 vaccine.The couple spent most of the past year at Windsor Castle, having moved away from Buckingham Palace during the first wave of the Covid-19 pandemic in spring 2020. CNN's Katharina Krebs contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince Philip\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The husband of Queen Elizabeth II, currently being treated for an infection at King Edward VII's Hospital.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Queen Elizabeth II\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The wife of Prince Philip, who has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"King Edward VII's Hospital\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The hospital where Prince Philip is receiving treatment for an infection.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Buckingham Palace\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The official residence of the British monarch, where updates on Prince Philip's condition are provided.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince William\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The grandson of Prince Philip, who visited a vaccination center in Norfolk and provided an update on his grandfather's condition.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Prince Edward\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The youngest son of Prince Philip, who provided an update on his father's condition to the UK's Sky News.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Covid-19\",\n", + "\n", + "\"entity_type\": \"DISEASE\",\n", + "\n", + "\"description\": \"The disease that both Prince Philip and Queen Elizabeth II have received vaccines for.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Windsor Castle\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The residence where Prince Philip and Queen Elizabeth II spent most of the past year during the Covid-19 pandemic.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"Queen Elizabeth II\",\n", + "\n", + "\"description\": \"Prince Philip is the husband of Queen Elizabeth II.\",\n", + "\n", + "\"weight\": 1.0,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"King Edward VII's Hospital\",\n", + "\n", + "\"description\": \"Prince Philip is being treated for an infection at King Edward VII's Hospital.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Philip\",\n", + "\n", + "\"tgt_id\": \"Covid-19\",\n", + "\n", + "\"description\": \"Prince Philip has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Queen Elizabeth II\",\n", + "\n", + "\"tgt_id\": \"Covid-19\",\n", + "\n", + "\"description\": \"Queen Elizabeth II has received at least one dose of a Covid-19 vaccine.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince William\",\n", + "\n", + "\"tgt_id\": \"Prince Philip\",\n", + "\n", + "\"description\": \"Prince William is the grandson of Prince Philip and provided an update on his condition.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 2\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Prince Edward\",\n", + "\n", + "\"tgt_id\": \"Prince Philip\",\n", + "\n", + "\"description\": \"Prince Edward is the youngest son of Prince Philip and provided an update on his condition.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 2\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, perform the following steps to produce the `entities_relationships`: 1) Identify all entities within the `input_text` that match the specified `entity_types`. 2) For each identified entity, provide a detailed description and an importance score based on its relevance to the text. 3) Determine the relationships between the identified entities, considering their interactions and dependencies within the context of the `input_text`. 4) For each relationship, provide a description, a weight indicating the strength of the relationship, and an order indicating the sequence of interactions. 5) Format the entities and relationships according to the specified JSON schema, ensuring all fields adhere to the required types and structures.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and `entity_types`, perform the following steps to produce the `entities_relationships`: 1) Identify all entities within the `input_text` that match the specified `entity_types`. 2) For each identified entity, provide a detailed description and an importance score based on its relevance to the text. 3) Determine the relationships between the identified entities, considering their interactions and dependencies within the context of the `input_text`. 4) For each relationship, provide a description, a weight indicating the strength of the relationship, and an order indicating the sequence of interactions. 5) Format the entities and relationships according to the specified JSON schema, ensuring all fields adhere to the required types and structures.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of entity and relationship extraction from text. It works by first defining a signature that specifies the input fields (input_text and entity_types) and the output fields (reasoning and entities_relationships). The program then uses a TypedPredictor class to enforce type annotations in the signature and handle retries and error explanations if the output is invalid. The TypedEntityRelationshipExtractor class further processes the input text to extract entities and relationships, ensuring that the output is formatted according to the specified JSON schema. The program handles retries and errors by providing detailed explanations and examples to guide the language model in producing the correct output format.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: (CNN)This weekend either Lewis Hamilton or Max Verstappen will be crowned the Formula One world champion. Either way, the sport won big in 2021. This year's success starts with a boom in the sport's fanbase that can be pinned to Netflix's \"Drive to Survive\" docuseries. The show is a crash course on the 10 teams, the 20 drivers, the rivalries between them and the race competition structure. A word-of-mouth campaign for the series along with short, easy viewing episodes made it a perfect pandemic-time binge watch for millions around the world.The series' intimate interviews combined with the best action from races and the high level drama is a strong pitch for the real-life spectacle of Formula One.The multifaceted storytelling lets fans focus on the aspects of the sport that most appeal to them. And most importantly, \"Drive to Survive\" primed fans for an easy transition to the track when the 2021 season started.Read MoreThe popularity of the Netflix program, notably in the US, has paid dividends. ESPN says its race viewership in 2021 is up 56% over 2020. There was record attendance at the US Grand Prix in Austin this October -- drivers also credit the docuseries for the sport's growth, including Hamilton himself. READ: Title rivals wary of being on collision course in Abu Dhabi showdownMax Verstappen and Lewis Hamilton have created a thrilling title race. Title race enthralls New fans are important but getting them to stick with the sport is critical. This year's constant action on the track anchored by the Hamilton-Verstappen title fight has satisfied the second part of that equation. Eyes across the globe will be on all 58 laps in Abu Dhabi on Sunday, the final race to determine who takes home the title. Up until the last lap, every second of the season has been and will be a nail biter.All the variables that brought the two drivers to tally the same exact score heading into the finale is what kept fans' attention throughout the season. Think Verstappen's car perched on top of Hamilton's after the crash in Monza or the smoke rising forebodingly from Hamilton's brakes seconds before the restart in Baku. Verstappen's car ended up on top of Hamilton's at the Italian Grand Prix in September.The best racing is when the two are at it, wheel-to-wheel, like they were through the opening laps of the British Grand Prix.Sprint Qualifying, a format introduced to three weekends this season was seen across the community as a controversial addition to the sport. Despite that, it provided something new for fans who welcomed the extra racing.F1 still has improvements to make on several fronts that would make the sport and viewing experience more inclusive.There is still an obvious need for greater diversity behind the wheel and in the institution itself. The league's first full-time driver from China, Guanyu Zhou, will be a much-welcomed addition to the track when he makes his debut next season. However, it is hard to ignore just how White and male dominated the sport is.JUST WATCHEDValtteri Bottas: F1 title race will go down to the wireReplayMore Videos ...MUST WATCHValtteri Bottas: F1 title race will go down to the wire 04:17The FIA, the governing body of the sport, also has some work to do to make rules more transparent and easier to follow. Commentators do their best to make sense of the calls being made by the stewards, but sometimes, even they are left scratching their heads -- driver and team penalties often seem totally arbitrary or ill-explained.Fan accessibility will remain a challenge. Part of that is built-in because F1 is a global sport with just over 20 events a year. Those constraints make attending a race weekend expensive. For American fans eager to make it to a race, there is some good news: a race in Miami next season means two races in the US for the first time since 1984.After the exciting culmination of this season comes the anticipation for the next, with new car designs and some changes in the driver lineup. Season four of \"Drive to Survive\" is imminent too. All of that adds up to a feeling that there is so much more to come from the sport. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the given entity types. We then determine the relationships between these entities, considering the context and interactions described in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Lewis Hamilton\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver who is competing for the world championship.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Max Verstappen\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver who is competing for the world championship.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Formula One\",\n", + "\n", + "\"entity_type\": \"SPORT\",\n", + "\n", + "\"description\": \"A motorsport that is the focus of the article, featuring drivers like Lewis Hamilton and Max Verstappen.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Netflix\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A streaming service that produced the 'Drive to Survive' docuseries, which has boosted the popularity of Formula One.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Drive to Survive\",\n", + "\n", + "\"entity_type\": \"TV_SHOW\",\n", + "\n", + "\"description\": \"A docuseries on Netflix that has increased the fanbase of Formula One.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"ESPN\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A sports network that reported a 56% increase in race viewership in 2021.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"US Grand Prix\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A Formula One race event held in Austin, Texas, with record attendance in October.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Abu Dhabi\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The location of the final race in the Formula One season, which will determine the world champion.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guanyu Zhou\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver from China who will make his debut next season.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"FIA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The governing body of Formula One, which needs to improve transparency in its rules.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Lewis Hamilton\",\n", + "\n", + "\"tgt_id\": \"Max Verstappen\",\n", + "\n", + "\"description\": \"Competing against each other for the Formula One world championship.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Drive to Survive\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A TV show that has increased the popularity of Formula One.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Netflix\",\n", + "\n", + "\"tgt_id\": \"Drive to Survive\",\n", + "\n", + "\"description\": \"A streaming service that produced the 'Drive to Survive' docuseries.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"ESPN\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A sports network that reported increased viewership of Formula One races.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"US Grand Prix\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A Formula One race event with record attendance.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Abu Dhabi\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"The location of the final race in the Formula One season.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Guanyu Zhou\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A new driver who will join Formula One next season.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"FIA\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"The governing body of Formula One that needs to improve its rules' transparency.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)In a five word tweet, Andy Murray made clear he had no sympathy for a fellow tennis player who had just been banned from tennis for doping.Follow @cnnsport\n", + "\n", + "After 29-year-old American Wayne Odesnik was handed a 15-year ban after testing positive for a range of performance enhancing drugs including anabolic steroids, the 2013 Wimbledon champion Murray tweeted: \"Bye bye Wayne... Good riddance.\"Bye bye Wayne... Good riddance— Andy Murray (@andy_murray) March 18, 2015\n", + "This isn't the first time Odesnik has been caught cheating -- he was initially given a two-year ban in 2010 when Australian customs officials discovered eight vials of human growth hormone in his luggage. As this is his second offense the International Tennis Federation increased his punishment to 15 years with Odesnik, who is ranked 267 in the world, subsequently announcing his retirement.Odesnik, who has made over $1m in prize money over his career, says the positive test results weren't his fault.Read MoreJUST WATCHEDThe power of tennis statisticsReplayMore Videos ...MUST WATCHThe power of tennis statistics 03:46\"In December 2014, I unknowingly ingested a contaminated over-the-counter supplement,\" Odesnik said in a statement.\"Upon learning of my positive test results I was immediately heartbroken as words could not describe my shock and disappointment,\" added the former world No. 77.\"Being the most tested American player on tour, I would never knowingly have taken any chance of consuming a banned substance.\"Fellow American tennis player Andy Roddick was also in no mood to forgive Odesnik, tweeting: \"I hate that he has a U.S. flag next to his name when he's cheating. Good riddance.\"According to the United States Anti-Doping Agency (USADA), Odesnik provided out-of-competition urine samples in December 2014 and January 2015 which tested positive.\"[It] was his second such violation, having been sanctioned in 2010 for the possession of human growth hormone. It was, therefore, determined that he is suspended from participation for a period of 15 years\" USADA said in a statement.\"It was also determined that Mr Odesnik's results at the Happy Valley Challenger event, Australian Open and Maui Challenger event should be disqualified, with forfeiture of the ranking points and prize money.\"Odesnik was born in Johannesburg but moved to the U.S. when he was three, before turning pro in 2004. Read: Thatch point! Roger Federer ribs Andy Murray's hairstyle on TwitterRead: The tournament that 'disgraced America'\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Andy Murray\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and the 2013 Wimbledon champion.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wayne Odesnik\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A 29-year-old American tennis player who was banned for 15 years for doping.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"International Tennis Federation\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The governing body that increased Odesnik's punishment to 15 years.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"United States Anti-Doping Agency\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The agency that tested Odesnik's samples and confirmed his second doping violation.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wimbledon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A prestigious tennis tournament where Murray won in 2013.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andy Murray\",\n", + "\n", + "\"tgt_id\": \"Wayne Odesnik\",\n", + "\n", + "\"description\": \"Andy Murray expressed no sympathy for Wayne Odesnik after his doping ban.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Wayne Odesnik\",\n", + "\n", + "\"tgt_id\": \"International Tennis Federation\",\n", + "\n", + "\"description\": \"Wayne Odesnik was banned for 15 years by the International Tennis Federation.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Wayne Odesnik\",\n", + "\n", + "\"tgt_id\": \"United States Anti-Doping Agency\",\n", + "\n", + "\"description\": \"Wayne Odesnik's doping violation was confirmed by the United States Anti-Doping Agency.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: Zurich Switzerland has chosen Lockheed Martin's F-35A Lightning II as its next-generation fighter jet, the government said on Wednesday, angering opponents who have pledged a new referendum to overturn what they dubbed an unnecessary \"Ferrari\" option.The $5.5 billion deal adds a 15th nation to the world's largest weapons project -- a family of interconnected, single-engine jets to be used by the United States and its allies and partners.It came alongside a $2.1 billion agreement to buy the Patriot surface-to-air missile system from US group Raytheon, with European competitors losing out on both deals.The F-35 has faced multiple budget overruns, delays and technical setbacks, but is building export momentum. Critics say the project, valued at trillions of dollars over its lifetime, has seen costs soar while failing to meet goals on capability.Neutral Switzerland will buy 36 F-35As after an evaluation found it had \"the highest overall benefit at the lowest overall cost,\" the government said.Read MoreBiden admin intends to move forward with $23B UAE weapons salesThe aircraft beat bids from Boeing's F/A-18 Super Hornet, the Rafale from France's Dassault and the four-nation Eurofighter built by Germany- and Spain-backed Airbus, Italy's Leonardo and Britain's BAE Systems.The decision drew immediate criticism from anti-armaments campaigners and left-wing parties who will now launch a campaign for a referendum on the issue, the third Swiss vote on buying fighter jets.Voters seven years ago rejected the purchase of Gripen jets from Sweden's Saab, while the 6 billion Swiss franc ($6.5 billion) funding, which led to the decision to buy the F-35As, was only narrowly approved last year.Opponents say Switzerland doesn't need cutting-edge warplanes to defend its Alpine territory, which a supersonic jet can cross in 10 minutes.US warplanes fly first combat missions off foreign aircraft carrier since World War II\"The decision is simply incomprehensible,\" said Priska Seiler Graf, a member of Parliament for the left-leaning Social Democrats (SP), who has raised concerns about the cost.\"It's not just about buying them, but the upkeep and operating costs,\" she added. \"We should seek a European solution ... we don't want to be dependent on the United States.\"The government picked the Patriot missile system over Franco-Italian group Eurosam.Defense Minister Viola Amherd said the F-35As were chosen after being the most impressive performer in an evaluation and offered best value for money.Total costs of 15.5 billion francs ($16.7 billion) came in 2 billion francs cheaper than the next lowest bidder, the government said, based on buying and operating the aircraft over 30 years.\"We would not have bought a Ferrari if a VW would do and the Ferrari would be three times more expensive,\" Amherd told a news conference.The Swiss Parliament now has to approve the funding for the purchase, with the debate scheduled for early next year. It can debate costs and terms but not revisit the model selection.Dozens of US Air Force F-35 fighters taxi on the runway in preparation for a combat power exercise on Nov. 19, 2018, at Hill Air Force Base, Utah. The fighter decision was closely watched as the first of three face-offs ahead of Finland and Canada.Lockheed's stealthy fifth-generation fighter recently added Poland to its list of European customers which includes Belgium, Denmark, Italy, the Netherlands, Norway, and Britain.US President Joe Biden had lobbied for American companies when meeting his Swiss counterpart while in Geneva for his summit with Russian President Vladimir Putin this month.JUST WATCHEDWatch US F-35 jet fighters arrive in Europe (2017)ReplayMore Videos ...MUST WATCHWatch US F-35 jet fighters arrive in Europe (2017) 00:55Analysts said the decision to snub both the European fighter jet candidates and surface-to-air missile offering could be seen as a Swiss rebuff to the European Union in a time of strained relations between Bern and Brussels after the collapse of talks over a new agreement governing trade and other matters.By doubling down on US suppliers the government could antagonize the 49.8% of voters who opposed funding last year.South Korea rolls out the KF-21, joining elite group of global supersonic fighter jet makersAnti-arms campaigners say Switzerland, which last fought a foreign war more than 200 years ago and has no discernable enemies, does not need cutting-edge fighters.But supporters have said Switzerland needs to be able to protect itself without relying on others.Jonas Kampus, political secretary of the Group for a Switzerland without an Army, said he was confident of winning a referendum against the F-35As.The government \"can expect a heavy defeat in the vote. The follow-up polls in September (2020) showed a clear rejection of the F-35 among the voting population,\" he said.Marionna Schlatter, a lawmaker with the Greens Party said the September poll was too close to ignore opposition concerns.\"The people don't want a Ferrari in the air,\" she said.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies described in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Zurich Switzerland\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A city and country where the decision to buy F-35A Lightning II was made.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Lockheed Martin\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that manufactures the F-35A Lightning II.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"F-35A Lightning II\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A next-generation fighter jet chosen by Zurich Switzerland.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A missile system bought alongside the F-35A Lightning II.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Raytheon\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A US group that manufactures the Patriot missile system.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Boeing\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that manufactures the F/A-18 Super Hornet.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"F/A-18 Super Hornet\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet bid by Boeing.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Dassault\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A French company that manufactures the Rafale.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rafale\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet bid by Dassault.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eurofighter\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet built by Germany- and Spain-backed Airbus, Italy's Leonardo, and Britain's BAE Systems.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Airbus\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Leonardo\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An Italian company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"BAE Systems\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A British company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Swiss Parliament\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The legislative body of Switzerland that will approve the funding for the F-35A purchase.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Priska Seiler Graf\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A member of Parliament for the left-leaning Social Democrats who raised concerns about the cost of the F-35A.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Viola Amherd\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Defense Minister of Switzerland who justified the choice of the F-35A.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Joe Biden\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The President of the United States who lobbied for American companies.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Vladimir Putin\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The President of Russia with whom Joe Biden had a summit.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jonas Kampus\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The political secretary of the Group for a Switzerland without an Army who is confident of winning a referendum against the F-35A.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Marionna Schlatter\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A lawmaker with the Greens Party who expressed opposition to the F-35A.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"F-35A Lightning II\",\n", + "\n", + "\"description\": \"Zurich Switzerland has chosen the F-35A Lightning II as its next-generation fighter jet.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"description\": \"Zurich Switzerland has agreed to buy the Patriot surface-to-air missile system.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"F-35A Lightning II\",\n", + "\n", + "\"tgt_id\": \"Lockheed Martin\",\n", + "\n", + "\"description\": \"The F-35A Lightning II is manufactured by Lockheed Martin.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"tgt_id\": \"Raytheon\",\n", + "\n", + "\"description\": \"The Patriot surface-to-air missile system is manufactured by Raytheon.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Boeing\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the F/A-18 Super Hornet bid by Boeing.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Dassault\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the Rafale bid by Dassault.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Eurofighter\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the Eurofighter bid by Airbus, Leonardo, and BAE Systems.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Swiss Parliament\",\n", + "\n", + "\"description\": \"The Swiss Parliament will approve the funding for the F-35A purchase.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Priska Seiler Graf\",\n", + "\n", + "\"description\": \"Priska Seiler Graf, a member of Parliament, raised concerns about the cost of the F-35A.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Viola Amherd\",\n", + "\n", + "\"description\": \"Viola Amherd, the Defense Minister, justified the choice of the F-35A.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Joe Biden\",\n", + "\n", + "\"description\": \"Joe Biden lobbied for American companies during his meeting with the Swiss counterpart.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Vladimir Putin\",\n", + "\n", + "\"description\": \"Joe Biden met with Vladimir Putin during his summit in Geneva.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Jonas Kampus\",\n", + "\n", + "\"description\": \"Jonas Kampus is confident of winning a referendum against the F-35A.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Marionna Schlatter\",\n", + "\n", + "\"description\": \"Marionna Schlatter expressed opposition to the F-35A.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of entity and relationship extraction from text. It works by first defining a signature that specifies the input fields (input_text and entity_types) and the output fields (reasoning and entities_relationships). The program then uses a TypedPredictor class to enforce type annotations in the signature and handle retries and error explanations if the output is invalid. The TypedEntityRelationshipExtractor class further processes the input text to extract entities and relationships, ensuring that the output is formatted according to the specified JSON schema. The program handles retries and errors by providing detailed explanations and examples to guide the language model in producing the correct output format.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: (CNN)This weekend either Lewis Hamilton or Max Verstappen will be crowned the Formula One world champion. Either way, the sport won big in 2021. This year's success starts with a boom in the sport's fanbase that can be pinned to Netflix's \"Drive to Survive\" docuseries. The show is a crash course on the 10 teams, the 20 drivers, the rivalries between them and the race competition structure. A word-of-mouth campaign for the series along with short, easy viewing episodes made it a perfect pandemic-time binge watch for millions around the world.The series' intimate interviews combined with the best action from races and the high level drama is a strong pitch for the real-life spectacle of Formula One.The multifaceted storytelling lets fans focus on the aspects of the sport that most appeal to them. And most importantly, \"Drive to Survive\" primed fans for an easy transition to the track when the 2021 season started.Read MoreThe popularity of the Netflix program, notably in the US, has paid dividends. ESPN says its race viewership in 2021 is up 56% over 2020. There was record attendance at the US Grand Prix in Austin this October -- drivers also credit the docuseries for the sport's growth, including Hamilton himself. READ: Title rivals wary of being on collision course in Abu Dhabi showdownMax Verstappen and Lewis Hamilton have created a thrilling title race. Title race enthralls New fans are important but getting them to stick with the sport is critical. This year's constant action on the track anchored by the Hamilton-Verstappen title fight has satisfied the second part of that equation. Eyes across the globe will be on all 58 laps in Abu Dhabi on Sunday, the final race to determine who takes home the title. Up until the last lap, every second of the season has been and will be a nail biter.All the variables that brought the two drivers to tally the same exact score heading into the finale is what kept fans' attention throughout the season. Think Verstappen's car perched on top of Hamilton's after the crash in Monza or the smoke rising forebodingly from Hamilton's brakes seconds before the restart in Baku. Verstappen's car ended up on top of Hamilton's at the Italian Grand Prix in September.The best racing is when the two are at it, wheel-to-wheel, like they were through the opening laps of the British Grand Prix.Sprint Qualifying, a format introduced to three weekends this season was seen across the community as a controversial addition to the sport. Despite that, it provided something new for fans who welcomed the extra racing.F1 still has improvements to make on several fronts that would make the sport and viewing experience more inclusive.There is still an obvious need for greater diversity behind the wheel and in the institution itself. The league's first full-time driver from China, Guanyu Zhou, will be a much-welcomed addition to the track when he makes his debut next season. However, it is hard to ignore just how White and male dominated the sport is.JUST WATCHEDValtteri Bottas: F1 title race will go down to the wireReplayMore Videos ...MUST WATCHValtteri Bottas: F1 title race will go down to the wire 04:17The FIA, the governing body of the sport, also has some work to do to make rules more transparent and easier to follow. Commentators do their best to make sense of the calls being made by the stewards, but sometimes, even they are left scratching their heads -- driver and team penalties often seem totally arbitrary or ill-explained.Fan accessibility will remain a challenge. Part of that is built-in because F1 is a global sport with just over 20 events a year. Those constraints make attending a race weekend expensive. For American fans eager to make it to a race, there is some good news: a race in Miami next season means two races in the US for the first time since 1984.After the exciting culmination of this season comes the anticipation for the next, with new car designs and some changes in the driver lineup. Season four of \"Drive to Survive\" is imminent too. All of that adds up to a feeling that there is so much more to come from the sport. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the given entity types. We then determine the relationships between these entities, considering the context and interactions described in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Lewis Hamilton\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver who is competing for the world championship.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Max Verstappen\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver who is competing for the world championship.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Formula One\",\n", + "\n", + "\"entity_type\": \"SPORT\",\n", + "\n", + "\"description\": \"A motorsport that is the focus of the article, featuring drivers like Lewis Hamilton and Max Verstappen.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Netflix\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A streaming service that produced the 'Drive to Survive' docuseries, which has boosted the popularity of Formula One.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Drive to Survive\",\n", + "\n", + "\"entity_type\": \"TV_SHOW\",\n", + "\n", + "\"description\": \"A docuseries on Netflix that has increased the fanbase of Formula One.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"ESPN\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A sports network that reported a 56% increase in race viewership in 2021.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"US Grand Prix\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A Formula One race event held in Austin, Texas, with record attendance in October.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Abu Dhabi\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The location of the final race in the Formula One season, which will determine the world champion.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Guanyu Zhou\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A Formula One driver from China who will make his debut next season.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"FIA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The governing body of Formula One, which needs to improve transparency in its rules.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Lewis Hamilton\",\n", + "\n", + "\"tgt_id\": \"Max Verstappen\",\n", + "\n", + "\"description\": \"Competing against each other for the Formula One world championship.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Drive to Survive\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A TV show that has increased the popularity of Formula One.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Netflix\",\n", + "\n", + "\"tgt_id\": \"Drive to Survive\",\n", + "\n", + "\"description\": \"A streaming service that produced the 'Drive to Survive' docuseries.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"ESPN\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A sports network that reported increased viewership of Formula One races.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"US Grand Prix\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A Formula One race event with record attendance.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Abu Dhabi\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"The location of the final race in the Formula One season.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Guanyu Zhou\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"A new driver who will join Formula One next season.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"FIA\",\n", + "\n", + "\"tgt_id\": \"Formula One\",\n", + "\n", + "\"description\": \"The governing body of Formula One that needs to improve its rules' transparency.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)In a five word tweet, Andy Murray made clear he had no sympathy for a fellow tennis player who had just been banned from tennis for doping.Follow @cnnsport\n", + "\n", + "After 29-year-old American Wayne Odesnik was handed a 15-year ban after testing positive for a range of performance enhancing drugs including anabolic steroids, the 2013 Wimbledon champion Murray tweeted: \"Bye bye Wayne... Good riddance.\"Bye bye Wayne... Good riddance— Andy Murray (@andy_murray) March 18, 2015\n", + "This isn't the first time Odesnik has been caught cheating -- he was initially given a two-year ban in 2010 when Australian customs officials discovered eight vials of human growth hormone in his luggage. As this is his second offense the International Tennis Federation increased his punishment to 15 years with Odesnik, who is ranked 267 in the world, subsequently announcing his retirement.Odesnik, who has made over $1m in prize money over his career, says the positive test results weren't his fault.Read MoreJUST WATCHEDThe power of tennis statisticsReplayMore Videos ...MUST WATCHThe power of tennis statistics 03:46\"In December 2014, I unknowingly ingested a contaminated over-the-counter supplement,\" Odesnik said in a statement.\"Upon learning of my positive test results I was immediately heartbroken as words could not describe my shock and disappointment,\" added the former world No. 77.\"Being the most tested American player on tour, I would never knowingly have taken any chance of consuming a banned substance.\"Fellow American tennis player Andy Roddick was also in no mood to forgive Odesnik, tweeting: \"I hate that he has a U.S. flag next to his name when he's cheating. Good riddance.\"According to the United States Anti-Doping Agency (USADA), Odesnik provided out-of-competition urine samples in December 2014 and January 2015 which tested positive.\"[It] was his second such violation, having been sanctioned in 2010 for the possession of human growth hormone. It was, therefore, determined that he is suspended from participation for a period of 15 years\" USADA said in a statement.\"It was also determined that Mr Odesnik's results at the Happy Valley Challenger event, Australian Open and Maui Challenger event should be disqualified, with forfeiture of the ranking points and prize money.\"Odesnik was born in Johannesburg but moved to the U.S. when he was three, before turning pro in 2004. Read: Thatch point! Roger Federer ribs Andy Murray's hairstyle on TwitterRead: The tournament that 'disgraced America'\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and the context provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Andy Murray\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professional tennis player and the 2013 Wimbledon champion.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wayne Odesnik\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A 29-year-old American tennis player who was banned for 15 years for doping.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"International Tennis Federation\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The governing body that increased Odesnik's punishment to 15 years.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"United States Anti-Doping Agency\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The agency that tested Odesnik's samples and confirmed his second doping violation.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Wimbledon\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"A prestigious tennis tournament where Murray won in 2013.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Andy Murray\",\n", + "\n", + "\"tgt_id\": \"Wayne Odesnik\",\n", + "\n", + "\"description\": \"Andy Murray expressed no sympathy for Wayne Odesnik after his doping ban.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Wayne Odesnik\",\n", + "\n", + "\"tgt_id\": \"International Tennis Federation\",\n", + "\n", + "\"description\": \"Wayne Odesnik was banned for 15 years by the International Tennis Federation.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Wayne Odesnik\",\n", + "\n", + "\"tgt_id\": \"United States Anti-Doping Agency\",\n", + "\n", + "\"description\": \"Wayne Odesnik's doping violation was confirmed by the United States Anti-Doping Agency.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: Zurich Switzerland has chosen Lockheed Martin's F-35A Lightning II as its next-generation fighter jet, the government said on Wednesday, angering opponents who have pledged a new referendum to overturn what they dubbed an unnecessary \"Ferrari\" option.The $5.5 billion deal adds a 15th nation to the world's largest weapons project -- a family of interconnected, single-engine jets to be used by the United States and its allies and partners.It came alongside a $2.1 billion agreement to buy the Patriot surface-to-air missile system from US group Raytheon, with European competitors losing out on both deals.The F-35 has faced multiple budget overruns, delays and technical setbacks, but is building export momentum. Critics say the project, valued at trillions of dollars over its lifetime, has seen costs soar while failing to meet goals on capability.Neutral Switzerland will buy 36 F-35As after an evaluation found it had \"the highest overall benefit at the lowest overall cost,\" the government said.Read MoreBiden admin intends to move forward with $23B UAE weapons salesThe aircraft beat bids from Boeing's F/A-18 Super Hornet, the Rafale from France's Dassault and the four-nation Eurofighter built by Germany- and Spain-backed Airbus, Italy's Leonardo and Britain's BAE Systems.The decision drew immediate criticism from anti-armaments campaigners and left-wing parties who will now launch a campaign for a referendum on the issue, the third Swiss vote on buying fighter jets.Voters seven years ago rejected the purchase of Gripen jets from Sweden's Saab, while the 6 billion Swiss franc ($6.5 billion) funding, which led to the decision to buy the F-35As, was only narrowly approved last year.Opponents say Switzerland doesn't need cutting-edge warplanes to defend its Alpine territory, which a supersonic jet can cross in 10 minutes.US warplanes fly first combat missions off foreign aircraft carrier since World War II\"The decision is simply incomprehensible,\" said Priska Seiler Graf, a member of Parliament for the left-leaning Social Democrats (SP), who has raised concerns about the cost.\"It's not just about buying them, but the upkeep and operating costs,\" she added. \"We should seek a European solution ... we don't want to be dependent on the United States.\"The government picked the Patriot missile system over Franco-Italian group Eurosam.Defense Minister Viola Amherd said the F-35As were chosen after being the most impressive performer in an evaluation and offered best value for money.Total costs of 15.5 billion francs ($16.7 billion) came in 2 billion francs cheaper than the next lowest bidder, the government said, based on buying and operating the aircraft over 30 years.\"We would not have bought a Ferrari if a VW would do and the Ferrari would be three times more expensive,\" Amherd told a news conference.The Swiss Parliament now has to approve the funding for the purchase, with the debate scheduled for early next year. It can debate costs and terms but not revisit the model selection.Dozens of US Air Force F-35 fighters taxi on the runway in preparation for a combat power exercise on Nov. 19, 2018, at Hill Air Force Base, Utah. The fighter decision was closely watched as the first of three face-offs ahead of Finland and Canada.Lockheed's stealthy fifth-generation fighter recently added Poland to its list of European customers which includes Belgium, Denmark, Italy, the Netherlands, Norway, and Britain.US President Joe Biden had lobbied for American companies when meeting his Swiss counterpart while in Geneva for his summit with Russian President Vladimir Putin this month.JUST WATCHEDWatch US F-35 jet fighters arrive in Europe (2017)ReplayMore Videos ...MUST WATCHWatch US F-35 jet fighters arrive in Europe (2017) 00:55Analysts said the decision to snub both the European fighter jet candidates and surface-to-air missile offering could be seen as a Swiss rebuff to the European Union in a time of strained relations between Bern and Brussels after the collapse of talks over a new agreement governing trade and other matters.By doubling down on US suppliers the government could antagonize the 49.8% of voters who opposed funding last year.South Korea rolls out the KF-21, joining elite group of global supersonic fighter jet makersAnti-arms campaigners say Switzerland, which last fought a foreign war more than 200 years ago and has no discernable enemies, does not need cutting-edge fighters.But supporters have said Switzerland needs to be able to protect itself without relying on others.Jonas Kampus, political secretary of the Group for a Switzerland without an Army, said he was confident of winning a referendum against the F-35As.The government \"can expect a heavy defeat in the vote. The follow-up polls in September (2020) showed a clear rejection of the F-35 among the voting population,\" he said.Marionna Schlatter, a lawmaker with the Greens Party said the September poll was too close to ignore opposition concerns.\"The people don't want a Ferrari in the air,\" she said.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies described in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Zurich Switzerland\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A city and country where the decision to buy F-35A Lightning II was made.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Lockheed Martin\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that manufactures the F-35A Lightning II.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"F-35A Lightning II\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A next-generation fighter jet chosen by Zurich Switzerland.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A missile system bought alongside the F-35A Lightning II.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Raytheon\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A US group that manufactures the Patriot missile system.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Boeing\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that manufactures the F/A-18 Super Hornet.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"F/A-18 Super Hornet\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet bid by Boeing.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Dassault\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A French company that manufactures the Rafale.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Rafale\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet bid by Dassault.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Eurofighter\",\n", + "\n", + "\"entity_type\": \"PRODUCT\",\n", + "\n", + "\"description\": \"A fighter jet built by Germany- and Spain-backed Airbus, Italy's Leonardo, and Britain's BAE Systems.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Airbus\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Leonardo\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An Italian company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"BAE Systems\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A British company that is part of the consortium building the Eurofighter.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Swiss Parliament\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The legislative body of Switzerland that will approve the funding for the F-35A purchase.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Priska Seiler Graf\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A member of Parliament for the left-leaning Social Democrats who raised concerns about the cost of the F-35A.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Viola Amherd\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Defense Minister of Switzerland who justified the choice of the F-35A.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Joe Biden\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The President of the United States who lobbied for American companies.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Vladimir Putin\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The President of Russia with whom Joe Biden had a summit.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jonas Kampus\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The political secretary of the Group for a Switzerland without an Army who is confident of winning a referendum against the F-35A.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Marionna Schlatter\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A lawmaker with the Greens Party who expressed opposition to the F-35A.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"F-35A Lightning II\",\n", + "\n", + "\"description\": \"Zurich Switzerland has chosen the F-35A Lightning II as its next-generation fighter jet.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"description\": \"Zurich Switzerland has agreed to buy the Patriot surface-to-air missile system.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"F-35A Lightning II\",\n", + "\n", + "\"tgt_id\": \"Lockheed Martin\",\n", + "\n", + "\"description\": \"The F-35A Lightning II is manufactured by Lockheed Martin.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Patriot surface-to-air missile system\",\n", + "\n", + "\"tgt_id\": \"Raytheon\",\n", + "\n", + "\"description\": \"The Patriot surface-to-air missile system is manufactured by Raytheon.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Boeing\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the F/A-18 Super Hornet bid by Boeing.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Dassault\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the Rafale bid by Dassault.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Eurofighter\",\n", + "\n", + "\"description\": \"Zurich Switzerland considered the Eurofighter bid by Airbus, Leonardo, and BAE Systems.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Swiss Parliament\",\n", + "\n", + "\"description\": \"The Swiss Parliament will approve the funding for the F-35A purchase.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Priska Seiler Graf\",\n", + "\n", + "\"description\": \"Priska Seiler Graf, a member of Parliament, raised concerns about the cost of the F-35A.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Viola Amherd\",\n", + "\n", + "\"description\": \"Viola Amherd, the Defense Minister, justified the choice of the F-35A.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Joe Biden\",\n", + "\n", + "\"description\": \"Joe Biden lobbied for American companies during his meeting with the Swiss counterpart.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Vladimir Putin\",\n", + "\n", + "\"description\": \"Joe Biden met with Vladimir Putin during his summit in Geneva.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Jonas Kampus\",\n", + "\n", + "\"description\": \"Jonas Kampus is confident of winning a referendum against the F-35A.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Zurich Switzerland\",\n", + "\n", + "\"tgt_id\": \"Marionna Schlatter\",\n", + "\n", + "\"description\": \"Marionna Schlatter expressed opposition to the F-35A.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously identify and extract all entities present in the text that match the provided entity types. Subsequently, determine the relationships between these entities based on their interactions and the context provided. Ensure that the entities and relationships are formatted according to the specified JSON schema, including providing detailed descriptions and importance scores for each entity, as well as defining the source and target IDs, description, weight, and order for each relationship.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, meticulously identify and extract all entities present in the text that match the provided entity types. Subsequently, determine the relationships between these entities based on their interactions and the context provided. Ensure that the entities and relationships are formatted according to the specified JSON schema, including providing detailed descriptions and importance scores for each entity, as well as defining the source and target IDs, description, weight, and order for each relationship.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from text, specifically focusing on identifying entities based on predefined types and determining the relationships between these entities. It works by using a language model to process the input text and entity types, generating a step-by-step reasoning process to identify and relate the entities, and finally outputting the entities and their relationships in a structured JSON format. The program ensures type annotations are enforced and includes mechanisms for handling errors and retries to improve the accuracy of the output.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "task_demos Input Text: (CNN)A ban on cellphones in classrooms was announced in the Australian state of Victoria on Wednesday. The ban, which will apply to primary and secondary students, is designed to crack down on cyber-bulling and classroom distractions, the Victorian government said in a statement. While some schools in Australia have already banned cell-phone use, this marks the first statewide ban in the country. It will begin from the first school term of 2020.\"Students will be required to switch off their phones and store them securely in lockers from the start of the school day until the final bell. When emergencies occur, parents or guardians can reach their child by calling the school,\" the statement said.Victoria Education Minister James Merlino said he didn't expect the move to be \"universally popular\" but said it was the \"right thing to do\" on his official social media. Read MoreA report says young people are growing horns on their skulls. Critics don't buy itThe move isn't without precedent. Australia's largest state, New South Wales, banned mobile phones in primary schools in 2018, and France has banned all smart phones and smart devices from schools.Australian Education Minister Dan Tehan said in a statement that he supported the Victoria ban.\"When I talk to parents and teachers the overwhelming majority want action on banning mobile phones in the classroom,\" Tehan said in a statement. In a 2019 survey of about 2,000 Australian adults, by Monash University professor Neil Selywn, almost 80% supported a restriction on cell-phone use in schools, while one-third supported an outright ban.Ann Marie, a secondary school principal at a college in Victoria, told ABC Radio Melbourne that pupils at her school had agreed to restrict their mobile phone use during the day, explaining that they hoped to reduce \"bullying, distraction and one-upmanship.\"But there have been some concerns. The Australian Education Union Victoria said in a statement that while it supports the ban: \"We can't lose sight of the benefits that technology, including mobile phones, can bring to high quality teaching and learning.\"Some pupils with special needs will be exempt from the ban, including those who require telephones to monitor health conditions or those who have received permission from a teacher. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering the context and descriptions provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Victoria\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The Australian state where the cellphone ban in classrooms was announced.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"James Merlino\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Victoria Education Minister who announced the cellphone ban.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New South Wales\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Australia's largest state that previously banned mobile phones in primary schools.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"France\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country that has banned all smart phones and smart devices from schools.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Dan Tehan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Australian Education Minister who supported the Victoria ban.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Monash University\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The university where a survey on cell-phone use in schools was conducted.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neil Selwyn\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professor at Monash University who conducted the survey on cell-phone use in schools.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ann Marie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A secondary school principal in Victoria who discussed the impact of the cellphone ban.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australian Education Union Victoria\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An organization that supports the cellphone ban but highlights the benefits of technology in education.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"James Merlino\",\n", + "\n", + "\"description\": \"James Merlino, the Victoria Education Minister, announced the cellphone ban in Victoria.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"New South Wales\",\n", + "\n", + "\"description\": \"The cellphone ban in Victoria follows a similar ban in New South Wales.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"France\",\n", + "\n", + "\"description\": \"The cellphone ban in Victoria is similar to a ban in France.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Dan Tehan\",\n", + "\n", + "\"description\": \"Dan Tehan, the Australian Education Minister, supported the Victoria ban.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Monash University\",\n", + "\n", + "\"description\": \"A survey conducted at Monash University showed support for the Victoria ban.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Neil Selwyn\",\n", + "\n", + "\"description\": \"Neil Selwyn, a professor at Monash University, conducted the survey on cell-phone use in schools.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Ann Marie\",\n", + "\n", + "\"description\": \"Ann Marie, a secondary school principal in Victoria, discussed the impact of the cellphone ban.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Australian Education Union Victoria\",\n", + "\n", + "\"description\": \"The Australian Education Union Victoria supports the cellphone ban but highlights the benefits of technology in education.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: Manchester, England (CNN)Boris Johnson wants to keep his Brexit plan private. But the Prime Minister isn't having much luck these days. On Monday night, Irish broadcaster RTE reported what it claimed were details of part of Johnson's plan to break the Brexit deadlock. According to RTE, the UK's plan to avoid post-Brexit infrastructure on the frontier between Northern Ireland and the Republic of Ireland involve \"customs clearance centers\" several miles from the border -- and on both sides of it. The women of Boris Johnson's past are starting to catch up with himThat, it hopes, will remove the need for something called the Irish border backstop from any final Brexit deal. The backstop is an emergency measure which would keep the two Irish nations in regulatory alignment, removing the need for any border or \"clearance centers\" at all. It's deeply unpopular with many in Johnson's Conservative party and is one of the main reasons his predecessor, Theresa May, failed to get her Withdrawal Agreement through Parliament.Ireland quickly dismissed the idea as a \"non starter\". And Johnson himself told the BBC on Tuesday, \"that's not what we're proposing at all... you'll forgive me, I would like to veil our proposals in decent obscurity until we've been able to share them properly with our friends\". Read MoreSo why is this story a big deal? Keeping the government's formal proposals from Brussels for the time being might be wise. The crunch EU summit, at which any Brexit deal will be struck, is now under three weeks away. JUST WATCHEDBoris Johnson tells opponents to hold no-confidence voteReplayMore Videos ...MUST WATCHBoris Johnson tells opponents to hold no-confidence vote 02:41The EU is a notoriously tough institution to negotiate with and has a history of tearing apart any Brexit proposal from the moment it leaves the door. Johnson has repeatedly said that the only way to make the EU budge on ditching the backstop is to show that he is both deadly serious about getting a deal but also prepared to leave without one. Some of the ideas reported by RTE chime with speculation about the UK government's plans in recent weeks. And anyone who has followed Brexit for the last three years will tell you, the EU and Ireland will refuse to sign a deal that involves infrastructure between Ireland and Northern Ireland. So, even though the Prime Minister has distanced himself from the proposals, these headlines will have done little to reassure Brussels that Johnson and his government are serious about getting a deal. It's also a bad look for Johnson back home. He is currently out of London and at his Conservative party's annual conference in Manchester, in the north of England. Johnson will close the conference with a keynote speech on Wednesday afternoon. British politics has been taken over by trollsIt's widely thought that he wanted to trail his final Brexit plan during this speech before formally submitting something to Brussels in the subsequent days. The problem with the leak of any Brexit plan now is that it increases the risk of Johnson being greeted by a hostile audience at his conference on Wednesday. Despite being the Brexit hero of 2016, Conservative hard-Brexiteers are deeply worried that Johnson, if he succeeds in getting rid of the backstop, will simply try and push through the rest May's Brexit deal.For Brexiteers, this would be a huge sellout. There are people sitting in Johnson's cabinet who resigned from Theresa May's government to vote against her Brexit deal. And Johnson himself resigned as May's Foreign Secretary over her general approach to Brexit. Up here in Manchester, the mood among party members is generally positive. Johnson has always been a popular Conservative MP and he seems to have broad support for his hard Brexit language. That could turn sour if he's seen to make concessions to Brussels. Which would all be somewhat ironic, for a man who has spent the past few days accusing his opponents of surrendering. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Boris Johnson\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Prime Minister of the United Kingdom who is leading the Brexit negotiations.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brexit\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"The process of the United Kingdom withdrawing from the European Union.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"European Union\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A political and economic union of 27 European countries.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Irish border backstop\",\n", + "\n", + "\"entity_type\": \"LAW\",\n", + "\n", + "\"description\": \"An emergency measure to keep the two Irish nations in regulatory alignment, removing the need for any border or 'clearance centers'.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Theresa May\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Prime Minister of the United Kingdom and predecessor of Boris Johnson.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Conservative party\",\n", + "\n", + "\"entity_type\": \"POLITICAL_PARTY\",\n", + "\n", + "\"description\": \"The political party in the United Kingdom to which Boris Johnson and Theresa May belong.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"RTE\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"Irish broadcaster that reported details of Johnson's Brexit plan.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Brexit\",\n", + "\n", + "\"description\": \"Boris Johnson is leading the Brexit negotiations as the Prime Minister of the United Kingdom.\",\n", + "\n", + "\"weight\": 1.0,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brexit\",\n", + "\n", + "\"tgt_id\": \"European Union\",\n", + "\n", + "\"description\": \"Brexit involves the United Kingdom withdrawing from the European Union.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Irish border backstop\",\n", + "\n", + "\"description\": \"Boris Johnson aims to remove the Irish border backstop from any final Brexit deal.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Theresa May\",\n", + "\n", + "\"tgt_id\": \"Brexit\",\n", + "\n", + "\"description\": \"Theresa May failed to get her Withdrawal Agreement through Parliament due to the Irish border backstop.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Conservative party\",\n", + "\n", + "\"description\": \"Boris Johnson is a member of the Conservative party and its leader.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"RTE\",\n", + "\n", + "\"tgt_id\": \"Boris Johnson\",\n", + "\n", + "\"description\": \"RTE reported details of part of Johnson's plan to break the Brexit deadlock.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Across the sports world, athletes took a stand for Black Lives Matter on Wednesday night by holding an unprecedented wildcat strike and refusing to play their regularly scheduled games.The decision started with the NBA's Milwaukee Bucks and stemmed from the police shooting of Jacob Blake in Kenosha, Wisconsin, on Sunday. The stand quickly spread across a number of sports -- from basketball courts in Florida to baseball diamonds in California to soccer fields in places like Miami and Salt Lake City and even into the broadcast booth, where TNT's \"Inside the NBA\" analyst Kenny Smith took off his mic and walked off stage.But the issue is bigger than Jacob Blake or George Floyd or Breonna Taylor. Malcolm Jenkins of the NFL's New Orleans Saints positioned the protests as a response to the systemic racism baked into America, including police violence, cycles of incarceration, and the plunder of generational wealth.\"Until we continue to demand it, until 'Black Lives Matter' goes from just an idea or a goal that we're trying to attain as a society and is actually realized in the streets, we won't see any peace,\" he said. \"And I think we'll continue to see athletes, entertainers as well as citizens disrupt the status quo until that's recognized.\"Here's a look at the teams and athletes who pulled back in order to have their voices be heard.Read MoreNBA Photos: The sports world has been taking a standMembers of the Los Angeles Lakers and the Miami Heat kneel during the National Anthem before Game 1 of the NBA Finals on Wednesday, September 30.Hide Caption 1 of 42 Photos: The sports world has been taking a standThe Philadelphia Eagles and the Washington Football Team stand together before their NFL season opener on September 13.Hide Caption 2 of 42 Photos: The sports world has been taking a standTyler Wright, a two-time World Surf League Champion, takes a knee before competing in an event in Tweed Heads South, Australia, on September 13.Hide Caption 3 of 42 Photos: The sports world has been taking a standTennis star Naomi Osaka wears a face mask with Tamir Rice's name before winning the US Open final on September 12. Osaka wore a different name for each of her seven matches. Rice, a 12-year-old boy, was killed by police gunfire in Cleveland while he was holding a toy replica pistol in 2014.Hide Caption 4 of 42 Photos: The sports world has been taking a standChelsea's Pernille Harder walks out for a warm-up before a soccer match in Leigh, England, on September 6.Hide Caption 5 of 42 Photos: The sports world has been taking a standThe University of Alabama football team, led by coach Nick Saban, marches on campus in support of the Black Lives Matter movement.Hide Caption 6 of 42 Photos: The sports world has been taking a standBlack Lives Matter signage is seen in New York's Louis Armstrong Stadium as Cori Gauff plays against Anastasija Sevastova at the US Open.Hide Caption 7 of 42 Photos: The sports world has been taking a standMembers of the WNBA's Indiana Fever wear Black Lives Matter shirts before their game against the Chicago Sky on August 31.Hide Caption 8 of 42 Photos: The sports world has been taking a standMiami Marlins outfielder Lewis Brinson walks off the field August 27 after placing a Black Lives Matter T-shirt on home plate in New York. The Marlins and New York Mets walked off the field after a moment of silence, choosing not to play their scheduled baseball game. Hide Caption 9 of 42 Photos: The sports world has been taking a standNew Orleans Saints wide receiver Emmanuel Sanders wears Jacob Blake's name on his helmet during a practice on August 27.Hide Caption 10 of 42 Photos: The sports world has been taking a standThe Washington Mystics pay tribute to Jacob Blake after their WNBA game was postponed in Palmetto, Florida, on August 26.Hide Caption 11 of 42 Photos: The sports world has been taking a standA grounds crew covers the field at San Francisco's Oracle Park after a Major League Baseball game was postponed on August 26.Hide Caption 12 of 42 Photos: The sports world has been taking a standThe Milwaukee Bucks make a statement to the media after boycotting a playoff game on August 26.Hide Caption 13 of 42 Photos: The sports world has been taking a standDuring a practice round for the BMW Championship, pro golfer Cameron Champ used his shoes to show his support for Blake and the Black Lives Matter movement.Hide Caption 14 of 42 Photos: The sports world has been taking a standNBA superstar LeBron James has been one of the most outspoken athletes in recent months. \"Having two boys of my own and me being African American in America and to see what continues to happen with the police brutality towards my kind, continue to see what goes on with just the (unjust), it's very troubling. It's very troubling,\" James told Turner Sports' Jared Greenberg. \"My prayers go to (the Blake) family, and hopefully we can have some change.\" Hide Caption 15 of 42 Photos: The sports world has been taking a stand\"End racism\" banners are shown in Toronto's Scotiabank Arena before an NHL playoff game on August 26.Hide Caption 16 of 42 Photos: The sports world has been taking a standDoc Rivers, head coach of the NBA's Los Angeles Clippers, became emotional while talking about the Blake shooting and the Republican National Convention. \"All you hear is Donald Trump and all of them talking about fear,\" Rivers said. \"We're the ones getting killed. We're the ones getting shot. We're the ones who were denied to live in certain communities. We've been hung. We've been shot.\"Hide Caption 17 of 42 Photos: The sports world has been taking a standDetroit Lions defensive end Trey Flowers addresses the media with his teammates outside their practice facility on August 25. The Lions canceled their practice in light of the Blake shooting, and they discussed the incident in a team meeting that lasted hours. \"We're going to spread our message; we're going to do it as a team,\" Flowers said. \"We understand that somebody's going to try to skew the narrative ... get the message lost, but we're going to stay focused on the topic.\"Hide Caption 18 of 42 Photos: The sports world has been taking a standA Real Salt Lake fan sits in the stands after the Major League Soccer team had its game postponed on August 26.Hide Caption 19 of 42 Photos: The sports world has been taking a standNASCAR driver Bubba Wallace tweeted this selfie before a Cup Series race in Talladega, Alabama, on June 22. Fellow drivers and pit crew members walked alongside Wallace's car to show their support for him. Wallace, the only Black driver in NASCAR's top circuit, has been an outspoken advocate of the Black Lives Matter movement. Hide Caption 20 of 42 Photos: The sports world has been taking a standMembers of the NHL's Colorado Avalanche, Vegas Golden Knights, Dallas Stars and Vancouver Canucks gather together after their playoff games were postponed in Edmonton, Alberta.Hide Caption 21 of 42 Photos: The sports world has been taking a standThe Black Lives Matter protests haven't been limited to just North America. Here, professional soccer players from Aston Villa and Sheffield United take a knee as their match kicked off in Birmingham, England, on June 17. Premier League teams sported the words \"Black Lives Matter\" on the back of their jerseys when their seasons resumed.Hide Caption 22 of 42 Photos: The sports world has been taking a standFormula One champion Lewis Hamilton raises his fist before a race in Northampton, England, on August 9.Hide Caption 23 of 42 Photos: The sports world has been taking a standMatt Dumba of the Minnesota Wild kneels during the US National Anthem as it is played before an NHL game in Edmonton, Alberta, on August 1. Dumba, a member of the Hockey Diversity Alliance, gave a short speech before kneeling. He is circled by members of the Edmonton Oilers and the Chicago Blackhawks, who were about to play on the first day of the NHL's return.Hide Caption 24 of 42 Photos: The sports world has been taking a standSoccer players from PSG and Lyon take a knee before a Champions League semifinal in Bilbao, Spain, on August 26.Hide Caption 25 of 42 Photos: The sports world has been taking a standBeach volleyball player April Ross wears a temporary Black Lives Matter tattoo during a match in Long Beach, California, on July 19.Hide Caption 26 of 42 Photos: The sports world has been taking a standTennis players Dan Evans and Kyle Edmund join match umpire James Keothavong in taking a knee in London on June 28.Hide Caption 27 of 42 Photos: The sports world has been taking a standCricketers with the Kent Spitfires take a knee during a match in Canterbury, England.Hide Caption 28 of 42 Photos: The sports world has been taking a standFormula One driver Kimi Raikkonen wears an \"end racism\" shirt before a race in Barcelona, Spain, on August 16.Hide Caption 29 of 42 Photos: The sports world has been taking a standMembers of the Trinbago Knight Riders kneel before a cricket match in Port of Spain, Trinidad and Tobago.Hide Caption 30 of 42 Photos: The sports world has been taking a standThe Washington Nationals observe a moment of silence before Major League Baseball's opening game on July 23. Their opponents, The New York Yankees, also took a knee, and the initials BLM were on the pitcher's mound for the game.Hide Caption 31 of 42 Photos: The sports world has been taking a standMajor League Soccer players participate in a pregame ceremony before the league's restart on July 8.Hide Caption 32 of 42 Photos: The sports world has been taking a standHouston Dynamo goalkeeper Cody Cropper warms up in a Black Lives Matter T-shirt on August 25.Hide Caption 33 of 42 Photos: The sports world has been taking a standThe Boston Red Sox Foundation put this sign behind Fenway Park's famous \"Green Monster\" outfield wall.Hide Caption 34 of 42 Photos: The sports world has been taking a standRugby players from Bath and Northampton show their support before a game in Northampton, England.Hide Caption 35 of 42 Photos: The sports world has been taking a standKansas City outfielder Jorge Soler catches a ball in front of a George Floyd sign in Minneapolis on August 15.Hide Caption 36 of 42 Photos: The sports world has been taking a standA Houston Dash player wears a Black Lives Matter armband before a NWSL match in Herriman, Utah, on July 17.Hide Caption 37 of 42 Photos: The sports world has been taking a standAfter winning the Champions League, Bayern Munich defender David Alaba wears a shirt that says \"Black Lives Still Matter.\"Hide Caption 38 of 42 Photos: The sports world has been taking a standA Black Lives Matter sign is seen during a Major League Soccer match in Harrison, New Jersey, on August 24.Hide Caption 39 of 42 Photos: The sports world has been taking a standMembers of the San Antonio Spurs huddle before a game against Sacramento on July 31.Hide Caption 40 of 42 Photos: The sports world has been taking a standKilmarnock manager Alex Dyer, second from right, takes a knee before a soccer match in Glasgow, Scotland.Hide Caption 41 of 42 Photos: The sports world has been taking a standMembers of the Orlando Magic and Brooklyn Nets kneel during the National Anthem before the start of an NBA game on July 31.Hide Caption 42 of 42The Milwaukee Bucks did not emerge from their locker room before the scheduled tip at 4 p.m. Wednesday as they decided to refuse to play their playoff game.Soon after, the NBA announced it would postpone Game 5 of three different playoff series -- the Bucks vs. Orlando Magic, Houston Rockets vs. Oklahoma City Thunder and Los Angeles Lakers vs. Portland Trail Blazers.The postponed games have not yet been rescheduled.Three playoff games on Thursday also were postponed but the league said it hoped to resume games Friday or Saturday. Strikes are banned under the NBA's collective bargaining agreement, which means the Bucks players broke their own contract to protest racial injustice and police violence. In a statement issued on Wednesday afternoon, the Bucks players said they are \"calling for justice for Jacob Blake and demand the officers be held accountable. \"Full statement from the Milwaukee Bucks: pic.twitter.com/jjGEyVcCmB— Milwaukee Bucks (@Bucks) August 26, 2020\n", + "\n", + "\"The past four months have shed a light on the ongoing racial injustices facing our African American communities. Citizens around the country have used their voices and platforms to speak out against these wrongdoings,\" they said in a statement.\"Despite the overwhelming plea for change, there has been no action, so our focus today cannot be on basketball.\"The Los Angeles Lakers also offered a statement in support of the players' decision.\"Eighty percent of NBA players are Black men. We cannot love them for the joy and entertainment that they bring to the world, yet sit in silence and fail to use our platforms and resources to amplify their voices when they demand the justice and equality that America has promised us all, but denied Black people for too long,\" the Lakers said.WNBAElizabeth Williams of the Atlanta Dream said WNBA players were standing in solidarity with \"our brothers in the NBA.\"Representing the six teams slated to play on Wednesday, Atlanta Dream player Elizabeth Williams announced that WNBA players were standing in solidarity with \"our brothers in the NBA\" and also would not play. The ESPN2 broadcast showed players from the six teams scheduled to take the court in locked arms and kneeling while wearing shirts spelling out Jacob Blake's name. The WNBA announced that the three games scheduled for Wednesday evening had been postponed. On Thursday, the league announced the postponement of the day's three scheduled games as well.Instead of playing Thursday, the entire league of WNBA players stood arm-in-arm in solidarity.\"It is important to note that this is not a strike,\" said Women's National Basketball Players Association (WNBPA) president and Los Angeles Sparks star Nneka Ogwumike, reading a statement from the players, to ESPN's Holly Rowe. \"This is not a boycott. This is affirmatively a day of reflection. A day of informed action and mobilization.\"Ogwumike said players plan to play in games that are scheduled for Friday.\"I feel like as a group we decided last night that we want to play,\" Ogwumike said. \"There are games scheduled tomorrow. That's what we're ready to do, but that doesn't come without, of course, demands of players to continue to amplify our voices in more ways than when we came here.\"We realize that the work is not easy, but we also understand that the work is never done. There are things that happen that just allow us times to take a moment -- that's fair; we can take a moment -- and that moment we came together and we decided we need time for ourselves to come back, regroup and continue to amplify our voices, and be there for our communities and demand change from the leaders that are representing us right now.\"MLBThe words 'Black Lives Matter' are displayed on the digital screen after the postponement of the game between the San Francisco Giants and the Los Angeles Dodgers Wednesday.Three MLB games were postponed Wednesday: Cincinnati Reds vs. Milwaukee Brewers, Seattle Mariners vs. San Diego Padres, Los Angeles Dodgers vs. San Francisco Giants.\"With our community and our nation in such pain, we wanted to draw as much attention to the issues that really matter, especially racial injustice and systemic oppression,\" a joint statement from the Brewers and Reds said.\"Given the pain in the communities of Wisconsin and beyond following the shooting of Jacob Blake, we respect the decisions of a number of players not to play tonight,\" a league statement Wednesday said. \"Major League Baseball remains united for change in our society and we will be allies in the fight to end racism and injustice.\"Mets player Dominic Smith, who is Black, was driven to tears on Wednesday, telling reporters after the Mets defeated the Marlins, \"I think the most difficult part is to see people still don't care. And for this to continuously happen it just shows just the hate in people's heart and that just sucks. Being a Black man in America is not easy.\"On Thursday, seven games would not take place: Minnesota Twins vs. Detroit Tigers, Colorado Rockies vs. Arizona Diamondbacks, Baltimore Orioles vs. Tampa Bay Rays, Oakland Athletics vs. Texas Rangers, Philadelphia Phillies vs. Washington Nationals, Boston Red Sox vs. Toronto Blue Jays and Miami Marlins vs. New York Mets.Athletes across US sports take a stand, as games are called off in solidarity with Bucks' boycottIn a powerful moment Thursday at Citi Field in New York, the Mets starting defense took the field as scheduled, and then the remaining players for both teams came out of their dugouts and a moment of silence was observed, which the SNY television broadcast said lasted 42 seconds, and then walked off the field.Before he left the field, Miami Marlins player Lewis Brinson laid a \"Black Lives Matter\" shirt at home plate.\"Enough,\" the Marlins tweeted.Enough. pic.twitter.com/25SnNMCfBu— Miami Marlins (@Marlins) August 27, 2020 MLB is scheduled to observe Jackie Robinson Day on Friday to honor the color-barrier-breaking Hall of Famer, who wore number 42.NFLSeveral National Football League teams canceled practice Thursday to focus on conversations about race.The Denver Broncos joined the Arizona Cardinals, Indianapolis Colts, Washington Football Team and New York Jets in forgoing practice to allow players to weigh in on what they can do to effect change.The Chicago Bears said in a statement Thursday that the organization had decided to pause football activities \"to voice to each other, our coaches and our staff where we stand on the real issues around race and police brutality in our country.\"\"We had a productive discussion, but we all agreed that talks and discussions are simply not enough anymore and we need action,\" the statement said. \"We are putting in plans to take action in our communities and together we believe we can make a real difference. We need action not only today, but in the days to come.\"MLSInter Miami midfielder Lee Nguyen exits the field after Inter Miami and Atlanta United players decided not to play.Five Major League Soccer games scheduled to take place on Wednesday night were postponed, the league announced.\"The entire Major League Soccer family unequivocally condemns racism and has always stood for equality, but we need to do more to take tangible steps to impact change,\" MLS said. \"We will continue to work without players, our clubs and the broader soccer community to harness our collective power to fight for equality and social justice.\"TennisTennis superstar Naomi Osaka announced Wednesday that she would not play in the Western & Southern Open semifinals on Thursday. \"Watching the continued genocide of Black people at the hand of the police is honestly making me sick to my stomach,\" she wrote.Later on Wednesday, after Osaka's post, it was announced that the Western & Southern Open would pause tournament play on Thursday, resuming on Friday.By Thursday, a statement by her management team to The Guardian and The New York Times indicated she planned to play Friday instead.\"They offered to postpone all matches until Friday and in my mind that brings more attention to the movement,\" Osaka said. She also thanked the WTA and the tournament for their support.A WTA spokesperson tells CNN that Osaka did not officially withdraw from the Western & Southern Open on Wednesday. She is scheduled to play Elise Mertens in the semifinals on Friday.NHLThe NHL's Stanley Cup Playoffs continued Wednesday with two games. However, the league opted to postpone two games scheduled for Thursday and two scheduled for Friday.\"After much discussion, NHL Players believe that the best course of action would be to take a step back and not play tonight's and tomorrow's games as scheduled,\" a Thursday joint statement from the league and the players association said. \"The NHL supports the Players' decision and will reschedule those four games beginning Saturday and adjust the remainder of the Second Round schedule accordingly. \"Black and Brown communities continue to face real, painful experiences. The NHL and NHLPA recognize that much work remains to be done before we can play an appropriate role in a discussion centered on diversity, inclusion and social justice.\"Before the start of Game 3 of the second-round series between the Tampa Bay Lightning and Boston Bruins on Wednesday, there was a moment of reflection for racism in response to the Blake shooting.CNN's Ray Sanchez, Eric Levenson and Cesar Marin contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We will first identify the entities in the text based on the provided entity types. Then, we will determine the relationships between these entities, focusing on the interactions and associations described in the text. Finally, we will format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Milwaukee Bucks\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An NBA team that initiated the wildcat strike.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jacob Blake\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A person who was shot by police, leading to the sports world's protest.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenosha, Wisconsin\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The location where Jacob Blake was shot by police.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"NBA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The National Basketball Association, which saw multiple teams participating in the strike.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Los Angeles Lakers\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An NBA team that supported the strike.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"WNBA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The Women's National Basketball Association, which also participated in the strike.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Major League Baseball (MLB)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional baseball organization where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"National Football League (NFL)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional football league where teams canceled practices to discuss racial issues.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Major League Soccer (MLS)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional soccer league where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Naomi Osaka\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A tennis superstar who refused to play in a tournament to protest racial injustice.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"National Hockey League (NHL)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional hockey league where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Milwaukee Bucks\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The Milwaukee Bucks initiated a wildcat strike in the NBA.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Milwaukee Bucks\",\n", + "\n", + "\"tgt_id\": \"Jacob Blake\",\n", + "\n", + "\"description\": \"The Milwaukee Bucks' strike was in response to the police shooting of Jacob Blake.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Los Angeles Lakers\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The Los Angeles Lakers supported the strike in the NBA.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"WNBA\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The WNBA stood in solidarity with the NBA during the strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Major League Baseball (MLB)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The MLB postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"National Football League (NFL)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The NFL canceled practices to discuss racial issues in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Major League Soccer (MLS)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The MLS postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Naomi Osaka\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"Naomi Osaka refused to play in a tournament in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"National Hockey League (NHL)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The NHL postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/beta/chat/completions \"HTTP/1.1 200 OK\"\n", + "/opt/homebrew/Caskroom/miniconda/base/envs/nano-graphrag/lib/python3.10/site-packages/optuna/samplers/_tpe/sampler.py:319: ExperimentalWarning: ``multivariate`` option is an experimental feature. The interface can change in the future.\n", + " warnings.warn(\n", + "[I 2024-09-20 18:37:04,086] A new study created in memory with name: no-name-80b7b333-e37b-4380-94c7-15183cc4517f\n", + "INFO:root:Starting trial num: 0\n", + "INFO:root:instruction_idx 1\n", + "INFO:root:demos_idx 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Use the information below to learn about a task that we are trying to solve using calls to an LM, then generate a new instruction that will be used to prompt a Language Model to better solve the task.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "DATASET SUMMARY: A description of the dataset that we are using.\n", + "\n", + "PROGRAM CODE: Language model program designed to solve a particular task.\n", + "\n", + "PROGRAM DESCRIPTION: Summary of the task the program is designed to solve, and how it goes about solving it.\n", + "\n", + "MODULE: The module to create an instruction for.\n", + "\n", + "TASK DEMO(S): Example inputs/outputs of our module.\n", + "\n", + "BASIC INSTRUCTION: Basic instruction.\n", + "\n", + "TIP: A suggestion for how to go about generating the new instruction.\n", + "\n", + "PROPOSED INSTRUCTION: Propose an instruction that will be used to prompt a Language Model to perform this task.\n", + "\n", + "---\n", + "\n", + "DATASET SUMMARY: The dataset consists of news articles from CNN, covering a wide range of topics with a structured format including headlines, summaries, and detailed bodies. Key entities are highlighted with importance scores, and multimedia elements are prevalent, making it suitable for natural language processing and information extraction tasks.\n", + "\n", + "PROGRAM CODE:\n", + "StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions='Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")\n", + "\n", + "\n", + "\n", + "class TypedPredictor(dspy.Module):\n", + " def __init__(self, signature, instructions=None, *, max_retries=3, wrap_json=False, explain_errors=False):\n", + " \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + " Args:\n", + " signature: The signature of the module. Can use type annotations.\n", + " instructions: A description of what the model should do.\n", + " max_retries: The number of times to retry the prediction if the output is invalid.\n", + " wrap_json: If True, json objects in the input will be wrapped in ```json ... ```\n", + " explain_errors: If True, the model will try to explain the errors it encounters.\n", + " \"\"\"\n", + " super().__init__()\n", + " self.signature = ensure_signature(signature, instructions)\n", + " self.predictor = dspy.Predict(signature)\n", + " self.max_retries = max_retries\n", + " self.wrap_json = wrap_json\n", + " self.explain_errors = explain_errors\n", + "\n", + " def copy(self) -> \"TypedPredictor\":\n", + " return TypedPredictor(\n", + " self.signature,\n", + " max_retries=self.max_retries,\n", + " wrap_json=self.wrap_json,\n", + " explain_errors=self.explain_errors,\n", + " )\n", + "\n", + " def __repr__(self):\n", + " \"\"\"Return a string representation of the TypedPredictor object.\"\"\"\n", + " return f\"TypedPredictor({self.signature})\"\n", + "\n", + " def _make_example(self, type_) -> str:\n", + " # Note: DSPy will cache this call so we only pay the first time TypedPredictor is called.\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " schema = \"```json\\n\" + schema + \"\\n```\\n\"\n", + " json_object = dspy.Predict(\n", + " make_signature(\n", + " \"json_schema -> json_object\",\n", + " \"Make a very succinct json object that validates with the following schema\",\n", + " ),\n", + " )(json_schema=schema).json_object\n", + " # We use the model_validate_json method to make sure the example is valid\n", + " try:\n", + " type_.model_validate_json(_unwrap_json(json_object, type_.model_validate_json))\n", + " except (pydantic.ValidationError, ValueError):\n", + " return \"\" # Unable to make an example\n", + " return json_object\n", + " # TODO: Another fun idea is to only (but automatically) do this if the output fails.\n", + " # We could also have a more general \"suggest solution\" prompt that tries to fix the output\n", + " # More directly.\n", + " # TODO: Instead of using a language model to create the example, we can also just use a\n", + " # library like https://pypi.org/project/polyfactory/ that's made exactly to do this.\n", + "\n", + " def _format_error(\n", + " self,\n", + " error: Exception,\n", + " task_description: Union[str, FieldInfo],\n", + " model_output: str,\n", + " lm_explain: bool,\n", + " ) -> str:\n", + " if isinstance(error, pydantic.ValidationError):\n", + " errors = []\n", + " for e in error.errors():\n", + " fields = \", \".join(map(str, e[\"loc\"]))\n", + " errors.append(f\"{e['msg']}: {fields} (error type: {e['type']})\")\n", + " error_text = \"; \".join(errors)\n", + " else:\n", + " error_text = repr(error)\n", + "\n", + " if self.explain_errors and lm_explain:\n", + " if isinstance(task_description, FieldInfo):\n", + " args = task_description.json_schema_extra\n", + " task_description = args[\"prefix\"] + \" \" + args[\"desc\"]\n", + " return (\n", + " error_text\n", + " + \"\\n\"\n", + " + self._make_explanation(\n", + " task_description=task_description,\n", + " model_output=model_output,\n", + " error=error_text,\n", + " )\n", + " )\n", + "\n", + " return error_text\n", + "\n", + " def _make_explanation(self, task_description: str, model_output: str, error: str) -> str:\n", + " class Signature(dspy.Signature):\n", + " \"\"\"I gave my language model a task, but it failed.\n", + "\n", + " Figure out what went wrong, and write instructions to help it avoid the error next time.\n", + " \"\"\"\n", + "\n", + " task_description: str = dspy.InputField(desc=\"What I asked the model to do\")\n", + " language_model_output: str = dspy.InputField(desc=\"The output of the model\")\n", + " error: str = dspy.InputField(desc=\"The validation error trigged by the models output\")\n", + " explanation: str = dspy.OutputField(desc=\"Explain what the model did wrong\")\n", + " advice: str = dspy.OutputField(\n", + " desc=\"Instructions for the model to do better next time. A single paragraph.\",\n", + " )\n", + "\n", + " # TODO: We could also try repair the output here. For example, if the output is a float, but the\n", + " # model returned a \"float + explanation\", the repair could be to remove the explanation.\n", + "\n", + " return dspy.Predict(Signature)(\n", + " task_description=task_description,\n", + " language_model_output=model_output,\n", + " error=error,\n", + " ).advice\n", + "\n", + " def _prepare_signature(self) -> dspy.Signature:\n", + " \"\"\"Add formats and parsers to the signature fields, based on the type annotations of the fields.\"\"\"\n", + " signature = self.signature\n", + " for name, field in self.signature.fields.items():\n", + " is_output = field.json_schema_extra[\"__dspy_field_type\"] == \"output\"\n", + " type_ = field.annotation\n", + " if is_output:\n", + " if type_ is bool:\n", + "\n", + " def parse(x):\n", + " x = x.strip().lower()\n", + " if x not in (\"true\", \"false\"):\n", + " raise ValueError(\"Respond with true or false\")\n", + " return x == \"true\"\n", + "\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\" (Respond with true or false)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=parse,\n", + " )\n", + " elif type_ in (str, int, float):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (f\" (Respond with a single {type_.__name__} value)\" if type_ != str else \"\"),\n", + " format=lambda x: x if isinstance(x, str) else str(x),\n", + " parser=type_,\n", + " )\n", + " elif False:\n", + " # TODO: I don't like forcing the model to write \"value\" in the output.\n", + " if not (inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel)):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()[9:-1] # {\"value\":\"123\"}\n", + " from_json = lambda x, type_=type_: type_.model_validate_json('{\"value\":' + x + \"}\").value\n", + " schema = json.dumps(type_.model_json_schema()[\"properties\"][\"value\"])\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " # Anything else we wrap in a pydantic object\n", + " if not (\n", + " inspect.isclass(type_)\n", + " and typing.get_origin(type_) not in (list, tuple) # To support Python 3.9\n", + " and issubclass(type_, pydantic.BaseModel)\n", + " ):\n", + " type_ = pydantic.create_model(\"Output\", value=(type_, ...), __base__=pydantic.BaseModel)\n", + " to_json = lambda x, type_=type_: type_(value=x).model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x).value\n", + " schema = json.dumps(type_.model_json_schema())\n", + " else:\n", + " to_json = lambda x: x.model_dump_json()\n", + " from_json = lambda x, type_=type_: type_.model_validate_json(x)\n", + " schema = json.dumps(type_.model_json_schema())\n", + " if self.wrap_json:\n", + " to_json = lambda x, inner=to_json: \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " schema = \"```json\\n\" + schema + \"\\n```\"\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=field.json_schema_extra.get(\"desc\", \"\")\n", + " + (\". Respond with a single JSON object. JSON Schema: \" + schema),\n", + " format=lambda x, to_json=to_json: (x if isinstance(x, str) else to_json(x)),\n", + " parser=lambda x, from_json=from_json: from_json(_unwrap_json(x, from_json)),\n", + " type_=type_,\n", + " )\n", + " else: # If input field\n", + " is_json = False\n", + " format_ = lambda x: x if isinstance(x, str) else str(x)\n", + " if type_ in (List[str], list[str], Tuple[str], tuple[str]):\n", + " format_ = passages2text\n", + " # Special formatting for lists of known types. Maybe the output fields sohuld have this too?\n", + " elif typing.get_origin(type_) in (List, list, Tuple, tuple):\n", + " (inner_type,) = typing.get_args(type_)\n", + " if inspect.isclass(inner_type) and issubclass(inner_type, pydantic.BaseModel):\n", + " format_ = (\n", + " lambda x: x if isinstance(x, str) else \"[\" + \",\".join(i.model_dump_json() for i in x) + \"]\"\n", + " )\n", + " else:\n", + " format_ = lambda x: x if isinstance(x, str) else json.dumps(x)\n", + " is_json = True\n", + " elif inspect.isclass(type_) and issubclass(type_, pydantic.BaseModel):\n", + " format_ = lambda x: x if isinstance(x, str) else x.model_dump_json()\n", + " is_json = True\n", + " if self.wrap_json and is_json:\n", + " format_ = lambda x, inner=format_: x if isinstance(x, str) else \"```json\\n\" + inner(x) + \"\\n```\\n\"\n", + " signature = signature.with_updated_fields(name, format=format_)\n", + "\n", + " return signature\n", + "\n", + " def forward(self, **kwargs) -> dspy.Prediction:\n", + " modified_kwargs = kwargs.copy()\n", + " # We have to re-prepare the signature on every forward call, because the base\n", + " # signature might have been modified by an optimizer or something like that.\n", + " signature = self._prepare_signature()\n", + " for try_i in range(self.max_retries):\n", + " result = self.predictor(**modified_kwargs, new_signature=signature)\n", + " errors = {}\n", + " parsed_results = []\n", + " # Parse the outputs\n", + " for completion in result.completions:\n", + " parsed = {}\n", + " for name, field in signature.output_fields.items():\n", + " try:\n", + " value = completion[name]\n", + " parser = field.json_schema_extra.get(\"parser\", lambda x: x)\n", + " parsed[name] = parser(value)\n", + " except (pydantic.ValidationError, ValueError) as e:\n", + " errors[name] = self._format_error(\n", + " e,\n", + " signature.fields[name],\n", + " value,\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + "\n", + " # If we can, we add an example to the error message\n", + " current_desc = field.json_schema_extra.get(\"desc\", \"\")\n", + " i = current_desc.find(\"JSON Schema: \")\n", + " if i == -1:\n", + " continue # Only add examples to JSON objects\n", + " suffix, current_desc = current_desc[i:], current_desc[:i]\n", + " prefix = \"You MUST use this format: \"\n", + " if (\n", + " try_i + 1 < self.max_retries\n", + " and prefix not in current_desc\n", + " and (example := self._make_example(field.annotation))\n", + " ):\n", + " signature = signature.with_updated_fields(\n", + " name,\n", + " desc=current_desc + \"\\n\" + prefix + example + \"\\n\" + suffix,\n", + " )\n", + " # No reason trying to parse the general signature, or run more completions, if we already have errors\n", + " if errors:\n", + " break\n", + " # Instantiate the actual signature with the parsed values.\n", + " # This allow pydantic to validate the fields defined in the signature.\n", + " try:\n", + " _ = self.signature(**kwargs, **parsed)\n", + " parsed_results.append(parsed)\n", + " except pydantic.ValidationError as e:\n", + " errors[\"general\"] = self._format_error(\n", + " e,\n", + " signature.instructions,\n", + " \"\\n\\n\".join(\n", + " \"> \" + field.json_schema_extra[\"prefix\"] + \" \" + completion[name]\n", + " for name, field in signature.output_fields.items()\n", + " ),\n", + " lm_explain=try_i + 1 < self.max_retries,\n", + " )\n", + " if errors:\n", + " # Add new fields for each error\n", + " for name, error in errors.items():\n", + " modified_kwargs[f\"error_{name}_{try_i}\"] = error\n", + " if name == \"general\":\n", + " error_prefix = \"General:\"\n", + " else:\n", + " error_prefix = signature.output_fields[name].json_schema_extra[\"prefix\"]\n", + " number = \"\" if try_i == 0 else f\" ({try_i+1})\"\n", + " signature = signature.append(\n", + " f\"error_{name}_{try_i}\",\n", + " dspy.InputField(\n", + " prefix=f\"Past Error{number} in {error_prefix}\",\n", + " desc=\"An error to avoid in the future\",\n", + " ),\n", + " )\n", + " else:\n", + " # If there are no errors, we return the parsed results\n", + " return Prediction.from_completions(\n", + " {key: [r[key] for r in parsed_results] for key in signature.output_fields},\n", + " )\n", + " raise ValueError(\n", + " \"Too many retries trying to get the correct output format. \" + \"Try simplifying the requirements.\",\n", + " errors,\n", + " )\n", + "\n", + "class TypedEntityRelationshipExtractorException(dspy.Module):\n", + " def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)):\n", + " super().__init__()\n", + " self.predictor = predictor\n", + " self.exception_types = exception_types\n", + "\n", + " def copy(self):\n", + " return TypedEntityRelationshipExtractorException(self.predictor)\n", + "\n", + " def forward(self, **kwargs):\n", + " try:\n", + " prediction = self.predictor(**kwargs)\n", + " return prediction\n", + "\n", + " except Exception as e:\n", + " if isinstance(e, self.exception_types):\n", + " return dspy.Prediction(entities_relationships=[])\n", + "\n", + " raise e\n", + "\n", + "class TypedEntityRelationshipExtractor(dspy.Module):\n", + " def __init__(self, instructions: str = None, reasoning: str = None, max_retries: int = 3, lm: dspy.LM = None):\n", + " super().__init__()\n", + " self.lm = lm\n", + " self.entity_types = ENTITY_TYPES\n", + " self.extractor = dspy.TypedChainOfThought(\n", + " signature=CombinedExtraction, \n", + " instructions=instructions, \n", + " reasoning=reasoning, \n", + " max_retries=max_retries\n", + " )\n", + " self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError,))\n", + "\n", + " def forward(self, input_text: str) -> dspy.Prediction:\n", + " with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm):\n", + " extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types)\n", + "\n", + " entities = [\n", + " {\n", + " \"entity_name\": clean_str(entity['entity_name'].upper()),\n", + " \"entity_type\": clean_str(entity['entity_type'].upper()),\n", + " \"description\": entity['description'],\n", + " \"importance_score\": float(entity['importance_score'])\n", + " }\n", + " for entity in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Entity)]\n", + " ]\n", + "\n", + " relationships = [\n", + " {\n", + " \"src_id\": clean_str(relationship['src_id'].upper()),\n", + " \"tgt_id\": clean_str(relationship['tgt_id'].upper()),\n", + " \"description\": relationship['description'],\n", + " \"weight\": float(relationship['weight']),\n", + " \"order\": int(relationship['order'])\n", + " }\n", + " for relationship in [item.model_dump() for item in extraction_result.entities_relationships if isinstance(item, Relationship)]\n", + " ]\n", + " return dspy.Prediction(entities=entities, relationships=relationships)\n", + "\n", + "\n", + "PROGRAM DESCRIPTION: The program is designed to solve the task of extracting entities and relationships from text, specifically focusing on identifying entities based on predefined types and determining the relationships between these entities. It works by using a language model to process the input text and entity types, generating a step-by-step reasoning process to identify and relate the entities, and finally outputting the entities and their relationships in a structured JSON format. The program ensures type annotations are enforced and includes mechanisms for handling errors and retries to improve the accuracy of the output.\n", + "\n", + "MODULE: \"\"\"Like dspy.Predict, but enforces type annotations in the signature.\n", + "\n", + "TASK DEMO(S):\n", + "Input Text: (CNN)A ban on cellphones in classrooms was announced in the Australian state of Victoria on Wednesday. The ban, which will apply to primary and secondary students, is designed to crack down on cyber-bulling and classroom distractions, the Victorian government said in a statement. While some schools in Australia have already banned cell-phone use, this marks the first statewide ban in the country. It will begin from the first school term of 2020.\"Students will be required to switch off their phones and store them securely in lockers from the start of the school day until the final bell. When emergencies occur, parents or guardians can reach their child by calling the school,\" the statement said.Victoria Education Minister James Merlino said he didn't expect the move to be \"universally popular\" but said it was the \"right thing to do\" on his official social media. Read MoreA report says young people are growing horns on their skulls. Critics don't buy itThe move isn't without precedent. Australia's largest state, New South Wales, banned mobile phones in primary schools in 2018, and France has banned all smart phones and smart devices from schools.Australian Education Minister Dan Tehan said in a statement that he supported the Victoria ban.\"When I talk to parents and teachers the overwhelming majority want action on banning mobile phones in the classroom,\" Tehan said in a statement. In a 2019 survey of about 2,000 Australian adults, by Monash University professor Neil Selywn, almost 80% supported a restriction on cell-phone use in schools, while one-third supported an outright ban.Ann Marie, a secondary school principal at a college in Victoria, told ABC Radio Melbourne that pupils at her school had agreed to restrict their mobile phone use during the day, explaining that they hoped to reduce \"bullying, distraction and one-upmanship.\"But there have been some concerns. The Australian Education Union Victoria said in a statement that while it supports the ban: \"We can't lose sight of the benefits that technology, including mobile phones, can bring to high quality teaching and learning.\"Some pupils with special needs will be exempt from the ban, including those who require telephones to monitor health conditions or those who have received permission from a teacher. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering the context and descriptions provided in the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Victoria\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The Australian state where the cellphone ban in classrooms was announced.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"James Merlino\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Victoria Education Minister who announced the cellphone ban.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"New South Wales\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"Australia's largest state that previously banned mobile phones in primary schools.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"France\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"A country that has banned all smart phones and smart devices from schools.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Dan Tehan\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Australian Education Minister who supported the Victoria ban.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Monash University\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The university where a survey on cell-phone use in schools was conducted.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Neil Selwyn\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A professor at Monash University who conducted the survey on cell-phone use in schools.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Ann Marie\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A secondary school principal in Victoria who discussed the impact of the cellphone ban.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Australian Education Union Victoria\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An organization that supports the cellphone ban but highlights the benefits of technology in education.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"James Merlino\",\n", + "\n", + "\"description\": \"James Merlino, the Victoria Education Minister, announced the cellphone ban in Victoria.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"New South Wales\",\n", + "\n", + "\"description\": \"The cellphone ban in Victoria follows a similar ban in New South Wales.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"France\",\n", + "\n", + "\"description\": \"The cellphone ban in Victoria is similar to a ban in France.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Dan Tehan\",\n", + "\n", + "\"description\": \"Dan Tehan, the Australian Education Minister, supported the Victoria ban.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Monash University\",\n", + "\n", + "\"description\": \"A survey conducted at Monash University showed support for the Victoria ban.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Neil Selwyn\",\n", + "\n", + "\"description\": \"Neil Selwyn, a professor at Monash University, conducted the survey on cell-phone use in schools.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Ann Marie\",\n", + "\n", + "\"description\": \"Ann Marie, a secondary school principal in Victoria, discussed the impact of the cellphone ban.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Victoria\",\n", + "\n", + "\"tgt_id\": \"Australian Education Union Victoria\",\n", + "\n", + "\"description\": \"The Australian Education Union Victoria supports the cellphone ban but highlights the benefits of technology in education.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: Manchester, England (CNN)Boris Johnson wants to keep his Brexit plan private. But the Prime Minister isn't having much luck these days. On Monday night, Irish broadcaster RTE reported what it claimed were details of part of Johnson's plan to break the Brexit deadlock. According to RTE, the UK's plan to avoid post-Brexit infrastructure on the frontier between Northern Ireland and the Republic of Ireland involve \"customs clearance centers\" several miles from the border -- and on both sides of it. The women of Boris Johnson's past are starting to catch up with himThat, it hopes, will remove the need for something called the Irish border backstop from any final Brexit deal. The backstop is an emergency measure which would keep the two Irish nations in regulatory alignment, removing the need for any border or \"clearance centers\" at all. It's deeply unpopular with many in Johnson's Conservative party and is one of the main reasons his predecessor, Theresa May, failed to get her Withdrawal Agreement through Parliament.Ireland quickly dismissed the idea as a \"non starter\". And Johnson himself told the BBC on Tuesday, \"that's not what we're proposing at all... you'll forgive me, I would like to veil our proposals in decent obscurity until we've been able to share them properly with our friends\". Read MoreSo why is this story a big deal? Keeping the government's formal proposals from Brussels for the time being might be wise. The crunch EU summit, at which any Brexit deal will be struck, is now under three weeks away. JUST WATCHEDBoris Johnson tells opponents to hold no-confidence voteReplayMore Videos ...MUST WATCHBoris Johnson tells opponents to hold no-confidence vote 02:41The EU is a notoriously tough institution to negotiate with and has a history of tearing apart any Brexit proposal from the moment it leaves the door. Johnson has repeatedly said that the only way to make the EU budge on ditching the backstop is to show that he is both deadly serious about getting a deal but also prepared to leave without one. Some of the ideas reported by RTE chime with speculation about the UK government's plans in recent weeks. And anyone who has followed Brexit for the last three years will tell you, the EU and Ireland will refuse to sign a deal that involves infrastructure between Ireland and Northern Ireland. So, even though the Prime Minister has distanced himself from the proposals, these headlines will have done little to reassure Brussels that Johnson and his government are serious about getting a deal. It's also a bad look for Johnson back home. He is currently out of London and at his Conservative party's annual conference in Manchester, in the north of England. Johnson will close the conference with a keynote speech on Wednesday afternoon. British politics has been taken over by trollsIt's widely thought that he wanted to trail his final Brexit plan during this speech before formally submitting something to Brussels in the subsequent days. The problem with the leak of any Brexit plan now is that it increases the risk of Johnson being greeted by a hostile audience at his conference on Wednesday. Despite being the Brexit hero of 2016, Conservative hard-Brexiteers are deeply worried that Johnson, if he succeeds in getting rid of the backstop, will simply try and push through the rest May's Brexit deal.For Brexiteers, this would be a huge sellout. There are people sitting in Johnson's cabinet who resigned from Theresa May's government to vote against her Brexit deal. And Johnson himself resigned as May's Foreign Secretary over her general approach to Brexit. Up here in Manchester, the mood among party members is generally positive. Johnson has always been a popular Conservative MP and he seems to have broad support for his hard Brexit language. That could turn sour if he's seen to make concessions to Brussels. Which would all be somewhat ironic, for a man who has spent the past few days accusing his opponents of surrendering. \n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. We then determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Boris Johnson\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"The Prime Minister of the United Kingdom who is leading the Brexit negotiations.\",\n", + "\n", + "\"importance_score\": 1.0\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Brexit\",\n", + "\n", + "\"entity_type\": \"EVENT\",\n", + "\n", + "\"description\": \"The process of the United Kingdom withdrawing from the European Union.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"European Union\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A political and economic union of 27 European countries.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Irish border backstop\",\n", + "\n", + "\"entity_type\": \"LAW\",\n", + "\n", + "\"description\": \"An emergency measure to keep the two Irish nations in regulatory alignment, removing the need for any border or 'clearance centers'.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Theresa May\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"Former Prime Minister of the United Kingdom and predecessor of Boris Johnson.\",\n", + "\n", + "\"importance_score\": 0.6\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Conservative party\",\n", + "\n", + "\"entity_type\": \"POLITICAL_PARTY\",\n", + "\n", + "\"description\": \"The political party in the United Kingdom to which Boris Johnson and Theresa May belong.\",\n", + "\n", + "\"importance_score\": 0.5\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"RTE\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"Irish broadcaster that reported details of Johnson's Brexit plan.\",\n", + "\n", + "\"importance_score\": 0.4\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Brexit\",\n", + "\n", + "\"description\": \"Boris Johnson is leading the Brexit negotiations as the Prime Minister of the United Kingdom.\",\n", + "\n", + "\"weight\": 1.0,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Brexit\",\n", + "\n", + "\"tgt_id\": \"European Union\",\n", + "\n", + "\"description\": \"Brexit involves the United Kingdom withdrawing from the European Union.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Irish border backstop\",\n", + "\n", + "\"description\": \"Boris Johnson aims to remove the Irish border backstop from any final Brexit deal.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Theresa May\",\n", + "\n", + "\"tgt_id\": \"Brexit\",\n", + "\n", + "\"description\": \"Theresa May failed to get her Withdrawal Agreement through Parliament due to the Irish border backstop.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Boris Johnson\",\n", + "\n", + "\"tgt_id\": \"Conservative party\",\n", + "\n", + "\"description\": \"Boris Johnson is a member of the Conservative party and its leader.\",\n", + "\n", + "\"weight\": 0.6,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"RTE\",\n", + "\n", + "\"tgt_id\": \"Boris Johnson\",\n", + "\n", + "\"description\": \"RTE reported details of part of Johnson's plan to break the Brexit deadlock.\",\n", + "\n", + "\"weight\": 0.5,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "Input Text: (CNN)Across the sports world, athletes took a stand for Black Lives Matter on Wednesday night by holding an unprecedented wildcat strike and refusing to play their regularly scheduled games.The decision started with the NBA's Milwaukee Bucks and stemmed from the police shooting of Jacob Blake in Kenosha, Wisconsin, on Sunday. The stand quickly spread across a number of sports -- from basketball courts in Florida to baseball diamonds in California to soccer fields in places like Miami and Salt Lake City and even into the broadcast booth, where TNT's \"Inside the NBA\" analyst Kenny Smith took off his mic and walked off stage.But the issue is bigger than Jacob Blake or George Floyd or Breonna Taylor. Malcolm Jenkins of the NFL's New Orleans Saints positioned the protests as a response to the systemic racism baked into America, including police violence, cycles of incarceration, and the plunder of generational wealth.\"Until we continue to demand it, until 'Black Lives Matter' goes from just an idea or a goal that we're trying to attain as a society and is actually realized in the streets, we won't see any peace,\" he said. \"And I think we'll continue to see athletes, entertainers as well as citizens disrupt the status quo until that's recognized.\"Here's a look at the teams and athletes who pulled back in order to have their voices be heard.Read MoreNBA Photos: The sports world has been taking a standMembers of the Los Angeles Lakers and the Miami Heat kneel during the National Anthem before Game 1 of the NBA Finals on Wednesday, September 30.Hide Caption 1 of 42 Photos: The sports world has been taking a standThe Philadelphia Eagles and the Washington Football Team stand together before their NFL season opener on September 13.Hide Caption 2 of 42 Photos: The sports world has been taking a standTyler Wright, a two-time World Surf League Champion, takes a knee before competing in an event in Tweed Heads South, Australia, on September 13.Hide Caption 3 of 42 Photos: The sports world has been taking a standTennis star Naomi Osaka wears a face mask with Tamir Rice's name before winning the US Open final on September 12. Osaka wore a different name for each of her seven matches. Rice, a 12-year-old boy, was killed by police gunfire in Cleveland while he was holding a toy replica pistol in 2014.Hide Caption 4 of 42 Photos: The sports world has been taking a standChelsea's Pernille Harder walks out for a warm-up before a soccer match in Leigh, England, on September 6.Hide Caption 5 of 42 Photos: The sports world has been taking a standThe University of Alabama football team, led by coach Nick Saban, marches on campus in support of the Black Lives Matter movement.Hide Caption 6 of 42 Photos: The sports world has been taking a standBlack Lives Matter signage is seen in New York's Louis Armstrong Stadium as Cori Gauff plays against Anastasija Sevastova at the US Open.Hide Caption 7 of 42 Photos: The sports world has been taking a standMembers of the WNBA's Indiana Fever wear Black Lives Matter shirts before their game against the Chicago Sky on August 31.Hide Caption 8 of 42 Photos: The sports world has been taking a standMiami Marlins outfielder Lewis Brinson walks off the field August 27 after placing a Black Lives Matter T-shirt on home plate in New York. The Marlins and New York Mets walked off the field after a moment of silence, choosing not to play their scheduled baseball game. Hide Caption 9 of 42 Photos: The sports world has been taking a standNew Orleans Saints wide receiver Emmanuel Sanders wears Jacob Blake's name on his helmet during a practice on August 27.Hide Caption 10 of 42 Photos: The sports world has been taking a standThe Washington Mystics pay tribute to Jacob Blake after their WNBA game was postponed in Palmetto, Florida, on August 26.Hide Caption 11 of 42 Photos: The sports world has been taking a standA grounds crew covers the field at San Francisco's Oracle Park after a Major League Baseball game was postponed on August 26.Hide Caption 12 of 42 Photos: The sports world has been taking a standThe Milwaukee Bucks make a statement to the media after boycotting a playoff game on August 26.Hide Caption 13 of 42 Photos: The sports world has been taking a standDuring a practice round for the BMW Championship, pro golfer Cameron Champ used his shoes to show his support for Blake and the Black Lives Matter movement.Hide Caption 14 of 42 Photos: The sports world has been taking a standNBA superstar LeBron James has been one of the most outspoken athletes in recent months. \"Having two boys of my own and me being African American in America and to see what continues to happen with the police brutality towards my kind, continue to see what goes on with just the (unjust), it's very troubling. It's very troubling,\" James told Turner Sports' Jared Greenberg. \"My prayers go to (the Blake) family, and hopefully we can have some change.\" Hide Caption 15 of 42 Photos: The sports world has been taking a stand\"End racism\" banners are shown in Toronto's Scotiabank Arena before an NHL playoff game on August 26.Hide Caption 16 of 42 Photos: The sports world has been taking a standDoc Rivers, head coach of the NBA's Los Angeles Clippers, became emotional while talking about the Blake shooting and the Republican National Convention. \"All you hear is Donald Trump and all of them talking about fear,\" Rivers said. \"We're the ones getting killed. We're the ones getting shot. We're the ones who were denied to live in certain communities. We've been hung. We've been shot.\"Hide Caption 17 of 42 Photos: The sports world has been taking a standDetroit Lions defensive end Trey Flowers addresses the media with his teammates outside their practice facility on August 25. The Lions canceled their practice in light of the Blake shooting, and they discussed the incident in a team meeting that lasted hours. \"We're going to spread our message; we're going to do it as a team,\" Flowers said. \"We understand that somebody's going to try to skew the narrative ... get the message lost, but we're going to stay focused on the topic.\"Hide Caption 18 of 42 Photos: The sports world has been taking a standA Real Salt Lake fan sits in the stands after the Major League Soccer team had its game postponed on August 26.Hide Caption 19 of 42 Photos: The sports world has been taking a standNASCAR driver Bubba Wallace tweeted this selfie before a Cup Series race in Talladega, Alabama, on June 22. Fellow drivers and pit crew members walked alongside Wallace's car to show their support for him. Wallace, the only Black driver in NASCAR's top circuit, has been an outspoken advocate of the Black Lives Matter movement. Hide Caption 20 of 42 Photos: The sports world has been taking a standMembers of the NHL's Colorado Avalanche, Vegas Golden Knights, Dallas Stars and Vancouver Canucks gather together after their playoff games were postponed in Edmonton, Alberta.Hide Caption 21 of 42 Photos: The sports world has been taking a standThe Black Lives Matter protests haven't been limited to just North America. Here, professional soccer players from Aston Villa and Sheffield United take a knee as their match kicked off in Birmingham, England, on June 17. Premier League teams sported the words \"Black Lives Matter\" on the back of their jerseys when their seasons resumed.Hide Caption 22 of 42 Photos: The sports world has been taking a standFormula One champion Lewis Hamilton raises his fist before a race in Northampton, England, on August 9.Hide Caption 23 of 42 Photos: The sports world has been taking a standMatt Dumba of the Minnesota Wild kneels during the US National Anthem as it is played before an NHL game in Edmonton, Alberta, on August 1. Dumba, a member of the Hockey Diversity Alliance, gave a short speech before kneeling. He is circled by members of the Edmonton Oilers and the Chicago Blackhawks, who were about to play on the first day of the NHL's return.Hide Caption 24 of 42 Photos: The sports world has been taking a standSoccer players from PSG and Lyon take a knee before a Champions League semifinal in Bilbao, Spain, on August 26.Hide Caption 25 of 42 Photos: The sports world has been taking a standBeach volleyball player April Ross wears a temporary Black Lives Matter tattoo during a match in Long Beach, California, on July 19.Hide Caption 26 of 42 Photos: The sports world has been taking a standTennis players Dan Evans and Kyle Edmund join match umpire James Keothavong in taking a knee in London on June 28.Hide Caption 27 of 42 Photos: The sports world has been taking a standCricketers with the Kent Spitfires take a knee during a match in Canterbury, England.Hide Caption 28 of 42 Photos: The sports world has been taking a standFormula One driver Kimi Raikkonen wears an \"end racism\" shirt before a race in Barcelona, Spain, on August 16.Hide Caption 29 of 42 Photos: The sports world has been taking a standMembers of the Trinbago Knight Riders kneel before a cricket match in Port of Spain, Trinidad and Tobago.Hide Caption 30 of 42 Photos: The sports world has been taking a standThe Washington Nationals observe a moment of silence before Major League Baseball's opening game on July 23. Their opponents, The New York Yankees, also took a knee, and the initials BLM were on the pitcher's mound for the game.Hide Caption 31 of 42 Photos: The sports world has been taking a standMajor League Soccer players participate in a pregame ceremony before the league's restart on July 8.Hide Caption 32 of 42 Photos: The sports world has been taking a standHouston Dynamo goalkeeper Cody Cropper warms up in a Black Lives Matter T-shirt on August 25.Hide Caption 33 of 42 Photos: The sports world has been taking a standThe Boston Red Sox Foundation put this sign behind Fenway Park's famous \"Green Monster\" outfield wall.Hide Caption 34 of 42 Photos: The sports world has been taking a standRugby players from Bath and Northampton show their support before a game in Northampton, England.Hide Caption 35 of 42 Photos: The sports world has been taking a standKansas City outfielder Jorge Soler catches a ball in front of a George Floyd sign in Minneapolis on August 15.Hide Caption 36 of 42 Photos: The sports world has been taking a standA Houston Dash player wears a Black Lives Matter armband before a NWSL match in Herriman, Utah, on July 17.Hide Caption 37 of 42 Photos: The sports world has been taking a standAfter winning the Champions League, Bayern Munich defender David Alaba wears a shirt that says \"Black Lives Still Matter.\"Hide Caption 38 of 42 Photos: The sports world has been taking a standA Black Lives Matter sign is seen during a Major League Soccer match in Harrison, New Jersey, on August 24.Hide Caption 39 of 42 Photos: The sports world has been taking a standMembers of the San Antonio Spurs huddle before a game against Sacramento on July 31.Hide Caption 40 of 42 Photos: The sports world has been taking a standKilmarnock manager Alex Dyer, second from right, takes a knee before a soccer match in Glasgow, Scotland.Hide Caption 41 of 42 Photos: The sports world has been taking a standMembers of the Orlando Magic and Brooklyn Nets kneel during the National Anthem before the start of an NBA game on July 31.Hide Caption 42 of 42The Milwaukee Bucks did not emerge from their locker room before the scheduled tip at 4 p.m. Wednesday as they decided to refuse to play their playoff game.Soon after, the NBA announced it would postpone Game 5 of three different playoff series -- the Bucks vs. Orlando Magic, Houston Rockets vs. Oklahoma City Thunder and Los Angeles Lakers vs. Portland Trail Blazers.The postponed games have not yet been rescheduled.Three playoff games on Thursday also were postponed but the league said it hoped to resume games Friday or Saturday. Strikes are banned under the NBA's collective bargaining agreement, which means the Bucks players broke their own contract to protest racial injustice and police violence. In a statement issued on Wednesday afternoon, the Bucks players said they are \"calling for justice for Jacob Blake and demand the officers be held accountable. \"Full statement from the Milwaukee Bucks: pic.twitter.com/jjGEyVcCmB— Milwaukee Bucks (@Bucks) August 26, 2020\n", + "\n", + "\"The past four months have shed a light on the ongoing racial injustices facing our African American communities. Citizens around the country have used their voices and platforms to speak out against these wrongdoings,\" they said in a statement.\"Despite the overwhelming plea for change, there has been no action, so our focus today cannot be on basketball.\"The Los Angeles Lakers also offered a statement in support of the players' decision.\"Eighty percent of NBA players are Black men. We cannot love them for the joy and entertainment that they bring to the world, yet sit in silence and fail to use our platforms and resources to amplify their voices when they demand the justice and equality that America has promised us all, but denied Black people for too long,\" the Lakers said.WNBAElizabeth Williams of the Atlanta Dream said WNBA players were standing in solidarity with \"our brothers in the NBA.\"Representing the six teams slated to play on Wednesday, Atlanta Dream player Elizabeth Williams announced that WNBA players were standing in solidarity with \"our brothers in the NBA\" and also would not play. The ESPN2 broadcast showed players from the six teams scheduled to take the court in locked arms and kneeling while wearing shirts spelling out Jacob Blake's name. The WNBA announced that the three games scheduled for Wednesday evening had been postponed. On Thursday, the league announced the postponement of the day's three scheduled games as well.Instead of playing Thursday, the entire league of WNBA players stood arm-in-arm in solidarity.\"It is important to note that this is not a strike,\" said Women's National Basketball Players Association (WNBPA) president and Los Angeles Sparks star Nneka Ogwumike, reading a statement from the players, to ESPN's Holly Rowe. \"This is not a boycott. This is affirmatively a day of reflection. A day of informed action and mobilization.\"Ogwumike said players plan to play in games that are scheduled for Friday.\"I feel like as a group we decided last night that we want to play,\" Ogwumike said. \"There are games scheduled tomorrow. That's what we're ready to do, but that doesn't come without, of course, demands of players to continue to amplify our voices in more ways than when we came here.\"We realize that the work is not easy, but we also understand that the work is never done. There are things that happen that just allow us times to take a moment -- that's fair; we can take a moment -- and that moment we came together and we decided we need time for ourselves to come back, regroup and continue to amplify our voices, and be there for our communities and demand change from the leaders that are representing us right now.\"MLBThe words 'Black Lives Matter' are displayed on the digital screen after the postponement of the game between the San Francisco Giants and the Los Angeles Dodgers Wednesday.Three MLB games were postponed Wednesday: Cincinnati Reds vs. Milwaukee Brewers, Seattle Mariners vs. San Diego Padres, Los Angeles Dodgers vs. San Francisco Giants.\"With our community and our nation in such pain, we wanted to draw as much attention to the issues that really matter, especially racial injustice and systemic oppression,\" a joint statement from the Brewers and Reds said.\"Given the pain in the communities of Wisconsin and beyond following the shooting of Jacob Blake, we respect the decisions of a number of players not to play tonight,\" a league statement Wednesday said. \"Major League Baseball remains united for change in our society and we will be allies in the fight to end racism and injustice.\"Mets player Dominic Smith, who is Black, was driven to tears on Wednesday, telling reporters after the Mets defeated the Marlins, \"I think the most difficult part is to see people still don't care. And for this to continuously happen it just shows just the hate in people's heart and that just sucks. Being a Black man in America is not easy.\"On Thursday, seven games would not take place: Minnesota Twins vs. Detroit Tigers, Colorado Rockies vs. Arizona Diamondbacks, Baltimore Orioles vs. Tampa Bay Rays, Oakland Athletics vs. Texas Rangers, Philadelphia Phillies vs. Washington Nationals, Boston Red Sox vs. Toronto Blue Jays and Miami Marlins vs. New York Mets.Athletes across US sports take a stand, as games are called off in solidarity with Bucks' boycottIn a powerful moment Thursday at Citi Field in New York, the Mets starting defense took the field as scheduled, and then the remaining players for both teams came out of their dugouts and a moment of silence was observed, which the SNY television broadcast said lasted 42 seconds, and then walked off the field.Before he left the field, Miami Marlins player Lewis Brinson laid a \"Black Lives Matter\" shirt at home plate.\"Enough,\" the Marlins tweeted.Enough. pic.twitter.com/25SnNMCfBu— Miami Marlins (@Marlins) August 27, 2020 MLB is scheduled to observe Jackie Robinson Day on Friday to honor the color-barrier-breaking Hall of Famer, who wore number 42.NFLSeveral National Football League teams canceled practice Thursday to focus on conversations about race.The Denver Broncos joined the Arizona Cardinals, Indianapolis Colts, Washington Football Team and New York Jets in forgoing practice to allow players to weigh in on what they can do to effect change.The Chicago Bears said in a statement Thursday that the organization had decided to pause football activities \"to voice to each other, our coaches and our staff where we stand on the real issues around race and police brutality in our country.\"\"We had a productive discussion, but we all agreed that talks and discussions are simply not enough anymore and we need action,\" the statement said. \"We are putting in plans to take action in our communities and together we believe we can make a real difference. We need action not only today, but in the days to come.\"MLSInter Miami midfielder Lee Nguyen exits the field after Inter Miami and Atlanta United players decided not to play.Five Major League Soccer games scheduled to take place on Wednesday night were postponed, the league announced.\"The entire Major League Soccer family unequivocally condemns racism and has always stood for equality, but we need to do more to take tangible steps to impact change,\" MLS said. \"We will continue to work without players, our clubs and the broader soccer community to harness our collective power to fight for equality and social justice.\"TennisTennis superstar Naomi Osaka announced Wednesday that she would not play in the Western & Southern Open semifinals on Thursday. \"Watching the continued genocide of Black people at the hand of the police is honestly making me sick to my stomach,\" she wrote.Later on Wednesday, after Osaka's post, it was announced that the Western & Southern Open would pause tournament play on Thursday, resuming on Friday.By Thursday, a statement by her management team to The Guardian and The New York Times indicated she planned to play Friday instead.\"They offered to postpone all matches until Friday and in my mind that brings more attention to the movement,\" Osaka said. She also thanked the WTA and the tournament for their support.A WTA spokesperson tells CNN that Osaka did not officially withdraw from the Western & Southern Open on Wednesday. She is scheduled to play Elise Mertens in the semifinals on Friday.NHLThe NHL's Stanley Cup Playoffs continued Wednesday with two games. However, the league opted to postpone two games scheduled for Thursday and two scheduled for Friday.\"After much discussion, NHL Players believe that the best course of action would be to take a step back and not play tonight's and tomorrow's games as scheduled,\" a Thursday joint statement from the league and the players association said. \"The NHL supports the Players' decision and will reschedule those four games beginning Saturday and adjust the remainder of the Second Round schedule accordingly. \"Black and Brown communities continue to face real, painful experiences. The NHL and NHLPA recognize that much work remains to be done before we can play an appropriate role in a discussion centered on diversity, inclusion and social justice.\"Before the start of Game 3 of the second-round series between the Tampa Bay Lightning and Boston Bruins on Wednesday, there was a moment of reflection for racism in response to the Blake shooting.CNN's Ray Sanchez, Eric Levenson and Cesar Marin contributed to this report.\n", + "Entity Types: ['PERSON', 'ORGANIZATION', 'LOCATION', 'DATE', 'TIME', 'MONEY', 'PERCENTAGE', 'PRODUCT', 'EVENT', 'LANGUAGE', 'NATIONALITY', 'RELIGION', 'TITLE', 'PROFESSION', 'ANIMAL', 'PLANT', 'DISEASE', 'MEDICATION', 'CHEMICAL', 'MATERIAL', 'COLOR', 'SHAPE', 'MEASUREMENT', 'WEATHER', 'NATURAL_DISASTER', 'AWARD', 'LAW', 'CRIME', 'TECHNOLOGY', 'SOFTWARE', 'HARDWARE', 'VEHICLE', 'FOOD', 'DRINK', 'SPORT', 'MUSIC_GENRE', 'INSTRUMENT', 'ARTWORK', 'BOOK', 'MOVIE', 'TV_SHOW', 'ACADEMIC_SUBJECT', 'SCIENTIFIC_THEORY', 'POLITICAL_PARTY', 'CURRENCY', 'STOCK_SYMBOL', 'FILE_TYPE', 'PROGRAMMING_LANGUAGE', 'MEDICAL_PROCEDURE', 'CELESTIAL_BODY']\n", + "Reasoning: Let's think step by step in order to produce the entities_relationships. We will first identify the entities in the text based on the provided entity types. Then, we will determine the relationships between these entities, focusing on the interactions and associations described in the text. Finally, we will format the entities and relationships according to the specified JSON schema.\n", + "Entities Relationships: ```json\n", + "\n", + "{\n", + "\n", + "\"value\": [\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Milwaukee Bucks\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An NBA team that initiated the wildcat strike.\",\n", + "\n", + "\"importance_score\": 0.9\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Jacob Blake\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A person who was shot by police, leading to the sports world's protest.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Kenosha, Wisconsin\",\n", + "\n", + "\"entity_type\": \"LOCATION\",\n", + "\n", + "\"description\": \"The location where Jacob Blake was shot by police.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"NBA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The National Basketball Association, which saw multiple teams participating in the strike.\",\n", + "\n", + "\"importance_score\": 0.8\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Los Angeles Lakers\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"An NBA team that supported the strike.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"WNBA\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"The Women's National Basketball Association, which also participated in the strike.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Major League Baseball (MLB)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional baseball organization where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"National Football League (NFL)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional football league where teams canceled practices to discuss racial issues.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Major League Soccer (MLS)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional soccer league where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"Naomi Osaka\",\n", + "\n", + "\"entity_type\": \"PERSON\",\n", + "\n", + "\"description\": \"A tennis superstar who refused to play in a tournament to protest racial injustice.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"entity_name\": \"National Hockey League (NHL)\",\n", + "\n", + "\"entity_type\": \"ORGANIZATION\",\n", + "\n", + "\"description\": \"A professional hockey league where games were postponed in solidarity.\",\n", + "\n", + "\"importance_score\": 0.7\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Milwaukee Bucks\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The Milwaukee Bucks initiated a wildcat strike in the NBA.\",\n", + "\n", + "\"weight\": 0.9,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Milwaukee Bucks\",\n", + "\n", + "\"tgt_id\": \"Jacob Blake\",\n", + "\n", + "\"description\": \"The Milwaukee Bucks' strike was in response to the police shooting of Jacob Blake.\",\n", + "\n", + "\"weight\": 0.8,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Los Angeles Lakers\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The Los Angeles Lakers supported the strike in the NBA.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"WNBA\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The WNBA stood in solidarity with the NBA during the strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Major League Baseball (MLB)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The MLB postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"National Football League (NFL)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The NFL canceled practices to discuss racial issues in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Major League Soccer (MLS)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The MLS postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"Naomi Osaka\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"Naomi Osaka refused to play in a tournament in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "},\n", + "\n", + "{\n", + "\n", + "\"src_id\": \"National Hockey League (NHL)\",\n", + "\n", + "\"tgt_id\": \"NBA\",\n", + "\n", + "\"description\": \"The NHL postponed games in solidarity with the NBA strike.\",\n", + "\n", + "\"weight\": 0.7,\n", + "\n", + "\"order\": 1\n", + "\n", + "}\n", + "\n", + "]\n", + "\n", + "}\n", + "\n", + "```\n", + "\n", + "\n", + "BASIC INSTRUCTION: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "TIP: Make sure your instruction is very informative and descriptive.\n", + "\n", + "Please provide the output field PROPOSED INSTRUCTION. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with only the field PROPOSED INSTRUCTION.\n", + "\n", + "\u001b[32mPROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, your task is to meticulously extract entities and their relationships from the text. First, identify all entities in the text that match the provided entity types. Then, analyze the context to determine the relationships between these entities, considering their interactions and dependencies within the text. Finally, format the identified entities and their relationships according to the specified JSON schema, ensuring all fields are accurately populated with relevant details such as entity names, types, descriptions, and importance scores for entities, and source IDs, target IDs, descriptions, weights, and orders for relationships.\u001b[0m\n", + "\n", + "\n", + "\n", + "PROPOSED INSTRUCTION: Given the `input_text` and a list of `entity_types`, your task is to meticulously extract entities and their relationships from the text. First, identify all entities in the text that match the provided entity types. Then, analyze the context to determine the relationships between these entities, considering their interactions and dependencies within the text. Finally, format the identified entities and their relationships according to the specified JSON schema, ensuring all fields are accurately populated with relevant details such as entity names, types, descriptions, and importance scores for entities, and source IDs, target IDs, descriptions, weights, and orders for relationships.\n", + "CANDIDATE PROGRAM:\n", + "Predictor 0\n", + "i: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\n", + "p: Entities Relationships:\n", + "\n", + "\n", + "...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/25 [00:00 reasoning, entities_relationships\n", + " instructions='Given the `input_text` and a list of `entity_types`, meticulously identify and extract all entities present in the text that match the provided entity types. Subsequently, determine the relationships between these entities based on their interactions and the context provided. Ensure that the entities and relationships are formatted according to the specified JSON schema, including providing detailed descriptions and importance scores for each entity, as well as defining the source and target IDs, description, weight, and order for each relationship.'\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + "))" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optimizer = MIPROv2(\n", + " prompt_model=deepseek,\n", + " task_model=qwen2,\n", + " metric=entity_recall_metric,\n", + " init_temperature=1.4,\n", + " num_candidates=10,\n", + " verbose=False\n", + ")\n", + "kwargs = dict(num_threads=os.cpu_count(), display_progress=True, display_table=0)\n", + "miprov2_model = optimizer.compile(\n", + " model, \n", + " trainset=trainset[:50],\n", + " valset=valset[:20],\n", + " requires_permission_to_run=False,\n", + " num_batches=20, \n", + " max_labeled_demos=5, \n", + " max_bootstrapped_demos=3, \n", + " eval_kwargs=kwargs\n", + ")\n", + "miprov2_model" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "---\n", + "\n", + "Follow the following format.\n", + "\n", + "Input Text: The text to extract entities and relationships from.\n", + "\n", + "Entity Types: List of entity types used for extraction.\n", + "\n", + "Reasoning: Let's think step by step in order to ${produce the entities_relationships}. We ...\n", + "\n", + "Entities Relationships: List of entities and relationships extracted from the text.. Respond with a single JSON object. JSON Schema: {\"$defs\": {\"Entity\": {\"properties\": {\"entity_name\": {\"description\": \"The name of the entity.\", \"title\": \"Entity Name\", \"type\": \"string\"}, \"entity_type\": {\"description\": \"The type of the entity.\", \"title\": \"Entity Type\", \"type\": \"string\"}, \"description\": {\"description\": \"The description of the entity, in details and comprehensive.\", \"title\": \"Description\", \"type\": \"string\"}, \"importance_score\": {\"description\": \"Importance score of the entity. Should be between 0 and 1 with 1 being the most important.\", \"maximum\": 1.0, \"minimum\": 0.0, \"title\": \"Importance Score\", \"type\": \"number\"}}, \"required\": [\"entity_name\", \"entity_type\", \"description\", \"importance_score\"], \"title\": \"Entity\", \"type\": \"object\"}, \"Relationship\": {\"properties\": {\"src_id\": {\"description\": \"The name of the source entity.\", \"title\": \"Src Id\", \"type\": \"string\"}, \"tgt_id\": {\"description\": \"The name of the target entity.\", \"title\": \"Tgt Id\", \"type\": \"string\"}, \"description\": {\"description\": \"The description of the relationship between the source and target entity, in details and comprehensive.\", \"title\": \"Description\", \"type\": \"string\"}, \"weight\": {\"description\": \"The weight of the relationship. Should be between 0 and 1 with 1 being the strongest relationship.\", \"maximum\": 1.0, \"minimum\": 0.0, \"title\": \"Weight\", \"type\": \"number\"}, \"order\": {\"description\": \"The order of the relationship. 1 for direct relationships, 2 for second-order, 3 for third-order.\", \"maximum\": 3, \"minimum\": 1, \"title\": \"Order\", \"type\": \"integer\"}}, \"required\": [\"src_id\", \"tgt_id\", \"description\", \"weight\", \"order\"], \"title\": \"Relationship\", \"type\": \"object\"}}, \"properties\": {\"value\": {\"items\": {\"anyOf\": [{\"$ref\": \"#/$defs/Entity\"}, {\"$ref\": \"#/$defs/Relationship\"}]}, \"title\": \"Value\", \"type\": \"array\"}}, \"required\": [\"value\"], \"title\": \"Output\", \"type\": \"object\"}\n", + "\n", + "---\n", + "\n", + "Input Text:\n", + "Melbourne, Australia (CNN)After spending part of the off-season training with Mike Tyson, Serena Williams is hoping to deliver a knockout punch at the Australian Open. Follow @cnnsport\n", + "\n", + "For Williams that would mean winning a record-tying 24th grand slam title, which has so far proved elusive despite getting close four times since returning to the tour after giving birth to daughter Alexis Olympia. Her preparation for the year's first major couldn't have gone much better, suggesting the mini grand slam drought for arguably tennis' greatest ever player is about to cease. Williams let rip into a punching bag in December -- drawing a compliment from former heavyweight boxing champion Tyson, whose daughter happens to be a budding tennis star -- and then won a buildup tournament in Auckland last week to incredibly land a title in a fourth straight decade. She also mingled with one of tennis' youngest sensations, Coco Gauff, as part of the training camp in Florida organized by her coach, Patrick Mouratoglou. JUST WATCHEDMouratoglou on Serena comeback and McEnroeReplayMore Videos ...MUST WATCHMouratoglou on Serena comeback and McEnroe 02:19Read MoreWilliams overcame singles opponents with differing styles, reached the doubles final with one of her best friends -- the soon-to-be retired Caroline Wozniacki -- and most crucially, ended a five-match losing streak in finals with her daughter and husband Alexis Ohanian looking on. No wonder the 38-year-old said following her straight-set victory over fellow American Jessica Pegula: \"It's pretty satisfying just to get a win in the final. That was really important for me. And I just want to build on it,\" added Williams, who donated her prize money check of $43,000 to bush fire relief efforts in Australia. \"It's just a step towards the next goal.\"Indeed. READ: Can Rafael Nadal match Roger Federer's all-time grand slam record?READ: Player brands Australian Open email a 'slap in the face'Eyes on bigger prizeYes, as nice as it was to be holding the winners' trophy in Auckland -- where Williams once hit 88 unforced errors in a loss to Madison Brengle -- she didn't make the long trip to simply prosper in New Zealand. The much bigger prize is the Australian Open, where Williams triumphed while in the early stages of pregnancy in 2017. If Williams makes the final in Melbourne -- and she might have to defeat the likes of twice grand slam winner Naomi Osaka and current world No. 1 Ashleigh Barty along the way -- she will probably have to then defeat someone with a heftier reputation than the 66th-ranked Pegula. Helping Williams, however, is that one of the main contenders, Bianca Andreescu, isn't in Melbourne because of another injury, this time to a knee. But winning any final -- after losses in grand slam finals to Osaka, Andreescu, Angelique Kerber and Simona Halep and retiring against Andreescu in the Rogers Cup finale last August -- could potentially be turning point as Williams attempts to draw level with the grand slam haul of Australia's Margaret Court. JUST WATCHEDSerena Williams falls short in major title chaseReplayMore Videos ...MUST WATCHSerena Williams falls short in major title chase 01:00\"Serena, she certainly looks hungry, and I think she's got a little momentum going into the Australian Open,\" Chris Evert, the 18-time grand slam champion, told reporters in an ESPN conference call last week. \"And it would probably be the least pressure, this grand slam, to win for her. \"I think every other tournament, the French Open, the clay isn't her best surface. Wimbledon is a lot of pressure, US Open is a lot of pressure. \"This one, the first one of the year, it's a 'Happy Slam,'\" referring to the Australian Open's nickname. \"I think if she just takes a little bit of pressure off herself and she can just play her brand of tennis, I think she's got a good shot at winning it.\"She's better at grand slams than any other player when you look at the last two years.\"The way Wozniacki put it, Williams has a \"big chance\" to match Court. pic.twitter.com/skoZilynH8— Serena Williams (@serenawilliams) January 12, 2020 Other high-profile players, such as Halep and former world No. 1 Karolina Pliskova -- who saved four match points last year in Melbourne against Williams and rallied from 5-1 down in the third set after the American rolled her ankle -- aren't discounting her chances, either, despite just falling short recently at grand slams. \"I'm very impressed about her, that she keeps playing at this level, with being a mother and also being a little bit older than us,\" said Halep, who made a mere three unforced errors in downing Williams in the 2019 Wimbledon final. \"It's impressive what she does.\"Challenges aheadBut Evert also underscored the stiffer challenges facing Williams. \"There are some darned good players out there that I have a lot of respect for, and the way that Osaka, Barty, Pliskova, Halep ... there's not one or two threats to Serena, there's probably about eight threats, eight players that can probably do some damage and that can compete against her.\"And one more thing is -- I always felt this way -- the older you get, I think the more bad days you may have, days when you feel burned out, days when you don't want to get out of bed, days when you don't have incentive. You don't want to have that day during a grand slam, but sometimes you can't help it.\"JUST WATCHEDSimona Halep on winning her first Wimbledon titleReplayMore Videos ...MUST WATCHSimona Halep on winning her first Wimbledon title 01:56Visit our tennis page for more news and videosWilliams figures to still be around towards the end of the Australian Open, but whether it is as the last person standing continues to be the question. \n", + "\n", + "Entity Types:\n", + "[1] «PERSON»\n", + "[2] «ORGANIZATION»\n", + "[3] «LOCATION»\n", + "[4] «DATE»\n", + "[5] «TIME»\n", + "[6] «MONEY»\n", + "[7] «PERCENTAGE»\n", + "[8] «PRODUCT»\n", + "[9] «EVENT»\n", + "[10] «LANGUAGE»\n", + "[11] «NATIONALITY»\n", + "[12] «RELIGION»\n", + "[13] «TITLE»\n", + "[14] «PROFESSION»\n", + "[15] «ANIMAL»\n", + "[16] «PLANT»\n", + "[17] «DISEASE»\n", + "[18] «MEDICATION»\n", + "[19] «CHEMICAL»\n", + "[20] «MATERIAL»\n", + "[21] «COLOR»\n", + "[22] «SHAPE»\n", + "[23] «MEASUREMENT»\n", + "[24] «WEATHER»\n", + "[25] «NATURAL_DISASTER»\n", + "[26] «AWARD»\n", + "[27] «LAW»\n", + "[28] «CRIME»\n", + "[29] «TECHNOLOGY»\n", + "[30] «SOFTWARE»\n", + "[31] «HARDWARE»\n", + "[32] «VEHICLE»\n", + "[33] «FOOD»\n", + "[34] «DRINK»\n", + "[35] «SPORT»\n", + "[36] «MUSIC_GENRE»\n", + "[37] «INSTRUMENT»\n", + "[38] «ARTWORK»\n", + "[39] «BOOK»\n", + "[40] «MOVIE»\n", + "[41] «TV_SHOW»\n", + "[42] «ACADEMIC_SUBJECT»\n", + "[43] «SCIENTIFIC_THEORY»\n", + "[44] «POLITICAL_PARTY»\n", + "[45] «CURRENCY»\n", + "[46] «STOCK_SYMBOL»\n", + "[47] «FILE_TYPE»\n", + "[48] «PROGRAMMING_LANGUAGE»\n", + "[49] «MEDICAL_PROCEDURE»\n", + "[50] «CELESTIAL_BODY»\n", + "\n", + "Please provide the output fields Reasoning then Entities Relationships. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\n", + "\n", + "\u001b[32mReasoning: Let's think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\n", + "\n", + "Entities Relationships:\n", + "```json\n", + "{\n", + " \"value\": [\n", + " {\n", + " \"entity_name\": \"Melbourne\",\n", + " \"entity_type\": \"LOCATION\",\n", + " \"description\": \"A city in Australia where the Australian Open is held.\",\n", + " \"importance_score\": 0.8\n", + " },\n", + " {\n", + " \"entity_name\": \"Australia\",\n", + " \"entity_type\": \"LOCATION\",\n", + " \"description\": \"The country where Melbourne is located and where the Australian Open takes place.\",\n", + " \"importance_score\": 0.7\n", + " },\n", + " {\n", + " \"entity_name\": \"Serena Williams\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A professional tennis player aiming to win the Australian Open.\",\n", + " \"importance_score\": 1.0\n", + " },\n", + " {\n", + " \"entity_name\": \"Mike Tyson\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A former heavyweight boxing champion who trained with Serena Williams.\",\n", + " \"importance_score\": 0.6\n", + " },\n", + " {\n", + " \"entity_name\": \"Australian Open\",\n", + " \"entity_type\": \"EVENT\",\n", + " \"description\": \"A major tennis tournament held annually in Melbourne, Australia.\",\n", + " \"importance_score\": 0.9\n", + " },\n", + " {\n", + " \"entity_name\": \"Alexis Olympia\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"The daughter of Serena Williams.\",\n", + " \"importance_score\": 0.5\n", + " },\n", + " {\n", + " \"entity_name\": \"Caroline Wozniacki\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A professional tennis player and friend of Serena Williams.\",\n", + " \"importance_score\": 0.6\n", + " },\n", + " {\n", + " \"entity_name\": \"Coco Gauff\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A young tennis sensation who trained with Serena Williams.\",\n", + " \"importance_score\": 0.5\n", + " },\n", + " {\n", + " \"entity_name\": \"Patrick Mouratoglou\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"The coach of Serena Williams who organized the training camp.\",\n", + " \"importance_score\": 0.6\n", + " },\n", + " {\n", + " \"entity_name\": \"Naomi Osaka\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A professional tennis player and grand slam winner who could face Serena Williams.\",\n", + " \"importance_score\": 0.7\n", + " },\n", + " {\n", + " \"entity_name\": \"Ashleigh Barty\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"The current world No. 1 tennis player who could face Serena Williams.\",\n", + " \"importance_score\": 0.7\n", + " },\n", + " {\n", + " \"entity_name\": \"Bianca Andreescu\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A professional tennis player who is a main contender but is not in Melbourne due to injury.\",\n", + " \"importance_score\": 0.6\n", + " },\n", + " {\n", + " \"entity_name\": \"Simona Halep\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A professional tennis player who defeated Serena Williams in the 2019 Wimbledon final.\",\n", + " \"importance_score\": 0.6\n", + " },\n", + " {\n", + " \"entity_name\": \"Chris Evert\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"An 18-time grand slam champion who commented on Serena Williams' chances at the Australian Open.\",\n", + " \"importance_score\": 0.5\n", + " },\n", + " {\n", + " \"entity_name\": \"Margaret Court\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"An Australian tennis player with a record-tying 24 grand slam titles.\",\n", + " \"importance_score\": 0.5\n", + " },\n", + " {\n", + " \"entity_name\": \"Jessica Pegula\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A fellow American tennis player who Serena Williams defeated in a match.\",\n", + " \"importance_score\": 0.4\n", + " },\n", + " {\n", + " \"entity_name\": \"Alexis Ohanian\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"The husband of Serena Williams.\",\n", + " \"importance_score\": 0.4\n", + " },\n", + " {\n", + " \"entity_name\": \"Karolina Pliskova\",\n", + " \"entity_type\": \"PERSON\",\n", + " \"description\": \"A former world No. 1 tennis player who saved match points against Serena Williams in Melbourne.\",\n", + " \"importance_score\": 0.5\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Mike Tyson\",\n", + " \"description\": \"Serena Williams trained with Mike Tyson during the off-season.\",\n", + " \"weight\": 0.7,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Australian Open\",\n", + " \"description\": \"Serena Williams is aiming to win the Australian Open.\",\n", + " \"weight\": 0.9,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Alexis Olympia\",\n", + " \"description\": \"Serena Williams is the mother of Alexis Olympia.\",\n", + " \"weight\": 0.6,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Caroline Wozniacki\",\n", + " \"description\": \"Serena Williams reached the doubles final with Caroline Wozniacki.\",\n", + " \"weight\": 0.6,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Coco Gauff\",\n", + " \"description\": \"Serena Williams trained with Coco Gauff.\",\n", + " \"weight\": 0.5,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Patrick Mouratoglou\",\n", + " \"description\": \"Patrick Mouratoglou is the coach of Serena Williams.\",\n", + " \"weight\": 0.6,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Naomi Osaka\",\n", + " \"description\": \"Serena Williams might have to defeat Naomi Osaka to reach the final of the Australian Open.\",\n", + " \"weight\": 0.7,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Ashleigh Barty\",\n", + " \"description\": \"Serena Williams might have to defeat Ashleigh Barty to reach the final of the Australian Open.\",\n", + " \"weight\": 0.7,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Bianca Andreescu\",\n", + " \"description\": \"Bianca Andreescu is a main contender but is not in Melbourne due to injury.\",\n", + " \"weight\": 0.6,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Simona Halep\",\n", + " \"description\": \"Simona Halep defeated Serena Williams in the 2019 Wimbledon final.\",\n", + " \"weight\": 0.6,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Chris Evert\",\n", + " \"description\": \"Chris Evert commented on Serena Williams' chances at the Australian Open.\",\n", + " \"weight\": 0.5,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Margaret Court\",\n", + " \"description\": \"Serena Williams is attempting to match Margaret Court's record of 24 grand slam titles.\",\n", + " \"weight\": 0.5,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Jessica Pegula\",\n", + " \"description\": \"Serena Williams defeated Jessica Pegula in a match.\",\n", + " \"weight\": 0.4,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Alexis Ohanian\",\n", + " \"description\": \"Alexis Ohanian is the husband of Serena Williams.\",\n", + " \"weight\": 0.4,\n", + " \"order\": 1\n", + " },\n", + " {\n", + " \"src_id\": \"Serena Williams\",\n", + " \"tgt_id\": \"Karolina Pliskova\",\n", + " \"description\": \"Karolina Pliskova saved match points against Serena Williams in Melbourne.\",\n", + " \"weight\": 0.5,\n", + " \"order\": 1\n", + " }\n", + " ]\n", + "}\n", + "```\u001b[0m\n", + "\n", + "\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "'\\n\\n\\nGiven the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\\n\\n---\\n\\nFollow the following format.\\n\\nInput Text: The text to extract entities and relationships from.\\n\\nEntity Types: List of entity types used for extraction.\\n\\nReasoning: Let\\'s think step by step in order to ${produce the entities_relationships}. We ...\\n\\nEntities Relationships: List of entities and relationships extracted from the text.. Respond with a single JSON object. JSON Schema: {\"$defs\": {\"Entity\": {\"properties\": {\"entity_name\": {\"description\": \"The name of the entity.\", \"title\": \"Entity Name\", \"type\": \"string\"}, \"entity_type\": {\"description\": \"The type of the entity.\", \"title\": \"Entity Type\", \"type\": \"string\"}, \"description\": {\"description\": \"The description of the entity, in details and comprehensive.\", \"title\": \"Description\", \"type\": \"string\"}, \"importance_score\": {\"description\": \"Importance score of the entity. Should be between 0 and 1 with 1 being the most important.\", \"maximum\": 1.0, \"minimum\": 0.0, \"title\": \"Importance Score\", \"type\": \"number\"}}, \"required\": [\"entity_name\", \"entity_type\", \"description\", \"importance_score\"], \"title\": \"Entity\", \"type\": \"object\"}, \"Relationship\": {\"properties\": {\"src_id\": {\"description\": \"The name of the source entity.\", \"title\": \"Src Id\", \"type\": \"string\"}, \"tgt_id\": {\"description\": \"The name of the target entity.\", \"title\": \"Tgt Id\", \"type\": \"string\"}, \"description\": {\"description\": \"The description of the relationship between the source and target entity, in details and comprehensive.\", \"title\": \"Description\", \"type\": \"string\"}, \"weight\": {\"description\": \"The weight of the relationship. Should be between 0 and 1 with 1 being the strongest relationship.\", \"maximum\": 1.0, \"minimum\": 0.0, \"title\": \"Weight\", \"type\": \"number\"}, \"order\": {\"description\": \"The order of the relationship. 1 for direct relationships, 2 for second-order, 3 for third-order.\", \"maximum\": 3, \"minimum\": 1, \"title\": \"Order\", \"type\": \"integer\"}}, \"required\": [\"src_id\", \"tgt_id\", \"description\", \"weight\", \"order\"], \"title\": \"Relationship\", \"type\": \"object\"}}, \"properties\": {\"value\": {\"items\": {\"anyOf\": [{\"$ref\": \"#/$defs/Entity\"}, {\"$ref\": \"#/$defs/Relationship\"}]}, \"title\": \"Value\", \"type\": \"array\"}}, \"required\": [\"value\"], \"title\": \"Output\", \"type\": \"object\"}\\n\\n---\\n\\nInput Text:\\nMelbourne, Australia (CNN)After spending part of the off-season training with Mike Tyson, Serena Williams is hoping to deliver a knockout punch at the Australian Open. Follow @cnnsport\\n\\nFor Williams that would mean winning a record-tying 24th grand slam title, which has so far proved elusive despite getting close four times since returning to the tour after giving birth to daughter Alexis Olympia. Her preparation for the year\\'s first major couldn\\'t have gone much better, suggesting the mini grand slam drought for arguably tennis\\' greatest ever player is about to cease. Williams let rip into a punching bag in December -- drawing a compliment from former heavyweight boxing champion Tyson, whose daughter happens to be a budding tennis star -- and then won a buildup tournament in Auckland last week to incredibly land a title in a fourth straight decade. She also mingled with one of tennis\\' youngest sensations, Coco Gauff, as part of the training camp in Florida organized by her coach, Patrick Mouratoglou. JUST WATCHEDMouratoglou on Serena comeback and McEnroeReplayMore Videos ...MUST WATCHMouratoglou on Serena comeback and McEnroe 02:19Read MoreWilliams overcame singles opponents with differing styles, reached the doubles final with one of her best friends -- the soon-to-be retired Caroline Wozniacki -- and most crucially, ended a five-match losing streak in finals with her daughter and husband Alexis Ohanian looking on. No wonder the 38-year-old said following her straight-set victory over fellow American Jessica Pegula: \"It\\'s pretty satisfying just to get a win in the final. That was really important for me. And I just want to build on it,\" added Williams, who donated her prize money check of $43,000 to bush fire relief efforts in Australia. \"It\\'s just a step towards the next goal.\"Indeed. READ: Can Rafael Nadal match Roger Federer\\'s all-time grand slam record?READ: Player brands Australian Open email a \\'slap in the face\\'Eyes on bigger prizeYes, as nice as it was to be holding the winners\\' trophy in Auckland -- where Williams once hit 88 unforced errors in a loss to Madison Brengle -- she didn\\'t make the long trip to simply prosper in New Zealand. The much bigger prize is the Australian Open, where Williams triumphed while in the early stages of pregnancy in 2017. If Williams makes the final in Melbourne -- and she might have to defeat the likes of twice grand slam winner Naomi Osaka and current world No. 1 Ashleigh Barty along the way -- she will probably have to then defeat someone with a heftier reputation than the 66th-ranked Pegula. Helping Williams, however, is that one of the main contenders, Bianca Andreescu, isn\\'t in Melbourne because of another injury, this time to a knee. But winning any final -- after losses in grand slam finals to Osaka, Andreescu, Angelique Kerber and Simona Halep and retiring against Andreescu in the Rogers Cup finale last August -- could potentially be turning point as Williams attempts to draw level with the grand slam haul of Australia\\'s Margaret Court. JUST WATCHEDSerena Williams falls short in major title chaseReplayMore Videos ...MUST WATCHSerena Williams falls short in major title chase 01:00\"Serena, she certainly looks hungry, and I think she\\'s got a little momentum going into the Australian Open,\" Chris Evert, the 18-time grand slam champion, told reporters in an ESPN conference call last week. \"And it would probably be the least pressure, this grand slam, to win for her. \"I think every other tournament, the French Open, the clay isn\\'t her best surface. Wimbledon is a lot of pressure, US Open is a lot of pressure. \"This one, the first one of the year, it\\'s a \\'Happy Slam,\\'\" referring to the Australian Open\\'s nickname. \"I think if she just takes a little bit of pressure off herself and she can just play her brand of tennis, I think she\\'s got a good shot at winning it.\"She\\'s better at grand slams than any other player when you look at the last two years.\"The way Wozniacki put it, Williams has a \"big chance\" to match Court. pic.twitter.com/skoZilynH8— Serena Williams (@serenawilliams) January 12, 2020 Other high-profile players, such as Halep and former world No. 1 Karolina Pliskova -- who saved four match points last year in Melbourne against Williams and rallied from 5-1 down in the third set after the American rolled her ankle -- aren\\'t discounting her chances, either, despite just falling short recently at grand slams. \"I\\'m very impressed about her, that she keeps playing at this level, with being a mother and also being a little bit older than us,\" said Halep, who made a mere three unforced errors in downing Williams in the 2019 Wimbledon final. \"It\\'s impressive what she does.\"Challenges aheadBut Evert also underscored the stiffer challenges facing Williams. \"There are some darned good players out there that I have a lot of respect for, and the way that Osaka, Barty, Pliskova, Halep ... there\\'s not one or two threats to Serena, there\\'s probably about eight threats, eight players that can probably do some damage and that can compete against her.\"And one more thing is -- I always felt this way -- the older you get, I think the more bad days you may have, days when you feel burned out, days when you don\\'t want to get out of bed, days when you don\\'t have incentive. You don\\'t want to have that day during a grand slam, but sometimes you can\\'t help it.\"JUST WATCHEDSimona Halep on winning her first Wimbledon titleReplayMore Videos ...MUST WATCHSimona Halep on winning her first Wimbledon title 01:56Visit our tennis page for more news and videosWilliams figures to still be around towards the end of the Australian Open, but whether it is as the last person standing continues to be the question. \\n\\nEntity Types:\\n[1] «PERSON»\\n[2] «ORGANIZATION»\\n[3] «LOCATION»\\n[4] «DATE»\\n[5] «TIME»\\n[6] «MONEY»\\n[7] «PERCENTAGE»\\n[8] «PRODUCT»\\n[9] «EVENT»\\n[10] «LANGUAGE»\\n[11] «NATIONALITY»\\n[12] «RELIGION»\\n[13] «TITLE»\\n[14] «PROFESSION»\\n[15] «ANIMAL»\\n[16] «PLANT»\\n[17] «DISEASE»\\n[18] «MEDICATION»\\n[19] «CHEMICAL»\\n[20] «MATERIAL»\\n[21] «COLOR»\\n[22] «SHAPE»\\n[23] «MEASUREMENT»\\n[24] «WEATHER»\\n[25] «NATURAL_DISASTER»\\n[26] «AWARD»\\n[27] «LAW»\\n[28] «CRIME»\\n[29] «TECHNOLOGY»\\n[30] «SOFTWARE»\\n[31] «HARDWARE»\\n[32] «VEHICLE»\\n[33] «FOOD»\\n[34] «DRINK»\\n[35] «SPORT»\\n[36] «MUSIC_GENRE»\\n[37] «INSTRUMENT»\\n[38] «ARTWORK»\\n[39] «BOOK»\\n[40] «MOVIE»\\n[41] «TV_SHOW»\\n[42] «ACADEMIC_SUBJECT»\\n[43] «SCIENTIFIC_THEORY»\\n[44] «POLITICAL_PARTY»\\n[45] «CURRENCY»\\n[46] «STOCK_SYMBOL»\\n[47] «FILE_TYPE»\\n[48] «PROGRAMMING_LANGUAGE»\\n[49] «MEDICAL_PROCEDURE»\\n[50] «CELESTIAL_BODY»\\n\\nPlease provide the output fields Reasoning then Entities Relationships. Do so immediately, without additional content before or after, and precisely as the format above shows. Begin with the field Reasoning.\\n\\n\\x1b[32mReasoning: Let\\'s think step by step in order to produce the entities_relationships. We first identify the entities in the text based on the provided entity types. Then, we determine the relationships between these entities, considering their interactions and dependencies within the context of the text. Finally, we format the entities and relationships according to the specified JSON schema.\\n\\nEntities Relationships:\\n```json\\n{\\n \"value\": [\\n {\\n \"entity_name\": \"Melbourne\",\\n \"entity_type\": \"LOCATION\",\\n \"description\": \"A city in Australia where the Australian Open is held.\",\\n \"importance_score\": 0.8\\n },\\n {\\n \"entity_name\": \"Australia\",\\n \"entity_type\": \"LOCATION\",\\n \"description\": \"The country where Melbourne is located and where the Australian Open takes place.\",\\n \"importance_score\": 0.7\\n },\\n {\\n \"entity_name\": \"Serena Williams\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A professional tennis player aiming to win the Australian Open.\",\\n \"importance_score\": 1.0\\n },\\n {\\n \"entity_name\": \"Mike Tyson\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A former heavyweight boxing champion who trained with Serena Williams.\",\\n \"importance_score\": 0.6\\n },\\n {\\n \"entity_name\": \"Australian Open\",\\n \"entity_type\": \"EVENT\",\\n \"description\": \"A major tennis tournament held annually in Melbourne, Australia.\",\\n \"importance_score\": 0.9\\n },\\n {\\n \"entity_name\": \"Alexis Olympia\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"The daughter of Serena Williams.\",\\n \"importance_score\": 0.5\\n },\\n {\\n \"entity_name\": \"Caroline Wozniacki\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A professional tennis player and friend of Serena Williams.\",\\n \"importance_score\": 0.6\\n },\\n {\\n \"entity_name\": \"Coco Gauff\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A young tennis sensation who trained with Serena Williams.\",\\n \"importance_score\": 0.5\\n },\\n {\\n \"entity_name\": \"Patrick Mouratoglou\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"The coach of Serena Williams who organized the training camp.\",\\n \"importance_score\": 0.6\\n },\\n {\\n \"entity_name\": \"Naomi Osaka\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A professional tennis player and grand slam winner who could face Serena Williams.\",\\n \"importance_score\": 0.7\\n },\\n {\\n \"entity_name\": \"Ashleigh Barty\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"The current world No. 1 tennis player who could face Serena Williams.\",\\n \"importance_score\": 0.7\\n },\\n {\\n \"entity_name\": \"Bianca Andreescu\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A professional tennis player who is a main contender but is not in Melbourne due to injury.\",\\n \"importance_score\": 0.6\\n },\\n {\\n \"entity_name\": \"Simona Halep\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A professional tennis player who defeated Serena Williams in the 2019 Wimbledon final.\",\\n \"importance_score\": 0.6\\n },\\n {\\n \"entity_name\": \"Chris Evert\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"An 18-time grand slam champion who commented on Serena Williams\\' chances at the Australian Open.\",\\n \"importance_score\": 0.5\\n },\\n {\\n \"entity_name\": \"Margaret Court\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"An Australian tennis player with a record-tying 24 grand slam titles.\",\\n \"importance_score\": 0.5\\n },\\n {\\n \"entity_name\": \"Jessica Pegula\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A fellow American tennis player who Serena Williams defeated in a match.\",\\n \"importance_score\": 0.4\\n },\\n {\\n \"entity_name\": \"Alexis Ohanian\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"The husband of Serena Williams.\",\\n \"importance_score\": 0.4\\n },\\n {\\n \"entity_name\": \"Karolina Pliskova\",\\n \"entity_type\": \"PERSON\",\\n \"description\": \"A former world No. 1 tennis player who saved match points against Serena Williams in Melbourne.\",\\n \"importance_score\": 0.5\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Mike Tyson\",\\n \"description\": \"Serena Williams trained with Mike Tyson during the off-season.\",\\n \"weight\": 0.7,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Australian Open\",\\n \"description\": \"Serena Williams is aiming to win the Australian Open.\",\\n \"weight\": 0.9,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Alexis Olympia\",\\n \"description\": \"Serena Williams is the mother of Alexis Olympia.\",\\n \"weight\": 0.6,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Caroline Wozniacki\",\\n \"description\": \"Serena Williams reached the doubles final with Caroline Wozniacki.\",\\n \"weight\": 0.6,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Coco Gauff\",\\n \"description\": \"Serena Williams trained with Coco Gauff.\",\\n \"weight\": 0.5,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Patrick Mouratoglou\",\\n \"description\": \"Patrick Mouratoglou is the coach of Serena Williams.\",\\n \"weight\": 0.6,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Naomi Osaka\",\\n \"description\": \"Serena Williams might have to defeat Naomi Osaka to reach the final of the Australian Open.\",\\n \"weight\": 0.7,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Ashleigh Barty\",\\n \"description\": \"Serena Williams might have to defeat Ashleigh Barty to reach the final of the Australian Open.\",\\n \"weight\": 0.7,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Bianca Andreescu\",\\n \"description\": \"Bianca Andreescu is a main contender but is not in Melbourne due to injury.\",\\n \"weight\": 0.6,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Simona Halep\",\\n \"description\": \"Simona Halep defeated Serena Williams in the 2019 Wimbledon final.\",\\n \"weight\": 0.6,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Chris Evert\",\\n \"description\": \"Chris Evert commented on Serena Williams\\' chances at the Australian Open.\",\\n \"weight\": 0.5,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Margaret Court\",\\n \"description\": \"Serena Williams is attempting to match Margaret Court\\'s record of 24 grand slam titles.\",\\n \"weight\": 0.5,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Jessica Pegula\",\\n \"description\": \"Serena Williams defeated Jessica Pegula in a match.\",\\n \"weight\": 0.4,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Alexis Ohanian\",\\n \"description\": \"Alexis Ohanian is the husband of Serena Williams.\",\\n \"weight\": 0.4,\\n \"order\": 1\\n },\\n {\\n \"src_id\": \"Serena Williams\",\\n \"tgt_id\": \"Karolina Pliskova\",\\n \"description\": \"Karolina Pliskova saved match points against Serena Williams in Melbourne.\",\\n \"weight\": 0.5,\\n \"order\": 1\\n }\\n ]\\n}\\n```\\x1b[0m\\n\\n\\n'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "deepseek.inspect_history(n=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcYAAAE8CAYAAABaaxFWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABJl0lEQVR4nO3deVhUZf8/8PeAMAzLoCKrgiCIoqIWLrmgqciiIj7uWom5tWiKmus3FNw1M9RMq8ew9AF3fTIXAst9LUMtzdRUXHDJRxkQgWHm/P7wxzQj6wzDLPB+XRdXzTn3ued9jsN8OPfZRIIgCCAiIiIAgIWxAxAREZkSFkYiIiI1LIxERERqWBiJiIjUsDASERGpYWEkIiJSw8JIRESkhoWRiIhIDQsjERGRGhZGIjOxYcMGiEQi/Pzzz8aOQlStsTASlaCoCKn/uLi4oFu3bti/f7/O/S5atAi7d+/WX1AtHTt2DBEREahfvz5sbGzg5eWFyMhIJCUlGS0TkalhYSQqw7x587Bx40Z8++23mD59Oh49eoRevXrh+++/16k/YxbGbdu2oUuXLnjw4AEmTZqE1atX480338STJ0/w1VdfGSUTkSmqZewARKYsIiICbdq0Ub0ePXo0XF1dkZycjD59+hgxmfbi4uLQrFkznDp1CtbW1hrzHj58aLAcgiAgLy8PEonEYO9JpA3uMRJpoXbt2pBIJKhVS/NvyuXLl6Njx45wcnKCRCJBUFAQtm/frtFGJBLh2bNn+Oabb1TDsyNHjlTNv3v3LkaPHg0PDw+IxWL4+PjgvffeQ0FBgUY/+fn5mDJlCpydnWFnZ4d//etfePToUbnZr1+/jrZt2xYrigDg4uKi8VqpVGLlypUIDAyEjY0NnJ2dER4ernF8s7CwEPPnz4evry/EYjG8vb0xe/Zs5Ofna/Tl7e2NPn36ICUlBW3atIFEIsEXX3wBAHj69CliYmLg6ekJsVgMPz8/LF26FEqlUqOPzZs3IygoCA4ODpBKpQgMDMTKlSvLXWciXXCPkagMWVlZ+PvvvyEIAh4+fIjVq1cjJycHb775pka7lStXom/fvnjjjTdQUFCAzZs3Y9CgQfj+++/Ru3dvAMDGjRsxZswYtGvXDuPGjQMA+Pr6AgDu3buHdu3a4enTpxg3bhyaNm2Ku3fvYvv27cjNzdUoZh988AHq1KmDuXPn4ubNm0hISMCECROwZcuWMtelYcOGOHjwIO7cuYMGDRqU2Xb06NHYsGEDIiIiMGbMGBQWFuLo0aM4deqUag96zJgx+OabbzBw4EBMnToVp0+fxuLFi3H58mXs2rVLo78rV65g2LBheOeddzB27Fg0adIEubm56Nq1K+7evYt33nkHXl5eOHHiBGbNmoXMzEwkJCQAAFJTUzFs2DD06NEDS5cuBQBcvnwZx48fx6RJk8pcDyKdCERUTGJiogCg2I9YLBY2bNhQrH1ubq7G64KCAqFFixZC9+7dNabb2dkJ0dHRxZYfMWKEYGFhIZw9e7bYPKVSqZEpJCRENU0QBGHy5MmCpaWl8PTp0zLXaf369QIAwdraWujWrZsQGxsrHD16VFAoFBrtfvzxRwGAMHHixFKzpKenCwCEMWPGaMz/8MMPBQDCjz/+qJrWsGFDAYBw4MABjbbz588X7OzshD///FNj+syZMwVLS0shIyNDEARBmDRpkiCVSoXCwsIy149IXziUSlSGNWvWIDU1Fampqdi0aRO6deuGMWPGYOfOnRrt1I+XPXnyBFlZWQgODsa5c+fKfQ+lUondu3cjMjJS43hmEZFIpPF63LhxGtOCg4OhUChw69atMt9n1KhROHDgAF5//XUcO3YM8+fPR3BwMBo3bowTJ06o2u3YsQMikQhz584tNcu+ffsAAFOmTNGYP3XqVADA3r17Nab7+PggLCxMY9q2bdsQHByMOnXq4O+//1b9hISEQKFQ4MiRIwBeDF8/e/YMqampZa4fkb5wKJWoDO3atdMoVsOGDcMrr7yCCRMmoE+fPqohzu+//x4LFixAenq6xjG2l4taSR49egSZTIYWLVpUKJOXl5fG6zp16gB4UZDLExYWhrCwMOTm5uKXX37Bli1bsG7dOvTp0wd//PEHXFxccP36dXh4eKBu3bql9nPr1i1YWFjAz89PY7qbmxtq165drEj7+PgU6+Pq1au4cOECnJ2dS3yPohOC3n//fWzdulV1mUloaCgGDx6M8PDwcteXSBcsjERasLCwQLdu3bBy5UpcvXoVzZs3x9GjR9G3b1906dIFn3/+Odzd3WFlZYXExMQquT7Q0tKyxOmCIFS4D1tbWwQHByM4OBj16tVDfHw89u/fj+joaK2yVKTwAyjxDFSlUomePXti+vTpJS7j7+8P4MWJQenp6UhJScH+/fuxf/9+JCYmYsSIEfjmm2+0yktUESyMRFoqLCwEAOTk5AB4MfRoY2ODlJQUiMViVbvExMRiy5ZUSJydnSGVSvHbb79VUeKyFe0RZ2ZmAnhxQlBKSgr+97//lbrX2LBhQyiVSly9ehUBAQGq6Q8ePMDTp0/RsGHDct/X19cXOTk5CAkJKbettbU1IiMjERkZCaVSiffffx9ffPEFYmNji+21ElUWjzESaUEul+OHH36AtbW1qiBYWlpCJBJBoVCo2t28ebPEC/nt7Ozw9OlTjWkWFhbo168f9uzZU+Lt3rTZEyzLwYMHS5xedLywSZMmAIABAwZAEATEx8eXmqVXr14AoDpztMiKFSsAQHUmblkGDx6MkydPIiUlpdi8p0+fqv4Aefz4scY8CwsLtGzZEgCKXRpCpA/cYyQqw/79+/HHH38AeHHMKykpCVevXsXMmTMhlUoBvCgCK1asQHh4OIYPH46HDx9izZo18PPzw4ULFzT6CwoKQlpaGlasWAEPDw/4+Pigffv2WLRoEX744Qd07doV48aNQ0BAADIzM7Ft2zYcO3YMtWvXrvS6REVFwcfHB5GRkfD19cWzZ8+QlpaGPXv2oG3btoiMjAQAdOvWDW+99RZWrVqFq1evIjw8HEqlEkePHkW3bt0wYcIEtGrVCtHR0fjyyy/x9OlTdO3aFWfOnME333yDfv36oVu3buXmmTZtGr777jv06dMHI0eORFBQEJ49e4aLFy9i+/btuHnzJurVq4cxY8bgf//7H7p3744GDRrg1q1bWL16NVq3bq2xt0qkN8Y9KZbINJV0uYaNjY3QunVrYe3atRqXSwjCi0shGjduLIjFYqFp06ZCYmKiMHfuXOHlX7E//vhD6NKliyCRSAQAGpdu3Lp1SxgxYoTg7OwsiMVioVGjRsL48eOF/Px8jUwvX9Lx008/CQCEn376qcx1Sk5OFoYOHSr4+voKEolEsLGxEZo1ayb83//9nyCTyTTaFhYWCh9//LHQtGlTwdraWnB2dhYiIiKEX375RdVGLpcL8fHxgo+Pj2BlZSV4enoKs2bNEvLy8jT6atiwodC7d+8SM2VnZwuzZs0S/Pz8BGtra6FevXpCx44dheXLlwsFBQWCIAjC9u3bhdDQUMHFxUWwtrYWvLy8hHfeeUfIzMwsc32JdCUSBD2N0xAREVUDPMZIRESkhoWRiIhIDQsjERGRGhZGIiIiNSyMREREalgYiYiI1FT7C/yVSiXu3bsHBweHCt/XkYiIqhdBEJCdnQ0PDw9YWJS9T1jtC+O9e/fg6elp7BhERGQCbt++Xe6Duqt9YXRwcADwYmMU3cJLF0X3yAwNDYWVlZW+4lU55jY8c83O3IZnrtnNMbdMJoOnp6eqJpSl2hfGouFTqVRa6cJoa2sLqVRqNh8EgLmNwVyzM7fhmWt2c80NVOxRaTz5hoiISA0LIxERkZpqP5RKRESGpVAqcDTjKDKzM+Hu4I5gr2BYWlgaO1aFsTASEZHe7Ly8E5MOTMId2R3VtAbSBlgZvhL9A/obMVnFcSiViIj0YuflnRi4daBGUQSAu7K7GLh1IHZe3mmkZNphYSQiokpTKBWYdGASBBR/xG/RtJgDMVAoFYaOpjWjFsbs7GzExMSgYcOGkEgk6NixI86ePauaLwgC5syZA3d3d0gkEoSEhODq1atGTExERCU5mnG02J6iOgECbstu42jGUQOm0o1RC+OYMWOQmpqKjRs34uLFiwgNDUVISAju3r0LAFi2bBlWrVqFdevW4fTp07Czs0NYWBjy8vKMGdskKJQKHLp5CMkXk3Ho5iGz+CuMiPTPVL4LMrMz9drOmIx28s3z58+xY8cO/Pe//0WXLl0AAHFxcdizZw/Wrl2L+fPnIyEhAR999BGioqIAAN9++y1cXV2xe/duDB061FjRja46HNwmosozpe8Cdwd3vbYzJqMVxsLCQigUCtjY2GhMl0gkOHbsGG7cuIH79+8jJCRENc/R0RHt27fHyZMnSy2M+fn5yM/PV72WyWQAXtypQS6X65y3aNnK9KEPe67swVu73oIAARILiWr6/3L+h7e2vwX8C4hsEqmabiq5tWWuuQHzzc7chleZ7Np+F+hTSblfc38Nfo5+uJd9r8TjjCKIUN+hPl5zf80o/1bavKdIEITia2AgHTt2hLW1NZKSkuDq6ork5GRER0fDz88PiYmJ6NSpE+7duwd393/+whg8eDBEIhG2bNlSYp9xcXGIj48vNj0pKQm2trZVti5ERGS6cnNzMXz4cGRlZZV7e1CjXse4ceNGjBo1CvXr14elpSVeffVVDBs2DL/88ovOfc6aNQtTpkxRvS66cWxoaGil75WampqKnj17Gu3egMcyjqF3Uu9y2+0dvhedvToDMI3cujDX3ID5Zmduw9M1uy7fBfpUVu49V/ZgRtoM3M2+q5rWwKEBloQsqbI92IooGj2sCKMWRl9fXxw+fBjPnj2DTCaDu7s7hgwZgkaNGsHNzQ0A8ODBA409xgcPHqB169al9ikWiyEWi4tNt7Ky0ssvjb760cX93Pt4rnxeoXYvZzRm7sow19yA+WZnbsPTNntlvgv0qaTc/Vv0R1SzKJO7840228Ek7nxjZ2cHOzs7PHnyBCkpKVi2bBl8fHzg5uaGgwcPqgqhTCbD6dOn8d577xk3sJFUp4PbRKQ7U/8usLSwxOverxvlvfXBqIUxJSUFgiCgSZMmuHbtGqZNm4amTZvi7bffhkgkQkxMDBYsWIDGjRvDx8cHsbGx8PDwQL9+/YwZ22iCvYLRQNoAd2V3Sz243UDaAMFewUZIR0SGwu+CqmXU6xizsrIwfvx4NG3aFCNGjEDnzp2RkpKi2uWdPn06PvjgA4wbNw5t27ZFTk4ODhw4UOxM1prC0sISK8NXAnjxwVdX9DohPMHoQxZEVLX4XVC1jFoYBw8ejOvXryM/Px+ZmZn47LPP4OjoqJovEokwb9483L9/H3l5eUhLS4O/v78RExtf/4D+2D54O+pL62tMbyBtgO2Dt/M6RqIagt8FVcckjjGSdvoH9EdUE9M7uE1EhsXvgqrBwmimzP3gNhHpB78L9I+FkYiIAJj/A4b1hYWRiIhM6r6rxsbnMRIR1XDV5QHD+sLCSERUg1WnBwzrCwsjEVENVp0eMKwvLIxERDVYdXrAsL6wMBIR1WCmft9VY2BhJCKqwYruu/ryreWKiCCCp9SzRt13lYWRiKgG431Xi2NhJDIwhVKBQzcPIfliMg7dPFSjzvYj08T7rmriBf5EBsSLqMlU8b6r/2BhJDKQoouoX75erOgi6pr4lzmZFt539QUOpRIZAC+iJjIfLIxEBlBTLqLm8VOqDjiUSmQANeEiah4/peqCe4xEBlDdL6LmTaipOmFhJDKA6nwRtTkcP+UQL2mDhZHIAKrzRdSmfvx05+Wd8F7pjW7fdMPwncPR7Ztu8F7pzb1YKhULI1EF6GOPw5QvolYoFTiWcQwAcCzjmFbrZ8rHTznES7rgyTdE5dDnSSWmeBF10fo9znmM5JbJ6J3UG072ThVeP1M9flreEK8IIsQciEFUkyiz3FOnqmPUPUaFQoHY2Fj4+PhAIpHA19cX8+fPhyD880EeOXIkRCKRxk94eLgRU1NNUhV7HEUXUQ8LHIbXvV83elGs7PqZ6vFTUx/iJdNl1MK4dOlSrF27Fp999hkuX76MpUuXYtmyZVi9erVGu/DwcGRmZqp+kpOTjZSYahJzOKmkMvS1fqZ6/NSUh3jJtBm1MJ44cQJRUVHo3bs3vL29MXDgQISGhuLMmTMa7cRiMdzc3FQ/derUMVLiyuGZcealuu9x6HP9TPH4qakO8ZLpM+oxxo4dO+LLL7/En3/+CX9/f5w/fx7Hjh3DihUrNNodOnQILi4uqFOnDrp3744FCxbAycmpxD7z8/ORn5+vei2TyQAAcrkccrlc56xFy+rax54rezAjbQbuZt9VTavvUB9LQ5YiskmkzrnKU9ncxmIKuTOzMiGxkFSonXpOU8heES+vX9H/v7zOL69faSL9ItHr/V44eeck7ufch5u9Gzo06ABLC8sq3Ralbe/X3F+Dn6Mf7mXfK3GvWAQR6jvUx2vurxnt38pcPisvM8fc2mQVCeoH9AxMqVRi9uzZWLZsGSwtLaFQKLBw4ULMmjVL1Wbz5s2wtbWFj48Prl+/jtmzZ8Pe3h4nT56EpWXxoZm4uDjEx8cXm56UlARbW9sqXR8iIjJNubm5GD58OLKysiCVSstsa9TCuHnzZkybNg0ff/wxmjdvjvT0dMTExGDFihWIjo4ucZm//voLvr6+SEtLQ48ePYrNL2mP0dPTE3///Xe5G6Mscrkcqamp6NmzJ6ysrCq8nEKpQODaQI09RXVFf7VeeO9ClRyD0TW3sZlC7qJ/u/L2OF7+tzOF7BXx8vpJLCT4usXXGPXbKDxXPq/yz6a+lLe9SxqtaeDQAEtCllTpaE1FmMtn5WXmmFsmk6FevXoVKoxGHUqdNm0aZs6ciaFDhwIAAgMDcevWLSxevLjUwtioUSPUq1cP165dK7EwisViiMXiYtOtrKz08g+obT/Hbx7HtaxrZba5mnUVpzJPVenjXvS1/oZmzNxWsMLSsKUYuHUgAGgUx6KTSpaELYGN2Kbk5as4u0KpqNRlHy+vX5HnyufIU+YBKHv9TE1p27t/i/6IamZal8i8jL+fVU+bnEYtjLm5ubCw0Dz/x9LSEkqlstRl7ty5g8ePH8Pd3TwOmPPMOPNWdFJJSdcxJoQnGO2ifH1dW6m+fo9zHmv0Zcz10zc+Z5C0YdTCGBkZiYULF8LLywvNmzfHr7/+ihUrVmDUqFEAgJycHMTHx2PAgAFwc3PD9evXMX36dPj5+SEsLMyY0SuMZ8aZP1O7KF/fDzwuWr8jN45A9psMe4fvRRefLia1R0VkSEYtjKtXr0ZsbCzef/99PHz4EB4eHnjnnXcwZ84cAC/2Hi9cuIBvvvkGT58+hYeHB0JDQzF//vwSh0tNUdHFz3dld0s9TtVA2sAsbx5dk5jKHkdV3c3F0sISnb06Y99v+9DZq7NJFMXKDhXXlEykf0YtjA4ODkhISEBCQkKJ8yUSCVJSUgwbSs+KLn4euHUgRBCVeJzKXG8eTYanzbWHplDIdWWKz3Y0xUxUNXgTcQMwxYufyTzVhGPWpnjjb1PMRFWHNxE3EFM7TlUVOMxU9ar7MWtTvPG3KWaiqsXCaECmcpyqKnCYyTCq+zFrUxwqNsVMVLU4lEqVxmEmwzHVG3briykOFZtiJqpaLIxUKdX9CRSmqDofszbFoWJTzERVi0OpVCkcZjKO6nrM2hSHik0xE1Ut7jFSpXCYyXhM6YHH+mKKQ8WmmImqFgsjVQqHmUjfTHGo2BQzUdXhUCpVCoeZqCqY4lCxKWaiqsHCSJXCO/tQVTHFy5tMMRPpH4dSqdI4zERE1Qn3GEkvOMxERNUFC2MFKJQKHMs4BgA4lnGMj+QpBYeZiKg64FBqOXZe3gnvld7ondQbANA7qTe8V3rzbi5ERAagUCpw6OYhJF9MxqGbhwxysxDuMZZB/YGwEguJarquD4QlIqKKM9Y9mLnHWAre6oyIyHiMeQ9mFsZSaHOrMyIi0h9j75iwMJaCtzojIjIOY++YsDCWgrc6IyIyDmPvmLAwlqLoVmcv3zS4iAgieEo9easzIiI9M/aOCQtjKXhHfSIi4zD2jolRC6NCoUBsbCx8fHwgkUjg6+uL+fPnQxD+OeAqCALmzJkDd3d3SCQShISE4OrVqwbJx1udEREZnrF3TIxaGJcuXYq1a9fis88+w+XLl7F06VIsW7YMq1evVrVZtmwZVq1ahXXr1uH06dOws7NDWFgY8vLyDJKxf0B/3Jx0E3uH7wUA7B2+Fzcm3WBRJCKqQsbcMTHqBf4nTpxAVFQUevd+cVcZb29vJCcn48yZMwBe7C0mJCTgo48+QlRUFADg22+/haurK3bv3o2hQ4caJKelhSU6e3XGvt/2obNXZw6fEhEZgLHuwWzUwtixY0d8+eWX+PPPP+Hv74/z58/j2LFjWLFiBQDgxo0buH//PkJCQlTLODo6on379jh58mSJhTE/Px/5+fmq1zKZDAAgl8shl8t1zlq0bGX6MAbmNjxzzc7chmeu2Q2du1P9Tqr/VyqUUCqUWvehTVaRoH5Az8CUSiVmz56NZcuWwdLSEgqFAgsXLsSsWbMAvNij7NSpE+7duwd393/OPho8eDBEIhG2bNlSrM+4uDjEx8cXm56UlARbW9uqWxkiIjJZubm5GD58OLKysiCVSstsa9Q9xq1bt+I///kPkpKS0Lx5c6SnpyMmJgYeHh6Ijo7Wqc9Zs2ZhypQpqtcymQyenp4IDQ0td2OURS6XIzU1FT179oSVlZXO/RgacxueuWZnbsMz1+zmmLto9LAijFoYp02bhpkzZ6qGRAMDA3Hr1i0sXrwY0dHRcHNzAwA8ePBAY4/xwYMHaN26dYl9isViiMXiYtOtrKz08g+or34MjbkNz1yzM7fhmWt2c8qtTU6jnpWam5sLCwvNCJaWllAqX4wf+/j4wM3NDQcPHlTNl8lkOH36NDp06GDQrEREVDMYdY8xMjISCxcuhJeXF5o3b45ff/0VK1aswKhRowAAIpEIMTExWLBgARo3bgwfHx/ExsbCw8MD/fr1M2Z0IiKqpoxaGFevXo3Y2Fi8//77ePjwITw8PPDOO+9gzpw5qjbTp0/Hs2fPMG7cODx9+hSdO3fGgQMHYGNjY8TkRERUXRm1MDo4OCAhIQEJCQmlthGJRJg3bx7mzZtnuGBERFRj8V6pREREalgYiYiI1LAwEhERqWFhJCIiUsPCSEREpIaFkYiISA0LIxERkRoWRiIiIjUsjERERGpYGImIiNSwMBIREalhYSQiIlLDwkhERKSmUoWxoKAAV65cQWFhob7yEBERGZVOhTE3NxejR4+Gra0tmjdvjoyMDADABx98gCVLlug1IBERkSHpVBhnzZqF8+fP49ChQxoPDA4JCcGWLVv0Fo6IiMjQdHpQ8e7du7Flyxa89tprEIlEqunNmzfH9evX9RaOiIjI0HTaY3z06BFcXFyKTX/27JlGoSQiIjI3OhXGNm3aYO/evarXRcXw3//+Nzp06KCfZEREREag01DqokWLEBERgUuXLqGwsBArV67EpUuXcOLECRw+fFjfGYmIiAxGpz3Gzp074/z58ygsLERgYCB++OEHuLi44OTJkwgKCtJ3RqphFEoFjmUcAwAcyzgGhVJh5EREVJNoXRjlcjlGjRoFkUiEr776CmfOnMGlS5ewadMmBAYGVkVGqkF2Xt4J75Xe6J3UGwDQO6k3vFd6Y+flnUZORkQ1hdaF0crKCjt27NDLm3t7e0MkEhX7GT9+PADg9ddfLzbv3Xff1ct7k+nZeXknBm4diDuyOxrT78ruYuDWgSyORGQQOg2l9uvXD7t37670m589exaZmZmqn9TUVADAoEGDVG3Gjh2r0WbZsmWVfl8yPQqlApMOTIIAodi8omkxB2I4rEpEVU6nk28aN26MefPm4fjx4wgKCoKdnZ3G/IkTJ1aoH2dnZ43XS5Ysga+vL7p27aqaZmtrCzc3twpny8/PR35+vuq1TCYD8GIIWC6XV7iflxUtW5k+jMFcch/LOIbHOY8hsZAAQLH/AsDfOX/jyI0j6OzV2SgZK8pctvnLmNvwzDW7OebWJqtIEITif6KXw8fHp/QORSL89ddf2naJgoICeHh4YMqUKZg9ezaAF0Opv//+OwRBgJubGyIjIxEbGwtbW9tS+4mLi0N8fHyx6UlJSWUuR0RE1Vdubi6GDx+OrKwsSKXSMtvqVBirwtatWzF8+HBkZGTAw8MDAPDll1+iYcOG8PDwwIULFzBjxgy0a9cOO3eWfqyppD1GT09P/P333+VujLLI5XKkpqaiZ8+esLKy0rkfQzOX3McyjqlOuAFe7Cl+3eJrjPptFJ4rn6um7x2+1yz2GM1hm7+MuQ3PXLObY26ZTIZ69epVqDDqNJSqrqiuVvaON+vXr0dERISqKALAuHHjVP8fGBgId3d39OjRA9evX4evr2+J/YjFYojF4mLTrays9PIPqK9+DM3Uc3fx6QIneyfcld3VOM74XPkcz5XPIYIIDaQN0MWnCywtLI2YtOJMfZuXhrkNz1yzm1NubXLq/Nipb7/9FoGBgZBIJJBIJGjZsiU2btyoU1+3bt1CWloaxowZU2a79u3bAwCuXbum0/uQ6bK0sMTK8JUAABE0/8gqep0QnmA2RZGIzJdOhXHFihV477330KtXL2zduhVbt25FeHg43n33XXz66ada95eYmAgXFxf07t27zHbp6ekAAHd3d11ik4nrH9Af2wdvR31pfY3pDaQNsH3wdvQP6G+kZERUk+g0lLp69WqsXbsWI0aMUE3r27cvmjdvjri4OEyePLnCfSmVSiQmJiI6Ohq1av0T5/r160hKSkKvXr3g5OSECxcuYPLkyejSpQtatmypS2wyA/0D+iOqSRSO3DgC2W8y7B2+16yGT4nI/Om0x5iZmYmOHTsWm96xY0dkZmZq1VdaWhoyMjIwatQojenW1tZIS0tDaGgomjZtiqlTp2LAgAHYs2ePLpHJjFhaWKpOsOns1ZlFkYgMSqc9Rj8/P2zdulV1WUWRLVu2oHHjxlr1FRoaipJOjPX09OQNyYmo2nr5nsAcGTEdOhXG+Ph4DBkyBEeOHEGnTp0AAMePH8fBgwexdetWvQYkIqpudl7eiUkHJuFxzmMkt0xG76TecLJ3wsrwlTyWbgJ0GkodMGAATp8+jXr16mH37t3YvXs36tWrhzNnzuBf//qXvjMSEVUbvCew6dP5OsagoCBs2rRJn1mIiKq18u4JLIIIMQdiENUkisOqRqTTHuO+ffuQkpJSbHpKSgr2799f6VBERNXR0YyjxfYU1QkQcFt2G0czjhowFb1Mp8I4c+ZMKBTFn3IgCAJmzpxZ6VBERNVRZnbFztqvaDuqGjoVxqtXr6JZs2bFpjdt2pR3pSEiKoW7Q8VuTlLRdlQ1dCqMjo6OJT5B49q1a8UeQUVERC8EewWjgbRBsdseFhFBBE+pJ4K9gg2cjNTpVBijoqIQExOD69evq6Zdu3YNU6dORd++ffUWjoioOuE9gc2DToVx2bJlsLOzQ9OmTeHj4wMfHx80bdoUTk5OWL58ub4zEhFVG7wnsOnT6XINR0dHnDhxAqmpqTh//jwkEglatWqF4GDu/hMRlYf3BDZtWu0xnjx5Et9//z2AF89fDA0NhYuLC5YvX44BAwZg3LhxGg8JJiKikvGewKZLq8I4b948/P7776rXFy9exNixY9GzZ0/MnDkTe/bsweLFi/UekoiIyFC0Kozp6eno0aOH6vXmzZvRrl07fPXVV5gyZQpWrVrFe6USEZFZ06owPnnyBK6urqrXhw8fRkREhOp127Ztcfv2bf2lIyIiMjCtCqOrqytu3LgBACgoKMC5c+fw2muvqeZnZ2fDyspKvwmJiIgMSKvC2KtXL8ycORNHjx7FrFmzYGtrq3Em6oULF+Dr66v3kERERIai1eUa8+fPR//+/dG1a1fY29vjm2++gbW1tWr+119/jdDQUL2HJCIiMhStCmO9evVw5MgRZGVlwd7eHpaWmqcXb9u2Dfb29noNSEREZEg6X+Bfkrp161YqDBERkbHpdEs4IiKi6oqFkYiISI1RC6O3tzdEIlGxn/HjxwMA8vLyMH78eDg5OcHe3h4DBgzAgwcPjBmZiIiqOaMWxrNnzyIzM1P1k5qaCgAYNGgQAGDy5MnYs2cPtm3bhsOHD+PevXvo3593nicioqqj08k3+uLs7KzxesmSJfD19UXXrl2RlZWF9evXIykpCd27dwcAJCYmIiAgAKdOndK4sQAREZG+GLUwqisoKMCmTZswZcoUiEQi/PLLL5DL5QgJCVG1adq0Kby8vHDy5MlSC2N+fr7GEz5kMhkAQC6XQy6X65yvaNnK9GEMzG145pqduQ3PXLObY25tsooEQRCqMEuFbd26FcOHD0dGRgY8PDyQlJSEt99+u9hjrNq1a4du3bph6dKlJfYTFxeH+Pj4YtOTkpJga2tbJdmJiMi05ebmYvjw4cjKyoJUKi2zrcnsMa5fvx4RERHw8PCoVD+zZs3ClClTVK9lMhk8PT0RGhpa7sYoi1wuR2pqKnr27GlW94NlbsMz1+zMbXjmmt0ccxeNHlaESRTGW7duIS0tDTt37lRNc3NzQ0FBAZ4+fYratWurpj948ABubm6l9iUWiyEWi4tNt7Ky0ss/oL76MTTmNjxzzc7chmeu2c0ptzY5TeI6xsTERLi4uKB3796qaUFBQbCyssLBgwdV065cuYKMjAx06NDBGDGJiKgGMPoeo1KpRGJiIqKjo1Gr1j9xHB0dMXr0aEyZMgV169aFVCrFBx98gA4dOvCMVCIiqjJGL4xpaWnIyMjAqFGjis379NNPYWFhgQEDBiA/Px9hYWH4/PPPjZCSiIhqCqMXxtDQUJR2YqyNjQ3WrFmDNWvWGDgVERHVVCZxjJGIiMhUsDASERGpYWEkIiJSw8JIRESkhoWRiIhIDQsjERGRGhZGIiIiNSyMREREalgYiYiI1LAwEhERqWFhJCIiUsPCSEREpIaFkYiISA0LIxERkRoWRiIiIjUsjERERGpYGImIiNSwMBIREalhYSQiIlLDwkhERKSGhZGIiEiN0Qvj3bt38eabb8LJyQkSiQSBgYH4+eefVfNHjhwJkUik8RMeHm7ExEREVJ3VMuabP3nyBJ06dUK3bt2wf/9+ODs74+rVq6hTp45Gu/DwcCQmJqpei8ViQ0clIqIawqiFcenSpfD09NQoej4+PsXaicViuLm5GTIaERHVUEYtjN999x3CwsIwaNAgHD58GPXr18f777+PsWPHarQ7dOgQXFxcUKdOHXTv3h0LFiyAk5NTiX3m5+cjPz9f9VomkwEA5HI55HK5zlmLlq1MH8bA3IZnrtmZ2/DMNbs55tYmq0gQBKEKs5TJxsYGADBlyhQMGjQIZ8+exaRJk7Bu3TpER0cDADZv3gxbW1v4+Pjg+vXrmD17Nuzt7XHy5ElYWloW6zMuLg7x8fHFpiclJcHW1rZqV4iIiExSbm4uhg8fjqysLEil0jLbGrUwWltbo02bNjhx4oRq2sSJE3H27FmcPHmyxGX++usv+Pr6Ii0tDT169Cg2v6Q9Rk9PT/z999/lboyyyOVypKamomfPnrCystK5H0NjbsMz1+zMbXjmmt0cc8tkMtSrV69ChdGoQ6nu7u5o1qyZxrSAgADs2LGj1GUaNWqEevXq4dq1ayUWRrFYXOLJOVZWVnr5B9RXP4bG3IZnrtmZ2/DMNbs55dYmp1Ev1+jUqROuXLmiMe3PP/9Ew4YNS13mzp07ePz4Mdzd3as6HhER1UBGLYyTJ0/GqVOnsGjRIly7dg1JSUn48ssvMX78eABATk4Opk2bhlOnTuHmzZs4ePAgoqKi4Ofnh7CwMGNGJyKiasqohbFt27bYtWsXkpOT0aJFC8yfPx8JCQl44403AACWlpa4cOEC+vbtC39/f4wePRpBQUE4evQor2UkIqIqYdRjjADQp08f9OnTp8R5EokEKSkpBk5EREQ1mdFvCUdERGRKWBiJiIjUsDASERGpYWEkIiJSw8JIRESkhoWRiIhIDQsjERGRGhZGIiIiNSyMREREalgYiYiI1LAwEhERqWFhJCIiUsPCSEREpIaFkYiISA0LIxERkRoWRiIiIjVGf1AxEZknQRBQWFgIhUJh1BxyuRy1atVCXl6e0bNoy1yzm2puKysrWFpaVrofFkYi0lpBQQEyMzORm5tr7CgQBAFubm64ffs2RCKRseNoxVyzm2pukUiEBg0awN7evlL9sDASkVaUSiVu3LgBS0tLeHh4wNra2qhfjkqlEjk5ObC3t4eFhXkdHTLX7KaYWxAEPHr0CHfu3EHjxo0rtefIwkhEWikoKIBSqYSnpydsbW2NHQdKpRIFBQWwsbExmS/pijLX7Kaa29nZGTdv3oRcLq9UYTSdNSIis2JKX4hEAPQ2csFPNhERkRqjF8a7d+/izTffhJOTEyQSCQIDA/Hzzz+r5guCgDlz5sDd3R0SiQQhISG4evWqERMTEVF1ZtTC+OTJE3Tq1AlWVlbYv38/Ll26hE8++QR16tRRtVm2bBlWrVqFdevW4fTp07Czs0NYWBjy8vKMmJyIyLQcP34cgYGBsLKyQr9+/Sq0TFxcHFq3bl2luSrD29sbCQkJBn9foxbGpUuXwtPTE4mJiWjXrh18fHwQGhoKX19fAC/2FhMSEvDRRx8hKioKLVu2xLfffot79+5h9+7dxoxORGbm0aNHeO+99+Dl5QWxWAw3NzeEhYXh+PHjxo6mF1OmTEHr1q1x48YNbNiwQS993rx5EyKRSPVjbW0NPz8/LFy4EIIgaNWXSCQym+9to56V+t133yEsLAyDBg3C4cOHUb9+fbz//vsYO3YsAODGjRu4f/8+QkJCVMs4Ojqiffv2OHnyJIYOHVqsz/z8fOTn56tey2QyAC8uSJXL5TpnLVq2Mn0YA3Mbnrlmr2huuVwOQRCgVCqhVCp1fj+FUoGjGUeRmZMJd3t3BHsFw9JC+zMJi76gizKVZsCAASgoKEBiYiIaNWqEBw8e4Mcff8SjR48qtR6VUdHsFXH9+nWMGzcOHh4eAFCh/orev7S2RdN/+OEHNG/eHPn5+Th27BjGjRuH2rVr4/3339cqty6fGW22jVKphCAIJZ6Vqs3vo1EL419//YW1a9diypQpmD17Ns6ePYuJEyfC2toa0dHRuH//PgDA1dVVYzlXV1fVvJctXrwY8fHxxab/8MMPejm1PDU1tdJ9GANzG565Zi8vd61ateDm5oacnBwUFBTo9B57ru3BzMMzcS/nnmqah70HlnRdgki/SJ36zM7OLnVeVlYWjh49iu+//x5BQUEAgDp16qBp06YAXvwBnZGRgVatWuHIkSMIDAxULeft7Y09e/agc+fOAIDLly8jLi4OJ0+ehCAIaNGiBT7//HP4+PgAADZt2oQ1a9bgr7/+Qp06dRAZGYmPP/5Y1V9sbCz27duHgoICtG7dGgsXLlS938WLFzF79mykp6dDJBKhUaNG+PTTT/HKK68gIyMD06dPx6lTpyCXy+Hl5YX4+Hg0bdoUrVq1AgCMGTMGY8aMwZo1awAAs2bNwq1bt1TbYe/evXjzzTfx5MkTAC92JBQKhWoH4mU5OTkAABsbG9ja2sLW1haRkZFo3749zp8/r9rm586dw/z583HhwgXI5XIEBgZi0aJFqlwtW7YE8OKPEwDw9PTEhQsXAAD79+/Hxx9/jEuXLsHOzg4dOnTApk2bALwodP/73/8wYsQI/Pe//4WjoyM+/PBDjBw5ssS8BQUFeP78OY4cOYLCwkKNedrcjMKohVGpVKJNmzZYtGgRAOCVV17Bb7/9hnXr1iE6OlqnPmfNmoUpU6aoXstkMnh6eiI0NBRSqVTnrHK5HKmpqejZsyesrKx07sfQanLuPVf2YEbaDNzNvquaVt+hPpaGLEVkE92+fCuium/zvLw83L59G/b29rCxsdH6fXZe3onovdEQoDkUl5mTiei90dg6cCv6B/SvcH+CICA7OxsODg6lnq5va2sLe3t7pKamonv37hCLxcXaFN0txc7OTvVdUbSnYmtrC6lUirt376JPnz7o2rUr0tLSIJVKcfz4cdjY2EAqlWLt2rWYNm0aFi9ejPDwcGRlZeHEiROq/gYOHAiJRIJ9+/bB0dERX3zxBfr164c//vgDTk5OeO+999C6dWt88cUXsLS0RHp6OmrXrg2pVIpZs2ZBoVDg8OHDsLOzw6VLlyCVShEQEIC7d+8iICAA8fHxGDx4MBwdHbFlyxaIRCKN7z2JRAIAqmlisRiWlpalfjeWtE1+/vlnnD9/HkOHDlVtc6VSibfffhtt2rSBIAhYsWIFhgwZgitXrsDBwQFnz56Fm5sb1q9fj/DwcNV77t27F2+99RZmz56NjRs3oqCgAPv371e9l4WFBT7//HPMmzcPc+bMwY4dOzB16lSEhYWhSZMmxfLm5eVBIpGgS5cuxT6bpRX/khi1MLq7u6NZs2Ya0wICArBjxw4AgJubGwDgwYMHcHd3V7V58OBBqQeMxWJxiR96KysrvXxJ6asfQ6tpuXde3omBOwYW+/K9nnUdA3cMxPbB27X68tVFdd3mCoUCIpEIFhYWWl/LqFAqMPmHycX+XQBAgAARRJjywxT8K+BfFR5WLSpeRZlKYm1tjQ0bNmDs2LH44osv8Oqrr6Jr164YOnSoam+maFn19Xp52tq1a1VFp2gbFe11AsCiRYswdepUxMTEqKa1b98eAHDs2DGcPXsWDx8+VH1HLV++HLt378aOHTvw7rvvIiMjA9OmTVN9L6p/+d++fRsDBgxQ7YX5+fmp5nl4eEAkEqF27dqqodSX16GkaUV/SJS23Yqmd+7cGRYWFigoKIBcLsfYsWMxdOhQ1TZXP9wFAF999RVq166No0ePok+fPqpRv7p166ryAS9G+IYOHYp58+appr3yyisaffXq1Qvjx48HAMycORMJCQk4fPgwAgICSswrEolK/Axr87to1JNvOnXqhCtXrmhM+/PPP9GwYUMAgI+PD9zc3HDw4EHVfJlMhtOnT6NDhw4GzUrmQ6FUYNKBSaV++QJAzIEYKJSmc/PjmuJoxlHckd0pdb4AAbdlt3E046je33vAgAG4d+8evvvuO4SHh+PQoUN49dVXtTpRJT09HcHBwSV+yT58+BD37t1Djx49Slz2/PnzyMnJgZOTE+zt7WFvbw+pVIpbt27hr7/+AvDiBJoxY8YgJCQES5YswfXr11XLT5w4EQsWLECnTp0wd+5c1VCkIWzZsgXp6ek4f/48tm7diu+++w5xcXGq+Q8ePMDYsWPRuHFjODo6QiqVIicnBxkZGWX2m56eXur2KlL0hwvwopC7ubnh4cOHlVqf8hi1ME6ePBmnTp3CokWLcO3aNSQlJeHLL79U/XUgEokQExODBQsW4LvvvsPFixcxYsQIeHh4VPh0ZKp5jPnlS2XLzM7Uaztt2djYoGfPnoiNjcWJEycwcuRIzJ07F8A/e0fqZ1u+fMJG0VBkScqaB7w4Xufu7o709HTVz7lz53D27Fl8+OGHAF5cPvH777+jd+/e+PHHH9GsWTPs2rULwIvjh3/99RfeeustXLx4EW3atMHq1atLfT8LC4tiZ47qekKYp6cn/Pz8EBAQgEGDBmHSpElYs2aN6rK56OhopKenY+XKlThx4gTS09Ph5ORU7jHo8rYZUHxPr2jotioZtTC2bdsWu3btQnJyMlq0aIH58+cjISEBb7zxhqrN9OnT8cEHH2DcuHFo27YtcnJycODAAZ2ObVDNYOwvXyqdu4N7+Y20aFdZzZo1w7NnzwC8uM8mAGRm/vO5SE9P12jfsmVLHD16tMQC4+DgAG9vb40RLnWvvvoq7t+/j1q1asHPz0/106hRI9SrV0/Vzt/fH5MnT8YPP/yA/v37IzExUTXP09MT7777Lnbu3ImpU6fiq6++KnXdnJ2dkZ2drVq/ktZHV5aWligsLFQVvuPHj2PixIno1asXmjdvDrFYjL///ltjGSsrq2KPqGrZsmWp28uYjH4T8T59+qBPnz6lzheJRJg3b57GGDRRWUzty5f+EewVjAbSBrgru1viULcIIjSQNkCwV7Be3/fx48cYNGgQRo0ahZYtW8LBwQE///wzli1bhqioKAAv9l5ee+01LFmyBD4+Pnj48CE++ugjjX4mTJiA1atXY+jQoZg1axYcHR1x6tQptGvXDk2aNEFcXBzeffdduLi4ICIiAtnZ2Th+/Dg++OADhISEoEOHDujXrx+WLVsGf39/3LlzB7t27cKQIUMQGBiIadOmYeDAgfDx8cGdO3dw9uxZ1ZmcMTExiIiIgL+/P548eYKffvqpxONsRdq3bw9bW1vMnj0bEydOxOnTp3W+vvHx48e4f/8+CgsLcfHiRaxatQrBwcGqk2QaN26MjRs3ok2bNpDJZJg2bVqxvcGiPxo6deoEsViMOnXqYO7cuejRowd8fX0xdOhQFBYWYt++fZgxY4ZOOfXF6LeEI+NSKBU4dPMQki8m49DNQ9XiuFvRl68IJZ+hKIIInlJPvX/5UvksLSyxMnwlABT79yl6nRCeoNP1jGWxt7dH+/bt8emnn6JLly5o0aIFYmNjMXbsWHz22Weqdl9//TUKCwsRFBSkOoyjzsnJCT/++CNycnLQtWtXBAUF4auvvlIN90VHRyMhIQGff/45mjdvjj59+qhuYSkSibBv3z506dIFb7/9Nvz9/TF8+HDcvn0brq6usLS0xOPHjzFixAj4+/tj8ODBiIiIUF1+plAoMH78eAQEBCA8PBz+/v74/PPPS13nunXrYtOmTdi3bx8CAwORnJyscVxQGyEhIXB3d4e3tzfGjRuHiIgIfP3116r569evx5MnT/Dqq6/irbfewsSJE+Hi4qLRxyeffILU1FR4enqqTrB5/fXXsW3bNnz33Xdo3bo1unfvjjNnzuiUUZ9Egra3LzAzMpkMjo6OyMrKqvTlGvv27UOvXr3M6kzDsnLvvLwTkw5M0jge10DaACvDV1b5GZvlqez23nl5JwZuHQgAGnsmRV++VXlWanX8rKjLy8vDjRs34OPjo/MhjZI+e55STySEJ2j976JUKiGTySCVSs3uiR/mmt1Uc5f12dSmFhh9KJWMo6hwvDycdVd2FwO3GuZyhqrUP6A/tg/eXmLh1+XLl/Srf0B/RDWJenHnm+xMuDvofucbIn1jYayByrucQQQRYg7EIKpJlFl/UfHL17RZWljide/XjR2DqBgWxhpIm8sZzP2Li1++RKQt0xkcJoPh5QxERKVjYayBeDkD6UM1P2+PzJC+PpMsjDUQL2egyig6Y1WbpxUQGULRDQdefuSUtniMsQYqupZs4NaBEEFU4uUMVXEtGVUPlpaWqF27tup+lba2tqU+1cIQlEolCgoKkJeXZ1KXDlSEuWY3xdxKpRKPHj2Cra0tatWqXGljYayheDkDVUbRk2+q+mbOFSEIAp4/fw6JRGLUAq0Lc81uqrktLCzg5eVV6UwsjDUYL2cgXYlEIri7u8PFxUXnG1Pri1wux5EjR9ClSxezuqECYL7ZTTW3tbW1XvZgWRhrOF7OQJVhaWlZ6eM5+shQWFgIGxsbk/qSrghzzW6uuSvKNAaHiYiITAQLIxERkRoWRiIiIjXV/hhj0QWfMpmsUv3I5XLk5uZCJpOZ1Zg6cxueuWZnbsMz1+zmmLuoBlTkJgDVvjBmZ2cDePHkayIiqtmys7Ph6OhYZptq/zxGpVKJe/fuwcHBoVLXtshkMnh6euL27duVeq6joTG34ZlrduY2PHPNbo65BUFAdnY2PDw8yr2ko9rvMVpYWKBBgwZ6608qlZrNB0EdcxueuWZnbsMz1+zmlru8PcUiPPmGiIhIDQsjERGRGhbGChKLxZg7dy7EYrGxo2iFuQ3PXLMzt+GZa3ZzzV1R1f7kGyIiIm1wj5GIiEgNCyMREZEaFkYiIiI1LIxERERqWBj/vzVr1sDb2xs2NjZo3749zpw5U2b7bdu2oWnTprCxsUFgYCD27dtnoKT/WLx4Mdq2bQsHBwe4uLigX79+uHLlSpnLbNiwASKRSOPHxsbGQIlfiIuLK5ahadOmZS5jCtsbALy9vYtlF4lEGD9+fIntjbW9jxw5gsjISHh4eEAkEmH37t0a8wVBwJw5c+Du7g6JRIKQkBBcvXq13H61/T3RZ265XI4ZM2YgMDAQdnZ28PDwwIgRI3Dv3r0y+9Tl86bv7AAwcuTIYjnCw8PL7deY2xxAiZ93kUiEjz/+uNQ+DbXNqwoLI4AtW7ZgypQpmDt3Ls6dO4dWrVohLCwMDx8+LLH9iRMnMGzYMIwePRq//vor+vXrh379+uG3334zaO7Dhw9j/PjxOHXqFFJTUyGXyxEaGopnz56VuZxUKkVmZqbq59atWwZK/I/mzZtrZDh27FipbU1lewPA2bNnNXKnpqYCAAYNGlTqMsbY3s+ePUOrVq2wZs2aEucvW7YMq1atwrp163D69GnY2dkhLCwMeXl5pfap7e+JvnPn5ubi3LlziI2Nxblz57Bz505cuXIFffv2LbdfbT5vVZG9SHh4uEaO5OTkMvs09jYHoJE3MzMTX3/9NUQiEQYMGFBmv4bY5lVGIKFdu3bC+PHjVa8VCoXg4eEhLF68uMT2gwcPFnr37q0xrX379sI777xTpTnL8/DhQwGAcPjw4VLbJCYmCo6OjoYLVYK5c+cKrVq1qnB7U93egiAIkyZNEnx9fQWlUlnifFPY3gCEXbt2qV4rlUrBzc1N+Pjjj1XTnj59KojFYiE5ObnUfrT9PdF37pKcOXNGACDcunWr1Dbaft70oaTs0dHRQlRUlFb9mOI2j4qKErp3715mG2Nsc32q8XuMBQUF+OWXXxASEqKaZmFhgZCQEJw8ebLEZU6ePKnRHgDCwsJKbW8oWVlZAIC6deuW2S4nJwcNGzaEp6cnoqKi8PvvvxsinoarV6/Cw8MDjRo1whtvvIGMjIxS25rq9i4oKMCmTZswatSoMm9QbwrbW92NGzdw//59jW3q6OiI9u3bl7pNdfk9MYSsrCyIRCLUrl27zHbafN6q0qFDh+Di4oImTZrgvffew+PHj0tta4rb/MGDB9i7dy9Gjx5dbltT2ea6qPGF8e+//4ZCoYCrq6vGdFdXV9y/f7/EZe7fv69Ve0NQKpWIiYlBp06d0KJFi1LbNWnSBF9//TX++9//YtOmTVAqlejYsSPu3LljsKzt27fHhg0bcODAAaxduxY3btxAcHCw6hFhLzPF7Q0Au3fvxtOnTzFy5MhS25jC9n5Z0XbTZpvq8ntS1fLy8jBjxgwMGzaszBtZa/t5qyrh4eH49ttvcfDgQSxduhSHDx9GREQEFApFie1NcZt/8803cHBwQP/+/ctsZyrbXFfV/ukaNcX48ePx22+/lTuO36FDB3To0EH1umPHjggICMAXX3yB+fPnV3VMAEBERITq/1u2bIn27dujYcOG2Lp1a4X+EjUV69evR0REBDw8PEptYwrbuzqSy+UYPHgwBEHA2rVry2xrKp+3oUOHqv4/MDAQLVu2hK+vLw4dOoQePXoYLEdlfP3113jjjTfKPYHMVLa5rmr8HmO9evVgaWmJBw8eaEx/8OAB3NzcSlzGzc1Nq/ZVbcKECfj+++/x008/af2ILSsrK7zyyiu4du1aFaUrX+3ateHv719qBlPb3gBw69YtpKWlYcyYMVotZwrbu2i7abNNdfk9qSpFRfHWrVtITU3V+rFH5X3eDKVRo0aoV69eqTlMaZsDwNGjR3HlyhWtP/OA6WzziqrxhdHa2hpBQUE4ePCgappSqcTBgwc1/tJX16FDB432AJCamlpq+6oiCAImTJiAXbt24ccff4SPj4/WfSgUCly8eBHu7u5VkLBicnJycP369VIzmMr2VpeYmAgXFxf07t1bq+VMYXv7+PjAzc1NY5vKZDKcPn261G2qy+9JVSgqilevXkVaWhqcnJy07qO8z5uh3LlzB48fPy41h6ls8yLr169HUFAQWrVqpfWyprLNK8zYZ/+Ygs2bNwtisVjYsGGDcOnSJWHcuHFC7dq1hfv37wuCIAhvvfWWMHPmTFX748ePC7Vq1RKWL18uXL58WZg7d65gZWUlXLx40aC533vvPcHR0VE4dOiQkJmZqfrJzc1VtXk5e3x8vJCSkiJcv35d+OWXX4ShQ4cKNjY2wu+//26w3FOnThUOHTok3LhxQzh+/LgQEhIi1KtXT3j48GGJmU1lexdRKBSCl5eXMGPGjGLzTGV7Z2dnC7/++qvw66+/CgCEFStWCL/++qvq7M0lS5YItWvXFv773/8KFy5cEKKiogQfHx/h+fPnqj66d+8urF69WvW6vN+Tqs5dUFAg9O3bV2jQoIGQnp6u8ZnPz88vNXd5nzdDZM/OzhY+/PBD4eTJk8KNGzeEtLQ04dVXXxUaN24s5OXllZrd2Nu8SFZWlmBrayusXbu2xD6Mtc2rCgvj/7d69WrBy8tLsLa2Ftq1ayecOnVKNa9r165CdHS0RvutW7cK/v7+grW1tdC8eXNh7969Bk784tTqkn4SExNVbV7OHhMTo1pPV1dXoVevXsK5c+cMmnvIkCGCu7u7YG1tLdSvX18YMmSIcO3atVIzC4JpbO8iKSkpAgDhypUrxeaZyvb+6aefSvxsFGVTKpVCbGys4OrqKojFYqFHjx7F1qdhw4bC3LlzNaaV9XtS1blv3LhR6mf+p59+KjV3eZ83Q2TPzc0VQkNDBWdnZ8HKykpo2LChMHbs2GIFztS2eZEvvvhCkEgkwtOnT0vsw1jbvKrwsVNERERqavwxRiIiInUsjERERGpYGImIiNSwMBIREalhYSQiIlLDwkhERKSGhZGIiEgNCyMREZEaFkYiE7dhw4ZynzdoikaOHIl+/foZOwaR1lgYiSpg5MiREIlEqh8nJyeEh4fjwoULWvUTFxeH1q1bV01INTdv3oRIJIKLi0uxZ+C1bt0acXFxVZ6ByFyxMBJVUHh4ODIzM5GZmYmDBw+iVq1a6NOnj7FjlSk7OxvLly83dgy9EQQBhYWFxo5B1RwLI1EFicViuLm5wc3NDa1bt8bMmTNx+/ZtPHr0SNVmxowZ8Pf3h62tLRo1aoTY2FjI5XIAL4ZE4+Pjcf78edWe54YNGwAAT58+xTvvvANXV1fY2NigRYsW+P777zXePyUlBQEBAbC3t1cV6fJ88MEHWLFiBR4+fFhqG5FIhN27d2tMq127tipb0d7n1q1bERwcDIlEgrZt2+LPP//E2bNn0aZNG9jb2yMiIkJjWxSJj4+Hs7MzpFIp3n33XRQUFKjmKZVKLF68GD4+PpBIJGjVqhW2b9+umn/o0CGIRCLs378fQUFBEIvF5T6Mm6iyahk7AJE5ysnJwaZNm+Dn56fxTEAHBwds2LABHh4euHjxIsaOHQsHBwdMnz4dQ4YMwW+//YYDBw4gLS0NAODo6AilUomIiAhkZ2dj06ZN8PX1xaVLl2BpaanqNzc3F8uXL8fGjRthYWGBN998Ex9++CH+85//lJlz2LBhSE1Nxbx58/DZZ59Vap3nzp2LhIQEeHl5YdSoURg+fDgcHBywcuVK2NraYvDgwZgzZw7Wrl2rWubgwYOwsbHBoUOHcPPmTbz99ttwcnLCwoULAQCLFy/Gpk2bsG7dOjRu3BhHjhzBm2++CWdnZ3Tt2lXVz8yZM7F8+XI0atQIderUqdR6EJXLyE/3IDIL0dHRgqWlpWBnZyfY2dkJAAR3d3fhl19+KXO5jz/+WAgKClK9njt3rtCqVSuNNikpKYKFhUWJj7ESBEFITEwUAGg8tmfNmjWCq6trqe9b9IimX3/9VThw4IBgZWWlWr5Vq1YajwgCIOzatUtjeUdHR9Xjy4r6+ve//62an5ycLAAQDh48qJq2ePFioUmTJqrX0dHRQt26dYVnz56ppq1du1awt7cXFAqFkJeXJ9ja2gonTpzQeO/Ro0cLw4YNEwThn0ci7d69u9R1JdI37jESVVC3bt1Ue0NPnjzB559/joiICJw5cwYNGzYEAGzZsgWrVq3C9evXkZOTg8LCQkil0jL7TU9PR4MGDeDv719qG1tbW/j6+qpeu7u7lzk8qi4sLAydO3dGbGwskpKSKrRMSVq2bKn6f1dXVwBAYGCgxrSXM7Vq1Qq2traq1x06dEBOTg5u376NnJwc5ObmomfPnhrLFBQU4JVXXtGY1qZNG51zE2mLhZGoguzs7ODn56d6/e9//xuOjo746quvsGDBApw8eRJvvPEG4uPjERYWBkdHR2zevBmffPJJmf1KJJJy39vKykrjtUgkgqDFo1SXLFmCDh06YNq0acXmldRX0XHR0jKIRKISpymVygpnysnJAQDs3bsX9evX15gnFos1XtvZ2VW4X6LKYmEk0pFIJIKFhQWeP38OADhx4gQaNmyI//u//1O1uXXrlsYy1tbWUCgUGtNatmyJO3fu4M8//yxzr7Ey2rVrh/79+2PmzJnF5jk7O2ucyHP16lXk5ubq5X3Pnz+P58+fq4r/qVOnYG9vD09PT9StWxdisRgZGRkaxxOJjI2FkaiC8vPzcf/+fQAvhlI/++wz5OTkIDIyEgDQuHFjZGRkYPPmzWjbti327t2LXbt2afTh7e2NGzduqIZPHRwc0LVrV3Tp0gUDBgzAihUr4Ofnhz/++AMikQjh4eF6y79w4UI0b94ctWpp/tp3794dn332GTp06ACFQoEZM2YU20PVVUFBAUaPHo2PPvoIN2/exNy5czFhwgRYWFjAwcEBH374ISZPngylUonOnTsjKysLx48fh1QqRXR0tF4yEGmLl2sQVdCBAwfg7u4Od3d3tG/fHmfPnsW2bdvw+uuvAwD69u2LyZMnY8KECWjdujVOnDiB2NhYjT4GDBiA8PBwdOvWDc7OzkhOTgYA7NixA23btsWwYcPQrFkzTJ8+vdieZWX5+/tj1KhRyMvL05j+ySefwNPTE8HBwRg+fDg+/PBDjeOCldGjRw80btwYXbp0wZAhQ9C3b1+NmwvMnz8fsbGxWLx4MQICAhAeHo69e/fCx8dHL+9PpAuRoM2BCiIiomqOe4xERERqWBiJiIjUsDASERGpYWEkIiJSw8JIRESkhoWRiIhIDQsjERGRGhZGIiIiNSyMREREalgYiYiI1LAwEhERqfl/1iPUxfzqDdwAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "trial_logs = miprov2_model.trial_logs\n", + "trial_numbers = list(trial_logs.keys())\n", + "scores = [trial_logs[trial]['score'] for trial in trial_numbers]\n", + "pruning_status = [trial_logs[trial]['pruned'] for trial in trial_numbers]\n", + "\n", + "plt.figure(figsize=(5, 3))\n", + "for trial_number, score, pruned in zip(trial_numbers, scores, pruning_status):\n", + " if pruned:\n", + " plt.scatter(trial_number, score, color='grey', label='Pruned Batch' if 'Pruned Batch' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n", + " else:\n", + " plt.scatter(trial_number, score, color='green', label='Successful Batch' if 'Successful Batch' not in plt.gca().get_legend_handles_labels()[1] else \"\")\n", + "\n", + "plt.xlabel('Batch Number')\n", + "plt.ylabel('Score')\n", + "plt.title('Batch Scores')\n", + "plt.grid(True)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Baseline program | Score: 0:\n", + "Prompt 1 Instruction: Given the fields `input_text`, `entity_types`, produce the fields `entities_relationships`.\n", + "\n", + "----------------\n", + "Best program after 0 batches | Score: 81.23:\n", + "Prompt 1 Instruction: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\n", + "\n", + "Best program after 5 batches | Score: 81.23:\n", + "Prompt 1 Instruction: Given the `input_text` and a list of `entity_types`, meticulously extract and identify all entities and their relationships within the text. Ensure that each entity is accurately classified according to the provided entity types, and clearly define the relationships between entities, including their descriptions, weights, and orders. Provide a step-by-step reasoning process to justify the extraction and classification of each entity and relationship.\n", + "\n", + "Best program after 10 batches | Score: 85.16:\n", + "Prompt 1 Instruction: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide its `entity_name`, `entity_type`, a concise `description`, and an `importance_score`. Subsequently, determine and list the relationships between these entities, specifying the `src_id` (source entity), `tgt_id` (target entity), a `description` of the relationship, a `weight` indicating the strength of the relationship, and an `order` to indicate the sequence of relationships. Ensure that the relationships are coherent and directly derived from the context provided in the `input_text`.\n", + "\n", + "Best program after 15 batches | Score: 85.16:\n", + "Prompt 1 Instruction: Given the `input_text` and `entity_types`, meticulously identify and extract entities from the text. For each identified entity, provide its `entity_name`, `entity_type`, a concise `description`, and an `importance_score`. Subsequently, determine and list the relationships between these entities, specifying the `src_id` (source entity), `tgt_id` (target entity), a `description` of the relationship, a `weight` indicating the strength of the relationship, and an `order` to indicate the sequence of relationships. Ensure that the relationships are coherent and directly derived from the context provided in the `input_text`.\n", + "\n" + ] + } + ], + "source": [ + "best_score = 0\n", + "\n", + "def get_signature(predictor):\n", + " if (hasattr(predictor, 'extended_signature')):\n", + " return predictor.extended_signature\n", + " elif (hasattr(predictor, 'signature')):\n", + " return predictor.signature\n", + "\n", + "print(f\"Baseline program | Score: {best_score}:\")\n", + "for i,predictor in enumerate(model.predictors()):\n", + " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n", + "print()\n", + "\n", + "print(\"----------------\")\n", + "\n", + "for trial_num in miprov2_model.trial_logs:\n", + " program_score = miprov2_model.trial_logs[trial_num][\"score\"]\n", + " program_pruned = miprov2_model.trial_logs[trial_num][\"pruned\"]\n", + " if program_score > best_score and not program_pruned and miprov2_model.trial_logs[trial_num][\"full_eval\"]:\n", + " best_score = program_score\n", + " best_program_so_far = miprov2_model.trial_logs[trial_num][\"program\"]\n", + " if trial_num % 5 == 0:\n", + " print(f\"Best program after {trial_num} batches | Score: {best_score}:\")\n", + " for i,predictor in enumerate(best_program_so_far.predictors()):\n", + " print(f\"Prompt {i+1} Instruction: {get_signature(predictor).instructions}\")\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/20 [00:00\n", + "#T_a2fb4 th {\n", + " text-align: left;\n", + "}\n", + "#T_a2fb4 td {\n", + " text-align: left;\n", + "}\n", + "#T_a2fb4_row0_col0, #T_a2fb4_row0_col1, #T_a2fb4_row0_col2, #T_a2fb4_row0_col3, #T_a2fb4_row0_col4, #T_a2fb4_row0_col5, #T_a2fb4_row1_col0, #T_a2fb4_row1_col1, #T_a2fb4_row1_col2, #T_a2fb4_row1_col3, #T_a2fb4_row1_col4, #T_a2fb4_row1_col5, #T_a2fb4_row2_col0, #T_a2fb4_row2_col1, #T_a2fb4_row2_col2, #T_a2fb4_row2_col3, #T_a2fb4_row2_col4, #T_a2fb4_row2_col5, #T_a2fb4_row3_col0, #T_a2fb4_row3_col1, #T_a2fb4_row3_col2, #T_a2fb4_row3_col3, #T_a2fb4_row3_col4, #T_a2fb4_row3_col5, #T_a2fb4_row4_col0, #T_a2fb4_row4_col1, #T_a2fb4_row4_col2, #T_a2fb4_row4_col3, #T_a2fb4_row4_col4, #T_a2fb4_row4_col5 {\n", + " text-align: left;\n", + " white-space: pre-wrap;\n", + " word-wrap: break-word;\n", + " max-width: 400px;\n", + "}\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 input_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsentity_recall_metric
0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about restricting sales of assault rifles.', 'importance_score':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Students from Marjory Stoneman Douglas High School are located in the state of Florida.', 'weight': 0.9,...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where a mass shooting occurred, leading to demands for restricting assault...[{'src_id': 'NIKOLAS CRUZ', 'tgt_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'description': 'Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', 'weight': 0.9,...✔️ [1.0]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels that offers solutions to improve train operations.', 'importance_score': 0.9}, {'entity_name': 'CHRISTIAN SPRAUER', 'entity_type': 'PERSON',...[{'src_id': 'CHRISTIAN SPRAUER', 'tgt_id': 'RAILNOVA', 'description': 'Christian Sprauer is the CEO and founder of Railnova.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description':...✔️ [0.8888888888888888]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...[{'entity_name': 'YONGHUI SUPERSTORES CO LTD', 'entity_type': 'ORGANIZATION', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'importance_score': 1.0}, {'entity_name':...[{'src_id': 'YONGHUI SUPERSTORES CO LTD', 'tgt_id': '166.3 MILLION SHARES', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'weight':...[{'entity_name': 'SHANGHAI STOCK EXCHANGE', 'entity_type': 'ORGANIZATION', 'description': \"A stock exchange where the block trade of YONGHUI SUPERSTORES Co Ltd's shares took place.\", 'importance_score': 0.8}, {'entity_name':...[{'src_id': 'SHANGHAI STOCK EXCHANGE', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': \"The block trade of YONGHUI SUPERSTORES Co Ltd's shares took place at the Shanghai Stock...✔️ [0.8]
3LONDON (Reuters) - Britain’s economy was weaker than previously thought in 2017, official data showed on Thursday, leaving the country lagging further behind the global...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BANK OF ENGLAND', 'tgt_id': 'INTEREST RATES',...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.8, 'order': 1}, {'src_id': 'BRITAIN', 'tgt_id': 'BANK OF ENGLAND', 'description':...✔️ [1.0]
4Trump taps White House doctor as new VA secretary 2 Hours Ago CNBC's Kayla Tausche reports President Trump has tapped White House physician Rear Admiral...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson as new VA secretary.', 'importance_score': 1.0}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'President Trump taps Ronny Jackson as new VA secretary.', 'weight': 1.0, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'VA', 'description':...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'The President who tapped Rear Admiral Ronny Jackson as the new VA secretary.', 'importance_score': 0.9}, {'entity_name': 'REAR ADMIRAL RONNY JACKSON',...[{'src_id': 'TRUMP', 'tgt_id': 'REAR ADMIRAL RONNY JACKSON', 'description': 'President Trump tapped Rear Admiral Ronny Jackson to run the Department of Veterans Affairs.', 'weight': 0.9, 'order':...✔️ [0.5714285714285714]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " ... 15 more rows not displayed ...\n", + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/20 [00:00\n", + "#T_a56c4 th {\n", + " text-align: left;\n", + "}\n", + "#T_a56c4 td {\n", + " text-align: left;\n", + "}\n", + "#T_a56c4_row0_col0, #T_a56c4_row0_col1, #T_a56c4_row0_col2, #T_a56c4_row0_col3, #T_a56c4_row0_col4, #T_a56c4_row0_col5, #T_a56c4_row1_col0, #T_a56c4_row1_col1, #T_a56c4_row1_col2, #T_a56c4_row1_col3, #T_a56c4_row1_col4, #T_a56c4_row1_col5, #T_a56c4_row2_col0, #T_a56c4_row2_col1, #T_a56c4_row2_col2, #T_a56c4_row2_col3, #T_a56c4_row2_col4, #T_a56c4_row2_col5, #T_a56c4_row3_col0, #T_a56c4_row3_col1, #T_a56c4_row3_col2, #T_a56c4_row3_col3, #T_a56c4_row3_col4, #T_a56c4_row3_col5, #T_a56c4_row4_col0, #T_a56c4_row4_col1, #T_a56c4_row4_col2, #T_a56c4_row4_col3, #T_a56c4_row4_col4, #T_a56c4_row4_col5 {\n", + " text-align: left;\n", + " white-space: pre-wrap;\n", + " word-wrap: break-word;\n", + " max-width: 400px;\n", + "}\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 input_textexample_entitiesexample_relationshipspred_entitiespred_relationshipsrelationships_similarity_metric
0As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about restricting sales of assault rifles.', 'importance_score':...[{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Students from Marjory Stoneman Douglas High School are located in the state of Florida.', 'weight': 0.9,...[{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where a mass shooting occurred, leading to demands for restricting assault...[{'src_id': 'NIKOLAS CRUZ', 'tgt_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'description': 'Nikolas Cruz carried out a mass shooting at Marjory Stoneman Douglas High School.', 'weight': 0.9,...✔️ [0.4]
1From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE...[{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology...[{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels that offers solutions to improve train operations.', 'importance_score': 0.9}, {'entity_name': 'CHRISTIAN SPRAUER', 'entity_type': 'PERSON',...[{'src_id': 'CHRISTIAN SPRAUER', 'tgt_id': 'RAILNOVA', 'description': 'Christian Sprauer is the CEO and founder of Railnova.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description':...✔️ [0.85]
2Jan 22 (Reuters) - Shanghai Stock Exchange Filing * SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68...[{'entity_name': 'YONGHUI SUPERSTORES CO LTD', 'entity_type': 'ORGANIZATION', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'importance_score': 1.0}, {'entity_name':...[{'src_id': 'YONGHUI SUPERSTORES CO LTD', 'tgt_id': '166.3 MILLION SHARES', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'weight':...[{'entity_name': 'SHANGHAI STOCK EXCHANGE', 'entity_type': 'ORGANIZATION', 'description': \"A stock exchange where the block trade of YONGHUI SUPERSTORES Co Ltd's shares took place.\", 'importance_score': 0.8}, {'entity_name':...[{'src_id': 'SHANGHAI STOCK EXCHANGE', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': \"The block trade of YONGHUI SUPERSTORES Co Ltd's shares took place at the Shanghai Stock...✔️ [0.3]
3LONDON (Reuters) - Britain’s economy was weaker than previously thought in 2017, official data showed on Thursday, leaving the country lagging further behind the global...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BANK OF ENGLAND', 'tgt_id': 'INTEREST RATES',...[{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The...[{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.8, 'order': 1}, {'src_id': 'BRITAIN', 'tgt_id': 'BANK OF ENGLAND', 'description':...✔️ [0.4]
4Trump taps White House doctor as new VA secretary 2 Hours Ago CNBC's Kayla Tausche reports President Trump has tapped White House physician Rear Admiral...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson as new VA secretary.', 'importance_score': 1.0}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White...[{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'President Trump taps Ronny Jackson as new VA secretary.', 'weight': 1.0, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'VA', 'description':...[{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'The President who tapped Rear Admiral Ronny Jackson as the new VA secretary.', 'importance_score': 0.9}, {'entity_name': 'REAR ADMIRAL RONNY JACKSON',...[{'src_id': 'TRUMP', 'tgt_id': 'REAR ADMIRAL RONNY JACKSON', 'description': 'President Trump tapped Rear Admiral Ronny Jackson to run the Department of Veterans Affairs.', 'weight': 0.9, 'order':...✔️ [0.65]
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " ... 15 more rows not displayed ...\n", + "
\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "metrics = [entity_recall_metric, relationships_similarity_metric]\n", + "for metric in metrics:\n", + " evaluate = Evaluate(\n", + " devset=devset[:20], \n", + " metric=metric, \n", + " num_threads=os.cpu_count(), \n", + " display_progress=True,\n", + " display_table=5,\n", + " )\n", + " evaluate(miprov2_model)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[('extractor.predictor', Predict(StringSignature(input_text, entity_types -> reasoning, entities_relationships\n", + " instructions=\"You are a meticulous entity relationship extractor tasked with analyzing a given text and extracting detailed entities and their relationships based on specified entity types. Your process involves:\\n\\n1. Identifying entities in the text that match the provided entity types, ensuring comprehensive descriptions and ignoring duplicates or generic terms.\\n\\n2. Extracting relationships between these entities using 'src_id' and 'tgt_id' keys from the text descriptions.\\n\\n3. Classifying relationships as direct (order:0), second-order (order:2), or third-order (order:3) based on their proximity and context in the text.\\n\\n4. Ensuring all entities and relationships are organized into a structured JSON object following the specified schema, providing clear and detailed descriptions for each entity and relationship.\"\n", + " input_text = Field(annotation=str required=True json_schema_extra={'desc': 'The text to extract entities and relationships from.', '__dspy_field_type': 'input', 'prefix': 'Input Text:'})\n", + " entity_types = Field(annotation=list[str] required=True json_schema_extra={'desc': 'List of entity types used for extraction.', '__dspy_field_type': 'input', 'prefix': 'Entity Types:'})\n", + " reasoning = Field(annotation=str required=True json_schema_extra={'prefix': \"Reasoning: Let's think step by step in order to\", 'desc': '${produce the entities_relationships}. We ...', '__dspy_field_type': 'output'})\n", + " entities_relationships = Field(annotation=list[Union[Entity, Relationship]] required=True json_schema_extra={'desc': 'List of entities and relationships extracted from the text.', '__dspy_field_type': 'output', 'prefix': 'Entities Relationships:'})\n", + ")))]\n" + ] + } + ], + "source": [ + "miprov2_model.save(entity_relationship_miprov2_path)" ] } ], diff --git a/examples/generate_entity_relationship_dspy.ipynb b/examples/generate_entity_relationship_dspy.ipynb new file mode 100644 index 0000000..9c5545e --- /dev/null +++ b/examples/generate_entity_relationship_dspy.ipynb @@ -0,0 +1,2062 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Generate Examples for Entity Relationship Extraction\n", + "\n", + "- Taking datasets from Huggingface containing news articles and generate entities and relationships out of each news article.\n", + "- Save them as DSPy examples locally to be used for fine-tuning prompt instructions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import nest_asyncio\n", + "nest_asyncio.apply()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniconda/base/envs/nano-graphrag/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import dspy\n", + "import asyncio\n", + "import os\n", + "import numpy as np\n", + "from dotenv import load_dotenv\n", + "from datasets import load_dataset\n", + "import logging\n", + "import pickle\n", + "\n", + "from nano_graphrag._utils import compute_mdhash_id\n", + "from nano_graphrag.entity_extraction.extract import generate_dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "WORKING_DIR = \"./nano_graphrag_cache_generate_dspy_examples\"\n", + "\n", + "load_dotenv()\n", + "\n", + "logging.basicConfig(level=logging.WARNING)\n", + "logging.getLogger(\"nano-graphrag\").setLevel(logging.DEBUG)\n", + "\n", + "np.random.seed(1337)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "system_prompt = \"\"\"\n", + " You are a world-class AI system, capable of complex reasoning and reflection. \n", + " Reason through the query, and then provide your final response. \n", + " If you detect that you made a mistake in your reasoning at any point, correct yourself.\n", + " Think carefully.\n", + "\"\"\"\n", + "lm = dspy.OpenAI(\n", + " model=\"deepseek-chat\", \n", + " model_type=\"chat\", \n", + " api_key=os.environ[\"DEEPSEEK_API_KEY\"], \n", + " base_url=os.environ[\"DEEPSEEK_BASE_URL\"], \n", + " system_prompt=system_prompt, \n", + " temperature=1.0,\n", + " top_p=1.0,\n", + " max_tokens=4096\n", + ")\n", + "dspy.settings.configure(lm=lm, experimental=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "train_len = 100\n", + "val_len = 100\n", + "dev_len = 200" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(WORKING_DIR, exist_ok=True)\n", + "entity_relationship_trainset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_trainset.pkl\")\n", + "entity_relationship_valset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_valset.pkl\")\n", + "entity_relationship_devset_path = os.path.join(WORKING_DIR, \"entity_relationship_extraction_news_devset.pkl\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniconda/base/envs/nano-graphrag/lib/python3.10/site-packages/datasets/table.py:1421: FutureWarning: promote has been superseded by promote_options='default'.\n", + " table = cls._concat_blocks(blocks, axis=0)\n" + ] + } + ], + "source": [ + "fin_news = load_dataset(\"ashraq/financial-news-articles\")\n", + "cnn_news = load_dataset(\"AyoubChLin/CNN_News_Articles_2011-2022\")\n", + "fin_shuffled_indices = np.random.permutation(len(fin_news['train']))\n", + "cnn_train_shuffled_indices = np.random.permutation(len(cnn_news['train']))\n", + "cnn_test_shuffled_indices = np.random.permutation(len(cnn_news['test']))\n", + "train_data = cnn_news['train'].select(cnn_train_shuffled_indices[:train_len])\n", + "val_data = cnn_news['test'].select(cnn_test_shuffled_indices[:val_len])\n", + "dev_data = fin_news['train'].select(fin_shuffled_indices[:dev_len])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 100 chunks, 907 entities(duplicated), 667 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:nano-graphrag:Saved 100 examples with keys: ['input_text', 'entities', 'relationships'], filtered 0 examples\n" + ] + } + ], + "source": [ + "train_chunks = {compute_mdhash_id(text, prefix=f\"chunk-trainset-\"): {\"content\": text} for text in train_data[\"text\"]}\n", + "trainset = asyncio.run(generate_dataset(chunks=train_chunks, filepath=entity_relationship_trainset_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Example({'input_text': ' (CNN)Former Russian spy Sergei Skripal and his daughter, Yulia, are continuing to recover from a nerve agent attack, and their rehabilitation has been \"slow and extremely painful,\" the daughter said Wednesday. Sergei, 66, and Yulia Skripal were found March 4 slumped on a bench in Salisbury, England, after being exposed to novichok, a military-grade nerve agent.While Yulia Skripal, 33, spent 20 days in a coma before she was released from the hospital in April and taken to a safe location, her father remained hospitalized until last week. \"We are so lucky to have both survived this attempted assassination,\" she said Wednesday from an undisclosed location, her first public appearance since the attack. \"I don\\'t want to describe the details, but the clinical treatment was invasive, painful and depressing. Our recovery has been slow and extremely painful.\"JUST WATCHEDYulia Skripal released from hospitalReplayMore Videos ...MUST WATCHYulia Skripal released from hospital 01:48They both are progressing, she said, but her life has been turned upside down, and she\\'s experienced physical and emotional changes, on which she did not elaborate. Read More\"I take one day at a time and want to help care for my father until his full recovery. In the longer term, I hope to return home to my country,\" she said. In requesting privacy and emphasizing that no one speaks for either her or her father, Yulia Skripal thanked the Salisbury District Hospital staff and others involved in their care. She added, \"I\\'m grateful for the offers of assistance from the Russian Embassy, but at the moment I do not wish to avail myself of their services.\"Former Russian spy Sergei Skripal and his daughter, Yulia Skripal, at a restaurant in Salisbury, UK.Lorna Wilkinson, director of nursing at the hospital, said last week that Sergei Skripal would continue his recovery outside the hospital, the UK Press Association reported. The poisoning of the Skripals sparked a diplomatic row between the UK and Russia, which has consistently denied allegations it was behind the poisoningThe UK expelled 23 Russian diplomats who had been declared as unidentified intelligence officers.The United States, Canada, Australia and 18 European Union states kicked out Russian diplomats in a show of support for the UK.Detectives with London\\'s Metropolitan Police believe the Skripals first came into contact with a nerve agent at Sergei Skripal\\'s home.JUST WATCHEDWho is Sergei Skripal? ReplayMore Videos ...MUST WATCHWho is Sergei Skripal? 01:56In late March, police identified the highest concentration of the nerve agent on the home\\'s front door.Upon Sergei Skripal\\'s release, Russian President Vladimir Putin wished him good health, but questioned the British claim that a military-grade nerve agent was responsible for his illness. \"A military-grade poisonous substance is so powerful that the person dies within seconds or minutes,\" Putin said at a news conference alongside German Chancellor Angela Merkel in Sochi, Russia.\"We repeatedly offered UK authorities our help, and we asked to be given access to the investigation, but there is no response,\" he said. \"Our offer stands.\"CNN\\'s James Masters contributed to this report.', 'entities': [{'entity_name': 'SERGEI SKRIPAL', 'entity_type': 'PERSON', 'description': 'Former Russian spy who was attacked with a nerve agent.', 'importance_score': 1.0}, {'entity_name': 'YULIA SKRIPAL', 'entity_type': 'PERSON', 'description': 'Daughter of Sergei Skripal, also attacked with a nerve agent.', 'importance_score': 0.9}, {'entity_name': 'SALISBURY', 'entity_type': 'LOCATION', 'description': 'City in England where the Skripals were found after the attack.', 'importance_score': 0.8}, {'entity_name': 'UK', 'entity_type': 'LOCATION', 'description': 'Country where the attack on the Skripals took place.', 'importance_score': 0.7}, {'entity_name': 'RUSSIA', 'entity_type': 'LOCATION', 'description': 'Country of origin for Sergei Skripal and the suspected source of the attack.', 'importance_score': 0.8}, {'entity_name': 'VLADIMIR PUTIN', 'entity_type': 'PERSON', 'description': 'Russian President who commented on the Skripal case.', 'importance_score': 0.7}, {'entity_name': 'ANGELA MERKEL', 'entity_type': 'PERSON', 'description': 'German Chancellor who was alongside Putin during a news conference.', 'importance_score': 0.6}, {'entity_name': 'SOCHI', 'entity_type': 'LOCATION', 'description': 'City in Russia where Putin and Merkel held a news conference.', 'importance_score': 0.5}, {'entity_name': 'NOVICHOK', 'entity_type': 'CHEMICAL', 'description': 'Military-grade nerve agent used in the attack on the Skripals.', 'importance_score': 0.9}], 'relationships': [{'src_id': 'SERGEI SKRIPAL', 'tgt_id': 'YULIA SKRIPAL', 'description': 'Father-daughter relationship between Sergei Skripal and Yulia Skripal.', 'weight': 1.0, 'order': 1}, {'src_id': 'SERGEI SKRIPAL', 'tgt_id': 'SALISBURY', 'description': 'Sergei Skripal was found in Salisbury after the attack.', 'weight': 0.8, 'order': 1}, {'src_id': 'YULIA SKRIPAL', 'tgt_id': 'SALISBURY', 'description': 'Yulia Skripal was found in Salisbury after the attack.', 'weight': 0.8, 'order': 1}, {'src_id': 'SERGEI SKRIPAL', 'tgt_id': 'UK', 'description': 'Sergei Skripal was attacked in the UK.', 'weight': 0.7, 'order': 1}, {'src_id': 'YULIA SKRIPAL', 'tgt_id': 'UK', 'description': 'Yulia Skripal was attacked in the UK.', 'weight': 0.7, 'order': 1}, {'src_id': 'SERGEI SKRIPAL', 'tgt_id': 'RUSSIA', 'description': 'Sergei Skripal is from Russia and the attack is suspected to be from Russia.', 'weight': 0.8, 'order': 2}, {'src_id': 'YULIA SKRIPAL', 'tgt_id': 'RUSSIA', 'description': 'Yulia Skripal is from Russia and the attack is suspected to be from Russia.', 'weight': 0.8, 'order': 2}, {'src_id': 'VLADIMIR PUTIN', 'tgt_id': 'SERGEI SKRIPAL', 'description': 'Vladimir Putin commented on the Skripal case.', 'weight': 0.7, 'order': 2}, {'src_id': 'VLADIMIR PUTIN', 'tgt_id': 'ANGELA MERKEL', 'description': 'Vladimir Putin and Angela Merkel held a news conference together.', 'weight': 0.6, 'order': 1}, {'src_id': 'VLADIMIR PUTIN', 'tgt_id': 'SOCHI', 'description': 'Vladimir Putin and Angela Merkel held a news conference in Sochi.', 'weight': 0.5, 'order': 1}, {'src_id': 'SERGEI SKRIPAL', 'tgt_id': 'NOVICHOK', 'description': 'Sergei Skripal was exposed to novichok in the attack.', 'weight': 0.9, 'order': 1}, {'src_id': 'YULIA SKRIPAL', 'tgt_id': 'NOVICHOK', 'description': 'Yulia Skripal was exposed to novichok in the attack.', 'weight': 0.9, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': ' (CNN)Roger Federer thinks the professional tennis circuit won\\'t return for a while due to the coronavirus pandemic but, when the time does come, the Swiss superstar said he would find it difficult to play without fans. While European football\\'s Bundesliga resurfaced last week behind closed doors and Spain\\'s La Liga is set to resume the middle of next month, the last official word from tennis authorities essentially saw all action suspended through July. The hiatus began in March, with Federer already sidelined since he was recuperating from knee surgery. Sunday would have marked the first day of the French Open in its usual spot on the tennis calendar -- in March, though, it was rescheduled for September -- and another grand slam, Wimbledon in July, was called off. \"I\\'m not training at the moment because I don\\'t see a reason for that to be honest,\" Federer told three-time French Open champion Gustavo Kuerten -- who is raising funds for coronavirus relief efforts in his native Brazil -- in a video interview reported by Tennis.com.Read More\"I am happy with my body now and I still believe that the return of the tour\\xa0is a long way off,\" continued the 38-year-old. \"And I think it\\'s important\\xa0mentally to enjoy this break, having played so much tennis.\\xa0\"When I\\'m getting towards returning and have a goal to train for, I think I will be super motivated.\"We should be sliding into @rolandgarros right now, thinking of our mates in Paris 👋 pic.twitter.com/0PLKryyIjj— #AusOpen (@AustralianOpen) May 24, 2020 Federer is arguably tennis\\' best supported player ever, and the prospect of competing without spectators doesn\\'t appeal to him. \"Most of the time when we are training, there is no one,\" said the men\\'s record 20-time grand slam champion. \"For us, of course, it is possible to play if there are no fans. But on the other hand, I really hope that the circuit can return as it normally is. \"And hold off till the time is appropriate, minimum\\xa0a third of the stadium or half full. But, for me, completely empty when playing big tournaments is very difficult.\"Federer has been active on social media during the lockdown, sparking a public discussion on the merging of the men\\'s and women\\'s tours with a tweet last month and embarking on a funny Instagram Live with tennis rival Rafael Nadal.Nadal, unlike Federer, has started practicing, though only very recently. The Spaniard would have been favored to win a 20th major and tie Federer had the French Open been played as usual given he has collected a record 12 titles at Roland Garros. Here I am, the first pictures I am posting for you on court. This is my practice earlier today at @rnadalacademy #BackOnCourt #BabolatFamily 🎾👍🏻💪🏻😉 pic.twitter.com/x7tzgLj9pc— Rafa Nadal (@RafaelNadal) May 22, 2020 The next grand slam is scheduled to be late August\\'s US Open in New York, with organizers expected to announce in June if it will go ahead. ', 'entities': [{'entity_name': 'ROGER FEDERER', 'entity_type': 'PERSON', 'description': \"Swiss tennis superstar who thinks the professional tennis circuit won't return for a while due to the coronavirus pandemic.\", 'importance_score': 1.0}, {'entity_name': 'GUSTAVO KUERTEN', 'entity_type': 'PERSON', 'description': 'Three-time French Open champion who is raising funds for coronavirus relief efforts in his native Brazil.', 'importance_score': 0.8}, {'entity_name': 'RAFAEL NADAL', 'entity_type': 'PERSON', 'description': 'Tennis rival of Roger Federer who has started practicing recently.', 'importance_score': 0.9}, {'entity_name': 'FRENCH OPEN', 'entity_type': 'EVENT', 'description': 'A grand slam tennis tournament that was rescheduled for September.', 'importance_score': 0.7}, {'entity_name': 'WIMBLEDON', 'entity_type': 'EVENT', 'description': 'A grand slam tennis tournament in July that was called off.', 'importance_score': 0.7}, {'entity_name': 'US OPEN', 'entity_type': 'EVENT', 'description': 'A grand slam tennis tournament scheduled for late August in New York.', 'importance_score': 0.7}, {'entity_name': 'ROLAND GARROS', 'entity_type': 'LOCATION', 'description': 'The location where the French Open is usually held.', 'importance_score': 0.6}], 'relationships': [{'src_id': 'ROGER FEDERER', 'tgt_id': 'GUSTAVO KUERTEN', 'description': 'Roger Federer discussed the return of the tennis tour with Gustavo Kuerten in a video interview.', 'weight': 0.8, 'order': 1}, {'src_id': 'ROGER FEDERER', 'tgt_id': 'RAFAEL NADAL', 'description': 'Roger Federer has engaged in social media discussions and a funny Instagram Live with tennis rival Rafael Nadal.', 'weight': 0.9, 'order': 1}, {'src_id': 'ROGER FEDERER', 'tgt_id': 'FRENCH OPEN', 'description': 'Roger Federer mentioned the rescheduling of the French Open in September.', 'weight': 0.7, 'order': 1}, {'src_id': 'ROGER FEDERER', 'tgt_id': 'WIMBLEDON', 'description': 'Roger Federer mentioned the cancellation of Wimbledon.', 'weight': 0.7, 'order': 1}, {'src_id': 'RAFAEL NADAL', 'tgt_id': 'FRENCH OPEN', 'description': 'Rafael Nadal would have been favored to win the French Open if it had been played as usual.', 'weight': 0.8, 'order': 1}, {'src_id': 'RAFAEL NADAL', 'tgt_id': 'US OPEN', 'description': \"Rafael Nadal's practice is in preparation for the upcoming US Open.\", 'weight': 0.7, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'London (CNN)Top archbishops in the Church of England have apologized for guidance issued by the church last week that said only married heterosexuals should have sex -- while same-sex or heterosexual Christians in civil partnerships should remain abstinent. Archbishop of Canterbury Justin Welby and Archbishop of York John Sentamu said in a statement Thursday that they took responsibility for last week\\'s announcement, \"which we acknowledge has jeopardized trust.\" \"We are very sorry and recognize the division and hurt this has caused,\" Welby and Sentamu wrote.The statement stopped short of retracting the \"pastoral guidance\" issued by the bishops of the Church of England, which was in response to the extension of civil partnerships to heterosexual couples in the UK.Church of England backs climate-friendly stock index that excludes BP and Exxon MobilThe guidance, published January 22, said \"for Christians, marriage -- that is the lifelong union between a man and a woman, contracted with the making of vows -- remains the proper context for sexual activity.\"Read MoreWhen it comes to civil partnerships, the church sought \"to affirm the value of committed, sexually abstinent friendships.\" It highlighted division between conservatives -- who want the church to stick to a traditional biblical teachings on marriage -- and the church\\'s progressive members. The guidance was described as \"a laughingstock to a nation that believes it (the church) is obsessed with sex,\" in an open letter to the archbishops. The letter, signed by more than 3,000 people, including dozens of members of the clergy, said the bishops\\' guidance \"has significantly damaged the mission of the church and it has broken the trust of those it seeks to serve.\"It goes on to express dismay that a public pronouncement on sex and marriage was made while the church was still undergoing a review of of the issue, called the \"Living in Love and Faith\" project. The letter said its signatories had not expected an announcement on the matter until the project\\'s report was published. \"It seems our trust has been misplaced and we feel badly let down.\" ', 'entities': [{'entity_name': 'LONDON', 'entity_type': 'LOCATION', 'description': 'The capital city of the United Kingdom, mentioned as the location of the news source.', 'importance_score': 0.8}, {'entity_name': 'JUSTIN WELBY', 'entity_type': 'PERSON', 'description': \"The Archbishop of Canterbury, who apologized for the church's guidance on sexual activity.\", 'importance_score': 0.9}, {'entity_name': 'JOHN SENTAMU', 'entity_type': 'PERSON', 'description': \"The Archbishop of York, who also apologized for the church's guidance on sexual activity.\", 'importance_score': 0.9}, {'entity_name': 'CHURCH OF ENGLAND', 'entity_type': 'ORGANIZATION', 'description': 'The established church of England, which issued controversial guidance on sexual activity.', 'importance_score': 0.9}, {'entity_name': 'JANUARY 22', 'entity_type': 'DATE', 'description': 'The date when the pastoral guidance was published by the Church of England.', 'importance_score': 0.7}], 'relationships': [{'src_id': 'JUSTIN WELBY', 'tgt_id': 'CHURCH OF ENGLAND', 'description': \"Justin Welby, as the Archbishop of Canterbury, is responsible for the Church of England's guidance on sexual activity.\", 'weight': 0.9, 'order': 1}, {'src_id': 'JOHN SENTAMU', 'tgt_id': 'CHURCH OF ENGLAND', 'description': \"John Sentamu, as the Archbishop of York, is responsible for the Church of England's guidance on sexual activity.\", 'weight': 0.9, 'order': 1}, {'src_id': 'CHURCH OF ENGLAND', 'tgt_id': 'JUSTIN WELBY', 'description': 'The Church of England issued guidance on sexual activity under the leadership of Justin Welby.', 'weight': 0.9, 'order': 1}, {'src_id': 'CHURCH OF ENGLAND', 'tgt_id': 'JOHN SENTAMU', 'description': 'The Church of England issued guidance on sexual activity under the leadership of John Sentamu.', 'weight': 0.9, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': ' (CNN)Germans will file into polling stations to vote in an unpredictable federal election on Sunday -- but for the first time in nearly two decades, their longtime leader won\\'t be in contention.Chancellor Angela Merkel has been a symbol of stability in Europe since she took on the role in 2005; the chemist-turned-political mainstay has withstood a wave of populism, a financial crisis, a pandemic and Brexit to carve out an impressive legacy as the world\\'s most successful female leader.But Merkel, 67, will step down once the repercussions of Sunday\\'s vote become clear, a move that has cast a sense of uncertainty over the weekend\\'s election.Compared to the previous votes in 2017 and 2013, there is \"much more chance of a significant shift in German politics and policy after the election,\" according to Pepijn Bergsen, a research fellow who monitors the country for international think tank Chatham House.The race to become Merkel\\'s successor is tight, and the ultimate victor may not be known for days or even weeks after polls close. Read MoreBut for the first time in a generation, Germans will be deciding what post-Merkel Germany will look like. Whoever they turn to will face a catalog of challenges, both at home and abroad.Merkel has provided a steady hand domestically and abroad, but Germans must now decide on her successor.Where is Merkel?Merkel\\'s departure from the front lines of global politics has been a long time coming; she first announced in 2018 that she would not seek re-election at the end of her term, following a series of setbacks in regional elections.In her time in office, she has dealt with five UK prime ministers, four French presidents, seven Italian prime ministers, and four American commanders-in-chief. Her period in power has been a remarkably eventful one, and Merkel\\'s imperturbable presence throughout has earned her an international reputation for stability and level-headedness. \"That worked very well politically for her in Germany, and on the world stage,\" Bergsen told CNN. \"Germany has done very well over the last 15 years from an economic perspective ... (and) Germany didn\\'t do that badly during the financial crisis, but the realization has crept in that that won\\'t last.\"The European refugee crisis of the mid-2010s proved a major challenge to Merkel\\'s party, the Christian Democratic Union (CDU), and she has also earned detractors over her close relationship with China. But after a pandemic which saw Germany fare better than many of its neighbors, analysts and polling suggest Merkel will leave office with the respect of most Germans. \"She\\'s seen very positively in Germany, because she\\'s associated with stability -- people know what they\\'re getting,\" said Ben Schreer, from the International Institute for Strategic Studies\\' (IISS) Berlin-based Europe office.Who\\'s in the race to replace her?German politics is dominated by two parties -- the center-right CDU and the left-leaning Social Democratic Party, or SPD -- who have governed together in a coalition for the past eight years. But other parties have grown in popularity over the past decade as the CDU and SPD have lost ground. This election is particularly close; the CDU and SPD have both held polling advantages, and the Green Party has also emerged as a serious contender.Merkel\\'s successor at the helm of the CDU is Armin Laschet, 60, a long-time ally of the Chancellor and the party\\'s deputy leader since 2012. A devout Catholic whose father was at one point a coal mining engineer, he was selected as the party\\'s candidate after a torturous leadership tussle.Laschet has a background in law and journalism, and was elected to the German Bundestag in 1994.Laschet won a protracted leadership campaign to replace Merkel, but he is struggling to attract voters on the national stage.Merkel has voiced her support for Laschet, but despite her efforts to persuade Germans to stick with the CDU, polling suggests her replacement as the party\\'s leader has struggled to win over Germans.His foremost opponent is the SPD\\'s Olaf Scholz, who has taken a surprise lead in the polls in recent weeks, leaving him as the marginal frontrunner heading into Sunday\\'s vote. Like Laschet, Scholz has a long history as a political player in Germany. He has been Merkel\\'s finance minister and vice chancellor since 2018, placing him arguably in a better position to run as her natural successor than her own party\\'s candidate.Scholz has earned increased visibility as he navigated Germany\\'s economic response to the pandemic, and cleared the last electoral hurdle with an assured performance in the final television debate. But polls nonetheless suggest a huge number of undecided voters late in the campaign, increasing the unpredictability of the vote.The Green Party\\'s leader Annalena Baerbock caused a brief sensation in German politics when she surged in the polls early in the campaign, prompting voters to wonder whether she could become the country\\'s first ever Green chancellor.Olaf Scholz has taken a surprise lead in polling in recent weeks.A 40-year-old former professional trampolinist, Baerbock stands out in a field of mostly male political leaders. And although her star has faded somewhat in the closing stretch, she has capitalized on voters\\' climate concerns to establish her group as the third party in the race. The far-right AfD remains a stubborn presence on the political scene, scrapping with the liberal Free Democratic Party for fourth place. The refugee crisis that sparked the AfD\\'s surge in German politics has subsided as a pressing political issue, but the party remains an outlet for voters angered by immigration issues. In March, they became the first German party since the Nazi era to be put under government surveillance. How does the voting work?German elections to the Bundestag are run on a system of proportional representation, meaning that each party\\'s vote share relates directly to how many seats they get in parliament. That principle makes it virtually impossible for a party to lead a government alone; coalitions must instead be formed after the vote, and these often contain more than two groups.Many Germans have already cast their ballots; the pandemic has increased the amount of postal voting that took place before polling day. Regardless of how they choose to vote, Germans are asked to pick their local lawmaker, and also their preferred overall party. Once the results come in, a race will start to put together enough seats to govern -- meaning smaller parties can become kingmakers.\"Whoever wins on paper on Sunday night probably can\\'t be sure that he or she will actually lead the government, because there\\'s going to be so many permutations,\" Schreer explained, adding: \"We may not know until November, if we\\'re lucky.\"What are the issues?All the candidates are caught in a Merkel-sized conundrum, as they attempt to define their own agendas while allaying Germans\\' fears over a change in leadership. Climate change has been a major factor in the country\\'s national debate, particularly after devastating flooding hit the country in July. In Canada and Germany, the climate crisis is finally on the ballot. But can it win?A push from Merkel has put environmental issues at the heart of German politics, and virtually all parties have stressed their green credentials. In this campaign the Green Party has called for a 70% cut in greenhouse gas emissions from 1990 levels by 2030, compared to the current government goal of a 55% cut.Economic worries have also come to the fore; in a last-gasp pitch to voters, Laschet said Monday that a left-wing coalition led by the SPD would cause a \"severe economic crisis,\" Reuters reported. Laschet has also followed Merkel\\'s line regarding the European Union; in the final pre-election debate, he emphasized European cohesion as one of his flagship policies. But the campaign has been mostly defined by domestic matters; a minimum wage hike and pension reforms are at the heart of Scholz\\'s campaign, and he stressed those plans again in the debate.Will a Merkel-less Germany still lead on the world stage?The global consequences of Sunday\\'s vote are clear; Merkel\\'s longevity saw her become Europe\\'s de facto leader, and it\\'s unclear whether her successor will fill the same role.\"Germany\\'s going to be faced with some significant foreign policy challenges which the new government has to take on,\" Schreer said. \"The question is, who\\'s going to replace (Merkel), and will that person have the same charisma and ability that she did?\" he added. \"Allies are skeptical, and Germans as well are quite cautious in that regard.\"A key part of Merkel\\'s role was her steadfast determination to maintain European cohesion and paper over the cracks between EU member states. Merkel has outlasted dozens of major leaders during her 16-year period in power.\"Macron will try to usurp Merkel\\'s position in Europe,\" predicted Bergsen, signaling a possible shift in the balance of power towards France, Germany\\'s western neighbor. \"The German position won\\'t necessarily change, but whoever now comes to power will have to deal with a broader (domestic) coalition so they will find it slightly harder to lead on the international stage.\"Looking further afield, Germany\\'s new leader will also have to balance the country\\'s relationships with the United States and China, two nations with whom Merkel attempted to maintain close ties.And keeping the United Kingdom close after its departure from the EU is key. \"The UK remains an important partner in strategic terms, and Germany knows that if the UK isn\\'t engaged in the European continent, then you will split the Europeans,\" said Schreer.\"(Germany) is a well-respected country at the international stage -- that is undoubtedly the case,\" he added. \"The question is: Does that now enable Germany to weather those international storms that are certainly coming?\"', 'entities': [{'entity_name': 'ANGELA MERKEL', 'entity_type': 'PERSON', 'description': 'Chancellor of Germany, symbol of stability in Europe, chemist-turned-political mainstay', 'importance_score': 1.0}, {'entity_name': 'GERMANY', 'entity_type': 'LOCATION', 'description': 'Country where the federal election is taking place', 'importance_score': 0.9}, {'entity_name': 'CHRISTIAN DEMOCRATIC UNION (CDU)', 'entity_type': 'POLITICAL_PARTY', 'description': \"Merkel's party, faced challenges during the European refugee crisis\", 'importance_score': 0.8}, {'entity_name': 'ARMIN LASCHET', 'entity_type': 'PERSON', 'description': \"Merkel's successor at the helm of the CDU, long-time ally of the Chancellor\", 'importance_score': 0.8}, {'entity_name': 'OLAF SCHOLZ', 'entity_type': 'PERSON', 'description': \"SPD's candidate, took a surprise lead in the polls, Merkel's finance minister and vice chancellor\", 'importance_score': 0.8}, {'entity_name': 'SOCIAL DEMOCRATIC PARTY (SPD)', 'entity_type': 'POLITICAL_PARTY', 'description': 'Left-leaning party, governed together with CDU in a coalition', 'importance_score': 0.7}, {'entity_name': 'ANNALENA BAERBOCK', 'entity_type': 'PERSON', 'description': \"Green Party's leader, former professional trampolinist\", 'importance_score': 0.7}, {'entity_name': 'GREEN PARTY', 'entity_type': 'POLITICAL_PARTY', 'description': 'Serious contender in the election, focused on climate change', 'importance_score': 0.7}, {'entity_name': 'AFD', 'entity_type': 'POLITICAL_PARTY', 'description': 'Far-right party, remains a presence on the political scene', 'importance_score': 0.6}], 'relationships': [{'src_id': 'ANGELA MERKEL', 'tgt_id': 'ARMIN LASCHET', 'description': \"Merkel's successor at the helm of the CDU\", 'weight': 0.9, 'order': 1}, {'src_id': 'ANGELA MERKEL', 'tgt_id': 'OLAF SCHOLZ', 'description': \"Merkel's finance minister and vice chancellor\", 'weight': 0.8, 'order': 1}, {'src_id': 'ARMIN LASCHET', 'tgt_id': 'CHRISTIAN DEMOCRATIC UNION (CDU)', 'description': 'Laschet is the candidate for the CDU', 'weight': 0.9, 'order': 1}, {'src_id': 'OLAF SCHOLZ', 'tgt_id': 'SOCIAL DEMOCRATIC PARTY (SPD)', 'description': 'Scholz is the candidate for the SPD', 'weight': 0.9, 'order': 1}, {'src_id': 'ANNALENA BAERBOCK', 'tgt_id': 'GREEN PARTY', 'description': 'Baerbock is the leader of the Green Party', 'weight': 0.9, 'order': 1}, {'src_id': 'GERMANY', 'tgt_id': 'CHRISTIAN DEMOCRATIC UNION (CDU)', 'description': 'CDU is a major political party in Germany', 'weight': 0.8, 'order': 1}, {'src_id': 'GERMANY', 'tgt_id': 'SOCIAL DEMOCRATIC PARTY (SPD)', 'description': 'SPD is a major political party in Germany', 'weight': 0.8, 'order': 1}, {'src_id': 'GERMANY', 'tgt_id': 'GREEN PARTY', 'description': 'Green Party is a serious contender in the German election', 'weight': 0.7, 'order': 1}, {'src_id': 'GERMANY', 'tgt_id': 'AFD', 'description': 'AfD is a far-right party in Germany', 'weight': 0.6, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'Story highlightsBrazil beat Spain 3-0 in Confederations Cup finalBrazilian striker Neymar named as the best player of Confederations Cup Tournament permeated by protests over staging of 2014 World Cup in BrazilBrazil\\'s 3-0 win over Spain in the Confederations Cup final brought to an end an event that was designed as a test run for the 2014 World Cup hosts.It was a tournament that was permeated by social unrest, with protesters partly unhappy over the degree of public money devoted to the staging of the World Cup.CNN looks at five things we learned from the event.1. Neymar is the real dealNeymar has been touted as the latest in a line of Brazilian superstars that stretches all the way back to the great Pele, but outside Brazil many were beginning to believe the hype surrounding the young star was just that. Prior to the Confederations Cup his performances for the national team had been lacklustre, particular in the 2-2 draw against Chile in April, when he was singled out for jeers by Brazil\\'s own fans.JUST WATCHEDTour Brazilian football\\'s spiritual homeReplayMore Videos ...MUST WATCHTour Brazilian football\\'s spiritual home 02:24JUST WATCHEDBrazil wins Confederations CupReplayMore Videos ...MUST WATCHBrazil wins Confederations Cup 01:57JUST WATCHEDSpotlight on Paulinho ReplayMore Videos ...MUST WATCHSpotlight on Paulinho 00:46JUST WATCHEDThiago Silva targets PSG dominance ReplayMore Videos ...MUST WATCHThiago Silva targets PSG dominance 03:07Read: Maracana magic by Brazil to claim Confederations CupWhile most would argue that he\\'s in no position to judge, Marseille midfielder Joey Barton\\'s Tweet likening Neymar to Justin Bieber, and something rather unpleasant relating to cats, seemed to sum up the views of many European observers.Following the Brazilian\\'s $75 million transfer to Barcelona -- just before the tournament kicked off -- some were even suggesting the Catalan club had been fleeced by Brazilian side Santos. Not any more: this was no tantalizing glimpse of what La Liga fans might be watching next season, it was incontrovertible proof of what Neymar can bring. The 21-year-old produced a string of vibrant, flamboyant, but above all effective performances throughout the tournament, his goals and guile serving notice that he is every inch a superstar.2. Time may be up for Buffon and companyAfter winning friends at Euro 2012, the Azzurri continued to show that the days of \"catenaccio\" are long gone, with some fast attacking football. Italy\\'s run in the tournament was certainly invigorating, lurching from near disaster in an extraordinary first-half capitulation against Japan, to a sublime (albeit goalless) first-half domination of Spain.Read: Spain edge past Italy In the end they were well worth their third place, but it could and possibly should have been more. Gianluigi Buffon is widely and justifiably regarded as one of the world\\'s finest goalkeepers; yet his penalty saves in the third place play-off against Uruguay masked his culpability in goals conceded at key moments in the tournament.This was not the dominant, steely-eyed Buffon of old -- particularly against Brazil, where he looked anything but solid. Likewise, there are other old hands whose time in the famous blue shirt may be nearing its end. Given Cesare Prandelli\\'s reluctance to start AC Milan\\'s exciting midfielder Stephan El Shaarawy, major surgery is unlikely. Photos: Brazil\\'s most painful moment Photos: Brazil\\'s most painful momentA national tragedy – Moacyr Barbosa Nascimento\\'s life was forever changed after the 1950 World Cup. With Brazil needing just a draw against Uruguay in its final game to lift the trophy for the first time, the team lost 2-1 and he was blamed for the second goal. The goalkeeper\\'s perceived mistake haunted him. Twenty years later he overheard a woman in a supermarket say to her son, \"There is the man who made Brazil cry.\"Hide Caption 1 of 8 Photos: Brazil\\'s most painful momentThe Maracana – The Maracana Stadium in Rio de Janeiro was the venue for the 1950 final, with 200,000 spectators packed into the purpose-built arena. The stadium has been redeveloped and a crowd of 78,000 people will watch the final of 2014 World Cup at the iconic ground.Hide Caption 2 of 8 Photos: Brazil\\'s most painful momentBrazil\\'s golden boy – All eyes will be on Neymar during both June\\'s Confederations Cup and next year\\'s World Cup. The attacker, who recently signed for Barcelona in a deal reportedly worth in excess of $80 million, is Brazil\\'s star player and must perform to his best if \"La Selecao\" are to satisfy an expectant public.Hide Caption 3 of 8 Photos: Brazil\\'s most painful momentThe greatest ever? – The Brazil team of 1970, which beat Italy 4-1 in the World Cup final in Mexico, is widely regarded as the greatest of all time. Pele, a three-time World Cup winner seen here leaping on his teammates, says Brazil must recover from the failure of 63 years ago.Hide Caption 4 of 8 Photos: Brazil\\'s most painful momentThree-peat – Carlos Alberto, captain of the 1970 team, lifts the Jules Rimet trophy which Brazil was allowed to keep after becoming the first nation to win the World Cup three times. The former fullback thinks next year\\'s World Cup will come too soon for Brazil\\'s inexperienced team.Hide Caption 5 of 8 Photos: Brazil\\'s most painful momentThe second coming – Luiz Felipe Scolari was the coach of the last Brazil team to lift the World Cup, in Japan and South Korea in 2002. The veteran has been reappointed in a bid to inject life into an ailing Brazil team. His results have so far left much to be desired: two wins, one defeat and four draws since November 2012.Hide Caption 6 of 8 Photos: Brazil\\'s most painful momentGrand reopening – England was Brazil\\'s first opponent at a refurbished Maracana earlier this month. A half-volley from midfielder Paulinho, pictured, rescued a 2-2 draw for the 2014 World Cup host.Hide Caption 7 of 8 Photos: Brazil\\'s most painful momentA flourish against France – Brazil\\'s most recent match, the last before the Confederations Cup starts, ended in a comfortable 3-0 defeat of France. A penalty from Lucas Moura, right, completed the scoring.Hide Caption 8 of 8 Photos: Brazil\\'s greatest footballers Photos: Brazil\\'s greatest footballersHeleno – Actor Rodrigo Santoro signs a poster for the film \"Heleno\", in which he plays the mercurial striker. A destructive personality, together with illness and drug problems prevented Heleno from becoming one of Brazil\\'s greatest ever players. But he helped pave the way for some of the world\\'s greatest soccer icons...Hide Caption 1 of 8 Photos: Brazil\\'s greatest footballersPele – Ask many Brazilians who is the greatest footballer of all time and their answer will be simple: \"Pele.\" The striker won three World Cups with Brazil between 1958 and 1970 and is his country\\'s leading goalscorer with 77 goals from 92 caps.Hide Caption 2 of 8 Photos: Brazil\\'s greatest footballersGarrincha – Most football fans would say Argentina\\'s Diego Maradona is the only player who can rival Pele for the title of greatest ever. In Brazil, however, Garrincha is regarded as the only player who comes close to the great man. The tricky winger was a key part of Brazil\\'s World Cup triumphs in 1958 and 1962. Sadly, Garrincha struggled with alcohol problems and died of liver cirrhosis aged 49.Hide Caption 3 of 8 Photos: Brazil\\'s greatest footballersZico – After a Pele-inspired triumph in 1970, Brazil would wait 24 years before lifting the World Cup again. Although the 1980s was a barren decade in terms of trophies for Brazil, the team which the South Americans sent to the 1982 World Cup is heralded as one of the most entertaining in history. Central to its free-flowing, attacking style was Zico, a midfielder of considerable craft and guile who collected 72 caps between 1976 and 1988.Hide Caption 4 of 8 Photos: Brazil\\'s greatest footballersRomario – When Brazil finally won the World Cup for a fourth time in 1994 in the U.S., the team was derided by some for being too functional. In a team short of star quality, striker Romario was the shining light, scoring five goals as Brazil lifted the trophy thanks to a penalty-shootout victory over Italy.Hide Caption 5 of 8 Photos: Brazil\\'s greatest footballersRonaldo – Ronaldo watched on as Romario fired Brazil to victory in 1994, four years later he was the star man at France 1998. Brazil lost the final 3-0 to the hosts, with mystery surrounding their starting 11 as Ronaldo was left out of, then reinstated to, the team for the deciding match at the Stade de France. Ronaldo\\'s redemption arrived in 2002, when he scored both goals as Brazil beat Germany 2-0 to lift the World Cup for a fifth time.Hide Caption 6 of 8 Photos: Brazil\\'s greatest footballersRonaldinho – While Ronaldo was the star man in Japan and South Korea, he was ably supported by flamboyant playmaker Ronaldinho. Ronaldinho\\'s performance in the World Cup earned him a move to Barcelona in 2003, where he went on to win the European Champions League in 2006. He was twice named FIFA World Player of the Year.Hide Caption 7 of 8 Photos: Brazil\\'s greatest footballersNeymar? – The latest Brazilian tipped for stardom is Neymar, who recently followed in Ronaldinho\\'s footsteps by joining Barcelona. All eyes will be on the forward when Brazil host the World Cup in 2014. Neymar has made a good start to Brazil\\'s Confederations Cup campaign, scoring two goals in two matches.Hide Caption 8 of 8 Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protests Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protestsGolden Boy – Hours after declaring himself saddened by the need for protests against Brazil\\'s social conditions, Neymar brought joy to his compatriots with the opening goal in a 2-0 win over Mexico. Hide Caption 1 of 5 Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protestsNo Ordinary Game – A family of four negotiate their way to the game as riot police prevent protesters from gaining access to Fortaleza\\'s Castelao Stadium. Hide Caption 2 of 5 Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protestsStalemate – Protesters confront riot police officers on the distant outskirts of the Castelao Stadium, which has been newly built for next year\\'s World Cup at a cost of $240 million.Hide Caption 3 of 5 Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protestsLone Marksman – A police officer holds his weapon as protesters continue their demonstration ahead of Brazil\\'s Group A match with Mexico in the ongoing Confederations Cup.Hide Caption 4 of 5 Photos: Confederations Cup: Brazil beats Mexico as Fortaleza protestsPower Play – Fans hold up banners - which state that they are protesting against corruption, rather than the national team - ahead of Brazil\\'s 2-0 win over Mexico.Hide Caption 5 of 5The Italian national side has always been resistant to change, as its ageing spine suggests, but it may need to pick up the pace of its evolution. The likes of Buffon and the peerless Andrea Pirlo may have one more big tournament in them, but Prandelli would do well to blood some fresh talent in the meantime, just in case -- especially given the physical demands next year\\'s World Cup will present. 3. Protests are a wake-up call to FIFA and BrazilFor a country that is synonymous with football, there were times during this tournament when Brazil seemed to have fallen well and truly out of love with the game -- or at least, with FIFA and the Brazilian government\\'s interpretation of what a World Cup should look like. As simmering social unrest threatened to boil into something more serious, the tournament\\'s detractors grew in volume and number. Even former star striker Romario joined a critical chorus that cited ticket prices, infrastructure costs and a questionable legacy as reasons why the Confederations Cup, and next year\\'s World Cup, were bad for Brazil. Read: A fair World Cup deal for BrazilIn anticipation of trouble, a reported 10,000 police were on duty in Rio de Janeiro ahead of Brazil\\'s clash with Spain, but in the end protests were relatively low-key. Following their team\\'s emphatic triumph over the world and European champions, crowds spilled out onto the street and into a carnival atmosphere.Clearly this story is not over, and there is much work to do; for now, however, football\\'s primacy has been restored. In fact there was much cause for optimism on the pitch. This was a well-deserved win from a team that looks to be gaining in stature, and the atmosphere inside the Maracana for the final was profoundly imposing.Of course Brazil won the 2009 Confederations Cup before limping out in the quarterfinals in South Africa; but this time they will be at home. The World Cup will be a tougher test; but don\\'t bet against them.4. Spain needs a re-bootFinally, after a seemingly interminable period of dominance, Spain\\'s champions look to have been found out. For clues as to how, it is hard to see beyond the Germans. Italy really should have beaten the Spaniards after playing them off the park for significant chunks of their semi-final; Brazil\\'s 3-0 defeat of the world champions was a muscular and ruthless final execution.Both owed much to the approaches of Bayern Munich and Borussia Dortmund in their UEFA Champions League defeats of Barcelona and Real Madrid.Spain were hustled and hassled, with attempts to impose their intricate passing game met by fiercely committed opponents, closing them down until the Europeans simply folded.Fernando Torres only won the golden boot because of his goals against Tahiti\\'s part-timers. Xavi and Iniesta are still fabulously creative, but elsewhere Spain\\'s weak links were exploited with a kind of physicality this team now seems incapable of countering. Drained of confidence, or possibly just appetite, they looked listless and in dire need of fresh ideas. The sight of a defender, Sergio Ramos, taking a penalty that could have brought them back into the final spoke volumes. True Brazil had had an extra day of rest before the final, but the mask of invincibility has slipped; Spain has 12 months to work out how to set it back in place.5. The Confederations Cup comes of ageIn theory at least, the Confederations Cup is the unloved second cousin of the World Cup, the Euros and the Copa America. Traditionally these games are seen by the cynics as little more than jumped up exhibition matches -- a mere aperitif before the main meal of the World Cup. By and large, no one really cares who wins. This time, however, something seemed to click. Maybe it was the location. Brazil may have some serious issues to confront, but there is something about the host country that elevated this tournament onto a different plane.In football terms, this is about as educated as any crowd can get. The enthusiastic way in which local fans adopted teams such as Japan, Uruguay, Tahiti and Italy conveyed atmosphere and meaning to fixtures where normally none would exist, and more than made up for the absence of traveling support. The outcome, for the neutrals at least, was first class entertainment. Brazil 2013 served up some of the most memorable international games of recent memory, played in front of passionate crowds in some spectacular arenas. Read: Brazil\\'s beautiful game?If Brazil 2014 can pick up where this rehearsal left off, then we are truly in for a treat.', 'entities': [{'entity_name': 'BRAZIL', 'entity_type': 'LOCATION', 'description': 'Brazil is a country that hosted the Confederations Cup and is preparing to host the 2014 World Cup. It experienced social unrest during the Confederations Cup due to public money being devoted to the World Cup.', 'importance_score': 1.0}, {'entity_name': 'SPAIN', 'entity_type': 'LOCATION', 'description': 'Spain is a country whose national football team was defeated 3-0 by Brazil in the Confederations Cup final. The Spanish team is known for its dominance in international football.', 'importance_score': 0.9}, {'entity_name': 'NEYMAR', 'entity_type': 'PERSON', 'description': 'Neymar is a Brazilian footballer who was named the best player of the Confederations Cup. He is considered a rising superstar in the football world and recently transferred to Barcelona for $75 million.', 'importance_score': 1.0}, {'entity_name': 'CONFEDERATIONS CUP', 'entity_type': 'EVENT', 'description': 'The Confederations Cup is an international football tournament held in Brazil, serving as a test run for the 2014 World Cup. It was marked by social unrest and protests.', 'importance_score': 0.9}, {'entity_name': '2014 WORLD CUP', 'entity_type': 'EVENT', 'description': 'The 2014 World Cup is an upcoming international football tournament to be hosted by Brazil. It is preceded by the Confederations Cup, which highlighted social issues and protests.', 'importance_score': 1.0}], 'relationships': [{'src_id': 'BRAZIL', 'tgt_id': 'SPAIN', 'description': 'Brazil defeated Spain 3-0 in the Confederations Cup final, showcasing their strength as potential hosts of the 2014 World Cup.', 'weight': 1.0, 'order': 1}, {'src_id': 'NEYMAR', 'tgt_id': 'CONFEDERATIONS CUP', 'description': 'Neymar was named the best player of the Confederations Cup, demonstrating his skill and potential as a future football superstar.', 'weight': 1.0, 'order': 1}, {'src_id': 'CONFEDERATIONS CUP', 'tgt_id': '2014 WORLD CUP', 'description': 'The Confederations Cup served as a test run for the 2014 World Cup, highlighting both the potential and the challenges faced by Brazil as hosts.', 'weight': 0.9, 'order': 2}]}) (input_keys={'input_text'})]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trainset[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 99 chunks, 901 entities(duplicated), 631 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 400 Bad Request\"\n", + "ERROR:nano-graphrag:Error in TypedEntityRelationshipExtractor: Error code: 400 - {'error': {'message': 'Content Exists Risk', 'type': 'invalid_request_error', 'param': None, 'code': 'invalid_request_error'}}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 100 chunks, 901 entities(duplicated), 631 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:nano-graphrag:Saved 99 examples with keys: ['input_text', 'entities', 'relationships'], filtered 1 examples\n" + ] + } + ], + "source": [ + "val_chunks = {compute_mdhash_id(text, prefix=f\"chunk-valset-\"): {\"content\": text} for text in val_data[\"text\"]}\n", + "valset = asyncio.run(generate_dataset(chunks=val_chunks, filepath=entity_relationship_valset_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Example({'input_text': 'Story highlightsPortugal qualifies for Euro 2016 finalDefeats Wales 2-0 in semifinalCristiano Ronaldo and Nani on targetFrance or Germany awaits in the final (CNN)Cristiano Ronaldo proved himself the man for the big occasion once again by firing Portugal into the final of Euro 2016 Wednesday.Ronaldo\\'s second-half header and a strike by his former Manchester United teammate Nani secured a 2-0 win against Wales in Lyon.Follow @cnnsport\\n\\nPortugal, which last reached the final of the competition in 2004, will face either host nation France or world champion Germany in Paris on Sunday.This contest had been billed as clash between two of the world\\'s top players -- Ronaldo and his Real Madrid teammate Gareth Bale.But Bale, who has enjoyed a stellar tournament for a country which had never before qualified for the tournament, failed to match the exploits of his rival.Read MoreRonaldo, who equaled Michel Platini\\'s record of scoring nine career goals at the Euros, will now hope to exorcise the ghosts of 2004 when Portugal was beaten by Greece in a final held in its own backyard. When you know you\\'re going to Paris. 🎉 #PORWAL #EURO2016 pic.twitter.com/7FXah0E6QE— UEFA EURO 2016 (@UEFAEURO) July 6, 2016\\n\"I hope that [after the final] we\\'ll be smiling and that it will be tears of joy in the end,\" Ronaldo told the tournament\\'s official website.\"I\\'ve always said my dream was to win a trophy with Portugal. We\\'re closer to doing it and I believe that we\\'ll win. \"With a lot of work, humility and the spirit of self-sacrifice, which I have always shown in my career, these sorts of things are doable.\"Bale and Ronaldo embraced at the end of the game.Rocky roadFor Portugal, the road to the final has been long and arduous, often due to the team\\'s own inadequacies.While Wales enjoyed a rather enchanting run to its first major semifinal, Portugal\\'s journey to its fifth could scarcely have been more different.No other team had arrived at the last-four stage of the European Championship finals without winning a game in 90 minutes -- but Portugal, which drew all three of its group matches and only qualified in third place, has always managed to do just enough.A last-16 win over Croatia, secured with just three minutes of extra-time remaining, was then followed by a penalty shootout triumph over Poland after the game had finished 1-1.For all the exciting talent, and there is plenty even if this may not be a vintage Portuguese team, rarely has it looked impressive.Wales paid special attention to Cristiano Ronaldo in the opening stages of the contest.And yet, what it has done, is use its experience, its know-how and a resolute determination to ensure it will not be beaten.In Renato Sanches, an 18-year-old midfielder recently bought by Bayern Munich for $36 million, Portugal has a player who will surely go on to take on Ronaldo\\'s mantle.But even the youngest and most precocious need guidance.That guidance has come from Ronaldo, who despite failing to find his best form, has carried this team for long periods of the tournament.His two goals in the 3-3 draw with Hungary, a game in which it trailed on three separate occasions, enabled Portugal to qualify as the third of the four best third-placed teams.Cristiano Ronaldo: Is Euro 2016 the last chance for Portugal great?\"It\\'s what we have dreamed of since the beginning,\" Ronaldo added.\"We knew it would be a long road and we\\'re still in the tournament. We have believed right from the start. \"We had difficult moments, but it\\'s like I always say: it\\'s better to start poorly and have a positive ending. \"We haven\\'t won anything yet as I said a few days ago, but the dream is still alive.\"LeaderFor so long Ronaldo has been the one who has been responsible for pushing his team forward, but even he has appeared to be struggling both on and off the field of play.First there was his outburst after his side\\'s group draw against Iceland, when he accused the island team of having a \"small mentality\" and claiming it would never achieve anything in the tournament -- a prediction which proved rather wide of the mark.Then there was his penalty miss in the goalless draw with Austria which was followed a few days later by his decision to throw a reporter\\'s microphone into a lake while out on a prematch walk.But for all his histrionics and his ability to split public opinion, few can doubt his talents and the records he has broken.🎉 Bravo @selecaoportugal: #EURO2016 finalists! 🎉#POR #PORWAL pic.twitter.com/kizafyy7EI— UEFA EURO 2016 (@UEFAEURO) July 6, 2016\\nHis 61 goals in 132 games makes him Portugal\\'s top scorer, while he is the only player to have scored in four consecutive European Championship finals.But his struggles in France have been well documented -- his wayward shooting, the lack of usual poise on front of goal and his waning influence on games has become more noticeable.It is in sharp contrast to his Real Madrid teammate Bale, who was responsible for creating one of the only real opportunities of a tight and cagey first period.Bale, who scored in each of Wales\\' first three group games, picked the ball up in his own half and drove through the Portuguese midfield before unleashing a fierce effort straight at goalkeeper Rui Patricio.Turning pointWhile the first half may have been rather uninspiring, the second was anything but as Ronaldo and Portugal took control of the contest.Just five minutes of the second half had passed when Ronaldo rose highest at the far post to meet Raphael Guerreiro\\'s cross and power a header into the top corner.Wales, still dazed from conceding the opening goal, was then dealt another blow just three minutes later.FT #Por 2-0 #Wal. Cristiano Ronaldo leads Portugal into the #Euro2016 final! pic.twitter.com/Z8vl4YUfxR— CNN Football Club (@CNNFC) July 6, 2016\\nOnce again it was Ronaldo at the heart of the move, flashing a driven effort into the penalty area -- and Nani, who on Tuesday joined Valencia, diverted the ball past stranded goalkeeper Wayne Hennessey.Wales, pushing forward in search of a way back into the contest, began to leave holes at the back and was fortunate not to fall further behind.First, Hennessey failed to hold Nani\\'s shot and Joao Mario somehow contrived to fire the rebound wide of the post with the goal gaping.Moments later it was Danilo who came close, his short squirming under Hennessey\\'s body before the goalkeeper recovered to stop the ball on the line.Bale had a couple efforts from long-range but failed to find a way past the Portuguese defense.Wales huffed and puffed, with Bale unleashing a couple of efforts from long range as Portugal sat back and hit its opponent on the counter attack.Roared on by thousands of fans, Wales continued to move forward but it lacked the precision and quality which had been so evident during its run to the final four.In the immediate aftermath of this defeat, it is easy to to forget that when Chris Coleman took over this team four years ago that it sat 117th in the world rankings and had not qualified for a major tournament since 1958.Just to play in these finals was an achievement -- but what this team has done over the past month will change the landscape of football in Wales forever.Here\\'s another look at the Ronaldo header that gave #Por the lead. What a leap! #PORWAL pic.twitter.com/2tuSLXU9vI— CNN Football Club (@CNNFC) July 6, 2016\\nNot only did it qualify ahead of England and outlast it, it played with a sense of pride, passion and least of all, quality.Some had labeled Wales a \"one-man team\" with Bale cast as the superstar dragging 10 other players along with him.Yet, such an assertion was to prove folly. Read more: Full Euro 2016 coverageWales have been a joy to watch throughout the tournament. Like Iceland, it boasts a connection between the team and supporters which few others can match, bar the Irish of course.It has inspired, it has brought hope and it has revitalized football in the country -- but this was a fairy tale without the ending so many had yearned for.For Ronaldo, though, the end is not yet written.', 'entities': [{'entity_name': 'PORTUGAL', 'entity_type': 'ORGANIZATION', 'description': 'The national football team of Portugal that qualified for the Euro 2016 final.', 'importance_score': 1.0}, {'entity_name': 'CRISTIANO RONALDO', 'entity_type': 'PERSON', 'description': 'A Portuguese footballer who scored in the Euro 2016 semifinal against Wales.', 'importance_score': 1.0}, {'entity_name': 'NANI', 'entity_type': 'PERSON', 'description': 'A Portuguese footballer and former Manchester United teammate of Cristiano Ronaldo who scored in the Euro 2016 semifinal against Wales.', 'importance_score': 0.8}, {'entity_name': 'WALES', 'entity_type': 'ORGANIZATION', 'description': 'The national football team of Wales that was defeated by Portugal in the Euro 2016 semifinal.', 'importance_score': 0.9}, {'entity_name': 'EURO 2016', 'entity_type': 'EVENT', 'description': 'The 2016 UEFA European Championship, a major football tournament held in France.', 'importance_score': 1.0}, {'entity_name': 'FRANCE', 'entity_type': 'LOCATION', 'description': 'The host nation of Euro 2016 and a potential opponent for Portugal in the final.', 'importance_score': 0.8}, {'entity_name': 'GERMANY', 'entity_type': 'ORGANIZATION', 'description': 'The national football team of Germany, the world champion and a potential opponent for Portugal in the final.', 'importance_score': 0.8}], 'relationships': [{'src_id': 'PORTUGAL', 'tgt_id': 'WALES', 'description': 'Portugal defeated Wales 2-0 in the semifinal of Euro 2016.', 'weight': 1.0, 'order': 1}, {'src_id': 'CRISTIANO RONALDO', 'tgt_id': 'NANI', 'description': 'Cristiano Ronaldo and Nani, both Portuguese footballers, scored in the semifinal against Wales.', 'weight': 0.9, 'order': 1}, {'src_id': 'PORTUGAL', 'tgt_id': 'FRANCE', 'description': 'Portugal may face France in the final of Euro 2016.', 'weight': 0.8, 'order': 2}, {'src_id': 'PORTUGAL', 'tgt_id': 'GERMANY', 'description': 'Portugal may face Germany in the final of Euro 2016.', 'weight': 0.8, 'order': 2}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': ' (CNN)A Swedish train operator is investigating the forcible removal of a pregnant woman from a train in Stockholm.A video clip posted on social media shows security guards working for Stockholm public transport authority SL dragging the woman off the train and pushing her down onto a bench on the platform.The woman had allegedly refused to accept a fine after failing to show a valid ticket, according to transport officials. Security guards stepped in after she allegedly became aggressive.The woman\\'s daughter can be seen crying as she tries to hold on to her mother, before another security guard leads the child away.\"In these two videos you can see the victim dragged out of the train, her child taken from her and whilst she attempts to stand up she is held down,\" wrote popular blogger and activist Lovette Jallow, who posted mobile phone footage of the incident on Instagram. Read More View this post on Instagram Tomorrow all the witnesses that came forth with their evidence will be making their own reports. . . In these two videos you can see the victim dragged out of the train, her child taken from her and whilst she attempts to stand up she is held down. The full video is over 5 minutes long and I will try and edit and put it up tonight or tomorrow. . . After everything the Swedish public was shown in the recent @kallafaktatv4 documentary about how aftiswedes are racially profiled and mistreated this shouldn\\'t surprise anyone even if the victim is pregnant. The victim is currently in the hospital and all I can do is hope the baby is alright because if anything happens to that child. There will be hell to pay. A post shared by ᒪOᐯETTE ᒍᗩᒪᒪOᗯ (@action4humanity_se) on Jan 31, 2019 at 1:17pm PST\\nIn the post, Jallow also criticized how Swedes of African origin are \"racially profiled and mistreated.\"The woman visited hospital for treatment after the incident, which took place at Hötorget metro station on January 31, according to a police report.\"All I can do is hope the baby is alright because if anything happens to that child. There will be hell to pay,\" wrote Jallow.The post drew many comments from other Instagram users. Sofie Fisen Jakopsohn, @sofiejakopsohn, left a comment calling the incident a crime against human rights, while others said that things would have been different if the woman had listened to the guards\\' instructions.SL spokesman Henrik Palmer told CNN that the security guards are subject to rigorous training to prevent these kinds of incidents.An internal investigation is being carried out to work out what went wrong, he said.\"We take this very seriously,\" said Palmer, adding that the security guards have been suspended until the investigation is complete.There are cameras on the train and the platform, said Palmer, and police have access to all of the footage.Stockholm police are investigating the incident but could not be reached for comment.', 'entities': [{'entity_name': 'SWEDISH TRAIN OPERATOR', 'entity_type': 'ORGANIZATION', 'description': 'The organization responsible for operating trains in Sweden.', 'importance_score': 0.8}, {'entity_name': 'STOCKHOLM PUBLIC TRANSPORT AUTHORITY SL', 'entity_type': 'ORGANIZATION', 'description': 'The public transport authority in Stockholm responsible for managing public transportation.', 'importance_score': 0.9}, {'entity_name': 'LOVETTE JALLOW', 'entity_type': 'PERSON', 'description': 'A popular blogger and activist who posted the incident on Instagram.', 'importance_score': 0.7}, {'entity_name': 'HÖTORGET METRO STATION', 'entity_type': 'LOCATION', 'description': 'The metro station where the incident took place.', 'importance_score': 0.6}, {'entity_name': 'JANUARY 31, 2019', 'entity_type': 'DATE', 'description': 'The date when the incident occurred.', 'importance_score': 0.5}, {'entity_name': 'HENRIK PALMER', 'entity_type': 'PERSON', 'description': 'The spokesman for Stockholm public transport authority SL.', 'importance_score': 0.6}], 'relationships': [{'src_id': 'SWEDISH TRAIN OPERATOR', 'tgt_id': 'STOCKHOLM PUBLIC TRANSPORT AUTHORITY SL', 'description': 'The Swedish train operator is investigating an incident involving the Stockholm public transport authority SL.', 'weight': 0.8, 'order': 1}, {'src_id': 'STOCKHOLM PUBLIC TRANSPORT AUTHORITY SL', 'tgt_id': 'LOVETTE JALLOW', 'description': 'The Stockholm public transport authority SL is mentioned in a post by Lovette Jallow on Instagram.', 'weight': 0.7, 'order': 2}, {'src_id': 'LOVETTE JALLOW', 'tgt_id': 'HÖTORGET METRO STATION', 'description': 'Lovette Jallow posted about an incident that occurred at Hötorget metro station.', 'weight': 0.6, 'order': 2}, {'src_id': 'HÖTORGET METRO STATION', 'tgt_id': 'JANUARY 31, 2019', 'description': 'The incident at Hötorget metro station took place on January 31, 2019.', 'weight': 0.5, 'order': 1}, {'src_id': 'STOCKHOLM PUBLIC TRANSPORT AUTHORITY SL', 'tgt_id': 'HENRIK PALMER', 'description': 'Henrik Palmer is the spokesman for the Stockholm public transport authority SL.', 'weight': 0.6, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'Portrush, Northern Ireland (CNN)He grew up with a famous Gaelic football-playing dad, but Shane Lowry\\'s dream as one of the only golfers in a school of 500 was always to win the Open.He accomplished that in spectacular fashion Sunday with a momentous six-shot victory over England\\'s Tommy Fleetwood to clinch his first major title at Royal Portrush.The Open, the oldest of golf\\'s four majors, had not been held in Northern Ireland for 68 years because of the Troubles that dogged the nation, but Lowry\\'s charge to the Claret Jug united fans from both sides of the border.They roared him on in thunderous fashion and chanted his name in scenes reminiscent of the raucous atmosphere at the Ryder Cup.Judging by his nerve to build on a four-shot lead in the face of eye-popping pressure, he will be an asset to European captain Padraig Harrington\\'s side in the white-hot atmosphere of Whistling Straits, Wisconsin, in 2020. Read MoreThe 32-year-old described the experience of winning the Open as \"surreal,\" saying he can\\'t quite believe he is a major champion.\"To do it here in Portrush is even more special, it is a dream come true,\" Lowry told CNN Sport in the Royal Portrush clubhouse during a whirlwind tour of media engagements.\"I didn\\'t know if I would achieve anything like this and I have and I\\'m really going to enjoy it.\"READ: Shane Lowry clinches Claret Jug for first major titleShane Lowry was roared on by raucous home support at Royal Portrush. \\'Dream come true\\'Shane Lowry.... That is all https://t.co/B7T3bHR24z— Carl Frampton MBE (@RealCFrampton) July 21, 2019 Lowry joins an illustrious roll call of recent success stories from the Republic of Ireland and Northern Ireland, alongside three-time major champion Harrington, Portrush\\'s Graeme McDowell the 2010 US Open champion, 2011 Open winner Darren Clarke and former world No.1 Rory McIlroy, who bagged four majors between 2011 and 2014.And who should be one of his close group of family and friends watching behind the 18th green but Harrington, who won the Open in 2007 and 2008, and McDowell. \"Paddy and G-Mac (McDowell) are two really good friends of mine now and I\\'m just so happy I can add my name to the list of major champions,\" he said. \"Like, you go into Paddy\\'s house and the Claret Jug is sitting on the kitchen table, and I\\'m going to have one on my kitchen table as well.\" Now he has set his sights on making Harrington\\'s team in the biennial Europe against the USA match next year.\"The Ryder Cup, that\\'s the plan,\" he added. \"It was very kind of Paddy to wait for me on the 18th green.\"READ: Emotional McIlroy feels \\'love\\' after epic Open missREAD: Tiger Woods misses Open cut, yearns for \\'hot weeks\\'Shane Lowry joins an illustrious list of major champions from both sides of the border in Ireland. Golf is fickleLowry\\'s previous best finished in a major was tied second in the US Open at Oakmont in 2016 -- but the sting in the tail was blowing a four-shot overnight lead going into the final round.He suffered no such blips on a testing day at Portrush Sunday, feeding off the lively crowd and seemingly oblivious to the at times torrential rain and gusty winds.Lowry won his first European Tour title -- the 2009 Irish Open -- when still an amateur and added a fourth in Abu Dhabi in January.But he missed his fourth consecutive cut in the Open at Carnoustie last year and says he had fallen out of love with the game.\"Carnoustie, that just shows you how fickle golf is,\" said Lowry, whose dad Brendan followed his round Sunday. \"Golf is a weird sport and you never know what\\'s around the corner. That\\'s why you need to fight through the bad times. \"It was something that became very stressful and it was weighing on me and I just didn\\'t like doing it. What a difference a year makes.\"McDowell told a story outlining Lowry\\'s early potential as a member of the Irish amateur team alongside McIIroy. Renowned coach Pete Cowen, mentor now to multiple major champions, was asked down to Dublin to look at the squad and see what he thought. According to McDowell, he said: \"Rory McIlroy looks pretty good, but that slightly overweight kid with the glasses looks good, too.\" That was the young Lowry. READ: Why Holywood star McIlroy has always been box officeREAD: Golfers feud after caddie\\'s mom is hit by ballShane Lowry celebrates as he walks up the 18th fairway on his way to winning the Open.Lowry also lost his PGA Tour card last year but credits the turnaround in fortunes to the people around him, including new caddie Bo Martin and coach Neil Manchip, who delivered a crucial pep talk over coffee at a hotel in nearby Bushmills on the eve of the Open.\"I suppose hard work and belief in myself and belief from the people around me,\" he told CNN of his reversal.\"I wouldn\\'t be here without any of them.\"Lowry also says the perspective that comes from becoming a father to Iris, born in 2017, helps him to process the bad days on the golf course.\"It has a lot. If things didn\\'t go to plan [Sunday] I would have been unbelievably disappointed but at the end of the day you\\'re going back to a warm home with a family. \"It could be worse.\"He won the Open at Royal Portrush in front of an excited and proud home crowd.It couldn\\'t be much better.', 'entities': [{'entity_name': 'SHANE LOWRY', 'entity_type': 'PERSON', 'description': 'A professional golfer who won the Open at Royal Portrush.', 'importance_score': 1.0}, {'entity_name': 'TOMMY FLEETWOOD', 'entity_type': 'PERSON', 'description': 'A professional golfer from England who was runner-up in the Open.', 'importance_score': 0.8}, {'entity_name': 'ROYAL PORTRUSH', 'entity_type': 'LOCATION', 'description': 'A golf course in Northern Ireland where the Open was held.', 'importance_score': 0.9}, {'entity_name': 'PADRAIG HARRINGTON', 'entity_type': 'PERSON', 'description': 'A professional golfer and European captain who is a friend of Shane Lowry.', 'importance_score': 0.7}, {'entity_name': 'GRAEME MCDOWELL', 'entity_type': 'PERSON', 'description': 'A professional golfer from Portrush who is a friend of Shane Lowry.', 'importance_score': 0.6}, {'entity_name': 'RORY MCILROY', 'entity_type': 'PERSON', 'description': 'A professional golfer and former world No.1 who is from Northern Ireland.', 'importance_score': 0.7}, {'entity_name': 'DARREN CLARKE', 'entity_type': 'PERSON', 'description': 'A professional golfer who won the Open in 2011.', 'importance_score': 0.6}, {'entity_name': 'CLARET JUG', 'entity_type': 'AWARD', 'description': 'The trophy awarded to the winner of the Open.', 'importance_score': 0.8}, {'entity_name': 'RYDER CUP', 'entity_type': 'EVENT', 'description': 'A biennial golf match between Europe and the USA.', 'importance_score': 0.7}, {'entity_name': 'WHISTLING STRAITS', 'entity_type': 'LOCATION', 'description': 'A golf course in Wisconsin where the Ryder Cup will be held in 2020.', 'importance_score': 0.6}], 'relationships': [{'src_id': 'SHANE LOWRY', 'tgt_id': 'TOMMY FLEETWOOD', 'description': 'Shane Lowry won the Open with a six-shot victory over Tommy Fleetwood.', 'weight': 0.9, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'ROYAL PORTRUSH', 'description': 'Shane Lowry won the Open at Royal Portrush.', 'weight': 1.0, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'PADRAIG HARRINGTON', 'description': \"Shane Lowry is an asset to European captain Padraig Harrington's side in the Ryder Cup.\", 'weight': 0.8, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'GRAEME MCDOWELL', 'description': 'Shane Lowry is friends with Graeme McDowell.', 'weight': 0.7, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'RORY MCILROY', 'description': 'Shane Lowry joins an illustrious roll call of recent success stories from Ireland, alongside Rory McIlroy.', 'weight': 0.7, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'DARREN CLARKE', 'description': 'Shane Lowry joins an illustrious roll call of recent success stories from Ireland, alongside Darren Clarke.', 'weight': 0.6, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'CLARET JUG', 'description': 'Shane Lowry won the Claret Jug at the Open.', 'weight': 0.9, 'order': 1}, {'src_id': 'SHANE LOWRY', 'tgt_id': 'RYDER CUP', 'description': \"Shane Lowry has set his sights on making Padraig Harrington's team in the Ryder Cup.\", 'weight': 0.7, 'order': 1}, {'src_id': 'RYDER CUP', 'tgt_id': 'WHISTLING STRAITS', 'description': 'The Ryder Cup will be held at Whistling Straits in 2020.', 'weight': 0.6, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': ' (CNN)Tampa Bay Buccaneers head coach Bruce Arians says he cares about wide receiver Antonio Brown and hopes the best for him.\"I wish him well,\" Arians said at his weekly press conference on Monday remotely. \"I hope if he needs help, get some ... It\\'s very hard because I do care about him.\"While Brown\\'s future remains unclear, he was not released by the Buccaneers on Monday, according to the NFL\\'s transactions report.On Sunday, the seven-time Pro Bowler left the game against the New York Jets late in the third quarter. Following the Buccaneers\\' victory against the Jets, Arians said Brown was \"no longer a Buc.\"The broadcast showed the 33-year-old appearing to be upset on the sideline, removing his jersey and pads while teammates pleaded with him to stay. A shirtless Brown could be seen throwing his undershirt into the stands as he made his way into the stadium\\'s tunnel while waving a peace sign.Read MoreArians would not share the nature of the conversation he had with Brown and said the last time he saw the receiver was when Brown left the field. The Bucs head coach also said he has not heard from Brown or his representatives. When asked if there was any mental health evaluation performed on Brown before he left the stadium, Arians said he had no idea.Arians said he has no regrets on bringing Brown onto the team and \"hopes the best for him.\" Brown looks on against the New York Jets during the game.Before exiting the game, Brown had three receptions for 26 yards.After the game, Brown tweeted a photo of himself with the cryptic caption \"Super Gremlin.\" He also tweeted a link to his new single, \"Pit Not The Palace\" and posted a series of sponsored images showing himself wearing items from clothing company Fashion Nova on Instagram.The incident comes weeks after Brown said he was suspended in December for three games without pay following a league investigation, which determined he violated Covid-19 protocols.Visit CNN.com/sport for more news, features, and videosThe Pittsburgh Steelers drafted Brown in 2010, and he played for them the first nine seasons of his career. Brown was traded to the then-Oakland Raiders ahead of the 2019 season but was released before playing in a regular season game.Brown then signed with the New England Patriots and played one game before being released again, days after one of his former offseason athletic trainers filed a lawsuit accusing him of rape and assault in 2017 and 2018. Brown denied the allegations and said he would fight to clear his name. The NFL said at the time there was an ongoing investigation into Brown\\'s conduct.Brown signed a one-year deal with the Buccaneers in October 2020 and was part of the Bucs team that won the Super Bowl last season. CNN\\'s Amir Vera, Homero De la Fuente and Jacob Lev contributed to this report.', 'entities': [{'entity_name': 'BRUCE ARIANS', 'entity_type': 'PERSON', 'description': 'Tampa Bay Buccaneers head coach who expresses care for Antonio Brown and hopes for the best for him.', 'importance_score': 0.9}, {'entity_name': 'ANTONIO BROWN', 'entity_type': 'PERSON', 'description': 'Wide receiver for the Tampa Bay Buccaneers who left the game against the New York Jets and had a cryptic tweet after the game.', 'importance_score': 0.9}, {'entity_name': 'TAMPA BAY BUCCANEERS', 'entity_type': 'ORGANIZATION', 'description': 'NFL team that Antonio Brown plays for and Bruce Arians coaches.', 'importance_score': 0.8}, {'entity_name': 'NEW YORK JETS', 'entity_type': 'ORGANIZATION', 'description': 'NFL team that played against the Tampa Bay Buccaneers.', 'importance_score': 0.7}], 'relationships': [{'src_id': 'BRUCE ARIANS', 'tgt_id': 'ANTONIO BROWN', 'description': 'Bruce Arians, the head coach of the Tampa Bay Buccaneers, expresses care for Antonio Brown, the wide receiver, and hopes for the best for him.', 'weight': 0.9, 'order': 1}, {'src_id': 'ANTONIO BROWN', 'tgt_id': 'TAMPA BAY BUCCANEERS', 'description': 'Antonio Brown is a wide receiver for the Tampa Bay Buccaneers.', 'weight': 0.8, 'order': 1}, {'src_id': 'TAMPA BAY BUCCANEERS', 'tgt_id': 'NEW YORK JETS', 'description': 'The Tampa Bay Buccaneers played against the New York Jets in an NFL game.', 'weight': 0.7, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'Story highlights Death toll rises to more than 6,200Pemba Tamang, 15, shows no apparent signs of serious injury after rescueU.S. special forces helicopter 30, including 3 Americans, to safetyKathmandu, Nepal (CNN)On Day Six of Nepal\\'s tragedy, life triumphed as rescuers pulled an 15-year-old from the rubble of a multistory residential building in one of Kathmandu\\'s hard-hit neighborhoods.A large crowd erupted in cheers as Pemba Tamang was carried out on a stretcher. He was wearing a New York shirt and a blue neck brace, was blanketed by dust and had the look of a deer in the headlights. His rescuer, Inspector Lakshman Basnet of the Nepalese Armed Police Force, said Tamang was responsive and showed no apparent signs of serious injury. He was given an IV drip and rushed from the Gongapur area to a temporary emergency hospital run by an Israeli aid team. The Nepalese rescuers had been working for five hours to locate Tamang after they heard his voice coming from under the debris.Read MoreGroup helicoptered out by U.S. special forcesAlso Thursday, a U.S. special operations forces team rescued 30 people, including three Americans, by helicopter from an area of Nepal called Bamboo Village, according to the U.S. ambassador to Nepal.The group was trapped in the village and living in a makeshift shelter, Ambassador Peter Bodde said. The families had contacted the U.S. government to let officials know where their relatives were stuck, he said.They had no other way to get out of the area, Bodde said.\\'It\\'s dangerous, but it\\'s what we do\\'An American disaster response team was also involved in the rescue of the 15 year old boy. The team was at a nearby damaged bus station when it got word that someone might be alive. Andrew Olvera, the head of the U.S. team, said his men rushed over with search dogs and equipment ranging from breaching tools to sophisticated cameras that can probe under the rubble. Pemba Tamang being pulled alive from the rubble, five days after a huge earthquake hit Nepal.He said the operation carried enormous risk, as chunks of the collapsed building hung precariously on rebar. Entire floors of what used to be people\\'s homes were visible -- ceiling fans and beds still draped with cotton sheets. It was a mountain of loss and sorrow. \"It\\'s dangerous, but it\\'s what we do,\" said Olvera, who has a daughter and twin 11-year-old boys. \"It\\'s risk versus gain. To save a human life, we will risk almost everything. \"The way the building is, it\\'s definitely a miracle,\" he said. Man \\'survived by good faith\\' Tamang cried for water in a muffled voice. He had been buried for five days under a building that pancaked. He dodged death because of a motorcycle that shielded him from the pressure of the concrete and steel, according to Basnet. Photos: Powerful earthquake hits Nepal Photos: Powerful earthquake hits NepalNepalese police officers clear debris from Durbar Square in Kathmandu on Sunday, May 3. A magnitude-7.8 earthquake centered less than 50 miles from Kathmandu rocked Nepal with devastating force Saturday, April 25. The earthquake and its aftershocks have turned one of the world\\'s most scenic regions into a panorama of devastation, killing and injuring thousands.Hide Caption 1 of 64 Photos: Powerful earthquake hits NepalAn injured Nepalese woman is carried by villagers toward an Indian army helicopter to be airlifted from Philim village in Gorkha district in Nepal on May 3.Hide Caption 2 of 64 Photos: Powerful earthquake hits NepalMembers of the Tsayana family warm themselves next to a fire outside their damaged house on May 3 in Bhaktapur, Nepal.Hide Caption 3 of 64 Photos: Powerful earthquake hits NepalA woman receives comfort during the funeral of her mother, a victim of Nepal\\'s deadly earthquake, on Friday, May 1, in Kathmandu. Hide Caption 4 of 64 Photos: Powerful earthquake hits NepalHindu priests perform rituals during the cremations of victims at the Pashupatinath Temple on the banks of the Bagmati River in Kathmandu on May 1.Hide Caption 5 of 64 Photos: Powerful earthquake hits NepalPeople await aid from an Indian army helicopter in front of damaged homes in Kulgaun, Nepal, on May 1.Hide Caption 6 of 64 Photos: Powerful earthquake hits NepalAn injured woman gets carried on a stretcher at Kathmandu\\'s airport after being evacuated from Melamchi, Nepal, on May 1.Hide Caption 7 of 64 Photos: Powerful earthquake hits NepalA member of the Los Angeles County Fire Department guides his sniffing dog through a collapsed building in Kathmandu on Thursday, April 30.Hide Caption 8 of 64 Photos: Powerful earthquake hits NepalA teenage boy gets rushed to a hospital April 30 after being rescued from the debris of a building in Kathmandu days after the earthquake.Hide Caption 9 of 64 Photos: Powerful earthquake hits NepalA man is freed from the ruins of a hotel by French rescuers in the Gangabu area of Kathmandu on Tuesday, April 28. Reuters identified the man as Rishi Khanal.Hide Caption 10 of 64 Photos: Powerful earthquake hits NepalNepalese military police search through rubble outside Kathmandu on April 28.Hide Caption 11 of 64 Photos: Powerful earthquake hits NepalPeople rest April 28 in a temporary housing camp in Kathmandu. Large encampments of tents have sprung up in open areas, including a wide space belonging to the military in the center of the capital. Hide Caption 12 of 64 Photos: Powerful earthquake hits NepalA family collects belongings from their home in Bhaktapur, Nepal, on Monday, April 27. Hide Caption 13 of 64 Photos: Powerful earthquake hits NepalDamaged buildings lean to the side in Kathmandu on April 27.Hide Caption 14 of 64 Photos: Powerful earthquake hits NepalHide Caption 15 of 64 Photos: Powerful earthquake hits NepalMembers of the Nepalese army retrieve bodies from a collapsed building in Bhaktapur near Kathmandu on April 27.Hide Caption 16 of 64 Photos: Powerful earthquake hits NepalNepalese soldiers carry a wounded woman to a helicopter as they evacuate people from Trishuli Bazar, Nepal, on April 27.Hide Caption 17 of 64 Photos: Powerful earthquake hits NepalPeople charge their cell phones in an open area in Kathmandu on April 27.Hide Caption 18 of 64 Photos: Powerful earthquake hits NepalEmergency personnel evacuate an injured man to a waiting helicopter in Trishuli Bazar on April 27.Hide Caption 19 of 64 Photos: Powerful earthquake hits NepalAn aerial view of the devastation in Kathmandu on April 27. The destruction in Nepal\\'s capital is stark: revered temples reduced to rubble, people buried in the wreckage of their homes, hospitals short on medical supplies overflowing with patients.Hide Caption 20 of 64 Photos: Powerful earthquake hits NepalResidents rescue items from the debris of houses damaged in the quake in Kathmandu on April 27.Hide Caption 21 of 64 Photos: Powerful earthquake hits NepalAn aervial view shows ruined buildings in Trishuli Bazar on April 27.Hide Caption 22 of 64 Photos: Powerful earthquake hits NepalA woman prays at a ruined temple in Kathmandu on April 27.Hide Caption 23 of 64 Photos: Powerful earthquake hits NepalPeople rest in temporary shelters in Kathmandu on April 27.Hide Caption 24 of 64 Photos: Powerful earthquake hits NepalResidents cycle over damaged roads on the outskirts of Kathmandu on Sunday, April 26.Hide Caption 25 of 64 Photos: Powerful earthquake hits NepalFour-month-old Sonit Awal is held up by Nepalese army soldiers after being rescued from the rubble of his house in Bhaktapur, Nepal, on April 26.Hide Caption 26 of 64 Photos: Powerful earthquake hits NepalThe newspaper that provided photographs of the baby\\'s rescue says the Nepalese army initially left the site, thinking the baby had not survived. Hours later when the infant\\'s cries were heard, soldiers came back and rescued him.Hide Caption 27 of 64 Photos: Powerful earthquake hits NepalThe newspaper adds the Nepalese Army had initially failed to rescue the baby and left the site thinking the baby had not survived. Hours later when the baby\\'s cries were heard the army came back and rescued him.Hide Caption 28 of 64 Photos: Powerful earthquake hits NepalA woman cries after identifying the body of a relative in Bhaktapur on April 26.Hide Caption 29 of 64 Photos: Powerful earthquake hits NepalMen clear debris in Bhaktapur on April 26.Hide Caption 30 of 64 Photos: Powerful earthquake hits NepalA truck evacuates residents from Kathmandu on April 26. Hide Caption 31 of 64 Photos: Powerful earthquake hits NepalA Buddha statue is surrounded by debris on April 26 from a collapsed temple in the UNESCO world heritage site of Bhaktapur.Hide Caption 32 of 64 Photos: Powerful earthquake hits NepalAn elderly woman is helped to her home after being treated for her injuries in Bhaktapur on April 26. Hide Caption 33 of 64 Photos: Powerful earthquake hits NepalFamily members break down on April 26 during the cremation of a loved one killed in\\xa0Bhaktapur. Hide Caption 34 of 64 Photos: Powerful earthquake hits NepalSmoke from funeral pyres fills the air at the Pashupatinath temple on the banks of Bagmati River in Kathmandu on April 26.Hide Caption 35 of 64 Photos: Powerful earthquake hits NepalMembers of India\\'s National Disaster Response Force look for survivors in Kathmandu on April 26. Hide Caption 36 of 64 Photos: Powerful earthquake hits NepalRescue workers remove debris on April 26 as they search for victims in Bhaktapur.Hide Caption 37 of 64 Photos: Powerful earthquake hits NepalPeople look at the debris of one of the oldest temples in Kathmandu on April 26.Hide Caption 38 of 64 Photos: Powerful earthquake hits NepalPeople sleep on a street in Kathmandu, Nepal, on Saturday, April 25. A seemingly endless series of aftershocks continued to roil the area, further traumatizing survivors. Hide Caption 39 of 64 Photos: Powerful earthquake hits NepalCivilian rescuers carry a person on a stretcher in Kathmandu on April 25.Hide Caption 40 of 64 Photos: Powerful earthquake hits NepalPeople try to free a man from the rubble in Kathmandu on April 25. Cheers rose from the piles when people were found alive -- but mostly bodies turned up. Hide Caption 41 of 64 Photos: Powerful earthquake hits NepalPedestrians walk past collapsed buildings in Kathmandu on April 25.Hide Caption 42 of 64 Photos: Powerful earthquake hits NepalAzim Afif, of the Universiti Teknologi Malaysia climbing team, provided this photo of their Mount Everest base camp after it was ravaged by an avalanche triggered by the earthquake on April 25. All of Afif\\'s five-member team survived.Hide Caption 43 of 64 Photos: Powerful earthquake hits NepalRescuers clear rubble in Kathmandu\\'s Basantapur Durbar Square on April 25.Hide Caption 44 of 64 Photos: Powerful earthquake hits NepalA temple on Hanumandhoka Durbar Square lies in ruins after an earthquake in Kathmandu on April 25. Hide Caption 45 of 64 Photos: Powerful earthquake hits NepalDharahara, a tower dating back to 1832 that rose more than 60 meters (200 feet) and provided breathtaking views of Kathmandu and the surrounding Himalayas, collapsed in the earthquake on April 25.Hide Caption 46 of 64 Photos: Powerful earthquake hits NepalThe hand of a statue is seen under debris in Basantapur Durbar Square in Kathmandu.Hide Caption 47 of 64 Photos: Powerful earthquake hits NepalA Nepalese man and woman hold each other in Kathmandu\\'s Basantapur Durbar Square on April 25.Hide Caption 48 of 64 Photos: Powerful earthquake hits NepalA victim of Nepal\\'s earthquake lies in the debris of Dharahara after it collapsed on April 25 in Kathmandu, Nepal. Hide Caption 49 of 64 Photos: Powerful earthquake hits NepalRescuers look for victims under a collapsed building in Kathmandu on April 25. Hide Caption 50 of 64 Photos: Powerful earthquake hits NepalVolunteers carry a body recovered from the debris of a collapsed building in Kathmandu.Hide Caption 51 of 64 Photos: Powerful earthquake hits NepalA victim\\'s body is seen in the debris of the collapsed Dharahara on April 25. Hide Caption 52 of 64 Photos: Powerful earthquake hits NepalEmergency rescue workers carry a victim from Dharahara after the tower in Kathmandu collapsed on April 25. Hide Caption 53 of 64 Photos: Powerful earthquake hits NepalPeople free a man from the rubble of a destroyed building in Kathmandu.Hide Caption 54 of 64 Photos: Powerful earthquake hits NepalA man walks past a collapsed temple at Basantapur Durbar Square.Hide Caption 55 of 64 Photos: Powerful earthquake hits NepalRescue workers clear debris in Kathmandu while searching for survivors.Hide Caption 56 of 64 Photos: Powerful earthquake hits NepalPeople huddle together outside a hospital in Kathmandu. Eyewitnesses said residents were scared and waiting for aftershocks to end.Hide Caption 57 of 64 Photos: Powerful earthquake hits NepalPeople search for survivors stuck under the rubble of a destroyed building in Kathmandu.Hide Caption 58 of 64 Photos: Powerful earthquake hits NepalInjured people receive treatment in Kathmandu. A CNN reporter said medics were focused on treating the most severely injured.Hide Caption 59 of 64 Photos: Powerful earthquake hits NepalEmergency rescue workers search for survivors in the debris of Dharahara on April 25.Hide Caption 60 of 64 Photos: Powerful earthquake hits NepalAn injured child lies on the ground outside a hospital in Kathmandu on April 25.Hide Caption 61 of 64 Photos: Powerful earthquake hits NepalPeople help with rescue efforts at the site of a collapsed building in Kathmandu.Hide Caption 62 of 64 Photos: Powerful earthquake hits NepalAn injured child receives treatment outside Medicare Hospital in Kathmandu on April 25. Residents, after a relentless series of aftershocks, have been remaining outdoors.Hide Caption 63 of 64 Photos: Powerful earthquake hits NepalThe rubble of collapsed walls fills a street in Lalitpur, on the outskirts of Kathmandu, on April 25.Hide Caption 64 of 64And, Basnet said, he \"survived by good faith.\" Dennis Bautista, who went down to where Tamang was buried to administer medical aid, called the rescue amazing. \"It feels good to be able to help out. I can\\'t imagine what he went through,\" Bautista said. \"He is a brave young man.\" Basnet said once he got closer to Tamang, he tried to reassure him that he would be OK. \"I gave him water and talked to him regularly,\" Basnet said. Rescuers continue searchAfter Tamang was rushed to the hospital, the US Agency for International Development team continued the search. The possibility remained that someone else might have survived, like Tamang. Other search and rescue teams continued to scour through Kathmandu\\'s rubble Thursday. They are looking for survivors from the magnitude-7.8 earthquake that struck Saturday, killing at least 6,204 people and wounding almost 14,000, according to Nepali authorities. Another 72 people were reported dead in India and 25 in China.JUST WATCHEDEverest earthquake puts Nepal\\'s economy at riskReplayMore Videos ...MUST WATCHEverest earthquake puts Nepal\\'s economy at risk 02:22In Nepal, 19 of the deaths occurred on Mount Everest, where the quake triggered deadly avalanches. Despite the disaster, Nepali authorities plans to reopen routes up the mountain as soon as next week.Teams are clearing paths and and rebuilding ladders, Tourism Ministry spokesman Krishna Sapkota told CNN on Thursday. Officials are encouraging people who have already received permission to climb this season to go ahead with their plans.As the chances dimmed for finding people alive in the wreckage left by the quake across Nepal, Tamang\\'s rescue boosted hopes for all those who still have loved ones and friends missing. Other people have been saved from under collapsed buildings in previous days, including a 27-year-old man on Tuesday and a 4-month-old boy on Sunday. The Nepali military also released a photo of a dust-caked 11-year-old girl who they said was rescued Wednesday after 90 hours under the rubble.The rescuers, meanwhile, have no intention of giving up looking for more. The 2010 earthquake in Haiti, Olvera said, revised the benchmark for what was thought possible for survival. A man there was miraculously pulled from the rubble after 27 long days.360-degree view of earthquake zoneFollow @CNNbrk\\n\\n CNN\\'s Sugam Pokharel contributed to this report.', 'entities': [{'entity_name': 'PEMBA TAMANG', 'entity_type': 'PERSON', 'description': 'A 15-year-old boy rescued from the rubble of a multistory residential building in Kathmandu.', 'importance_score': 0.9}, {'entity_name': 'KATHMANDU', 'entity_type': 'LOCATION', 'description': 'A city in Nepal where the earthquake caused significant damage.', 'importance_score': 0.8}, {'entity_name': 'NEPAL', 'entity_type': 'LOCATION', 'description': 'The country where the earthquake occurred, causing widespread devastation.', 'importance_score': 0.9}, {'entity_name': 'U.S. SPECIAL FORCES', 'entity_type': 'ORGANIZATION', 'description': 'A team that rescued 30 people, including three Americans, by helicopter from Bamboo Village.', 'importance_score': 0.7}, {'entity_name': 'INSPECTOR LAKSHMAN BASNET', 'entity_type': 'PERSON', 'description': 'A rescuer from the Nepalese Armed Police Force who helped save Pemba Tamang.', 'importance_score': 0.7}, {'entity_name': 'BAMBOO VILLAGE', 'entity_type': 'LOCATION', 'description': 'An area in Nepal where a group of people, including Americans, were trapped and later rescued by U.S. special forces.', 'importance_score': 0.6}, {'entity_name': 'ANDREW OLVERA', 'entity_type': 'PERSON', 'description': 'The head of the U.S. disaster response team involved in the rescue of Pemba Tamang.', 'importance_score': 0.6}, {'entity_name': 'DENNIS BAUTISTA', 'entity_type': 'PERSON', 'description': 'A person who administered medical aid to Pemba Tamang after his rescue.', 'importance_score': 0.5}], 'relationships': [{'src_id': 'PEMBA TAMANG', 'tgt_id': 'INSPECTOR LAKSHMAN BASNET', 'description': 'Pemba Tamang was rescued by Inspector Lakshman Basnet.', 'weight': 0.9, 'order': 1}, {'src_id': 'U.S. SPECIAL FORCES', 'tgt_id': 'BAMBOO VILLAGE', 'description': 'U.S. special forces rescued people from Bamboo Village.', 'weight': 0.8, 'order': 1}, {'src_id': 'ANDREW OLVERA', 'tgt_id': 'PEMBA TAMANG', 'description': \"Andrew Olvera's team was involved in the rescue of Pemba Tamang.\", 'weight': 0.7, 'order': 1}, {'src_id': 'DENNIS BAUTISTA', 'tgt_id': 'PEMBA TAMANG', 'description': 'Dennis Bautista administered medical aid to Pemba Tamang after his rescue.', 'weight': 0.6, 'order': 1}]}) (input_keys={'input_text'})]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "valset[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 49 chunks, 320 entities(duplicated), 222 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 50 chunks, 323 entities(duplicated), 224 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 51 chunks, 328 entities(duplicated), 227 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 53 chunks, 338 entities(duplicated), 234 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 54 chunks, 345 entities(duplicated), 238 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 55 chunks, 352 entities(duplicated), 244 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 56 chunks, 359 entities(duplicated), 249 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 57 chunks, 367 entities(duplicated), 253 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 58 chunks, 377 entities(duplicated), 259 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 59 chunks, 387 entities(duplicated), 268 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 60 chunks, 397 entities(duplicated), 276 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 61 chunks, 413 entities(duplicated), 281 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 63 chunks, 421 entities(duplicated), 287 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 65 chunks, 430 entities(duplicated), 294 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 66 chunks, 445 entities(duplicated), 303 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 67 chunks, 460 entities(duplicated), 316 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 68 chunks, 468 entities(duplicated), 323 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 69 chunks, 474 entities(duplicated), 326 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 70 chunks, 490 entities(duplicated), 334 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 71 chunks, 505 entities(duplicated), 348 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 72 chunks, 510 entities(duplicated), 351 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 74 chunks, 521 entities(duplicated), 359 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 75 chunks, 528 entities(duplicated), 362 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 76 chunks, 537 entities(duplicated), 365 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 78 chunks, 558 entities(duplicated), 380 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 79 chunks, 565 entities(duplicated), 385 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 80 chunks, 573 entities(duplicated), 391 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 81 chunks, 577 entities(duplicated), 394 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 82 chunks, 582 entities(duplicated), 397 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 83 chunks, 593 entities(duplicated), 406 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 84 chunks, 601 entities(duplicated), 411 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 85 chunks, 608 entities(duplicated), 417 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 86 chunks, 618 entities(duplicated), 426 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 87 chunks, 623 entities(duplicated), 430 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 88 chunks, 633 entities(duplicated), 439 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 90 chunks, 646 entities(duplicated), 448 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 91 chunks, 657 entities(duplicated), 455 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 92 chunks, 662 entities(duplicated), 459 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 93 chunks, 667 entities(duplicated), 462 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 94 chunks, 673 entities(duplicated), 466 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 95 chunks, 677 entities(duplicated), 468 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 96 chunks, 688 entities(duplicated), 474 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 97 chunks, 692 entities(duplicated), 476 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 98 chunks, 703 entities(duplicated), 485 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 99 chunks, 736 entities(duplicated), 499 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 100 chunks, 746 entities(duplicated), 504 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 101 chunks, 760 entities(duplicated), 517 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 102 chunks, 765 entities(duplicated), 521 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 103 chunks, 770 entities(duplicated), 525 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 104 chunks, 776 entities(duplicated), 528 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 106 chunks, 788 entities(duplicated), 537 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 107 chunks, 794 entities(duplicated), 541 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 109 chunks, 809 entities(duplicated), 552 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 110 chunks, 813 entities(duplicated), 555 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 111 chunks, 820 entities(duplicated), 561 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 112 chunks, 823 entities(duplicated), 563 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 114 chunks, 838 entities(duplicated), 571 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 115 chunks, 844 entities(duplicated), 574 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 116 chunks, 855 entities(duplicated), 578 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 117 chunks, 858 entities(duplicated), 579 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 118 chunks, 871 entities(duplicated), 590 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 119 chunks, 876 entities(duplicated), 590 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 120 chunks, 882 entities(duplicated), 596 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 121 chunks, 890 entities(duplicated), 598 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 122 chunks, 893 entities(duplicated), 600 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 123 chunks, 900 entities(duplicated), 615 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 124 chunks, 915 entities(duplicated), 620 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 125 chunks, 921 entities(duplicated), 624 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 126 chunks, 924 entities(duplicated), 627 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 128 chunks, 937 entities(duplicated), 636 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 129 chunks, 950 entities(duplicated), 644 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 130 chunks, 961 entities(duplicated), 648 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 131 chunks, 967 entities(duplicated), 653 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 132 chunks, 972 entities(duplicated), 654 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 133 chunks, 977 entities(duplicated), 658 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 134 chunks, 983 entities(duplicated), 663 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 135 chunks, 988 entities(duplicated), 667 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 136 chunks, 992 entities(duplicated), 670 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 137 chunks, 996 entities(duplicated), 671 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 138 chunks, 1010 entities(duplicated), 680 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 139 chunks, 1014 entities(duplicated), 683 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 140 chunks, 1021 entities(duplicated), 689 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 141 chunks, 1024 entities(duplicated), 690 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 143 chunks, 1032 entities(duplicated), 695 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 144 chunks, 1039 entities(duplicated), 699 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 145 chunks, 1045 entities(duplicated), 704 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 146 chunks, 1050 entities(duplicated), 708 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 147 chunks, 1054 entities(duplicated), 710 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 148 chunks, 1062 entities(duplicated), 716 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 149 chunks, 1066 entities(duplicated), 719 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 150 chunks, 1073 entities(duplicated), 725 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 151 chunks, 1083 entities(duplicated), 730 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 152 chunks, 1089 entities(duplicated), 735 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 153 chunks, 1099 entities(duplicated), 740 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 154 chunks, 1103 entities(duplicated), 742 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 155 chunks, 1108 entities(duplicated), 745 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 156 chunks, 1112 entities(duplicated), 746 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 157 chunks, 1118 entities(duplicated), 750 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 158 chunks, 1124 entities(duplicated), 753 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 159 chunks, 1128 entities(duplicated), 756 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 160 chunks, 1139 entities(duplicated), 766 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 161 chunks, 1145 entities(duplicated), 769 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 162 chunks, 1152 entities(duplicated), 775 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 163 chunks, 1156 entities(duplicated), 778 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 164 chunks, 1174 entities(duplicated), 795 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 165 chunks, 1177 entities(duplicated), 797 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 166 chunks, 1184 entities(duplicated), 801 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 167 chunks, 1192 entities(duplicated), 805 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 168 chunks, 1219 entities(duplicated), 824 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 169 chunks, 1230 entities(duplicated), 832 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 170 chunks, 1235 entities(duplicated), 837 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠙ Processed 171 chunks, 1253 entities(duplicated), 842 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠹ Processed 172 chunks, 1255 entities(duplicated), 843 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠸ Processed 173 chunks, 1263 entities(duplicated), 849 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠴ Processed 175 chunks, 1285 entities(duplicated), 864 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠦ Processed 176 chunks, 1294 entities(duplicated), 868 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠧ Processed 177 chunks, 1301 entities(duplicated), 873 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠇ Processed 178 chunks, 1308 entities(duplicated), 875 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠏ Processed 179 chunks, 1316 entities(duplicated), 882 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 180 chunks, 1319 entities(duplicated), 882 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠼ Processed 184 chunks, 1352 entities(duplicated), 903 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n", + "INFO:httpx:HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "⠋ Processed 200 chunks, 1470 entities(duplicated), 982 relations(duplicated)\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:nano-graphrag:Saved 197 examples with keys: ['input_text', 'entities', 'relationships'], filtered 3 examples\n" + ] + } + ], + "source": [ + "dev_chunks = {compute_mdhash_id(text, prefix=f\"chunk-devset-\"): {\"content\": text} for text in dev_data[\"text\"]}\n", + "devset = asyncio.run(generate_dataset(chunks=dev_chunks, filepath=entity_relationship_devset_path))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Example({'input_text': \"As students from Marjory Stoneman Douglas High School confront lawmakers with demands to restrict sales of assault rifles, there were warnings by the president of the Florida school administration association that schools in the state were vulnerable to such an attack.\\nHighlights\\n“Nikolas Cruz was able to purchase an assault rifle before he was able to buy a beer,” said Stoneman Douglas student Laurenzo Prado, referring to a Florida law that allows people as young as 18 to buy assault weapons. Students galvanized by the deadly mass shooting at the Florida high school confronted lawmakers with demands to restrict sales of assault rifles, while President Donald Trump suggested arming teachers as a way to stop more U.S. rampages.\\nTwo weeks before a gunman fatally shot 17 people at a Florida high school, Bill Lee, the president of the state’s school administrators association, warned that Florida’s schools were vulnerable to just such an attack. “It’s not a matter of if, but when,” he wrote in the Orlando Sentinel on Jan. 29, calling on legislators to increase school security spending after two January school shootings in other states.\\nWorld\\nU.S. players pose for a photo as they celebrate their victory over Canada. REUTERS/Grigory Dukor She’s practiced the ‘Oops, I did it again’ thousands of times in training, and her signature trick was worth its weight in Olympic gold as American Jocelyne Lamoureux-Davidson’s shootout winner broke a Canadian 16-year stranglehold on the women’s ice hockey title. It was the perfect jaw-dropping finale to a game that was billed as a grudge match but swiftly developed into a classic for the ages at the Pyeongchang Winter Games.\\nWarplanes pounded the last rebel enclave near the Syrian capital for a fifth straight day, as the United Nations pleaded for a truce to halt one of the fiercest air assaults of the seven-year civil war and prevent a “massacre” .\\nBangladesh is racing to turn an uninhabited and muddy Bay of Bengal island into home for 100,000 Rohingya Muslims who have fled a military crackdown in Myanmar , amid conflicting signals from top Bangladeshi officials about whether the refugees would end up being stranded there.\\nCommentary\\nNorth Korea’s participation in the 2018 Winter Olympics has created an opening for renewed dialogue with the United States and South Korea, writes Peter App s. “By taking part so visibly alongside South Korea, Pyongyang has been able to present itself as a credible global and regional power in a way that has eluded North Korean leaders since the war that divided the peninsula. The real strategic winner, however, is the South Korean government, which has shrewdly used the games to reshape the diplomatic landscape.”\\nBusiness\\nLast March, executives at General Electric's power-plant business gave Wall Street a surprisingly bullish forecast for the year. Despite flat demand for new natural gas power plants, they said, GE Power’s revenue and profit would rise. But GE’s forecast turned out to be a mirage .\\nMexican buyers imported ten times more corn from Brazil last year amid concern that NAFTA renegotiations could disrupt their U.S. supplies, according to government data and top grains merchants.\\nWarren Buffett may use part of his annual letter to Berkshire Hathaway shareholders, due on Saturday , to renew his optimism about America, at a time economic growth is on the upswing and U.S. stocks sit near record highs despite rising interest rates.\\nWorld stocks tumbled to one-week lows after the U.S. Fed confirmed it was on track to raise interest rates several times this year, sending bond yields to new multi-year highs.\\nTop stories on Reuters TV\\n'Hand grenade' thrown at U.S. embassy in Montenegro\\nGM's potential Korea exit piles pressure on Moon Jae-in\\n \", 'entities': [{'entity_name': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'entity_type': 'ORGANIZATION', 'description': 'A high school in Florida where students confronted lawmakers about restricting sales of assault rifles.', 'importance_score': 0.9}, {'entity_name': 'NIKOLAS CRUZ', 'entity_type': 'PERSON', 'description': 'A person who was able to purchase an assault rifle before he was able to buy a beer, according to a Florida law.', 'importance_score': 0.8}, {'entity_name': 'LAURENZO PRADO', 'entity_type': 'PERSON', 'description': \"A student from Marjory Stoneman Douglas High School who mentioned Nikolas Cruz's ability to purchase an assault rifle.\", 'importance_score': 0.7}, {'entity_name': 'FLORIDA', 'entity_type': 'LOCATION', 'description': 'A state in the United States where the events related to the high school and gun control are taking place.', 'importance_score': 0.8}, {'entity_name': 'BILL LEE', 'entity_type': 'PERSON', 'description': \"The president of the Florida school administrators association who warned about the vulnerability of Florida's schools to attacks.\", 'importance_score': 0.7}, {'entity_name': 'ORLANDO SENTINEL', 'entity_type': 'ORGANIZATION', 'description': 'A newspaper where Bill Lee wrote about the need for increased school security spending.', 'importance_score': 0.6}, {'entity_name': 'DONALD TRUMP', 'entity_type': 'PERSON', 'description': 'The President of the United States who suggested arming teachers as a way to stop more U.S. rampages.', 'importance_score': 0.8}, {'entity_name': 'GENERAL ELECTRIC', 'entity_type': 'ORGANIZATION', 'description': 'A company whose power-plant business gave a surprisingly bullish forecast for the year, which turned out to be a mirage.', 'importance_score': 0.6}, {'entity_name': 'WARREN BUFFETT', 'entity_type': 'PERSON', 'description': 'An investor who may use his annual letter to Berkshire Hathaway shareholders to renew his optimism about America.', 'importance_score': 0.7}], 'relationships': [{'src_id': 'MARJORY STONEMAN DOUGLAS HIGH SCHOOL', 'tgt_id': 'FLORIDA', 'description': 'Students from Marjory Stoneman Douglas High School are located in the state of Florida.', 'weight': 0.9, 'order': 1}, {'src_id': 'NIKOLAS CRUZ', 'tgt_id': 'ASSAULT RIFLE', 'description': 'Nikolas Cruz was able to purchase an assault rifle before he was able to buy a beer.', 'weight': 0.8, 'order': 1}, {'src_id': 'LAURENZO PRADO', 'tgt_id': 'NIKOLAS CRUZ', 'description': \"Laurenzo Prado, a student, mentioned Nikolas Cruz's ability to purchase an assault rifle.\", 'weight': 0.7, 'order': 1}, {'src_id': 'BILL LEE', 'tgt_id': \"FLORIDA'S SCHOOLS\", 'description': \"Bill Lee warned that Florida's schools were vulnerable to attacks.\", 'weight': 0.7, 'order': 1}, {'src_id': 'DONALD TRUMP', 'tgt_id': 'ARMING TEACHERS', 'description': 'Donald Trump suggested arming teachers as a way to stop more U.S. rampages.', 'weight': 0.8, 'order': 1}, {'src_id': 'GENERAL ELECTRIC', 'tgt_id': 'POWER-PLANT BUSINESS', 'description': \"General Electric's power-plant business gave a surprisingly bullish forecast for the year.\", 'weight': 0.6, 'order': 1}, {'src_id': 'WARREN BUFFETT', 'tgt_id': 'BERKSHIRE HATHAWAY', 'description': 'Warren Buffett may use his annual letter to Berkshire Hathaway shareholders to renew his optimism about America.', 'weight': 0.7, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'From ferrying people to and from their place of work to transporting nuclear waste and coal, railways are not only an integral part of 21st century life but have played a crucial role in shaping the modern world.\\nIn Belgium, one business is looking to use innovation to drive further change in the sector. Based in Brussels, Railnova is a technology company whose clients include Deutsche Bahn and French rail operator SNCF.\\nAccording to its CEO and founder Christian Sprauer, Railnova offers train operators the ability to be alerted to issues before failures occur.\\nA device designed and manufactured by the business is fitted to a train. The box gathers a host of data, a subset of which is sent to the cloud where it can be analyzed.\\n\"Before, they would have to wait until the failure happens and bring the train to the workshop,\" Sprauer said. \"Now… they can be alerted up front and they can avoid certain failures and… react much faster to other failures.\"\\nInnovation is being used across the rail industry to improve services. Instead of buying paper tickets, rail users in London can now touch their debit or credit cards on to sensors to pay for their journey. In China and Japan, magnetic levitation — or Maglev — trains offer a tantalizing glimpse of the high-speed rail travel that could be just around the corner.\\nExamples of the solutions offered by the Railnova technology range from measuring engine speed and braking force, to measuring traction force. Energy consumption was another area of interest, with Sprauer stating it was a good predictor of whether there was a problem with a train or not.\\nThe wealth of data being mined could, Sprauer said, offer a number of benefits. Reducing energy consumption was one such example. On diesel trains, he said, energy consumption could be cut by coaching drivers on best driving practices.\\n\"We call this the \\'golden run\\'… every driver tries to achieve the golden run and, as a whole, the company reduces… fuel expenses by 5 percent.\"\\nFollow CNBC International on Twitter and Facebook .', 'entities': [{'entity_name': 'RAILNOVA', 'entity_type': 'ORGANIZATION', 'description': 'A technology company based in Brussels whose clients include Deutsche Bahn and French rail operator SNCF.', 'importance_score': 0.9}, {'entity_name': 'DEUTSCHE BAHN', 'entity_type': 'ORGANIZATION', 'description': 'A client of Railnova, a German railway company.', 'importance_score': 0.8}, {'entity_name': 'SNCF', 'entity_type': 'ORGANIZATION', 'description': 'A client of Railnova, a French rail operator.', 'importance_score': 0.8}, {'entity_name': 'CHRISTIAN SPRAUER', 'entity_type': 'PERSON', 'description': 'The CEO and founder of Railnova.', 'importance_score': 0.9}, {'entity_name': 'BRUSSELS', 'entity_type': 'LOCATION', 'description': 'The location where Railnova is based.', 'importance_score': 0.7}, {'entity_name': 'LONDON', 'entity_type': 'LOCATION', 'description': 'A location where rail users can now touch their debit or credit cards on sensors to pay for their journey.', 'importance_score': 0.7}, {'entity_name': 'CHINA', 'entity_type': 'LOCATION', 'description': 'A country where magnetic levitation trains are being used.', 'importance_score': 0.7}, {'entity_name': 'JAPAN', 'entity_type': 'LOCATION', 'description': 'A country where magnetic levitation trains are being used.', 'importance_score': 0.7}, {'entity_name': 'MAGLEV', 'entity_type': 'TECHNOLOGY', 'description': 'Magnetic levitation trains that offer high-speed rail travel.', 'importance_score': 0.8}], 'relationships': [{'src_id': 'RAILNOVA', 'tgt_id': 'DEUTSCHE BAHN', 'description': 'Railnova provides technology solutions to Deutsche Bahn.', 'weight': 0.9, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'SNCF', 'description': 'Railnova provides technology solutions to SNCF.', 'weight': 0.9, 'order': 1}, {'src_id': 'CHRISTIAN SPRAUER', 'tgt_id': 'RAILNOVA', 'description': 'Christian Sprauer is the CEO and founder of Railnova.', 'weight': 1.0, 'order': 1}, {'src_id': 'RAILNOVA', 'tgt_id': 'BRUSSELS', 'description': 'Railnova is based in Brussels.', 'weight': 0.8, 'order': 1}, {'src_id': 'LONDON', 'tgt_id': 'DEBIT CARD', 'description': 'In London, rail users can touch their debit cards on sensors to pay for their journey.', 'weight': 0.7, 'order': 2}, {'src_id': 'LONDON', 'tgt_id': 'CREDIT CARD', 'description': 'In London, rail users can touch their credit cards on sensors to pay for their journey.', 'weight': 0.7, 'order': 2}, {'src_id': 'CHINA', 'tgt_id': 'MAGLEV', 'description': 'In China, magnetic levitation trains are being used.', 'weight': 0.8, 'order': 1}, {'src_id': 'JAPAN', 'tgt_id': 'MAGLEV', 'description': 'In Japan, magnetic levitation trains are being used.', 'weight': 0.8, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': \"Jan 22 (Reuters) - Shanghai Stock Exchange Filing\\n* SHOWS BLOCK TRADE OF YONGHUI SUPERSTORES Co LTd's 166.3 MILLION SHARES INVOLVING 1.63 BILLION YUAN ($254.68 million) ON JAN 22 Source text in Chinese: bit.ly/2yJZikT Further company coverage: ($1 = 6.4003 Chinese yuan renminbi) (Reporting by Hong Kong newsroom)\\n \", 'entities': [{'entity_name': 'YONGHUI SUPERSTORES CO LTD', 'entity_type': 'ORGANIZATION', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'importance_score': 1.0}, {'entity_name': '166.3 MILLION SHARES', 'entity_type': 'MONEY', 'description': '166.3 million shares are being traded in the block trade involving YONGHUI SUPERSTORES Co LTd.', 'importance_score': 0.9}, {'entity_name': '1.63 BILLION YUAN', 'entity_type': 'MONEY', 'description': 'The block trade involves 1.63 billion yuan ($254.68 million).', 'importance_score': 0.9}, {'entity_name': 'JAN 22', 'entity_type': 'DATE', 'description': 'The block trade occurred on January 22.', 'importance_score': 0.8}, {'entity_name': 'SHANGHAI STOCK EXCHANGE', 'entity_type': 'ORGANIZATION', 'description': 'The block trade was filed with the Shanghai Stock Exchange.', 'importance_score': 0.8}], 'relationships': [{'src_id': 'YONGHUI SUPERSTORES CO LTD', 'tgt_id': '166.3 MILLION SHARES', 'description': 'YONGHUI SUPERSTORES Co LTd is involved in a block trade of 166.3 million shares.', 'weight': 1.0, 'order': 1}, {'src_id': '166.3 MILLION SHARES', 'tgt_id': '1.63 BILLION YUAN', 'description': 'The block trade of 166.3 million shares involves 1.63 billion yuan.', 'weight': 1.0, 'order': 1}, {'src_id': 'SHANGHAI STOCK EXCHANGE', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': 'The block trade involving YONGHUI SUPERSTORES Co LTd was filed with the Shanghai Stock Exchange.', 'weight': 0.9, 'order': 1}, {'src_id': 'JAN 22', 'tgt_id': 'YONGHUI SUPERSTORES CO LTD', 'description': 'The block trade involving YONGHUI SUPERSTORES Co LTd occurred on January 22.', 'weight': 0.8, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': 'LONDON (Reuters) - Britain’s economy was weaker than previously thought in 2017, official data showed on Thursday, leaving the country lagging further behind the global recovery as it prepares to leave the European Union.\\nThe downgrade of the full-year and fourth-quarter growth rates also raised questions about the strength of the economy as the Bank of England prepares to raise interest rates.\\nGross domestic product growth slowed to a quarterly 0.4 percent from a previous estimate of 0.5 percent, wrong-footing economists and reducing 2017 growth as a whole to 1.7 percent, its lowest since 2012.\\nRelated Coverage UK retail sales growth eases further in February: CBI This was still stronger than most economists feared immediately after Britain voted to leave the EU in June 2016.\\nBut the country has relied heavily on the unexpectedly robust global economy to sustain its economic growth while consumers have been squeezed by higher inflation caused by the fall in the pound after the Brexit vote.\\nSterling was little changed after Thursday’s data and government bond prices rose slightly.\\nAlan Clarke, an economist at Scotiabank, said the figures showed Britain’s economy was growing at roughly the pace the BoE sees its new, lower speed limit, meaning a rate hike was still on the cards.\\nBut Samuel Tombs, at Pantheon Macroeconomics, said the data showed the central bank should delay any action for now.\\n“The latest GDP data suggest that the economy remains in a fragile state and does not need to be cooled with another rate rise as soon as May,” he said in a note to clients.\\nBoE Governor Mark Carney said this month that rates would probably need to rise sooner and by somewhat more than the central bank had thought in November, when it raised borrowing costs for the first time in a decade.\\nMost economists think rates will rise again in May, and financial markets expect a further increase, to 1 percent, by the end of the year.\\nLAGGARD UK Britain’s year-on-year economic growth of 1.4 percent in the last three months of 2017 was not just its weakest in five years but also the weakest of any of the economies in the Group of Seven, including long-term laggards such as Japan and Italy.\\nThere were only limited signs of a rebalancing of the economy away from consumer demand and toward business investment and net trade that Carney pointed to as positive signs in an appearance before lawmakers on Wednesday.\\nBusiness investment was flat on the quarter and 2.1 percent higher on the year, both readings coming in weaker than expected in a Reuters poll of economists. Net trade dragged on growth in most quarters of 2017, though it was positive on the year.\\nThe BoE said earlier this month that it expected the economy would grow by 1.8 percent this year, faster than its previous forecast of 1.6 percent, mostly because of the strength of the global economy.\\nReporting by David Milliken; Editing by Toby Chopra\\n ', 'entities': [{'entity_name': 'BRITAIN', 'entity_type': 'LOCATION', 'description': 'The country whose economy was weaker than previously thought in 2017.', 'importance_score': 0.9}, {'entity_name': 'EUROPEAN UNION', 'entity_type': 'ORGANIZATION', 'description': 'The political and economic union that Britain is preparing to leave.', 'importance_score': 0.8}, {'entity_name': 'BANK OF ENGLAND', 'entity_type': 'ORGANIZATION', 'description': 'The central bank of the United Kingdom, preparing to raise interest rates.', 'importance_score': 0.8}, {'entity_name': 'ALAN CLARKE', 'entity_type': 'PERSON', 'description': 'An economist at Scotiabank, commenting on the economic figures.', 'importance_score': 0.7}, {'entity_name': 'SAMUEL TOMBS', 'entity_type': 'PERSON', 'description': 'An economist at Pantheon Macroeconomics, suggesting the central bank should delay any action.', 'importance_score': 0.7}, {'entity_name': 'MARK CARNEY', 'entity_type': 'PERSON', 'description': 'The Governor of the Bank of England, stating that rates would probably need to rise sooner.', 'importance_score': 0.8}, {'entity_name': 'SCOTIABANK', 'entity_type': 'ORGANIZATION', 'description': 'A financial institution where Alan Clarke works as an economist.', 'importance_score': 0.6}, {'entity_name': 'PANTHEON MACROECONOMICS', 'entity_type': 'ORGANIZATION', 'description': 'A macroeconomic research firm where Samuel Tombs works as an economist.', 'importance_score': 0.6}], 'relationships': [{'src_id': 'BRITAIN', 'tgt_id': 'EUROPEAN UNION', 'description': 'Britain is preparing to leave the European Union.', 'weight': 0.9, 'order': 1}, {'src_id': 'BANK OF ENGLAND', 'tgt_id': 'INTEREST RATES', 'description': 'The Bank of England is preparing to raise interest rates.', 'weight': 0.8, 'order': 1}, {'src_id': 'ALAN CLARKE', 'tgt_id': 'BANK OF ENGLAND', 'description': \"Alan Clarke, an economist at Scotiabank, comments on the Bank of England's potential rate hike.\", 'weight': 0.7, 'order': 2}, {'src_id': 'SAMUEL TOMBS', 'tgt_id': 'BANK OF ENGLAND', 'description': 'Samuel Tombs, an economist at Pantheon Macroeconomics, suggests the Bank of England should delay any action.', 'weight': 0.7, 'order': 2}, {'src_id': 'MARK CARNEY', 'tgt_id': 'BANK OF ENGLAND', 'description': 'Mark Carney, the Governor of the Bank of England, states that rates would probably need to rise sooner.', 'weight': 0.8, 'order': 1}]}) (input_keys={'input_text'}),\n", + " Example({'input_text': \"Trump taps White House doctor as new VA secretary 2 Hours Ago CNBC's Kayla Tausche reports President Trump has tapped White House physician Rear Admiral Ronny Jackson to run the Department of Veterans Affairs amid a shakeup at the White House.\", 'entities': [{'entity_name': 'TRUMP', 'entity_type': 'PERSON', 'description': 'President Trump who tapped Ronny Jackson as new VA secretary.', 'importance_score': 1.0}, {'entity_name': 'WHITE HOUSE', 'entity_type': 'ORGANIZATION', 'description': 'The White House where Ronny Jackson is a physician.', 'importance_score': 0.8}, {'entity_name': 'VA', 'entity_type': 'ORGANIZATION', 'description': 'Department of Veterans Affairs, the organization Ronny Jackson is tapped to run.', 'importance_score': 0.9}, {'entity_name': 'RONNY JACKSON', 'entity_type': 'PERSON', 'description': 'Rear Admiral Ronny Jackson, the White House physician tapped to run the VA.', 'importance_score': 0.9}, {'entity_name': 'CNBC', 'entity_type': 'ORGANIZATION', 'description': 'CNBC, the news organization reporting on the event.', 'importance_score': 0.7}, {'entity_name': 'KAYLA TAUSCHE', 'entity_type': 'PERSON', 'description': 'Kayla Tausche, the CNBC reporter covering the story.', 'importance_score': 0.7}, {'entity_name': '2 HOURS AGO', 'entity_type': 'TIME', 'description': 'The time when the report was made.', 'importance_score': 0.5}], 'relationships': [{'src_id': 'TRUMP', 'tgt_id': 'RONNY JACKSON', 'description': 'President Trump taps Ronny Jackson as new VA secretary.', 'weight': 1.0, 'order': 1}, {'src_id': 'RONNY JACKSON', 'tgt_id': 'VA', 'description': 'Ronny Jackson is tapped to run the Department of Veterans Affairs.', 'weight': 1.0, 'order': 1}, {'src_id': 'KAYLA TAUSCHE', 'tgt_id': 'TRUMP', 'description': \"CNBC's Kayla Tausche reports President Trump has tapped Ronny Jackson.\", 'weight': 0.8, 'order': 2}]}) (input_keys={'input_text'})]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "devset[:5]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "loaded_trainset = pickle.load(open(entity_relationship_trainset_path, \"rb\"))\n", + "loaded_valset = pickle.load(open(entity_relationship_valset_path, \"rb\"))\n", + "loaded_devset = pickle.load(open(entity_relationship_devset_path, \"rb\"))\n", + "assert loaded_trainset == trainset\n", + "assert loaded_valset == valset\n", + "assert loaded_devset == devset" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "nano-graphrag", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/using_dspy_entity_extraction.py b/examples/using_dspy_entity_extraction.py index 7e5a29c..096eea1 100644 --- a/examples/using_dspy_entity_extraction.py +++ b/examples/using_dspy_entity_extraction.py @@ -138,14 +138,14 @@ def query(): """ lm = dspy.OpenAI( model="deepseek-chat", - model_type="chat", + model_type="chat", + api_provider="openai", api_key=os.environ["DEEPSEEK_API_KEY"], base_url=os.environ["DEEPSEEK_BASE_URL"], system_prompt=system_prompt, temperature=1.0, - top_p=1, - max_tokens=4096 + max_tokens=8192 ) - dspy.settings.configure(lm=lm) + dspy.settings.configure(lm=lm, experimental=True) insert() query() diff --git a/nano_graphrag/entity_extraction/extract.py b/nano_graphrag/entity_extraction/extract.py index 0462b59..d19c433 100644 --- a/nano_graphrag/entity_extraction/extract.py +++ b/nano_graphrag/entity_extraction/extract.py @@ -1,6 +1,7 @@ from typing import Union import pickle import asyncio +from openai import BadRequestError from collections import defaultdict import dspy from nano_graphrag._storage import BaseGraphStorage @@ -11,40 +12,67 @@ ) from nano_graphrag.prompt import PROMPTS from nano_graphrag._utils import logger, compute_mdhash_id -from nano_graphrag.entity_extraction.module import EntityRelationshipExtractor +from nano_graphrag.entity_extraction.module import TypedEntityRelationshipExtractor from nano_graphrag._op import _merge_edges_then_upsert, _merge_nodes_then_upsert async def generate_dataset( chunks: dict[str, TextChunkSchema], filepath: str, - save_dataset: bool = True + save_dataset: bool = True, + global_config: dict = {} ) -> list[dspy.Example]: - entity_extractor = EntityRelationshipExtractor() + entity_extractor = TypedEntityRelationshipExtractor() + + if global_config.get("use_compiled_dspy_entity_relationship", False): + entity_extractor.load(global_config["entity_relationship_module_path"]) + ordered_chunks = list(chunks.items()) + already_processed = 0 + already_entities = 0 + already_relations = 0 async def _process_single_content(chunk_key_dp: tuple[str, TextChunkSchema]) -> dspy.Example: + nonlocal already_processed, already_entities, already_relations chunk_dp = chunk_key_dp[1] content = chunk_dp["content"] - prediction = await asyncio.to_thread( - entity_extractor, input_text=content - ) + try: + prediction = await asyncio.to_thread( + entity_extractor, input_text=content + ) + entities, relationships = prediction.entities, prediction.relationships + except BadRequestError as e: + logger.error(f"Error in TypedEntityRelationshipExtractor: {e}") + entities, relationships = [], [] example = dspy.Example( input_text=content, - entities=prediction.entities, - relationships=prediction.relationships + entities=entities, + relationships=relationships ).with_inputs("input_text") + already_entities += len(entities) + already_relations += len(relationships) + already_processed += 1 + now_ticks = PROMPTS["process_tickers"][ + already_processed % len(PROMPTS["process_tickers"]) + ] + print( + f"{now_ticks} Processed {already_processed} chunks, {already_entities} entities(duplicated), {already_relations} relations(duplicated)\r", + end="", + flush=True, + ) return example examples = await asyncio.gather( *[_process_single_content(c) for c in ordered_chunks] ) + filtered_examples = [example for example in examples if len(example.entities) > 0 and len(example.relationships) > 0] + num_filtered_examples = len(examples) - len(filtered_examples) if save_dataset: with open(filepath, 'wb') as f: - pickle.dump(examples, f) - logger.info(f"Saved {len(examples)} examples with keys: {examples[0].keys()}") + pickle.dump(filtered_examples, f) + logger.info(f"Saved {len(filtered_examples)} examples with keys: {filtered_examples[0].keys()}, filtered {num_filtered_examples} examples") - return examples + return filtered_examples async def extract_entities_dspy( @@ -53,7 +81,7 @@ async def extract_entities_dspy( entity_vdb: BaseVectorStorage, global_config: dict, ) -> Union[BaseGraphStorage, None]: - entity_extractor = EntityRelationshipExtractor() + entity_extractor = TypedEntityRelationshipExtractor() if global_config.get("use_compiled_dspy_entity_relationship", False): entity_extractor.load(global_config["entity_relationship_module_path"]) @@ -68,23 +96,26 @@ async def _process_single_content(chunk_key_dp: tuple[str, TextChunkSchema]): chunk_key = chunk_key_dp[0] chunk_dp = chunk_key_dp[1] content = chunk_dp["content"] - prediction = await asyncio.to_thread( - entity_extractor, input_text=content - ) - + try: + prediction = await asyncio.to_thread( + entity_extractor, input_text=content + ) + entities, relationships = prediction.entities, prediction.relationships + except BadRequestError as e: + logger.error(f"Error in TypedEntityRelationshipExtractor: {e}") + entities, relationships = [], [] + maybe_nodes = defaultdict(list) maybe_edges = defaultdict(list) - for entity in prediction.entities.context: - entity_dict = entity.dict() - entity_dict["source_id"] = chunk_key - maybe_nodes[entity_dict['entity_name']].append(entity_dict) + for entity in entities: + entity["source_id"] = chunk_key + maybe_nodes[entity['entity_name']].append(entity) already_entities += 1 - for relationship in prediction.relationships.context: - relationship_dict = relationship.dict() - relationship_dict["source_id"] = chunk_key - maybe_edges[(relationship_dict['src_id'], relationship_dict['tgt_id'])].append(relationship_dict) + for relationship in relationships: + relationship["source_id"] = chunk_key + maybe_edges[(relationship['src_id'], relationship['tgt_id'])].append(relationship) already_relations += 1 already_processed += 1 diff --git a/nano_graphrag/entity_extraction/metric.py b/nano_graphrag/entity_extraction/metric.py index 7b9712c..0b41413 100644 --- a/nano_graphrag/entity_extraction/metric.py +++ b/nano_graphrag/entity_extraction/metric.py @@ -1,35 +1,42 @@ import dspy -import numpy as np - - -class AssessRelationship(dspy.Signature): - """Assess the similarity of two relationships.""" - gold_relationship = dspy.InputField() - predicted_relationship = dspy.InputField() - similarity_score = dspy.OutputField(desc="Similarity score between 0 and 1") - - -def relationship_similarity_metric(gold: dspy.Example, pred: dspy.Prediction, trace=None) -> float: - similarity_scores = [] - - for gold_rel, pred_rel in zip(gold.relationships.context, pred.relationships.context): - assessment = dspy.Predict(AssessRelationship)( - gold_relationship=gold_rel, - predicted_relationship=pred_rel - ) - - try: - score = float(assessment.similarity_score) - similarity_scores.append(score) - except ValueError: - similarity_scores.append(0.0) - - return np.mean(similarity_scores) if similarity_scores else 0.0 +from nano_graphrag.entity_extraction.module import Relationship + + +class AssessRelationships(dspy.Signature): + """ + Assess the similarity between gold and predicted relationships: + 1. Match relationships based on src_id and tgt_id pairs, allowing for slight variations in entity names. + 2. For matched pairs, compare: + a) Description similarity (semantic meaning) + b) Weight similarity + c) Order similarity + 3. Consider unmatched relationships as penalties. + 4. Aggregate scores, accounting for precision and recall. + 5. Return a final similarity score between 0 (no similarity) and 1 (perfect match). + + Key considerations: + - Prioritize matching based on entity pairs over exact string matches. + - Use semantic similarity for descriptions rather than exact matches. + - Weight the importance of different aspects (e.g., entity matching, description, weight, order). + - Balance the impact of matched and unmatched relationships in the final score. + """ + + gold_relationships: list[Relationship] = dspy.InputField(desc="The gold-standard relationships to compare against.") + predicted_relationships: list[Relationship] = dspy.InputField(desc="The predicted relationships to compare against the gold-standard relationships.") + similarity_score: float = dspy.OutputField(desc="Similarity score between 0 and 1, with 1 being the highest similarity.") + + +def relationships_similarity_metric(gold: dspy.Example, pred: dspy.Prediction, trace=None) -> float: + model = dspy.TypedChainOfThought(AssessRelationships) + gold_relationships = [Relationship(**item) for item in gold['relationships']] + predicted_relationships = [Relationship(**item) for item in pred['relationships']] + similarity_score = float(model(gold_relationships=gold_relationships, predicted_relationships=predicted_relationships).similarity_score) + return similarity_score def entity_recall_metric(gold: dspy.Example, pred: dspy.Prediction, trace=None) -> float: - true_set = set(item.entity_name for item in gold.entities.context) - pred_set = set(item.entity_name for item in pred.entities.context) + true_set = set(item['entity_name'] for item in gold['entities']) + pred_set = set(item['entity_name'] for item in pred['entities']) true_positives = len(pred_set.intersection(true_set)) false_negatives = len(true_set - pred_set) recall = true_positives / (true_positives + false_negatives) if (true_positives + false_negatives) > 0 else 0 diff --git a/nano_graphrag/entity_extraction/module.py b/nano_graphrag/entity_extraction/module.py index fbcd31e..99efe3e 100644 --- a/nano_graphrag/entity_extraction/module.py +++ b/nano_graphrag/entity_extraction/module.py @@ -1,218 +1,134 @@ import dspy from pydantic import BaseModel, Field -from nano_graphrag._utils import logger, clean_str - - -class EntityTypes(BaseModel): - """ - Obtained from: - https://github.com/SciPhi-AI/R2R/blob/6e958d1e451c1cb10b6fc868572659785d1091cb/r2r/providers/prompts/defaults.jsonl - """ - context: list[str] = Field( - default=[ - "PERSON", "ORGANIZATION", "LOCATION", "DATE", "TIME", "MONEY", - "PERCENTAGE", "PRODUCT", "EVENT", "LANGUAGE", "NATIONALITY", - "RELIGION", "TITLE", "PROFESSION", "ANIMAL", "PLANT", "DISEASE", - "MEDICATION", "CHEMICAL", "MATERIAL", "COLOR", "SHAPE", - "MEASUREMENT", "WEATHER", "NATURAL_DISASTER", "AWARD", "LAW", - "CRIME", "TECHNOLOGY", "SOFTWARE", "HARDWARE", "VEHICLE", - "FOOD", "DRINK", "SPORT", "MUSIC_GENRE", "INSTRUMENT", - "ARTWORK", "BOOK", "MOVIE", "TV_SHOW", "ACADEMIC_SUBJECT", - "SCIENTIFIC_THEORY", "POLITICAL_PARTY", "CURRENCY", - "STOCK_SYMBOL", "FILE_TYPE", "PROGRAMMING_LANGUAGE", - "MEDICAL_PROCEDURE", "CELESTIAL_BODY" - ], - description="List of entity types used for extraction." - ) +from nano_graphrag._utils import clean_str + + +""" +Obtained from: +https://github.com/SciPhi-AI/R2R/blob/6e958d1e451c1cb10b6fc868572659785d1091cb/r2r/providers/prompts/defaults.jsonl +""" +ENTITY_TYPES = [ + "PERSON", "ORGANIZATION", "LOCATION", "DATE", "TIME", "MONEY", + "PERCENTAGE", "PRODUCT", "EVENT", "LANGUAGE", "NATIONALITY", + "RELIGION", "TITLE", "PROFESSION", "ANIMAL", "PLANT", "DISEASE", + "MEDICATION", "CHEMICAL", "MATERIAL", "COLOR", "SHAPE", + "MEASUREMENT", "WEATHER", "NATURAL_DISASTER", "AWARD", "LAW", + "CRIME", "TECHNOLOGY", "SOFTWARE", "HARDWARE", "VEHICLE", + "FOOD", "DRINK", "SPORT", "MUSIC_GENRE", "INSTRUMENT", + "ARTWORK", "BOOK", "MOVIE", "TV_SHOW", "ACADEMIC_SUBJECT", + "SCIENTIFIC_THEORY", "POLITICAL_PARTY", "CURRENCY", + "STOCK_SYMBOL", "FILE_TYPE", "PROGRAMMING_LANGUAGE", + "MEDICAL_PROCEDURE", "CELESTIAL_BODY" +] class Entity(BaseModel): - entity_name: str = Field(..., description="Cleaned and uppercased entity name, strictly upper case") - entity_type: str = Field(..., description="Cleaned and uppercased entity type, strictly upper case") - description: str = Field(..., description="Detailed and specific description of the entity") - importance_score: float = Field(ge=0.0, le=1.0, description="0 to 1, with 1 being most important") + entity_name: str = Field(..., description="The name of the entity.") + entity_type: str = Field(..., description="The type of the entity.") + description: str = Field(..., description="The description of the entity, in details and comprehensive.") + importance_score: float = Field(..., ge=0, le=1, description="Importance score of the entity. Should be between 0 and 1 with 1 being the most important.") class Relationship(BaseModel): - src_id: str = Field(..., description="Cleaned and uppercased source entity, strictly upper case") - tgt_id: str = Field(..., description="Cleaned and uppercased target entity, strictly upper case") - description: str = Field(..., description="Detailed and specific description of the relationship") - weight: float = Field(ge=0.0, le=1.0, description="0 to 1, with 1 being most important") - order: int = Field(..., description="1 for direct relationships, 2 for second-order, 3 for third-order, etc") + src_id: str = Field(..., description="The name of the source entity.") + tgt_id: str = Field(..., description="The name of the target entity.") + description: str = Field(..., description="The description of the relationship between the source and target entity, in details and comprehensive.") + weight: float = Field(..., ge=0, le=1, description="The weight of the relationship. Should be between 0 and 1 with 1 being the strongest relationship.") + order: int = Field(..., ge=1, le=3, description="The order of the relationship. 1 for direct relationships, 2 for second-order, 3 for third-order.") + + +class CombinedExtraction(dspy.Signature): + """ + Given a text document that is potentially relevant to this activity and a list of entity types, + identify all entities of those types from the text and all relationships among the identified entities. + + Entity Guidelines: + 1. Each entity name should be an actual atomic word from the input text. + 2. Avoid duplicates and generic terms. + 3. Make sure descriptions are detailed and comprehensive. Use multiple complete sentences for each point below: + a). The entity's role or significance in the context + b). Key attributes or characteristics + c). Relationships to other entities (if applicable) + d). Historical or cultural relevance (if applicable) + e). Any notable actions or events associated with the entity + 4. All entity types from the text must be included. + 5. IMPORTANT: Only use entity types from the provided 'entity_types' list. Do not introduce new entity types. + + Relationship Guidelines: + 1. Make sure relationship descriptions are detailed and comprehensive. Use multiple complete sentences for each point below: + a). The nature of the relationship (e.g., familial, professional, causal) + b). The impact or significance of the relationship on both entities + c). Any historical or contextual information relevant to the relationship + d). How the relationship evolved over time (if applicable) + e). Any notable events or actions that resulted from this relationship + 2. Include direct relationships (order 1) as well as higher-order relationships (order 2 and 3): + a). Direct relationships: Immediate connections between entities. + b). Second-order relationships: Indirect effects or connections that result from direct relationships. + c). Third-order relationships: Further indirect effects that result from second-order relationships. + 3. The "src_id" and "tgt_id" fields must exactly match entity names from the extracted entities list. + """ + input_text: str = dspy.InputField(desc="The text to extract entities and relationships from.") + entity_types: list[str] = dspy.InputField(desc="List of entity types used for extraction.") + entities_relationships: list[Entity | Relationship] = dspy.OutputField(desc="List of entities and relationships extracted from the text.") -class Entities(BaseModel): - context: list[Entity] +class TypedEntityRelationshipExtractorException(dspy.Module): + def __init__(self, predictor: dspy.Module, exception_types: tuple[type[Exception]] = (Exception,)): + super().__init__() + self.predictor = predictor + self.exception_types = exception_types -class Relationships(BaseModel): - context: list[Relationship] + def copy(self): + return TypedEntityRelationshipExtractorException(self.predictor) + def forward(self, **kwargs): + try: + prediction = self.predictor(**kwargs) + return prediction -class CombinedExtraction(dspy.Signature): - """Signature for extracting both entities and relationships from input text.""" + except Exception as e: + if isinstance(e, self.exception_types): + return dspy.Prediction(entities_relationships=[]) + + raise e - input_text: str = dspy.InputField(desc="The text to extract entities and relationships from.") - entity_types: EntityTypes = dspy.InputField() - entities: Entities = dspy.OutputField( - desc=""" - Format: - { - "context": [ - { - "entity_name": "ENTITY NAME", - "entity_type": "ENTITY TYPE", - "description": "Detailed description", - "importance_score": 0.8 - }, - ... - ] - } - Each entity name should be an actual atomic word from the input text. Avoid duplicates and generic terms. - Make sure descriptions are detailed and comprehensive, including: - 1. The entity's role or significance in the context - 2. Key attributes or characteristics - 3. Relationships to other entities (if applicable) - 4. Historical or cultural relevance (if applicable) - 5. Any notable actions or events associated with the entity - All entity types from the text must be included. - Entities must have an importance score greater than 0.5. - IMPORTANT: Only use entity types from the provided 'entity_types' list. Do not introduce new entity types. - Ensure the output is strictly JSON formatted without any trailing text or comments. - """ - ) - relationships: Relationships = dspy.OutputField( - desc=""" - Format: - { - "context": [ - { - "src_id": "SOURCE ENTITY", - "tgt_id": "TARGET ENTITY", - "description": "Detailed description of the relationship", - "weight": 0.7, - "order": 1 # 1 for direct relationships, 2 for second-order, 3 for third-order, etc. - }, - ... - ] - } - Make sure relationship descriptions are detailed and comprehensive, including: - 1. The nature of the relationship (e.g., familial, professional, causal) - 2. The impact or significance of the relationship on both entities - 3. Any historical or contextual information relevant to the relationship - 4. How the relationship evolved over time (if applicable) - 5. Any notable events or actions that resulted from this relationship - Include direct relationships (order 1) as well as higher-order relationships (order 2 and 3): - - Direct relationships: Immediate connections between entities. - - Second-order relationships: Indirect effects or connections that result from direct relationships. - - Third-order relationships: Further indirect effects that result from second-order relationships. - IMPORTANT: Only include relationships between existing entities from the extracted entities. Do not introduce new entities here. - The "src_id" and "tgt_id" fields must exactly match entity names from the extracted entities list. - Ensure the output is strictly JSON formatted without any trailing text or comments. - """ - ) - - -class CombinedSelfReflection(dspy.Signature): - """ - Signature for combined self-reflection on extracted entities and relationships. - Self-reflection is on the completeness and quality of both the extracted entities and relationships. - """ - input_text: str = dspy.InputField(desc="The original input text.") - entity_types: EntityTypes = dspy.InputField() - entities: Entities = dspy.InputField(desc="List of extracted entities.") - relationships: Relationships = dspy.InputField(desc="List of extracted relationships.") - missing_entities: Entities = dspy.OutputField( - desc=""" - Format: - { - "context": [ - { - "entity_name": "ENTITY NAME", - "entity_type": "ENTITY TYPE", - "description": "Detailed description", - "importance_score": 0.8 - }, - ... - ] - } - More specifically: - 1. Entities mentioned in the text but not captured in the initial extraction. - 2. Implicit entities that are crucial to the context but not explicitly mentioned. - 3. Entities that belong to the identified entity types but were overlooked. - 4. Subtypes or more specific instances of the already extracted entities. - Ensure the output is strictly JSON formatted without any trailing text or comments. - """ - ) - missing_relationships: Relationships = dspy.OutputField( - desc=""" - Format: - { - "context": [ - { - "src_id": "SOURCE ENTITY", - "tgt_id": "TARGET ENTITY", - "description": "Detailed description of the relationship", - "weight": 0.7, - "order": 1 # 1 for direct, 2 for second-order, 3 for third-order - }, - ... - ] - } - More specifically: - 1. Direct relationships (order 1) between entities that were not captured initially. - 2. Second-order relationships (order 2): Indirect effects or connections resulting from direct relationships. - 3. Third-order relationships (order 3): Further indirect effects resulting from second-order relationships. - 4. Implicit relationships that can be inferred from the context. - 5. Hierarchical, causal, or temporal relationships that may have been overlooked. - 6. Relationships involving the newly identified missing entities. - Only include relationships between entities in the combined entities list (extracted + missing). - Ensure the output is strictly JSON formatted without any trailing text or comments. - """ - ) - - -class EntityRelationshipExtractor(dspy.Module): - def __init__(self): +class TypedEntityRelationshipExtractor(dspy.Module): + def __init__(self, lm: dspy.LM = None, reasoning: dspy.OutputField = None, max_retries: int = 3): super().__init__() - self.entity_types = EntityTypes() - self.extractor = dspy.TypedPredictor(CombinedExtraction) - self.self_reflection = dspy.TypedPredictor(CombinedSelfReflection) + self.lm = lm + self.entity_types = ENTITY_TYPES + self.extractor = dspy.TypedChainOfThought( + signature=CombinedExtraction, + reasoning=reasoning, + max_retries=max_retries + ) + self.extractor = TypedEntityRelationshipExtractorException(self.extractor, exception_types=(ValueError, )) def forward(self, input_text: str) -> dspy.Prediction: - extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types) - reflection_result = self.self_reflection( - input_text=input_text, - entity_types=self.entity_types, - entities=extraction_result.entities, - relationships=extraction_result.relationships - ) - entities = extraction_result.entities - missing_entities = reflection_result.missing_entities - relationships = extraction_result.relationships - missing_relationships = reflection_result.missing_relationships - all_entities = Entities(context=entities.context + missing_entities.context) - all_relationships = Relationships(context=relationships.context + missing_relationships.context) - logger.debug(f"Entities: {len(entities.context)} | Missed Entities: {len(missing_entities.context)} | Total Entities: {len(all_entities.context)}") - logger.debug(f"Relationships: {len(relationships.context)} | Missed Relationships: {len(missing_relationships.context)} | Total Relationships: {len(all_relationships.context)}") + with dspy.context(lm=self.lm if self.lm is not None else dspy.settings.lm): + extraction_result = self.extractor(input_text=input_text, entity_types=self.entity_types) + + entities = [ + dict( + entity_name=clean_str(entity.entity_name.upper()), + entity_type=clean_str(entity.entity_type.upper()), + description=clean_str(entity.description), + importance_score=float(entity.importance_score) + ) + for entity in extraction_result.entities_relationships if isinstance(entity, Entity) + ] + + relationships = [ + dict( + src_id=clean_str(relationship.src_id.upper()), + tgt_id=clean_str(relationship.tgt_id.upper()), + description=clean_str(relationship.description), + weight=float(relationship.weight), + order=int(relationship.order) + ) + for relationship in extraction_result.entities_relationships if isinstance(relationship, Relationship) + ] - for entity in all_entities.context: - entity.entity_name = clean_str(entity.entity_name.upper()) - entity.entity_type = clean_str(entity.entity_type.upper()) - entity.description = clean_str(entity.description) - entity.importance_score = float(entity.importance_score) - - for relationship in all_relationships.context: - relationship.src_id = clean_str(relationship.src_id.upper()) - relationship.tgt_id = clean_str(relationship.tgt_id.upper()) - relationship.description = clean_str(relationship.description) - relationship.weight = float(relationship.weight) - relationship.order = int(relationship.order) - - direct_relationships = sum(1 for r in all_relationships.context if r.order == 1) - second_order_relationships = sum(1 for r in all_relationships.context if r.order == 2) - third_order_relationships = sum(1 for r in all_relationships.context if r.order == 3) - logger.debug(f"Direct Relationships: {direct_relationships} | Second-order: {second_order_relationships} | Third-order: {third_order_relationships} | Total Relationships: {len(all_relationships.context)}") - return dspy.Prediction(entities=all_entities, relationships=all_relationships) - \ No newline at end of file + return dspy.Prediction(entities=entities, relationships=relationships) diff --git a/tests/entity_extraction/test_extract.py b/tests/entity_extraction/test_extract.py index 10d3a6e..58e009b 100644 --- a/tests/entity_extraction/test_extract.py +++ b/tests/entity_extraction/test_extract.py @@ -1,12 +1,10 @@ import pytest import dspy +from openai import BadRequestError from unittest.mock import Mock, patch, AsyncMock -from nano_graphrag.entity_extraction.module import ( - Entities, - Relationships, -) from nano_graphrag.entity_extraction.extract import generate_dataset, extract_entities_dspy from nano_graphrag.base import TextChunkSchema, BaseGraphStorage, BaseVectorStorage +import httpx @pytest.fixture @@ -19,7 +17,7 @@ def mock_chunks(): @pytest.fixture def mock_entity_extractor(): - with patch('nano_graphrag.entity_extraction.extract.EntityRelationshipExtractor') as mock: + with patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock: mock_instance = Mock() mock.return_value = mock_instance yield mock_instance @@ -39,35 +37,110 @@ def mock_vector_storage(): def mock_global_config(): return { "use_compiled_dspy_entity_relationship": False, - "entity_relationship_module_path": "path/to/module" + "entity_relationship_module_path": "path/to/module.json" } @pytest.mark.asyncio -async def test_generate_dataset(mock_chunks, mock_entity_extractor, tmp_path): +@pytest.mark.parametrize("use_compiled,save_dataset", [ + (True, True), (False, True), (True, False), (False, False) +]) +async def test_generate_dataset(mock_chunks, mock_entity_extractor, tmp_path, use_compiled, save_dataset): mock_prediction = Mock( - entities=Mock(context=[{"entity_name": "APPLE", "entity_type": "ORGANIZATION"}]), - relationships=Mock(context=[{"src_id": "APPLE", "tgt_id": "IPHONE"}]) + entities=[{"entity_name": "APPLE", "entity_type": "ORGANIZATION"}], + relationships=[{"src_id": "APPLE", "tgt_id": "IPHONE"}] ) mock_entity_extractor.return_value = mock_prediction filepath = tmp_path / "test_dataset.pkl" - - with patch('nano_graphrag.entity_extraction.extract.pickle.dump') as mock_dump: - result = await generate_dataset(mock_chunks, str(filepath)) - + + mock_global_config = { + "use_compiled_dspy_entity_relationship": use_compiled, + "entity_relationship_module_path": "test/path.json" if use_compiled else None + } + + with patch('nano_graphrag.entity_extraction.extract.pickle.dump') as mock_dump, \ + patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock_extractor_class: + + mock_extractor_instance = Mock() + mock_extractor_instance.return_value = mock_prediction + mock_extractor_class.return_value = mock_extractor_instance + + if use_compiled: + mock_extractor_instance.load = Mock() + + result = await generate_dataset(chunks=mock_chunks, filepath=str(filepath), + save_dataset=save_dataset, global_config=mock_global_config) + assert len(result) == 2 assert isinstance(result[0], dspy.Example) assert hasattr(result[0], 'input_text') assert hasattr(result[0], 'entities') assert hasattr(result[0], 'relationships') - assert result[0].input_text == "Apple announced a new iPhone model." - assert result[0].entities.context == [{"entity_name": "APPLE", "entity_type": "ORGANIZATION"}] - assert result[0].relationships.context == [{"src_id": "APPLE", "tgt_id": "IPHONE"}] + + if save_dataset: + mock_dump.assert_called_once() + else: + mock_dump.assert_not_called() + + mock_extractor_class.assert_called_once() + assert mock_extractor_instance.call_count == len(mock_chunks) + + if use_compiled: + mock_extractor_instance.load.assert_called_once_with("test/path.json") + else: + assert not hasattr(mock_extractor_instance, 'load') or not mock_extractor_instance.load.called + + +@pytest.mark.asyncio +async def test_generate_dataset_with_empty_chunks(): + chunks = {} + filepath = "test_empty_dataset.pkl" + result = await generate_dataset(chunks, filepath, save_dataset=False) + assert len(result) == 0 + + +@pytest.mark.asyncio +async def test_generate_dataset_with_bad_request_error(): + chunks = {"chunk1": TextChunkSchema(content="Test content")} + filepath = "test_error_dataset.pkl" + + # Create a mock response object + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 400 + mock_response.headers = {"x-request-id": "test-request-id"} + mock_response.request = Mock(spec=httpx.Request) + + with patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock_extractor_class: + mock_extractor_instance = Mock() + mock_extractor_instance.side_effect = BadRequestError( + message="Test Error", + response=mock_response, + body={"error": {"message": "Test Error", "type": "invalid_request_error"}} + ) + mock_extractor_class.return_value = mock_extractor_instance + + with patch('nano_graphrag.entity_extraction.extract.asyncio.to_thread', new_callable=AsyncMock) as mock_to_thread: + mock_to_thread.side_effect = BadRequestError( + message="Test Error", + response=mock_response, + body={"error": {"message": "Test Error", "type": "invalid_request_error"}} + ) + + result = await generate_dataset(chunks, filepath, save_dataset=False) + + assert len(result) == 0 + mock_to_thread.assert_called_once() @pytest.mark.asyncio -async def test_extract_entities_dspy(mock_chunks, mock_graph_storage, mock_vector_storage, mock_global_config): +@pytest.mark.parametrize("use_compiled,entity_vdb", [ + (True, Mock(spec=BaseVectorStorage)), + (False, Mock(spec=BaseVectorStorage)), + (True, None), + (False, None) +]) +async def test_extract_entities_dspy(mock_chunks, mock_graph_storage, entity_vdb, mock_global_config, use_compiled): mock_entity = { "entity_name": "APPLE", "entity_type": "ORGANIZATION", @@ -82,23 +155,104 @@ async def test_extract_entities_dspy(mock_chunks, mock_graph_storage, mock_vecto "order": 1 } mock_prediction = Mock( - entities=Entities(context=[mock_entity]), - relationships=Relationships(context=[mock_relationship]) + entities=[mock_entity], + relationships=[mock_relationship] ) - with patch('nano_graphrag.entity_extraction.extract.EntityRelationshipExtractor') as mock_extractor_class: + mock_global_config.update({ + "use_compiled_dspy_entity_relationship": use_compiled, + "entity_relationship_module_path": "test/path.json" if use_compiled else None + }) + + with patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock_extractor_class: mock_extractor_instance = Mock() mock_extractor_instance.return_value = mock_prediction mock_extractor_class.return_value = mock_extractor_instance + if use_compiled: + mock_extractor_instance.load = Mock() + with patch('nano_graphrag.entity_extraction.extract._merge_nodes_then_upsert', new_callable=AsyncMock) as mock_merge_nodes, \ patch('nano_graphrag.entity_extraction.extract._merge_edges_then_upsert', new_callable=AsyncMock) as mock_merge_edges: mock_merge_nodes.return_value = mock_entity - result = await extract_entities_dspy(mock_chunks, mock_graph_storage, mock_vector_storage, mock_global_config) + result = await extract_entities_dspy(mock_chunks, mock_graph_storage, entity_vdb, mock_global_config) assert result == mock_graph_storage mock_extractor_class.assert_called_once() mock_extractor_instance.assert_called() mock_merge_nodes.assert_called() mock_merge_edges.assert_called() - mock_vector_storage.upsert.assert_called_once() + + if entity_vdb: + entity_vdb.upsert.assert_called_once() + else: + assert not hasattr(entity_vdb, 'upsert') or not entity_vdb.upsert.called + + assert mock_extractor_instance.call_count == len(mock_chunks) + + if use_compiled: + mock_extractor_instance.load.assert_called_once_with("test/path.json") + else: + assert not hasattr(mock_extractor_instance, 'load') or not mock_extractor_instance.load.called + + +@pytest.mark.asyncio +async def test_extract_entities_dspy_with_empty_chunks(): + chunks = {} + mock_graph_storage = Mock(spec=BaseGraphStorage) + mock_vector_storage = Mock(spec=BaseVectorStorage) + global_config = {} + + result = await extract_entities_dspy(chunks, mock_graph_storage, mock_vector_storage, global_config) + + assert result is None + + +@pytest.mark.asyncio +async def test_extract_entities_dspy_with_no_entities(): + chunks = {"chunk1": TextChunkSchema(content="Test content")} + mock_graph_storage = Mock(spec=BaseGraphStorage) + mock_vector_storage = Mock(spec=BaseVectorStorage) + global_config = {} + + with patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock_extractor: + mock_extractor.return_value.return_value = Mock(entities=[], relationships=[]) + result = await extract_entities_dspy(chunks, mock_graph_storage, mock_vector_storage, global_config) + + assert result is None + mock_vector_storage.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_extract_entities_dspy_with_bad_request_error(): + chunks = {"chunk1": TextChunkSchema(content="Test content")} + mock_graph_storage = Mock(spec=BaseGraphStorage) + mock_vector_storage = Mock(spec=BaseVectorStorage) + global_config = {} + + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 400 + mock_response.headers = {"x-request-id": "test-request-id"} + mock_response.request = Mock(spec=httpx.Request) + + with patch('nano_graphrag.entity_extraction.extract.TypedEntityRelationshipExtractor') as mock_extractor_class: + mock_extractor_instance = Mock() + mock_extractor_instance.side_effect = BadRequestError( + message="Test Error", + response=mock_response, + body={"error": {"message": "Test Error", "type": "invalid_request_error"}} + ) + mock_extractor_class.return_value = mock_extractor_instance + + with patch('nano_graphrag.entity_extraction.extract.asyncio.to_thread', new_callable=AsyncMock) as mock_to_thread: + mock_to_thread.side_effect = BadRequestError( + message="Test Error", + response=mock_response, + body={"error": {"message": "Test Error", "type": "invalid_request_error"}} + ) + + result = await extract_entities_dspy(chunks, mock_graph_storage, mock_vector_storage, global_config) + + assert result is None + mock_to_thread.assert_called_once() + mock_vector_storage.upsert.assert_not_called() diff --git a/tests/entity_extraction/test_metric.py b/tests/entity_extraction/test_metric.py index 8c1fea4..3606878 100644 --- a/tests/entity_extraction/test_metric.py +++ b/tests/entity_extraction/test_metric.py @@ -1,89 +1,82 @@ import pytest -import numpy as np import dspy from unittest.mock import Mock, patch from nano_graphrag.entity_extraction.metric import ( - relationship_similarity_metric, + relationships_similarity_metric, entity_recall_metric, ) @pytest.fixture -def relationship(): - class Relationship: - def __init__(self, src_id, tgt_id, description=None): - self.src_id = src_id - self.tgt_id = tgt_id - self.description = description - return Relationship - - -@pytest.fixture -def entity(): - class Entity: - def __init__(self, entity_name): - self.entity_name = entity_name - return Entity +def mock_dspy_predict(): + with patch('nano_graphrag.entity_extraction.metric.dspy.TypedChainOfThought') as mock_predict: + mock_instance = Mock() + mock_instance.return_value = dspy.Prediction(similarity_score=0.75) + mock_predict.return_value = mock_instance + yield mock_predict @pytest.fixture -def example(): - class Example: - def __init__(self, items): - self.relationships = type('obj', (object,), {'context': items}) - self.entities = type('obj', (object,), {'context': items}) - return Example +def sample_relationship(): + return { + "src_id": "ENTITY1", + "tgt_id": "ENTITY2", + "description": "Example relationship", + "weight": 0.8, + "order": 1 + } @pytest.fixture -def prediction(): - class Prediction: - def __init__(self, items): - self.relationships = type('obj', (object,), {'context': items}) - self.entities = type('obj', (object,), {'context': items}) - return Prediction +def sample_entity(): + return { + "entity_name": "EXAMPLE_ENTITY", + "entity_type": "PERSON", + "description": "An example entity", + "importance_score": 0.8 + } @pytest.fixture -def sample_texts(): - return ["Hello", "World", "Test"] +def example(): + def _example(items): + return {"relationships": items} if "src_id" in (items[0] if items else {}) else {"entities": items} + return _example @pytest.fixture -def mock_dspy_predict(): - with patch('nano_graphrag.entity_extraction.metric.dspy.Predict') as mock_predict: - mock_instance = Mock() - mock_instance.return_value = dspy.Prediction(similarity_score="0.75") - mock_predict.return_value = mock_instance - yield mock_predict +def prediction(): + def _prediction(items): + return {"relationships": items} if "src_id" in (items[0] if items else {}) else {"entities": items} + return _prediction @pytest.mark.asyncio -async def test_relationship_similarity_metric(relationship, example, prediction, mock_dspy_predict): +async def test_relationship_similarity_metric(sample_relationship, example, prediction, mock_dspy_predict): gold = example([ - relationship("1", "2", "is related to"), - relationship("2", "3", "is connected with"), + {**sample_relationship, "src_id": "ENTITY1", "tgt_id": "ENTITY2", "description": "is related to"}, + {**sample_relationship, "src_id": "ENTITY2", "tgt_id": "ENTITY3", "description": "is connected with"}, ]) pred = prediction([ - relationship("1", "2", "is connected to"), - relationship("2", "3", "is linked with"), + {**sample_relationship, "src_id": "ENTITY1", "tgt_id": "ENTITY2", "description": "is connected to"}, + {**sample_relationship, "src_id": "ENTITY2", "tgt_id": "ENTITY3", "description": "is linked with"}, ]) - similarity = relationship_similarity_metric(gold, pred) - assert np.isclose(similarity, 0.75, atol=1e-6) + similarity = relationships_similarity_metric(gold, pred) + assert 0 <= similarity <= 1 @pytest.mark.asyncio -async def test_entity_recall_metric(entity, example, prediction): +async def test_entity_recall_metric(sample_entity, example, prediction): gold = example([ - entity("Entity1"), - entity("Entity2"), - entity("Entity3"), + {**sample_entity, "entity_name": "ENTITY1"}, + {**sample_entity, "entity_name": "ENTITY2"}, + {**sample_entity, "entity_name": "ENTITY3"}, ]) - pred = prediction([ - entity("Entity1"), - entity("Entity3"), - entity("Entity4"), + pred = example([ + {**sample_entity, "entity_name": "ENTITY1"}, + {**sample_entity, "entity_name": "ENTITY3"}, + {**sample_entity, "entity_name": "ENTITY4"}, ]) recall = entity_recall_metric(gold, pred) @@ -91,35 +84,45 @@ async def test_entity_recall_metric(entity, example, prediction): @pytest.mark.asyncio -async def test_relationship_similarity_metric_no_common_keys(relationship, example, prediction, mock_dspy_predict): - gold = example([relationship("1", "2", "is related to")]) - pred = prediction([relationship("3", "4", "is connected with")]) +async def test_relationship_similarity_metric_no_common_keys(sample_relationship, example, prediction, mock_dspy_predict): + gold = example([{**sample_relationship, "src_id": "ENTITY1", "tgt_id": "ENTITY2", "description": "is related to"}]) + pred = prediction([{**sample_relationship, "src_id": "ENTITY3", "tgt_id": "ENTITY4", "description": "is connected with"}]) - similarity = relationship_similarity_metric(gold, pred) - assert similarity == 0.75 # The mocked value + similarity = relationships_similarity_metric(gold, pred) + assert 0 <= similarity <= 1 @pytest.mark.asyncio -async def test_entity_recall_metric_no_true_positives(entity, example, prediction): - gold = example([entity("Entity1"), entity("Entity2")]) - pred = prediction([entity("Entity3"), entity("Entity4")]) +async def test_entity_recall_metric_no_true_positives(sample_entity, example, prediction): + gold = example([ + {**sample_entity, "entity_name": "ENTITY1"}, + {**sample_entity, "entity_name": "ENTITY2"} + ]) + pred = prediction([ + {**sample_entity, "entity_name": "ENTITY3"}, + {**sample_entity, "entity_name": "ENTITY4"} + ]) recall = entity_recall_metric(gold, pred) assert recall == 0 + @pytest.mark.asyncio -async def test_relationship_similarity_metric_identical_descriptions(relationship, example, prediction, mock_dspy_predict): - gold = example([relationship("1", "2", "is related to")]) - pred = prediction([relationship("1", "2", "is related to")]) +async def test_relationship_similarity_metric_identical_descriptions(sample_relationship, example, prediction, mock_dspy_predict): + gold = example([{**sample_relationship, "src_id": "ENTITY1", "tgt_id": "ENTITY2", "description": "is related to"}]) + pred = prediction([{**sample_relationship, "src_id": "ENTITY1", "tgt_id": "ENTITY2", "description": "is related to"}]) - similarity = relationship_similarity_metric(gold, pred) - assert np.isclose(similarity, 0.75, atol=1e-6) + similarity = relationships_similarity_metric(gold, pred) + assert similarity == 0.75 @pytest.mark.asyncio -async def test_entity_recall_metric_perfect_recall(entity, example, prediction): - entities = [entity("Entity1"), entity("Entity2")] +async def test_entity_recall_metric_perfect_recall(sample_entity, example, prediction): + entities = [ + {**sample_entity, "entity_name": "ENTITY1"}, + {**sample_entity, "entity_name": "ENTITY2"} + ] gold = example(entities) pred = prediction(entities) @@ -132,19 +135,5 @@ async def test_relationship_similarity_metric_no_relationships(example, predicti gold = example([]) pred = prediction([]) - similarity = relationship_similarity_metric(gold, pred) - assert similarity == 0.0 - - -@pytest.mark.asyncio -async def test_relationship_similarity_metric_invalid_score(relationship, example, prediction): - with patch('nano_graphrag.entity_extraction.metric.dspy.Predict') as mock_predict: - mock_instance = Mock() - mock_instance.return_value = dspy.Prediction(similarity_score="invalid") - mock_predict.return_value = mock_instance - - gold = example([relationship("1", "2", "is related to")]) - pred = prediction([relationship("1", "2", "is connected to")]) - - similarity = relationship_similarity_metric(gold, pred) - assert similarity == 0.0 + with pytest.raises(KeyError): + similarity = relationships_similarity_metric(gold, pred) diff --git a/tests/entity_extraction/test_module.py b/tests/entity_extraction/test_module.py index 1d9ace3..f52cf84 100644 --- a/tests/entity_extraction/test_module.py +++ b/tests/entity_extraction/test_module.py @@ -1,19 +1,13 @@ +import dspy from unittest.mock import Mock, patch -from nano_graphrag.entity_extraction.module import ( - EntityRelationshipExtractor, - Entities, - Relationships, - Entity, - Relationship -) +from nano_graphrag.entity_extraction.module import TypedEntityRelationshipExtractor, Relationship, Entity def test_entity_relationship_extractor(): - with patch('nano_graphrag.entity_extraction.module.dspy.TypedPredictor') as mock_typed_predictor: + with patch('nano_graphrag.entity_extraction.module.dspy.TypedChainOfThought') as mock_chain_of_thought: input_text = "Apple announced a new iPhone model." mock_extractor = Mock() - mock_self_reflection = Mock() - mock_typed_predictor.side_effect = [mock_extractor, mock_self_reflection] + mock_chain_of_thought.return_value = mock_extractor mock_entities = [ Entity(entity_name="APPLE", entity_type="ORGANIZATION", description="A technology company", importance_score=1), @@ -22,24 +16,12 @@ def test_entity_relationship_extractor(): mock_relationships = [ Relationship(src_id="APPLE", tgt_id="IPHONE", description="Apple manufactures iPhone", weight=1, order=1) ] - mock_missing_entities = [ - Entity(entity_name="TIM_COOK", entity_type="PERSON", description="CEO of Apple", importance_score=0.8) - ] - mock_missing_relationships = [ - Relationship(src_id="TIM_COOK", tgt_id="APPLE", description="Tim Cook is the CEO of Apple", weight=0.9, order=1), - Relationship(src_id="APPLE", tgt_id="IPHONE", description="Apple announces new iPhone model", weight=1, order=1) - ] - mock_extractor.return_value = Mock( - entities=Entities(context=mock_entities), - relationships=Relationships(context=mock_relationships) - ) - mock_self_reflection.return_value = Mock( - missing_entities=Entities(context=mock_missing_entities), - missing_relationships=Relationships(context=mock_missing_relationships) + mock_extractor.return_value = dspy.Prediction( + entities_relationships=mock_entities + mock_relationships ) - - extractor = EntityRelationshipExtractor() + + extractor = TypedEntityRelationshipExtractor() result = extractor.forward(input_text=input_text) mock_extractor.assert_called_once_with( @@ -47,44 +29,21 @@ def test_entity_relationship_extractor(): entity_types=extractor.entity_types ) - mock_self_reflection.assert_called_once_with( - input_text=input_text, - entity_types=extractor.entity_types, - entities=mock_extractor.return_value.entities, - relationships=mock_extractor.return_value.relationships - ) - - assert len(result.entities.context) == 3 - assert len(result.relationships.context) == 3 - - assert result.entities.context[0].entity_name == "APPLE" - assert result.entities.context[0].entity_type == "ORGANIZATION" - assert result.entities.context[0].description == "A technology company" - - assert result.entities.context[1].entity_name == "IPHONE" - assert result.entities.context[1].entity_type == "PRODUCT" - assert result.entities.context[1].description == "A smartphone" - assert result.entities.context[1].importance_score == 1 - - assert result.entities.context[2].entity_name == "TIM_COOK" - assert result.entities.context[2].entity_type == "PERSON" - assert result.entities.context[2].description == "CEO of Apple" - assert result.entities.context[2].importance_score == 0.8 + assert len(result.entities) == 2 + assert len(result.relationships) == 1 - assert result.relationships.context[0].src_id == "APPLE" - assert result.relationships.context[0].tgt_id == "IPHONE" - assert result.relationships.context[0].description == "Apple manufactures iPhone" - assert result.relationships.context[0].weight == 1 - assert result.relationships.context[0].order == 1 + assert result.entities[0]["entity_name"] == "APPLE" + assert result.entities[0]["entity_type"] == "ORGANIZATION" + assert result.entities[0]["description"] == "A technology company" + assert result.entities[0]["importance_score"] == 1 - assert result.relationships.context[1].src_id == "TIM_COOK" - assert result.relationships.context[1].tgt_id == "APPLE" - assert result.relationships.context[1].description == "Tim Cook is the CEO of Apple" - assert result.relationships.context[1].weight == 0.9 - assert result.relationships.context[1].order == 1 + assert result.entities[1]["entity_name"] == "IPHONE" + assert result.entities[1]["entity_type"] == "PRODUCT" + assert result.entities[1]["description"] == "A smartphone" + assert result.entities[1]["importance_score"] == 1 - assert result.relationships.context[2].src_id == "APPLE" - assert result.relationships.context[2].tgt_id == "IPHONE" - assert result.relationships.context[2].description == "Apple announces new iPhone model" - assert result.relationships.context[2].weight == 1 - assert result.relationships.context[2].order == 1 + assert result.relationships[0]["src_id"] == "APPLE" + assert result.relationships[0]["tgt_id"] == "IPHONE" + assert result.relationships[0]["description"] == "Apple manufactures iPhone" + assert result.relationships[0]["weight"] == 1 + assert result.relationships[0]["order"] == 1