Skip to content
This repository has been archived by the owner on Sep 15, 2024. It is now read-only.

Commit

Permalink
fix: Fixed ruff formatter errors.
Browse files Browse the repository at this point in the history
  • Loading branch information
@anirbanbasu
anirbanbasu committed May 4, 2024
1 parent cebcd11 commit 1816b58
Show file tree
Hide file tree
Showing 2 changed files with 131 additions and 89 deletions.
132 changes: 69 additions & 63 deletions pages/ingest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,8 @@ async def ingest_arxiv_data():
ingested_documents.value = []
reader = ArxivReader()
show_status_message(
message=f"**Fetching** results of arXiv query `{arxiv_query.value}`.",
)
message=f"**Fetching** results of arXiv query `{arxiv_query.value}`.",
)
ingested_documents.value = reader.load_data(
papers_dir=global_state.md5_hash(
f"{datetime.datetime.now()}-{random.random()}"
Expand Down Expand Up @@ -371,6 +371,7 @@ async def ingest_pdfurl_data():
)
logger.error(f"Error ingesting PDF (URL). {str(e)}")


@task
async def load_existing_indices():
"""Load the existing indices from the storage context."""
Expand All @@ -390,7 +391,9 @@ async def load_existing_indices():
index_id=index_ids[0],
)
else:
raise NotImplementedError("Loading multiple indices is not yet supported.")
raise NotImplementedError(
"Loading multiple indices is not yet supported."
)
# Check: https://github.com/run-llama/llama_index/discussions/13233
show_status_message(
message=f"""
Expand Down Expand Up @@ -421,6 +424,7 @@ async def load_existing_indices():
)
logger.error(f"Error loading indice(s). {str(e)}")


def load_wikipedia_languages():
"""Load the list of supported Wikipedia languages."""
if len(wikipedia_languages.value) == 0:
Expand Down Expand Up @@ -455,6 +459,7 @@ def ExistingIndicesSourceComponent():
)
solara.ProgressLinear(load_existing_indices.pending)


@solara.component
def WikipediaSourceComponent():
"""Component for selecting a Wikipedia article as a data source."""
Expand Down Expand Up @@ -504,7 +509,11 @@ def WikipediaSourceComponent():
last_ingested_data_source.value == constants.SOURCE_TYPE_WIKIPEDIA
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(inset=True, hover=True, value=0 if len(ingested_documents.value) == 1 else None):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} Wikipedia {'articles' if len(ingested_documents.value) > 1 else 'article'}"
)
Expand Down Expand Up @@ -547,24 +556,24 @@ def ArxivSourceComponent():
)
solara.ProgressLinear(ingest_arxiv_data.pending)
if (
last_ingested_data_source.value == constants.SOURCE_TYPE_ARXIV
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} {'pages' if len(ingested_documents.value) > 1 else 'page'} of PDF as a result of the arXiv query `{arxiv_query.value}`."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**PDF page**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")
last_ingested_data_source.value == constants.SOURCE_TYPE_ARXIV
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} {'pages' if len(ingested_documents.value) > 1 else 'page'} of PDF as a result of the arXiv query `{arxiv_query.value}`."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**PDF page**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")


@solara.component
Expand All @@ -591,24 +600,24 @@ def PubmedSourceComponent():
)
solara.ProgressLinear(ingest_pubmed_data.pending)
if (
last_ingested_data_source.value == constants.SOURCE_TYPE_PUBMED
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} {'articles' if len(ingested_documents.value) > 1 else 'article'} as a result of the Pubmed query `{pubmed_query.value}`."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**Article**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")
last_ingested_data_source.value == constants.SOURCE_TYPE_PUBMED
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} {'articles' if len(ingested_documents.value) > 1 else 'article'} as a result of the Pubmed query `{pubmed_query.value}`."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**Article**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")


@solara.component
Expand Down Expand Up @@ -693,6 +702,7 @@ def WebpageSourceComponent():
f"[{doc.doc_id}]({doc.doc_id})"
)


@solara.component
def PDFURLSourceComponent():
"""Component for selecting a PDF (URL) as data source."""
Expand All @@ -717,42 +727,38 @@ def PDFURLSourceComponent():
)
solara.ProgressLinear(ingest_pdfurl_data.pending)
if (
last_ingested_data_source.value == constants.SOURCE_TYPE_PDF
and len(ingested_documents.value) > 0
last_ingested_data_source.value == constants.SOURCE_TYPE_PDF
and len(ingested_documents.value) > 0
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
with rv.ExpansionPanels(
inset=True,
hover=True,
value=0 if len(ingested_documents.value) == 1 else None,
):
solara.Text(
f"Fetched {len(ingested_documents.value)} {'pages' if len(ingested_documents.value) > 1 else 'page'} from the PDF at {pdf_url.value}."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**Page**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")
solara.Text(
f"Fetched {len(ingested_documents.value)} {'pages' if len(ingested_documents.value) > 1 else 'page'} from the PDF at {pdf_url.value}."
)
for doc in ingested_documents.value:
with rv.ExpansionPanel():
with rv.ExpansionPanelHeader():
solara.Markdown(f"**Page**: {doc.doc_id}")
with rv.ExpansionPanelContent():
with solara.Column():
solara.Markdown(f"{doc.text}")


@solara.component
def Page():
"""Main settings page."""
# Remove the "This website runs on Solara" message
solara.Style(
constants.UI_SOLARA_NOTICE_REMOVE
)
solara.Style(constants.UI_SOLARA_NOTICE_REMOVE)
global_state.initialise_default_settings()

with solara.Head():
solara.Title("Data ingestion")

with solara.AppBarTitle():
solara.Markdown(
"# Ingest data", style={"color": "#FFFFFF"}
)
solara.Markdown("# Ingest data", style={"color": "#FFFFFF"})

with solara.AppBar():
solara.lab.ThemeToggle()
Expand Down
88 changes: 62 additions & 26 deletions utils/global_state.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,34 @@
global_settings_initialised: solara.Reactive[bool] = solara.reactive(False)

""" Language model settings """
global_settings__language_model_provider: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__cohere_api_key: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__cohere_model: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__openai_model: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__openai_api_key: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__ollama_url: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__ollama_model: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__language_model_provider: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__cohere_api_key: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__cohere_model: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__openai_model: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__openai_api_key: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__ollama_url: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__ollama_model: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__llm_temperature: solara.Reactive[float] = solara.reactive(0.0)
global_settings__llm_chunk_size: solara.Reactive[int] = solara.reactive(0)
global_settings__llm_chunk_overlap: solara.Reactive[int] = solara.reactive(0)
global_settings__llm_request_timeout: solara.Reactive[int] = solara.reactive(0)
global_settings__llm_system_message: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__llm_system_message: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)

""" Index and chat settings """

Expand All @@ -85,25 +101,41 @@
global_settings__index_include_embeddings: solara.Reactive[bool] = solara.reactive(
False
)
global_settings__index_chat_mode: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__index_chat_mode: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)

""" Neo4j settings """
global_settings__neo4j_disable: solara.Reactive[bool] = solara.reactive(False)
global_settings__neo4j_url: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__neo4j_username: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__neo4j_password: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__neo4j_db_name: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__neo4j_url: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__neo4j_username: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__neo4j_password: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__neo4j_db_name: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)

""" Redis settings """
global_settings__redis_disable: solara.Reactive[bool] = solara.reactive(False)
global_settings__redis_url: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__redis_namespace: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__redis_url: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)
global_settings__redis_namespace: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)

""" Knowledge graph visualisation settings """
global_settings__kg_vis_height: solara.Reactive[int] = solara.reactive(0)
global_settings__kg_vis_max_nodes: solara.Reactive[int] = solara.reactive(0)
global_settings__kg_vis_max_depth: solara.Reactive[int] = solara.reactive(0)
global_settings__kg_vis_layout: solara.Reactive[str] = solara.reactive(constants.EMPTY_STRING)
global_settings__kg_vis_layout: solara.Reactive[str] = solara.reactive(
constants.EMPTY_STRING
)

""" LlamaIndex Settings objects """
global_llamaindex_storage_context: solara.Reactive[StorageContext] = solara.reactive(
Expand All @@ -129,17 +161,20 @@ class MessageDict(TypedDict):
global_chat_engine: solara.Reactive[BaseChatEngine] = solara.reactive(None)
global_chat_messages: solara.Reactive[List[MessageDict]] = solara.reactive([])


def md5_hash(some_string):
return hashlib.md5(some_string.encode(constants.CHAR_ENCODING_UTF8)).hexdigest()


def setup_langfuse():
"""Setup (or, disable) Langfuse for performance evaluation."""
use_langfuse = bool(
os.getenv(
constants.ENV_KEY_EVAL_USE_LANGFUSE, constants.DEFAULT_SETTING_EVAL_USE_LANGFUSE
).lower()
in ["true", "yes", "t", "y", "on"]
)
os.getenv(
constants.ENV_KEY_EVAL_USE_LANGFUSE,
constants.DEFAULT_SETTING_EVAL_USE_LANGFUSE,
).lower()
in ["true", "yes", "t", "y", "on"]
)
if use_langfuse:
langfuse__secret_key = os.getenv(constants.ENV_KEY_LANGFUSE_SECRET_KEY)
langfuse__public_key = os.getenv(constants.ENV_KEY_LANGFUSE_PUBLIC_KEY)
Expand All @@ -152,7 +187,9 @@ def setup_langfuse():
)
langfuse.auth_check()
# Setup LangFuse tags for the handler, if necessary
env__langfuse_trace_tags = os.getenv(constants.ENV_KEY_LANGFUSE_TRACE_TAGS, None)
env__langfuse_trace_tags = os.getenv(
constants.ENV_KEY_LANGFUSE_TRACE_TAGS, None
)
langfuse_trace_tags = None
if env__langfuse_trace_tags is not None:
langfuse_trace_tags = [
Expand All @@ -165,9 +202,7 @@ def setup_langfuse():
host=langfuse__host,
tags=langfuse_trace_tags,
)
Settings.callback_manager = CallbackManager(
[langfuse_callback_handler]
)
Settings.callback_manager = CallbackManager([langfuse_callback_handler])
logger.warning(
f"Using Langfuse at {langfuse__host} for performance evaluation."
)
Expand Down Expand Up @@ -324,7 +359,8 @@ def initialise_default_settings():
)
global_settings__llm_chunk_size.value = int(
os.getenv(
constants.ENV_KEY_LLM_CHUNK_SIZE, constants.DEFAULT_SETTING_LLM_CHUNK_SIZE
constants.ENV_KEY_LLM_CHUNK_SIZE,
constants.DEFAULT_SETTING_LLM_CHUNK_SIZE,
)
)
global_settings__llm_chunk_overlap.value = int(
Expand Down

0 comments on commit 1816b58

Please sign in to comment.