Skip to content

Commit

Permalink
Adds missing tags and import catch statement
Browse files Browse the repository at this point in the history
  • Loading branch information
skrawcz committed Mar 11, 2024
1 parent 59edd9a commit 961d231
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 3 deletions.
21 changes: 18 additions & 3 deletions contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,17 @@
5. what this doesn't do is create embeddings -- but that would be easy to extend.
"""

import logging
import re

import requests
from langchain import text_splitter
from langchain_core import documents
logger = logging.getLogger(__name__)

from hamilton import contrib

with contrib.catch_import_errors(__name__, __file__, logger):
import requests
from langchain import text_splitter
from langchain_core import documents

from hamilton.htypes import Collect, Parallelizable

Expand Down Expand Up @@ -160,3 +166,12 @@ def collect_chunked_url_text(url_result: Collect[dict]) -> list:
.build()
)
dr.display_all_functions("dag.png")
result = dr.execute(
["collect_chunked_url_text"],
inputs={"chunk_size": 256, "chunk_overlap": 32},
)
# do something with the result...
import pprint

for chunk in result["collect_chunked_url_text"]:
pprint.pprint(chunk)
Binary file modified contrib/hamilton/contrib/dagworks/sphinx_doc_chunking/dag.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,8 @@
"use_case_tags": ["data processing", "document chunking", "chunking", "langchain"],
"secondary_tags": {
"language": "English"
},
"driver_tags": {
"executor": "multithreading"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,8 @@
"secondary_tags": {
"dataframe-library" : "pandas",
"cloud-provider": "aws"
},
"driver_tags": {
"executor": "multithreading"
}
}

0 comments on commit 961d231

Please sign in to comment.