Added line to docstring for respect_robots_txt

DataCrawl-AI · Jun 19, 2024 · 6a67802 · 6a67802
1 parent 8c3ac23
commit 6a67802
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/src/tiny_web_crawler/core/spider.py b/src/tiny_web_crawler/core/spider.py
@@ -38,6 +38,7 @@ class Spider:
         include_body (bool): Whether or not to include the crawled page's body in crawl_result (default: False)
         internal_links_only (bool): Whether or not to crawl only internal links
         external_links_only (bool): Whether or not to crawl only external links
+        respect_robots_txt (bool): Whether or not to respect website's robots.txt files (defualt: True)
     """
 
     root_url: str