-
-
Notifications
You must be signed in to change notification settings - Fork 289
/
novelhi.py
78 lines (64 loc) · 2.76 KB
/
novelhi.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
# -*- coding: utf-8 -*-
import logging
from bs4 import Tag
from urllib.parse import quote
from lncrawl.core.crawler import Crawler
logger = logging.getLogger(__name__)
search_url = "%s/book/searchByPageInShelf?curr=1&limit=10&keyword=%s"
fetch_chapter_list_url = "%s/book/queryIndexList?bookId=%s&curr=1&limit=50000"
fetch_chapter_body_url = (
"%s/book/queryBookIndexContent?bookId=%s&bookIndexId=%s&autoUnlock=false"
)
class NovelHiCrawler(Crawler):
base_url = [
"https://novelhi.com/",
]
def search_novel(self, query):
data = self.get_json(search_url % (self.home_url, quote(query)))
results = []
for item in data["data"]["list"]:
results.append(
{
"title": item["bookName"],
"url": "%s/s/%s" % (self.home_url, item["simpleName"]),
"info": "Latest: %s (%s)"
% (item["lastIndexName"], item["lastIndexUpdateTime"]),
}
)
return results
def read_novel_info(self):
logger.debug("Visiting %s", self.novel_url)
soup = self.get_soup(self.novel_url)
possible_book_id = soup.select_one("input#bookId")
assert isinstance(possible_book_id, Tag), "No book id"
self.novel_id = possible_book_id["value"]
possible_image = soup.select_one("a.book_cover img.cover")
assert isinstance(possible_image, Tag), "No novel title"
self.novel_cover = self.absolute_url(possible_image["src"])
logger.info("Novel cover: %s", self.novel_cover)
self.novel_title = possible_image["alt"]
logger.info("Novel title: %s", self.novel_title)
for span in soup.select(".book_info ul.list span.item"):
if span.get_text().startswith("Author"):
author = span.find_next()
if author:
self.novel_author = author.text
logger.info("Novel author: %s", self.novel_author)
data = self.get_json(fetch_chapter_list_url % (self.home_url, self.novel_id))
for item in reversed(data["data"]["list"]):
chap_id = len(self.chapters) + 1
vol_id = len(self.chapters) // 100 + 1
if len(self.volumes) < vol_id:
self.volumes.append({"id": vol_id})
self.chapters.append(
{
"id": chap_id,
"volume": vol_id,
"title": item["indexName"],
"url": "%s/%s" % (self.novel_url.strip("/"), item["indexNum"]),
}
)
def download_chapter_body(self, chapter):
soup = self.get_soup(chapter["url"])
paras = soup.select("#showReading sent")
return "".join(["<p>%s</p>" % p.text for p in paras])