-
Notifications
You must be signed in to change notification settings - Fork 0
/
ChapterParser.py
126 lines (110 loc) · 4.22 KB
/
ChapterParser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/bin/python
# -*- coding: iso-8859-15 -*-
__author__ = 'valentinarho'
from HTMLParser import HTMLParser
import string
import re, urlparse
# create a subclass and override the handler methods
class ChapterParser(HTMLParser):
images = []
chapter = ""
is_h1 = 0
is_h3 = 0
is_p = 0
is_li = 0
is_code = 0
is_strong = 0
is_in_guide = 0
def urlEncodeNonAscii(self, b):
return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)
def iriToUri(self, iri):
parts = urlparse.urlparse(iri)
return urlparse.urlunparse(
part.encode('idna') if parti == 1 else re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)),
part.encode('utf-8'))
for parti, part in enumerate(parts)
)
def cleanString(self, data):
#print "\n\nCLEAN: "+data;
data = data.strip()
data = string.replace(data, "<", "<")
data = string.replace(data, ">", ">")
data = string.replace(data, ">", "&")
data = string.replace(data, "\"", "\\\"")
data = string.replace(data, "\'", "\\\'")
try:
data = string.replace(data, "″", "\\\'")
except UnicodeDecodeError:
pass
#print "\nCLEANED:"+data;
return data
def handle_starttag(self, tag, attrs):
if tag == 'div':
for name, value in attrs:
if name == 'class' and value.find('guide-item') != -1:
self.is_in_guide = 1
elif name == 'class' and value.find('responsive-grid') != -1:
self.is_in_guide = 0
elif tag == 'img' and self.is_in_guide:
for name, value in attrs:
if name == 'src':
self.images.append(value)
self.chapter += "![image](" + self.iriToUri(value) + ") \n\n"
elif tag == 'h1':
self.is_h1 = 1
self.chapter += "\n##"
elif tag == 'h3' and self.is_in_guide:
self.is_h3 = 1
self.chapter += "\n##"
elif (tag == 'p' or tag == 'span') and self.is_in_guide:
self.is_p = 1
for name, value in attrs:
if name == 'class' and value.find('codice') != -1:
self.chapter += "<code>"
self.is_code = 1;
elif tag == 'strong' and self.is_in_guide:
self.is_p = 1
self.chapter += " "
elif tag == 'ul' and self.is_in_guide:
self.chapter += "\n<ul>"
elif tag == 'li' and self.is_in_guide:
self.is_li = 1
self.chapter += "\n<li>"
elif tag == 'code' and self.is_in_guide:
self.chapter += "\n<code>"
elif tag == 'br' and self.is_code:
self.chapter += " <br> "
def handle_endtag(self, tag):
if (tag == 'p') and self.is_in_guide:
if self.is_code:
self.is_code = 0
self.chapter += "</code>"
self.is_p = 0
self.chapter += " \n\n"
elif tag == 'h1':
self.is_h1 = 0
self.chapter += "\n"
elif tag == 'h3':
self.is_h3 = 0
self.chapter += "\n"
elif tag == 'strong' and self.is_in_guide:
self.chapter += " "
elif tag == 'ul' and self.is_in_guide:
self.chapter += "\n</ul>"
elif tag == 'li' and self.is_in_guide:
self.is_li = 0
self.chapter += "\n</li>"
elif tag == 'code' and self.is_in_guide:
self.chapter += "</code> "
pass
def handle_data(self, data):
if self.is_h1 or self.is_h3 or self.is_p or self.is_li or self.is_code or self.is_strong:
# print "\n DATA " + data
data = self.cleanString(data)
self.chapter += " " + data
#elif self.is_in_guide and ( string.find(data, "“") != -1 or string.find(data, "\"")!= -1 ):
# ci sono delle virgolette
# print "IN DATA"+data
def handle_entityref(self, ref):
if self.is_h1 or self.is_h3 or self.is_p or self.is_li or self.is_code or self.is_strong:
self.chapter += "&"+ref+";";