Merge branch 'release/3.0.0'

.travis.yml

@@ -3,9 +3,7 @@ env:
   - REQUIREMENTS=true
   - REQUIREMENTS=false
 python:
-  - "2.6"
   - "2.7"
-  - "3.2"
   - "3.3"
   - "3.4"
   - "3.5"
@@ -14,7 +12,6 @@ python:
 # command to install dependencies
 install:
   - if [[ $REQUIREMENTS == true ]] ; then pip install -r requirements.txt ; fi
-  - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install coverage==3.7.1; fi
   - pip install coveralls pytest pytest-cov coverage codecov
   - pip install -e .
   - if [[ ! $TRAVIS_PYTHON_VERSION == 'pypy-5.4' ]] ; then pip install regex; fi

CHANGELOG.textile

@@ -1,5 +1,13 @@
 h1. Textile Changelog
 
+h2. Version 3.0.0
+* Drop support for Python 2.6 and 3.2.
+* Update to the current version of html5lib
+* Bugfixes:
+** Fix handling of HTML entities in extended pre blocks. ("#55":https://github.com/textile/python-textile/issues/55)
+** Empty definitions in definition lists raised an exception ("#56":https://github.com/textile/python-textile/issues/56)
+** Fix handling of unicode in img attributes ("#58":https://github.com/textile/python-textile/issues/58)
+
 h2. Version 2.3.16
 * Bugfixes:
 ** Fix processing of extended code blocks ("#50":https://github.com/textile/python-textile/issues/50)

README.textile

@@ -39,4 +39,4 @@ bc.. import textile
 
 h3. Notes:
 
-* Active development supports Python 2.6 or later (including Python 3.2+).
+* Active development supports Python 2.7 or later (including Python 3.3+).

requirements.txt

@@ -1,2 +1,3 @@
-html5lib==0.999
-Pillow==3.0.0
+html5lib>=1.0b10
+Pillow==3.0.0
+regex

setup.py

@@ -55,10 +55,15 @@ def get_version():
         'Programming Language :: Python :: 3.6',
     ],
     keywords='textile,text,html markup',
-    install_requires=['six',],
+    install_requires=[
+        'six',
+        'html5lib>=0.999999999',
+        ],
     extras_require={
         ':python_version=="2.6"': ['ordereddict>=1.1'],
-        'develop': ['regex', 'pytest', 'pytest-cov'],
+        'develop': ['pytest', 'pytest-cov'],
+        'imagesize': ['Pillow>=3.0.0'],
+        'regex': ['regex'],
     },
     entry_points={'console_scripts': ['pytextile=textile.__main__:main']},
     setup_requires=['pytest-runner'],

tests/fixtures/README.txt

@@ -42,5 +42,5 @@
 	<h3>Notes:</h3>
 
 	<ul>
-		<li>Active development supports Python 2.6 or later (including Python 3.2+).</li>
+		<li>Active development supports Python 2.7 or later (including Python 3.3+).</li>
 	</ul>

tests/test_github_issues.py

@@ -86,6 +86,11 @@ def test_github_issue_30():
     expect = '\t<p><a href="http://lala.com" title="Tëxtíle">Tëxtíle</a></p>'
     assert result == expect
 
+    text ='!http://lala.com/lol.gif(♡ imáges)!'
+    result = textile.textile(text)
+    expect = '\t<p><img alt="♡ imáges" src="http://lala.com/lol.gif" title="♡ imáges" /></p>'
+    assert result == expect
+
 def test_github_issue_36():
     text = '"Chögyam Trungpa":https://www.google.com/search?q=Chögyam+Trungpa'
     result = textile.textile(text)
@@ -199,3 +204,61 @@ def test_github_issue_52():
               '</td>\n\t\t\t<td style="text-align:center;">Second Header </td>'
               '\n\t\t</tr>\n\t</table>')
     assert result == expect
+
+def test_github_issue_55():
+    """Incorrect handling of quote entities in extended pre block"""
+    test = ('pre.. this is the first line\n\nbut "quotes" in an extended pre '
+            'block need to be handled properly.')
+    result = textile.textile(test)
+    expect = ('<pre>this is the first line\n\nbut &quot;quotes&quot; in an '
+              'extended pre block need to be handled properly.</pre>')
+    assert result == expect
+
+    # supplied input
+    test = ('pre.. import org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;'
+            '\nimport ru.onyma.job.Context;\nimport ru.onyma.job.'
+            'RescheduleTask;\n\nimport java.util.concurrent.'
+            'ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;'
+            '\n\n/**\n* @author ustits\n*/\npublic abstract class '
+            'MainService<T> extends RescheduleTask implements Context<T> {\n\n'
+            'private static final Logger log = LoggerFactory.getLogger('
+            'MainService.class);\nprivate final ScheduledExecutorService '
+            'scheduler;\n\nprivate boolean isFirstRun = true;\nprivate T '
+            'configs;\n\npublic MainService(final ScheduledExecutorService '
+            'scheduler) {\nsuper(scheduler);\nthis.scheduler = scheduler;\n}\n'
+            '\n@Override\npublic void setConfig(final T configs) {\nthis.'
+            'configs = configs;\nif (isFirstRun) {\nscheduler.schedule(this, '
+            '0, TimeUnit.SECONDS);\nisFirstRun = false;\n}\n}\n\n@Override\n'
+            'public void stop() {\nsuper.stop();\nscheduler.shutdown();\ntry {'
+            '\nscheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);\n} '
+            'catch (InterruptedException ie) {\nlog.warn("Unable to wait for '
+            'syncs termination", ie);\nThread.currentThread().interrupt();\n}'
+            '\n}\n\nprotected final T getConfigs() {\nreturn configs;\n}\n}')
+    result = textile.textile(test)
+    expect = ('<pre>import org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;'
+              '\nimport ru.onyma.job.Context;\nimport ru.onyma.job.'
+              'RescheduleTask;\n\nimport java.util.concurrent.'
+              'ScheduledExecutorService;\nimport java.util.concurrent.'
+              'TimeUnit;\n\n/**\n* @author ustits\n*/\npublic abstract class '
+              'MainService&lt;T&gt; extends RescheduleTask implements '
+              'Context&lt;T&gt; {\n\nprivate static final Logger log = '
+              'LoggerFactory.getLogger(MainService.class);\nprivate final '
+              'ScheduledExecutorService scheduler;\n\nprivate boolean '
+              'isFirstRun = true;\nprivate T configs;\n\npublic MainService('
+              'final ScheduledExecutorService scheduler) {\nsuper(scheduler);'
+              '\nthis.scheduler = scheduler;\n}\n\n@Override\npublic void '
+              'setConfig(final T configs) {\nthis.configs = configs;\nif ('
+              'isFirstRun) {\nscheduler.schedule(this, 0, TimeUnit.SECONDS);'
+              '\nisFirstRun = false;\n}\n}\n\n@Override\npublic void stop() {'
+              '\nsuper.stop();\nscheduler.shutdown();\ntry {\nscheduler.'
+              'awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);\n} catch '
+              '(InterruptedException ie) {\nlog.warn(&quot;Unable to wait '
+              'for syncs termination&quot;, ie);\nThread.currentThread().'
+              'interrupt();\n}\n}\n\nprotected final T getConfigs() {\n'
+              'return configs;\n}\n}</pre>')
+    assert result == expect
+
+def test_issue_56():
+    result = textile.textile("- :=\n-")
+    expect = '<dl>\n</dl>'
+    assert result == expect

tests/test_textile.py

@@ -67,22 +67,18 @@ def test_autolinking():
 def test_sanitize():
     test = "a paragraph of benign text"
     result = "\t<p>a paragraph of benign text</p>"
-    try:
-        expect = textile.Textile().parse(test, sanitize=True)
-        assert result == expect
-
-        test = """<p style="width: expression(alert('evil'));">a paragraph of evil text</p>"""
-        result = '<p style="">a paragraph of evil text</p>'
-        expect = textile.Textile().parse(test, sanitize=True)
-        assert result == expect
-
-        test = """<p>a paragraph of benign text<br />and more text</p>"""
-        result = '<p>a paragraph of benign text<br />\nand more text</p>'
-        expect = textile.Textile(html_type='html5').parse(test, sanitize=True)
-        assert result == expect
-    except Exception as e:
-        message = '{0}'.format(e)
-        assert "html5lib not available" in message
+    expect = textile.Textile().parse(test, sanitize=True)
+    assert result == expect
+
+    test = """<p style="width: expression(alert('evil'));">a paragraph of evil text</p>"""
+    result = '<p style="">a paragraph of evil text</p>'
+    expect = textile.Textile().parse(test, sanitize=True)
+    assert result == expect
+
+    test = """<p>a paragraph of benign text<br />and more text</p>"""
+    result = '<p>a paragraph of benign text<br />\nand more text</p>'
+    expect = textile.Textile(html_type='html5').parse(test, sanitize=True)
+    assert result == expect
 
 def test_imagesize():
     PIL = pytest.importorskip('PIL')

textile/__init__.py

@@ -9,12 +9,3 @@
 __all__ = ['textile', 'textile_restricted']
 
 __version__ = VERSION
-
-
-if sys.version_info[:2] == (2, 6):
-    warnings.warn(
-        "Python 2.6 is no longer supported by the Python core team, please "
-        "upgrade your Python. A future version of textile will drop support "
-        "for Python 2.6",
-        DeprecationWarning
-    )

textile/core.py

@@ -32,10 +32,7 @@
 from textile.objects import Block, Table
 
 
-try:
-    from collections import OrderedDict
-except ImportError:
-    from ordereddict import OrderedDict
+from collections import OrderedDict
 
 
 try:
@@ -506,7 +503,10 @@ def block(self, text):
                                 block.outer_atts)
                         line = "\t{0}".format(line)
                 else:
-                    line = self.graf(line)
+                    if block.tag == 'pre':
+                        line = self.shelve(encode_html(line, quotes=True))
+                    else:
+                        line = self.graf(line)
 
             line = self.doPBr(line)
             line = line.replace('<br>', '<br />')
@@ -1184,6 +1184,8 @@ def fRCList(self, match):
             # parse the attributes and content
             m = re.match(r'^[-]+({0})[ .](.*)$'.format(cls_re_s), line,
                     flags=re.M | re.S)
+            if not m:
+                continue
 
             atts, content = m.groups()
             # cleanup

textile/objects/block.py

@@ -1,10 +1,7 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-try:
-    from collections import OrderedDict
-except ImportError:
-    from ordereddict import OrderedDict
+from collections import OrderedDict
 try:
     import regex as re
 except ImportError:

textile/regex_strings.py

@@ -19,7 +19,6 @@
         'char': r'(?:[^\p{Zs}\v])',
         }
 except ImportError:
-    import re
     from sys import maxunicode
     upper_re_s = "".join(
             [six.unichr(c) for c in six.moves.range(maxunicode) if six.unichr(

textile/tools/sanitizer.py

@@ -3,18 +3,9 @@ def sanitize(string):
     Ensure that the text does not contain any malicious HTML code which might
     break the page.
     """
-    try:
-        import html5lib
-        from html5lib import sanitizer, serializer, treewalkers
-    except ImportError:
-        raise Exception("html5lib not available")
+    from html5lib import parseFragment, serialize
 
-    p = html5lib.HTMLParser(tokenizer=sanitizer.HTMLSanitizer)
-    tree = p.parseFragment(string)
-
-    walker = treewalkers.getTreeWalker("etree")
-    stream = walker(tree)
-
-    s = serializer.htmlserializer.HTMLSerializer(omit_optional_tags=False,
-            quote_attr_values=True)
-    return s.render(stream)
+    parsed = parseFragment(string)
+    clean = serialize(parsed, sanitize=True, omit_optional_tags=False,
+                      quote_attr_values='always')
+    return clean

textile/utils.py

@@ -10,10 +10,7 @@
 urlparse = urllib.parse.urlparse
 HTMLParser = html_parser.HTMLParser
 
-try:
-    from collections import OrderedDict
-except ImportError:
-    from ordereddict import OrderedDict
+from collections import OrderedDict
 
 from xml.etree import ElementTree
 
@@ -50,6 +47,8 @@ def generate_tag(tag, content, attributes=None):
     content are strings, the attributes argument is a dictionary.  As
     a convenience, if the content is ' /', a self-closing tag is generated."""
     content = six.text_type(content)
+    # In PY2, ElementTree tostringlist only works with bytes, not with
+    # unicode().
     enc = 'unicode'
     if six.PY2:
         enc = 'UTF-8'
@@ -60,21 +59,12 @@ def generate_tag(tag, content, attributes=None):
     # adding text by assigning it to element_tag.text.  That results in
     # non-ascii text being html-entity encoded.  Not bad, but not entirely
     # matching php-textile either.
-    try:
-        element_tag = ElementTree.tostringlist(element, encoding=enc,
-                method='html')
-        element_tag.insert(len(element_tag) - 1, content)
-        element_text = ''.join(element_tag)
-    except AttributeError:
-        # Python 2.6 doesn't have the tostringlist method, so we have to treat
-        # it differently.
-        element_tag = ElementTree.tostring(element, encoding=enc)
-        element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n",
-                '', element_tag)
-        if content != six.text_type(' /'):
-            element_text = element_text.rstrip(' />')
-            element_text = six.text_type('{0}>{1}</{2}>').format(six.text_type(
-                element_text), content, tag)
+    element_tag = ElementTree.tostringlist(element, encoding=enc,
+            method='html')
+    if six.PY2:
+        element_tag = [v.decode(enc) for v in element_tag]
+    element_tag.insert(len(element_tag) - 1, content)
+    element_text = ''.join(element_tag)
     return element_text
 
 def has_raw_text(text):

textile/version.py

@@ -1 +1 @@
-VERSION = '2.3.16'
+VERSION = '3.0.0'