From 8ecebbb817ff5eac3c792cca7c240b7b65238664 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Sep 2017 17:58:12 -0400 Subject: [PATCH 01/23] update to latest html5lib --- requirements.txt | 2 +- tests/test_textile.py | 6 +++--- textile/tools/sanitizer.py | 16 +++++----------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5cfb4428..d8cae33b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -html5lib==0.999 +html5lib>=1.0b10 Pillow==3.0.0 \ No newline at end of file diff --git a/tests/test_textile.py b/tests/test_textile.py index dd069fb8..ca6cbef2 100644 --- a/tests/test_textile.py +++ b/tests/test_textile.py @@ -71,8 +71,8 @@ def test_sanitize(): expect = textile.Textile().parse(test, sanitize=True) assert result == expect - test = """

a paragraph of evil text

""" - result = '

a paragraph of evil text

' + test = """

a paragraph of evil text

""" + result = '

a paragraph of evil text

' expect = textile.Textile().parse(test, sanitize=True) assert result == expect @@ -80,7 +80,7 @@ def test_sanitize(): result = '

a paragraph of benign text
\nand more text

' expect = textile.Textile(html_type='html5').parse(test, sanitize=True) assert result == expect - except Exception as e: + except ImportError as e: message = '{0}'.format(e) assert "html5lib not available" in message diff --git a/textile/tools/sanitizer.py b/textile/tools/sanitizer.py index 4fc8fb2c..6eada978 100644 --- a/textile/tools/sanitizer.py +++ b/textile/tools/sanitizer.py @@ -4,17 +4,11 @@ def sanitize(string): break the page. """ try: - import html5lib - from html5lib import sanitizer, serializer, treewalkers + from html5lib import parseFragment, serialize except ImportError: raise Exception("html5lib not available") - p = html5lib.HTMLParser(tokenizer=sanitizer.HTMLSanitizer) - tree = p.parseFragment(string) - - walker = treewalkers.getTreeWalker("etree") - stream = walker(tree) - - s = serializer.htmlserializer.HTMLSerializer(omit_optional_tags=False, - quote_attr_values=True) - return s.render(stream) + parsed = parseFragment(string) + clean = serialize(parsed, sanitize=True, omit_optional_tags=False, + quote_attr_values='always') + return clean From 5d5c25d8426d406373200b20f6d7e45921325d86 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Sep 2017 18:11:31 -0400 Subject: [PATCH 02/23] fix most of the breakage. --- tests/test_textile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_textile.py b/tests/test_textile.py index ca6cbef2..110de086 100644 --- a/tests/test_textile.py +++ b/tests/test_textile.py @@ -80,7 +80,7 @@ def test_sanitize(): result = '

a paragraph of benign text
\nand more text

' expect = textile.Textile(html_type='html5').parse(test, sanitize=True) assert result == expect - except ImportError as e: + except Exception as e: message = '{0}'.format(e) assert "html5lib not available" in message From 3fec61b3489fa24a5d7e2368e818be6b8b4ec4e9 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Sep 2017 18:28:49 -0400 Subject: [PATCH 03/23] fix setuptools on python 3.2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 844b19f4..d87a3cbb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ python: # command to install dependencies install: - if [[ $REQUIREMENTS == true ]] ; then pip install -r requirements.txt ; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install coverage==3.7.1; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install coverage==3.7.1 setuptools==29.0.1; fi - pip install coveralls pytest pytest-cov coverage codecov - pip install -e . - if [[ ! $TRAVIS_PYTHON_VERSION == 'pypy-5.4' ]] ; then pip install regex; fi From c54e2b1a2cb4b87909a9460823f32eef179f977e Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Sep 2017 18:31:26 -0400 Subject: [PATCH 04/23] will changing the order fix 3.2? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d87a3cbb..d3cd8025 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ python: # command to install dependencies install: - if [[ $REQUIREMENTS == true ]] ; then pip install -r requirements.txt ; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install coverage==3.7.1 setuptools==29.0.1; fi + - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install setuptools==29.0.1 coverage==3.7.1; fi - pip install coveralls pytest pytest-cov coverage codecov - pip install -e . - if [[ ! $TRAVIS_PYTHON_VERSION == 'pypy-5.4' ]] ; then pip install regex; fi From 5fef0334a6d474de937918ed484d90f584e50c3e Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 11 Sep 2017 22:19:26 -0400 Subject: [PATCH 05/23] urge to kill python 3.2 rising... --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d3cd8025..03a80eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ env: python: - "2.6" - "2.7" - - "3.2" - "3.3" - "3.4" - "3.5" @@ -14,7 +13,6 @@ python: # command to install dependencies install: - if [[ $REQUIREMENTS == true ]] ; then pip install -r requirements.txt ; fi - - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] ; then pip install setuptools==29.0.1 coverage==3.7.1; fi - pip install coveralls pytest pytest-cov coverage codecov - pip install -e . - if [[ ! $TRAVIS_PYTHON_VERSION == 'pypy-5.4' ]] ; then pip install regex; fi From 94224e1e799e11e92c4f0af07b4e10bb1394525c Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Tue, 12 Sep 2017 10:18:19 -0400 Subject: [PATCH 06/23] not sure why I had to change this test yesterday to make it work, but it works fine again today. --- tests/test_textile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_textile.py b/tests/test_textile.py index 110de086..dd069fb8 100644 --- a/tests/test_textile.py +++ b/tests/test_textile.py @@ -71,8 +71,8 @@ def test_sanitize(): expect = textile.Textile().parse(test, sanitize=True) assert result == expect - test = """

a paragraph of evil text

""" - result = '

a paragraph of evil text

' + test = """

a paragraph of evil text

""" + result = '

a paragraph of evil text

' expect = textile.Textile().parse(test, sanitize=True) assert result == expect From 54ac622e2bbaae845365e0081cc92d67f90753c3 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Sun, 17 Sep 2017 15:48:17 +0200 Subject: [PATCH 07/23] MAINT: Fix dependencies, versions see https://github.com/textile/python-textile/commit/8ecebbb817ff5eac3c792cca7c240b7b65238664 --- requirements.txt | 3 ++- setup.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index d8cae33b..a477d4b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ html5lib>=1.0b10 -Pillow==3.0.0 \ No newline at end of file +Pillow==3.0.0 +regex diff --git a/setup.py b/setup.py index c12d3e00..e2f49baf 100644 --- a/setup.py +++ b/setup.py @@ -55,10 +55,15 @@ def get_version(): 'Programming Language :: Python :: 3.6', ], keywords='textile,text,html markup', - install_requires=['six',], + install_requires=[ + 'six', + 'html5lib>=0.999999999', + ], extras_require={ ':python_version=="2.6"': ['ordereddict>=1.1'], - 'develop': ['regex', 'pytest', 'pytest-cov'], + 'develop': ['pytest', 'pytest-cov'], + 'imagesize': ['Pillow>=3.0.0'], + 'regex': ['regex'], }, entry_points={'console_scripts': ['pytextile=textile.__main__:main']}, setup_requires=['pytest-runner'], From 8938421ddb366925780b106373ff5917c43651e1 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Sun, 17 Sep 2017 10:36:08 -0400 Subject: [PATCH 08/23] increase coverage --- tests/test_textile.py | 28 +++++++++++-------------- textile/regex_strings.py | 43 +++++++++++--------------------------- textile/tools/sanitizer.py | 5 +---- 3 files changed, 25 insertions(+), 51 deletions(-) diff --git a/tests/test_textile.py b/tests/test_textile.py index dd069fb8..86a7d853 100644 --- a/tests/test_textile.py +++ b/tests/test_textile.py @@ -67,22 +67,18 @@ def test_autolinking(): def test_sanitize(): test = "a paragraph of benign text" result = "\t

a paragraph of benign text

" - try: - expect = textile.Textile().parse(test, sanitize=True) - assert result == expect - - test = """

a paragraph of evil text

""" - result = '

a paragraph of evil text

' - expect = textile.Textile().parse(test, sanitize=True) - assert result == expect - - test = """

a paragraph of benign text
and more text

""" - result = '

a paragraph of benign text
\nand more text

' - expect = textile.Textile(html_type='html5').parse(test, sanitize=True) - assert result == expect - except Exception as e: - message = '{0}'.format(e) - assert "html5lib not available" in message + expect = textile.Textile().parse(test, sanitize=True) + assert result == expect + + test = """

a paragraph of evil text

""" + result = '

a paragraph of evil text

' + expect = textile.Textile().parse(test, sanitize=True) + assert result == expect + + test = """

a paragraph of benign text
and more text

""" + result = '

a paragraph of benign text
\nand more text

' + expect = textile.Textile(html_type='html5').parse(test, sanitize=True) + assert result == expect def test_imagesize(): PIL = pytest.importorskip('PIL') diff --git a/textile/regex_strings.py b/textile/regex_strings.py index a1520726..4fa6cd0d 100644 --- a/textile/regex_strings.py +++ b/textile/regex_strings.py @@ -3,37 +3,18 @@ import six -try: - # Use regex module for matching uppercase characters if installed, - # otherwise fall back to finding all the uppercase chars in a loop. - import regex as re - upper_re_s = r'\p{Lu}' - regex_snippets = { - 'acr': r'\p{Lu}\p{Nd}', - 'abr': r'\p{Lu}', - 'nab': r'\p{Ll}', - 'wrd': r'(?:\p{L}|\p{M}|\p{N}|\p{Pc})', - 'cur': r'\p{Sc}', - 'digit': r'\p{N}', - 'space': r'(?:\p{Zs}|\v)', - 'char': r'(?:[^\p{Zs}\v])', - } -except ImportError: - import re - from sys import maxunicode - upper_re_s = "".join( - [six.unichr(c) for c in six.moves.range(maxunicode) if six.unichr( - c).isupper()]) - regex_snippets = { - 'acr': r'{0}0-9'.format(upper_re_s), - 'abr': r'{0}'.format(upper_re_s), - 'nab': r'a-z', - 'wrd': r'\w', - 'cur': r'', - 'digit': r'\d', - 'space': r'(?:\s|\v)', - 'char': r'\S', - } +import regex as re +upper_re_s = r'\p{Lu}' +regex_snippets = { + 'acr': r'\p{Lu}\p{Nd}', + 'abr': r'\p{Lu}', + 'nab': r'\p{Ll}', + 'wrd': r'(?:\p{L}|\p{M}|\p{N}|\p{Pc})', + 'cur': r'\p{Sc}', + 'digit': r'\p{N}', + 'space': r'(?:\p{Zs}|\v)', + 'char': r'(?:[^\p{Zs}\v])', +} halign_re_s = r'(?:\<(?!>)|(?|\<\>|\=|[()]+(?! ))' valign_re_s = r'[\-^~]' diff --git a/textile/tools/sanitizer.py b/textile/tools/sanitizer.py index 6eada978..3c7209c6 100644 --- a/textile/tools/sanitizer.py +++ b/textile/tools/sanitizer.py @@ -3,10 +3,7 @@ def sanitize(string): Ensure that the text does not contain any malicious HTML code which might break the page. """ - try: - from html5lib import parseFragment, serialize - except ImportError: - raise Exception("html5lib not available") + from html5lib import parseFragment, serialize parsed = parseFragment(string) clean = serialize(parsed, sanitize=True, omit_optional_tags=False, From 71d1962ef5a904a4f544bc282234ecbafac15476 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Sun, 17 Sep 2017 10:41:16 -0400 Subject: [PATCH 09/23] whoops. forgot about pypy. --- textile/regex_strings.py | 43 +++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/textile/regex_strings.py b/textile/regex_strings.py index 4fa6cd0d..a1520726 100644 --- a/textile/regex_strings.py +++ b/textile/regex_strings.py @@ -3,18 +3,37 @@ import six -import regex as re -upper_re_s = r'\p{Lu}' -regex_snippets = { - 'acr': r'\p{Lu}\p{Nd}', - 'abr': r'\p{Lu}', - 'nab': r'\p{Ll}', - 'wrd': r'(?:\p{L}|\p{M}|\p{N}|\p{Pc})', - 'cur': r'\p{Sc}', - 'digit': r'\p{N}', - 'space': r'(?:\p{Zs}|\v)', - 'char': r'(?:[^\p{Zs}\v])', -} +try: + # Use regex module for matching uppercase characters if installed, + # otherwise fall back to finding all the uppercase chars in a loop. + import regex as re + upper_re_s = r'\p{Lu}' + regex_snippets = { + 'acr': r'\p{Lu}\p{Nd}', + 'abr': r'\p{Lu}', + 'nab': r'\p{Ll}', + 'wrd': r'(?:\p{L}|\p{M}|\p{N}|\p{Pc})', + 'cur': r'\p{Sc}', + 'digit': r'\p{N}', + 'space': r'(?:\p{Zs}|\v)', + 'char': r'(?:[^\p{Zs}\v])', + } +except ImportError: + import re + from sys import maxunicode + upper_re_s = "".join( + [six.unichr(c) for c in six.moves.range(maxunicode) if six.unichr( + c).isupper()]) + regex_snippets = { + 'acr': r'{0}0-9'.format(upper_re_s), + 'abr': r'{0}'.format(upper_re_s), + 'nab': r'a-z', + 'wrd': r'\w', + 'cur': r'', + 'digit': r'\d', + 'space': r'(?:\s|\v)', + 'char': r'\S', + } halign_re_s = r'(?:\<(?!>)|(?|\<\>|\=|[()]+(?! ))' valign_re_s = r'[\-^~]' From e667bff45dbfe37b6207d1c31f7a791af1ba7d1c Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Sun, 17 Sep 2017 10:47:17 -0400 Subject: [PATCH 10/23] remove extraneous import. --- textile/regex_strings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/textile/regex_strings.py b/textile/regex_strings.py index a1520726..f7c6f127 100644 --- a/textile/regex_strings.py +++ b/textile/regex_strings.py @@ -19,7 +19,6 @@ 'char': r'(?:[^\p{Zs}\v])', } except ImportError: - import re from sys import maxunicode upper_re_s = "".join( [six.unichr(c) for c in six.moves.range(maxunicode) if six.unichr( From cb965a959119873488b60a201ed3488ed76a5104 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 25 Sep 2017 15:58:39 -0400 Subject: [PATCH 11/23] fix for issue #55. --- tests/test_github_issues.py | 111 ++++++++++++++++++++++++++++++++++++ textile/core.py | 5 +- 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/tests/test_github_issues.py b/tests/test_github_issues.py index 0aeba4d8..fab09346 100644 --- a/tests/test_github_issues.py +++ b/tests/test_github_issues.py @@ -199,3 +199,114 @@ def test_github_issue_52(): '\n\t\t\tSecond Header ' '\n\t\t\n\t') assert result == expect + +def test_github_issue_55(): + """Incorrect handling of quote entities in extended pre block""" + test = ('pre.. this is the first line\n\nbut "quotes" in an extended pre ' + 'block need to be handled properly.') + result = textile.textile(test) + expect = ('
this is the first line\n\nbut "quotes" in an '
+              'extended pre block need to be handled properly.
') + assert result == expect + + # supplied input + test = ('''pre.. import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import ru.onyma.job.Context; +import ru.onyma.job.RescheduleTask; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author ustits + */ +public abstract class MainService extends RescheduleTask implements Context { + + private static final Logger log = LoggerFactory.getLogger(MainService.class); + private final ScheduledExecutorService scheduler; + + private boolean isFirstRun = true; + private T configs; + + public MainService(final ScheduledExecutorService scheduler) { + super(scheduler); + this.scheduler = scheduler; + } + + @Override + public void setConfig(final T configs) { + this.configs = configs; + if (isFirstRun) { + scheduler.schedule(this, 0, TimeUnit.SECONDS); + isFirstRun = false; + } + } + + @Override + public void stop() { + super.stop(); + scheduler.shutdown(); + try { + scheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); + } catch (InterruptedException ie) { + log.warn("Unable to wait for syncs termination", ie); + Thread.currentThread().interrupt(); + } + } + + protected final T getConfigs() { + return configs; + } +}''') + result = textile.textile(test) + expect = ('''
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import ru.onyma.job.Context;
+import ru.onyma.job.RescheduleTask;
+
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * @author ustits
+ */
+public abstract class MainService<T> extends RescheduleTask implements Context<T> {
+
+  private static final Logger log = LoggerFactory.getLogger(MainService.class);
+  private final ScheduledExecutorService scheduler;
+
+  private boolean isFirstRun = true;
+  private T configs;
+
+  public MainService(final ScheduledExecutorService scheduler) {
+    super(scheduler);
+    this.scheduler = scheduler;
+  }
+
+  @Override
+  public void setConfig(final T configs) {
+    this.configs = configs;
+    if (isFirstRun) {
+      scheduler.schedule(this, 0, TimeUnit.SECONDS);
+      isFirstRun = false;
+    }
+  }
+
+  @Override
+  public void stop() {
+    super.stop();
+    scheduler.shutdown();
+    try {
+      scheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
+    } catch (InterruptedException ie) {
+      log.warn("Unable to wait for syncs termination", ie);
+      Thread.currentThread().interrupt();
+    }
+  }
+
+  protected final T getConfigs() {
+    return configs;
+  }
+}
''') + assert result == expect diff --git a/textile/core.py b/textile/core.py index efb1c4dc..0e44234b 100644 --- a/textile/core.py +++ b/textile/core.py @@ -506,7 +506,10 @@ def block(self, text): block.outer_atts) line = "\t{0}".format(line) else: - line = self.graf(line) + if block.tag == 'pre': + line = self.shelve(encode_html(line, quotes=True)) + else: + line = self.graf(line) line = self.doPBr(line) line = line.replace('
', '
') From 823bac5af2c9575cf2bff6543f372ac160e29454 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Mon, 25 Sep 2017 16:41:46 -0400 Subject: [PATCH 12/23] I hate how tests like these bundling folding in vim. --- tests/test_github_issues.py | 138 +++++++++++------------------------- 1 file changed, 40 insertions(+), 98 deletions(-) diff --git a/tests/test_github_issues.py b/tests/test_github_issues.py index fab09346..41be3752 100644 --- a/tests/test_github_issues.py +++ b/tests/test_github_issues.py @@ -210,103 +210,45 @@ def test_github_issue_55(): assert result == expect # supplied input - test = ('''pre.. import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ru.onyma.job.Context; -import ru.onyma.job.RescheduleTask; - -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * @author ustits - */ -public abstract class MainService extends RescheduleTask implements Context { - - private static final Logger log = LoggerFactory.getLogger(MainService.class); - private final ScheduledExecutorService scheduler; - - private boolean isFirstRun = true; - private T configs; - - public MainService(final ScheduledExecutorService scheduler) { - super(scheduler); - this.scheduler = scheduler; - } - - @Override - public void setConfig(final T configs) { - this.configs = configs; - if (isFirstRun) { - scheduler.schedule(this, 0, TimeUnit.SECONDS); - isFirstRun = false; - } - } - - @Override - public void stop() { - super.stop(); - scheduler.shutdown(); - try { - scheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); - } catch (InterruptedException ie) { - log.warn("Unable to wait for syncs termination", ie); - Thread.currentThread().interrupt(); - } - } - - protected final T getConfigs() { - return configs; - } -}''') + test = ('pre.. import org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;' + '\nimport ru.onyma.job.Context;\nimport ru.onyma.job.' + 'RescheduleTask;\n\nimport java.util.concurrent.' + 'ScheduledExecutorService;\nimport java.util.concurrent.TimeUnit;' + '\n\n/**\n* @author ustits\n*/\npublic abstract class ' + 'MainService extends RescheduleTask implements Context {\n\n' + 'private static final Logger log = LoggerFactory.getLogger(' + 'MainService.class);\nprivate final ScheduledExecutorService ' + 'scheduler;\n\nprivate boolean isFirstRun = true;\nprivate T ' + 'configs;\n\npublic MainService(final ScheduledExecutorService ' + 'scheduler) {\nsuper(scheduler);\nthis.scheduler = scheduler;\n}\n' + '\n@Override\npublic void setConfig(final T configs) {\nthis.' + 'configs = configs;\nif (isFirstRun) {\nscheduler.schedule(this, ' + '0, TimeUnit.SECONDS);\nisFirstRun = false;\n}\n}\n\n@Override\n' + 'public void stop() {\nsuper.stop();\nscheduler.shutdown();\ntry {' + '\nscheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);\n} ' + 'catch (InterruptedException ie) {\nlog.warn("Unable to wait for ' + 'syncs termination", ie);\nThread.currentThread().interrupt();\n}' + '\n}\n\nprotected final T getConfigs() {\nreturn configs;\n}\n}') result = textile.textile(test) - expect = ('''
import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import ru.onyma.job.Context;
-import ru.onyma.job.RescheduleTask;
-
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-/**
- * @author ustits
- */
-public abstract class MainService<T> extends RescheduleTask implements Context<T> {
-
-  private static final Logger log = LoggerFactory.getLogger(MainService.class);
-  private final ScheduledExecutorService scheduler;
-
-  private boolean isFirstRun = true;
-  private T configs;
-
-  public MainService(final ScheduledExecutorService scheduler) {
-    super(scheduler);
-    this.scheduler = scheduler;
-  }
-
-  @Override
-  public void setConfig(final T configs) {
-    this.configs = configs;
-    if (isFirstRun) {
-      scheduler.schedule(this, 0, TimeUnit.SECONDS);
-      isFirstRun = false;
-    }
-  }
-
-  @Override
-  public void stop() {
-    super.stop();
-    scheduler.shutdown();
-    try {
-      scheduler.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
-    } catch (InterruptedException ie) {
-      log.warn("Unable to wait for syncs termination", ie);
-      Thread.currentThread().interrupt();
-    }
-  }
-
-  protected final T getConfigs() {
-    return configs;
-  }
-}
''') + expect = ('
import org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;'
+              '\nimport ru.onyma.job.Context;\nimport ru.onyma.job.'
+              'RescheduleTask;\n\nimport java.util.concurrent.'
+              'ScheduledExecutorService;\nimport java.util.concurrent.'
+              'TimeUnit;\n\n/**\n* @author ustits\n*/\npublic abstract class '
+              'MainService<T> extends RescheduleTask implements '
+              'Context<T> {\n\nprivate static final Logger log = '
+              'LoggerFactory.getLogger(MainService.class);\nprivate final '
+              'ScheduledExecutorService scheduler;\n\nprivate boolean '
+              'isFirstRun = true;\nprivate T configs;\n\npublic MainService('
+              'final ScheduledExecutorService scheduler) {\nsuper(scheduler);'
+              '\nthis.scheduler = scheduler;\n}\n\n@Override\npublic void '
+              'setConfig(final T configs) {\nthis.configs = configs;\nif ('
+              'isFirstRun) {\nscheduler.schedule(this, 0, TimeUnit.SECONDS);'
+              '\nisFirstRun = false;\n}\n}\n\n@Override\npublic void stop() {'
+              '\nsuper.stop();\nscheduler.shutdown();\ntry {\nscheduler.'
+              'awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);\n} catch '
+              '(InterruptedException ie) {\nlog.warn("Unable to wait '
+              'for syncs termination", ie);\nThread.currentThread().'
+              'interrupt();\n}\n}\n\nprotected final T getConfigs() {\n'
+              'return configs;\n}\n}
') assert result == expect From 064f07c0c5bca7337abcb0b5cd83278260e17e33 Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 3 Oct 2017 21:24:32 +0200 Subject: [PATCH 13/23] fix definition list parsing with empty definitions fixes textile/python-textile#56 --- CHANGELOG.textile | 4 ++++ tests/test_github_issues.py | 5 +++++ textile/core.py | 2 ++ 3 files changed, 11 insertions(+) diff --git a/CHANGELOG.textile b/CHANGELOG.textile index 5047e1af..63a4a8c3 100644 --- a/CHANGELOG.textile +++ b/CHANGELOG.textile @@ -1,5 +1,9 @@ h1. Textile Changelog +h2. Version 2.3.17 (in development) +* Bugfixes: +** Empty definitions in definition lists raised an exception ("#56":https://github.com/textile/python-textile/issues/56) + h2. Version 2.3.16 * Bugfixes: ** Fix processing of extended code blocks ("#50":https://github.com/textile/python-textile/issues/50) diff --git a/tests/test_github_issues.py b/tests/test_github_issues.py index 41be3752..901029d6 100644 --- a/tests/test_github_issues.py +++ b/tests/test_github_issues.py @@ -252,3 +252,8 @@ def test_github_issue_55(): 'interrupt();\n}\n}\n\nprotected final T getConfigs() {\n' 'return configs;\n}\n}') assert result == expect + +def test_issue_56(): + result = textile.textile("- :=\n-") + expect = '
\n
' + assert result == expect diff --git a/textile/core.py b/textile/core.py index 0e44234b..90a374b3 100644 --- a/textile/core.py +++ b/textile/core.py @@ -1187,6 +1187,8 @@ def fRCList(self, match): # parse the attributes and content m = re.match(r'^[-]+({0})[ .](.*)$'.format(cls_re_s), line, flags=re.M | re.S) + if not m: + continue atts, content = m.groups() # cleanup From e26c454cc62cd64c7e5f715e73207ed970e32733 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 14:25:09 -0500 Subject: [PATCH 14/23] fix issue #58 --- tests/test_github_issues.py | 5 +++++ textile/utils.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/tests/test_github_issues.py b/tests/test_github_issues.py index 901029d6..27befff7 100644 --- a/tests/test_github_issues.py +++ b/tests/test_github_issues.py @@ -86,6 +86,11 @@ def test_github_issue_30(): expect = '\t

Tëxtíle

' assert result == expect + text ='!http://lala.com/lol.gif(♡ imáges)!' + result = textile.textile(text) + expect = '\t

♡ imáges

' + assert result == expect + def test_github_issue_36(): text = '"Chögyam Trungpa":https://www.google.com/search?q=Chögyam+Trungpa' result = textile.textile(text) diff --git a/textile/utils.py b/textile/utils.py index c9750245..1087758d 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -50,6 +50,8 @@ def generate_tag(tag, content, attributes=None): content are strings, the attributes argument is a dictionary. As a convenience, if the content is ' /', a self-closing tag is generated.""" content = six.text_type(content) + # In PY2, ElementTree tostringlist only works with bytes, not with + # unicode(). enc = 'unicode' if six.PY2: enc = 'UTF-8' @@ -63,6 +65,8 @@ def generate_tag(tag, content, attributes=None): try: element_tag = ElementTree.tostringlist(element, encoding=enc, method='html') + if six.PY2: + element_tag = [v.decode(enc) for v in element_tag] element_tag.insert(len(element_tag) - 1, content) element_text = ''.join(element_tag) except AttributeError: From 426abbe0f69a20e39ba64c970627c1a77b15323c Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 14:38:15 -0500 Subject: [PATCH 15/23] fix #58 on py26 ?? --- textile/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/textile/utils.py b/textile/utils.py index 1087758d..f4f3a075 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -73,6 +73,7 @@ def generate_tag(tag, content, attributes=None): # Python 2.6 doesn't have the tostringlist method, so we have to treat # it differently. element_tag = ElementTree.tostring(element, encoding=enc) + element_tag = [v.decode(enc) for v in element_tag] element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', element_tag) if content != six.text_type(' /'): From d8070804998dbce61e069290845da094e9566dba Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 14:54:59 -0500 Subject: [PATCH 16/23] let's see if THIS does the trick. --- textile/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/textile/utils.py b/textile/utils.py index f4f3a075..8b78bf39 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -72,8 +72,10 @@ def generate_tag(tag, content, attributes=None): except AttributeError: # Python 2.6 doesn't have the tostringlist method, so we have to treat # it differently. + attributes = dict(map(lambda (k, v): (k, v.decode(enc)), + attributes.items())) + element = ElementTree.Element(tag, attrib=attributes) element_tag = ElementTree.tostring(element, encoding=enc) - element_tag = [v.decode(enc) for v in element_tag] element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", '', element_tag) if content != six.text_type(' /'): From 338e1800fcce9d1193ede9452b6089bf8ae69b86 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 15:11:37 -0500 Subject: [PATCH 17/23] syntax fix. --- textile/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/textile/utils.py b/textile/utils.py index 8b78bf39..3bad0068 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -72,8 +72,8 @@ def generate_tag(tag, content, attributes=None): except AttributeError: # Python 2.6 doesn't have the tostringlist method, so we have to treat # it differently. - attributes = dict(map(lambda (k, v): (k, v.decode(enc)), - attributes.items())) + attributes = dict(map(lambda k, v: (k, six.text_type(v)), + six.iteritems(attributes))) element = ElementTree.Element(tag, attrib=attributes) element_tag = ElementTree.tostring(element, encoding=enc) element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", From b7ed17ee11bb38f8415e6c70dc798d3767907dc4 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 15:42:09 -0500 Subject: [PATCH 18/23] one more try --- textile/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textile/utils.py b/textile/utils.py index 3bad0068..1771b3c4 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -72,7 +72,7 @@ def generate_tag(tag, content, attributes=None): except AttributeError: # Python 2.6 doesn't have the tostringlist method, so we have to treat # it differently. - attributes = dict(map(lambda k, v: (k, six.text_type(v)), + attributes = dict(map(lambda (k, v): (k, six.text_type(v)), six.iteritems(attributes))) element = ElementTree.Element(tag, attrib=attributes) element_tag = ElementTree.tostring(element, encoding=enc) From 5ac14d31035f9c923ed4b0dffa55a85816903904 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 15:46:47 -0500 Subject: [PATCH 19/23] maybe now's the time to remove py26 --- .travis.yml | 1 - textile/utils.py | 26 ++++++-------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03a80eb2..3602a983 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ env: - REQUIREMENTS=true - REQUIREMENTS=false python: - - "2.6" - "2.7" - "3.3" - "3.4" diff --git a/textile/utils.py b/textile/utils.py index 1771b3c4..f144afe6 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -62,26 +62,12 @@ def generate_tag(tag, content, attributes=None): # adding text by assigning it to element_tag.text. That results in # non-ascii text being html-entity encoded. Not bad, but not entirely # matching php-textile either. - try: - element_tag = ElementTree.tostringlist(element, encoding=enc, - method='html') - if six.PY2: - element_tag = [v.decode(enc) for v in element_tag] - element_tag.insert(len(element_tag) - 1, content) - element_text = ''.join(element_tag) - except AttributeError: - # Python 2.6 doesn't have the tostringlist method, so we have to treat - # it differently. - attributes = dict(map(lambda (k, v): (k, six.text_type(v)), - six.iteritems(attributes))) - element = ElementTree.Element(tag, attrib=attributes) - element_tag = ElementTree.tostring(element, encoding=enc) - element_text = re.sub(r"<\?xml version='1.0' encoding='UTF-8'\?>\n", - '', element_tag) - if content != six.text_type(' /'): - element_text = element_text.rstrip(' />') - element_text = six.text_type('{0}>{1}').format(six.text_type( - element_text), content, tag) + element_tag = ElementTree.tostringlist(element, encoding=enc, + method='html') + if six.PY2: + element_tag = [v.decode(enc) for v in element_tag] + element_tag.insert(len(element_tag) - 1, content) + element_text = ''.join(element_tag) return element_text def has_raw_text(text): From 0c7c3677afaf03e6cfdec0d8a2312e1e183c1ab7 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 15:56:10 -0500 Subject: [PATCH 20/23] remove more py26-specific code. --- textile/__init__.py | 9 --------- textile/core.py | 5 +---- textile/utils.py | 5 +---- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/textile/__init__.py b/textile/__init__.py index c019f413..bb7829f7 100644 --- a/textile/__init__.py +++ b/textile/__init__.py @@ -9,12 +9,3 @@ __all__ = ['textile', 'textile_restricted'] __version__ = VERSION - - -if sys.version_info[:2] == (2, 6): - warnings.warn( - "Python 2.6 is no longer supported by the Python core team, please " - "upgrade your Python. A future version of textile will drop support " - "for Python 2.6", - DeprecationWarning - ) diff --git a/textile/core.py b/textile/core.py index 90a374b3..7572a468 100644 --- a/textile/core.py +++ b/textile/core.py @@ -32,10 +32,7 @@ from textile.objects import Block, Table -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict try: diff --git a/textile/utils.py b/textile/utils.py index f144afe6..fa21f058 100644 --- a/textile/utils.py +++ b/textile/utils.py @@ -10,10 +10,7 @@ urlparse = urllib.parse.urlparse HTMLParser = html_parser.HTMLParser -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict from xml.etree import ElementTree From 8a3df2972212a4f1577c2ef3c11658f85a7a3dc5 Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 15:56:48 -0500 Subject: [PATCH 21/23] one more py26 removal --- textile/objects/block.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/textile/objects/block.py b/textile/objects/block.py index 5c613211..7002ecaf 100644 --- a/textile/objects/block.py +++ b/textile/objects/block.py @@ -1,10 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -try: - from collections import OrderedDict -except ImportError: - from ordereddict import OrderedDict +from collections import OrderedDict try: import regex as re except ImportError: From c718da233b51ff611f6f7c72e7734a66885635ed Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Wed, 15 Nov 2017 16:24:43 -0500 Subject: [PATCH 22/23] update CHANGELOG, README, and fixture to reflect recent changes. --- CHANGELOG.textile | 6 +++++- README.textile | 2 +- tests/fixtures/README.txt | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.textile b/CHANGELOG.textile index 63a4a8c3..c998dd02 100644 --- a/CHANGELOG.textile +++ b/CHANGELOG.textile @@ -1,8 +1,12 @@ h1. Textile Changelog -h2. Version 2.3.17 (in development) +h2. Version 3.0.0 +* Drop support for Python 2.6 and 3.2. +* Update to the current version of html5lib * Bugfixes: +** Fix handling of HTML entities in extended pre blocks. ("#55":https://github.com/textile/python-textile/issues/55) ** Empty definitions in definition lists raised an exception ("#56":https://github.com/textile/python-textile/issues/56) +** Fix handling of unicode in img attributes ("#58":https://github.com/textile/python-textile/issues/58) h2. Version 2.3.16 * Bugfixes: diff --git a/README.textile b/README.textile index a363721b..56155508 100644 --- a/README.textile +++ b/README.textile @@ -39,4 +39,4 @@ bc.. import textile h3. Notes: -* Active development supports Python 2.6 or later (including Python 3.2+). +* Active development supports Python 2.7 or later (including Python 3.3+). diff --git a/tests/fixtures/README.txt b/tests/fixtures/README.txt index 426c9aa4..ba867308 100644 --- a/tests/fixtures/README.txt +++ b/tests/fixtures/README.txt @@ -42,5 +42,5 @@

Notes:

    -
  • Active development supports Python 2.6 or later (including Python 3.2+).
    • +
  • Active development supports Python 2.7 or later (including Python 3.3+).
\ No newline at end of file From 0ca140f84d77c10a03715aac1c76b9d9f1c8c2db Mon Sep 17 00:00:00 2001 From: Dennis Burke Date: Thu, 16 Nov 2017 15:22:13 -0500 Subject: [PATCH 23/23] bump version number. --- textile/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/textile/version.py b/textile/version.py index beb7a8e4..aaa42644 100644 --- a/textile/version.py +++ b/textile/version.py @@ -1 +1 @@ -VERSION = '2.3.16' +VERSION = '3.0.0'