From be2eb91edb596e6408507c511735893e43be0023 Mon Sep 17 00:00:00 2001 From: Jean-Matthieu BARBIER Date: Mon, 14 Oct 2013 02:42:17 +0200 Subject: [PATCH] First working version - autogenerated with sphinx --- Makefile | 8 +- conf.py | 16 +-- index.rst | 124 ++++++++--------- make.bat | 4 +- parser/generate.py | 25 ---- parser/index.rst | 63 --------- parser/process.py | 32 ----- parser/store.py | 14 -- parser/test.py | 0 requirements.txt | 5 + rst2code/__init__.py | 112 ++++++++------- rst2code/rst2doc.initial.py | 231 +++++++++++++++++++++++++++++++ source/cmdline.rst | 87 ++++++++++++ source/index.rst | 264 ++++++++++++++++++++++++++++++++++++ source/misc.rst | 28 ++++ source/sphinx.rst | 32 +++++ 16 files changed, 786 insertions(+), 259 deletions(-) delete mode 100644 parser/generate.py delete mode 100644 parser/index.rst delete mode 100644 parser/process.py delete mode 100644 parser/store.py delete mode 100644 parser/test.py create mode 100644 requirements.txt create mode 100644 rst2code/rst2doc.initial.py create mode 100644 source/cmdline.rst create mode 100644 source/index.rst create mode 100644 source/misc.rst create mode 100644 source/sphinx.rst diff --git a/Makefile b/Makefile index a48764a..bcd616b 100644 --- a/Makefile +++ b/Makefile @@ -77,17 +77,17 @@ qthelp: @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rstlit.qhcp" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/rst2code.qhcp" @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rstlit.qhc" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rst2code.qhc" devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/rstlit" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rstlit" + @echo "# mkdir -p $$HOME/.local/share/devhelp/rst2code" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rst2code" @echo "# devhelp" epub: diff --git a/conf.py b/conf.py index 4e01c89..e1e2f01 100644 --- a/conf.py +++ b/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# rstlit documentation build configuration file, created by +# rst2code documentation build configuration file, created by # sphinx-quickstart on Sat Oct 12 23:22:07 2013. # # This file is execfile()d with the current directory set to its containing dir. @@ -40,7 +40,7 @@ master_doc = 'index' # General information about the project. -project = u'rstlit' +project = u'rst2code' copyright = u'2013, BARBIER Jean-Matthieu' # The version info for the project you're documenting, acts as replacement for @@ -164,7 +164,7 @@ #html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'rstlitdoc' +htmlhelp_basename = 'rst2codedoc' # -- Options for LaTeX output -------------------------------------------------- @@ -183,7 +183,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'rstlit.tex', u'rstlit Documentation', + ('index', 'rst2code.tex', u'rst2code Documentation', u'BARBIER Jean-Matthieu', 'manual'), ] @@ -213,7 +213,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'rstlit', u'rstlit Documentation', + ('index', 'rst2code', u'rst2code Documentation', [u'BARBIER Jean-Matthieu'], 1) ] @@ -227,8 +227,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'rstlit', u'rstlit Documentation', - u'BARBIER Jean-Matthieu', 'rstlit', 'One line description of project.', + ('index', 'rst2code', u'rst2code Documentation', + u'BARBIER Jean-Matthieu', 'rst2code', 'One line description of project.', 'Miscellaneous'), ] @@ -245,7 +245,7 @@ # -- Options for Epub output --------------------------------------------------- # Bibliographic Dublin Core info. -epub_title = u'rstlit' +epub_title = u'rst2code' epub_author = u'BARBIER Jean-Matthieu' epub_publisher = u'BARBIER Jean-Matthieu' epub_copyright = u'2013, BARBIER Jean-Matthieu' diff --git a/index.rst b/index.rst index df85d5b..3c7b608 100644 --- a/index.rst +++ b/index.rst @@ -1,50 +1,68 @@ -.. rstlit documentation master file, created by - sphinx-quickstart on Sat Oct 12 23:22:07 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +Welcome to rst2code's documentation! +==================================== -Welcome to rstlit's documentation! -================================== -**rstlit** est un outil de programmation littérale que je vais essayer -de créer pour répondre à un besoin que j'ai depuis des années et que -je n'ai pas pu combler en utilisant les outils existants. +**rst2code** is a literate programming tool. -Le principe ------------ +The goal +-------- -Le principe est d'écrire le code en même temps que la documentation, -et plus précisément de pouvoir générer le code à partir de la -documentation. Contrairement aux outils permettant de générer la -documentation à partir du code (docco, doxygen), ici l'idée est de -générer le code à partir de la documentation. +The goal is to be able to write code at the same time that documentation. More precisely, **rst2code** allows to +generate code from documentation, whereas many tools - pycco, docco, doxygen, sphinx, are designed to extract +documentation from code. -L'ordre de lecture et de compréhension d'un humain n'étant pas -forcément celui d'un ordinateur, la documentation doit pouvoir -présenter les morceaux de codes dans un ordre différent de celui du -code pour l'ordinateur. +As human reading and understanding order is not the same than computer's, **rst2code** allows us to write code +in any order, reassembling code blocks at "compile" time to obtain real source code suitable for computer use. -La documentation doit pouvoir être utilisable directement, sans avoir -à être transformée. Le code, en revanche, est généré par lecture de la -documentation. +**rst2code** is code agnostic (normally.. i tried it with python, html and coffee-script for now) -L'outil doit être multi-languages. +For the sake of readability, code blocks included in documentation should be easily syntax highligted (this means that +a code block should be more or less valid code) + + +Syntax +------ + + +Code blocks are written in ... well, reStructuredText code blocks directives ( code / code-block / sourcecode / :: ) + +Each code block is named, using code block name options (if available), or within a comment in the first line of +code block. Names are in @@MACRONAME@@ form (characters surrounded by two @) : -Les morceaux de code générés doivent pouvoir être valide avec un -"surligneur de syntaxe" directement. -Proposition de syntaxe .. code-block:: rest .. code-block:: langage - #MACRONAME definition - Code with #@@MACRONAME2@@ in comments + #@@MACRONAME@@ (within language comment) + Code + +or + +.. code-block:: rest + + .. code:: language + :name: @@MACRONAME@@ + + Code + +or + +.. code-block:: rest + + :: + + @@MACRONAME@@ (within language comment) + + +If **name** option is available, it is used to get block comment name. + +If not, or if no block name is found in **name** option, the first line of source code should +contain the name inside a comment (language dependant), using the format @@MACRONAME@@. + +If no block name is found, the block is not used in **rst2code** -If the first line of source code is a comment (language dependant) -containing the special (unescaped) syntax @@MACRONAME@@, the code -block have to be considered by the program generator. MACRONAMES definitions are : @@ -58,41 +76,22 @@ MACRONAMES definitions are : moment. -Inside the program lines, we detect comments containing @@MACRONAMES@@ +Inside the code blocks, **rst2code** detect comments containing @@MACRONAMES@@ strings, remove the comment while retaining indentation for every line -in macro content and then replace it. Circular references have to be -detected. - -L'invocation du code est faite par la ligne de commande suivante :: +in macro content and then replace it. - $ rst2code [OPTIONS] DEST_DIR RST_FILE1 RST_FILE2 RST_FILE3 ... +Usage +----- -Le programme se compose donc de la manière suivante :: +With "standard" reStructuredText :: - #@@/rst2code.py@@ - - #@@python headers@@ - #@@license@@ - #@@imports@@ - if __main__: - #@@main_function@@ - #@@command line arguments@@ - #@@rst file parser@@ - #@@blocks analysis@@ - #@@code generator@@ - - - -La license du programme est la license GPLv3 :: + $ rst2code [OPTIONS] DEST_DIR RST_FILE1 RST_FILE2 RST_FILE3 ... - #@@license@@ - # GPL v3 -La fonction principale :: +With "sphinx-flavoured" .rst files : just add "rst2code" to sphinx extensions, and +set rst2code_output_dir config option and launch any sphinx document generation. - #@@main_function@@ - def main(): - return True +**Current rst2code module have been written from this documentation** Contents: @@ -100,7 +99,10 @@ Contents: .. toctree:: :maxdepth: 2 - + source/index + source/cmdline + source/sphinx + source/misc Indices and tables ================== diff --git a/make.bat b/make.bat index 43fe034..053a35f 100644 --- a/make.bat +++ b/make.bat @@ -99,9 +99,9 @@ if "%1" == "qthelp" ( echo. echo.Build finished; now you can run "qcollectiongenerator" with the ^ .qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\rstlit.qhcp + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\rst2code.qhcp echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\rstlit.ghc + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\rst2code.ghc goto end ) diff --git a/parser/generate.py b/parser/generate.py deleted file mode 100644 index 2b9da1a..0000000 --- a/parser/generate.py +++ /dev/null @@ -1,25 +0,0 @@ -import re -import store - -macro = "(\s)*.*@@([@\w/.+! -]+)@@.*" - -def macrorepl(m): - cb = store.codeblocks - if cb.has_key(m.group(2)): - newc = [b['content'] for b in cb[m.group(2)]] - return m.group(1)+"\n".join(newc) - else: - return m.group(1) - - -def generate_code(fb,cb): - for codes in fb.values(): - for block in codes: - code = block['content'] - while True: - code, n = re.subn(macro, macrorepl, code, re.M) - print code - if n == 0: - break - - print fb diff --git a/parser/index.rst b/parser/index.rst deleted file mode 100644 index 103f1df..0000000 --- a/parser/index.rst +++ /dev/null @@ -1,63 +0,0 @@ -ReStructuredText parser -======================= - -Le travail est déjà fait par docutils - -.. code:: python - :name: @@/parser.py@@ - - from docutils.core import publish_doctree - #@@imports@/parser.py@@ - - #@@rest_text_parser@@ - -Il ne reste plus qu'à traverser le doctree et à processer les blocs de -texte littéral :: - - #@@rest_text_parser@@ - with open(sys.argv[1], "r") as f: - lines = f.read() - - out = publish_doctree(lines) - - for item in out.traverse(): - if item.tagname == "literal_block": - process_block(item) - -Le process des blocks est fait dans le module process - -.. code:: python - :name: @@imports@/parser.py@@ - - from process import process_block - - - - -TEST -==== - -Pour tester :: - - python test.py index.rst - from docutils.core import publish_doctree - from process import process_block - with open(sys.argv[1], "r") as f: - lines = f.read() - - out = publish_doctree(lines) - - for item in out.traverse(): - if item.tagname == "literal_block": - process_block(item) - from docutils.core import publish_doctree - from process import process_block - with open(sys.argv[1], "r") as f: - lines = f.read() - - out = publish_doctree(lines) - - for item in out.traverse(): - if item.tagname == "literal_block": - process_block(item) - {u'/parser.py': [{'content': u'from docutils.core import publish_doctree\n#@@imports@/parser.py@@\n\n#@@rest_text_parser@@', 'bname': u'/parser.py'}]} diff --git a/parser/process.py b/parser/process.py deleted file mode 100644 index 7c411a3..0000000 --- a/parser/process.py +++ /dev/null @@ -1,32 +0,0 @@ -import re - -from store import store_block - -macro = re.compile("@@([@\w/.+! -]+)@@") -incodemacro = re.compile("^(\s)*.*@@([@\w/.+! -]+)@@.*") - - -def process_block(item): - - block = {} - - for name in item.attributes['names']: - m1 = macro.match(name) - if m1 is not None: - # Got macro name from name attribute - block['bname'] = m1.group(1) - block['content'] = item.astext() - break - if block == {}: - # Have to search in code : search in 1st line - m2 = incodemacro.match(item.astext()) - if m2 is not None: - block['bname'] = m2.group(2) - block['content'] = "\n".join(item.astext().split("\n")[1:]) - if block == {}: - print "UNKNOWN BLOCK, PASSED" - else: - store_block(block) - - - diff --git a/parser/store.py b/parser/store.py deleted file mode 100644 index 8665fd1..0000000 --- a/parser/store.py +++ /dev/null @@ -1,14 +0,0 @@ -fileblocks = {} -codeblocks = {} - -def store_block(block): - b = block['bname'] - if b[0]=="/": - dest = fileblocks - else: - dest = codeblocks - if not dest.has_key(b): - dest[b] = [] - dest[b].append(block) - - diff --git a/parser/test.py b/parser/test.py deleted file mode 100644 index e69de29..0000000 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..fe7b3e9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +Jinja2==2.7.1 +MarkupSafe==0.18 +Pygments==1.6 +Sphinx==1.1.3 +docutils==0.11 diff --git a/rst2code/__init__.py b/rst2code/__init__.py index 581267a..aebd1fd 100644 --- a/rst2code/__init__.py +++ b/rst2code/__init__.py @@ -1,30 +1,37 @@ -__author__ = 'jmbarbier' -#!python -# SEE: http://fr.slideshare.net/doughellmann/better-documentation-through-automation-creating-docutils-sphinx-extensions +# encoding=UTF-8 +# +# rst2code : reStructuredText to code literal programming +# Copyright (C) 2013 JM Barbier +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + -from docutils.core import publish_doctree -import re -import sys -import getopt -import sys import argparse +import sys +from docutils.core import publish_doctree import logging -import os -import shutil +logging.basicConfig(level=logging.WARNING) +import re +import shutil, os -logging.basicConfig(level=logging.ERROR) -# GLOBALS +MAX_ITERATIONS = 10 BLOCKS = {} -DEBUG = 0 OUTPUT_DIR = "./src" -fileblocks = {} -codeblocks = {} -MAX_ITERATIONS = 10 -# REGULAR EXPRESSIONS macro = r"@@([@\w/.+! -]+)@@" -macro_in_code = "^(\s*).*?@@([@\w/.+! -]+)@@.*$" +macro_in_code = "^([ ]*).*?@@([@\w/.+! -]+)@@.*$" class CodeBlock(object): @@ -37,16 +44,22 @@ def __init__(self): self.iterations = 0 self.replacement_done = False + def macro_replace_step(self, blocks): logging.debug("Preparing blocks") logging.debug(self.content) if self.replacement_done: - return True - + return 0 + def macro_replace(match): indent = "" if match.group(1) is not None: indent = match.group(1) + if len(indent)>0 and indent[0]=="\n": + if len(indent)>1: + indent=indent[1:] + else: + indent="" logging.debug("INDENT:|%s|" % indent) logging.debug("Replacing %s", match.group(2)) if match.group(2) in BLOCKS.keys(): @@ -59,18 +72,18 @@ def macro_replace(match): return out else: # Macro not found, don't replace anything - logging.warning("Unknown macro - not replaced") + logging.warning("@@%s@@ : unknown macro - not replaced" % match.group(2)) return match.group(0) - + if self.iterations > MAX_ITERATIONS: logging.warning("Replacemement max iterations done") - return False + return 0 self.iterations += 1 self.content, n = re.subn(macro_in_code, macro_replace, self.content, flags=re.MULTILINE) if n==0: self.replacement_done = True - return self.replacement_done + return n @@ -81,6 +94,15 @@ def store_block(block): BLOCKS[block.name] = [] BLOCKS[block.name].append(block) +def process_blocks(): + logging.info("Generating code") + replaced = 1 + while replaced != 0: + replaced = 0 + for blocks in BLOCKS.values(): + for block in blocks: + result = block.macro_replace_step(BLOCKS) + replaced += result def get_block(item, filename): """ @@ -113,19 +135,6 @@ def get_block(item, filename): return True -def scan_doctree(doctree, filename=""): - for item in doctree.traverse(): - if item.tagname == "literal_block": - get_block(item, filename) - - - -def scan_file(filename): - with open(filename, "r") as f: - lines = f.read() - doctree = publish_doctree(lines) - scan_doctree(doctree, filename) - def clean_output_dir(): for root, dirs, files in os.walk(OUTPUT_DIR): for f in files: @@ -154,20 +163,24 @@ def write_files(): print("") -def process_blocks(): - logging.info("Generating code") - running = True - while running: - for blocks in BLOCKS.values(): - for block in blocks: - running = running and block.macro_replace_step(BLOCKS) +def scan_doctree(doctree, filename=""): + for item in doctree.traverse(): + if item.tagname == "literal_block": + get_block(item, filename) + + + +def scan_file(filename): + with open(filename, "r") as f: + lines = f.read() + doctree = publish_doctree(lines) + scan_doctree(doctree, filename) class Usage(Exception): def __init__(self, msg): self.msg = msg - def main(argv=None): if argv is None: argv = sys.argv[1:] @@ -202,22 +215,21 @@ def main(argv=None): print ("for help use --help", file=sys.stderr) return 2 +if __name__ == "__main__": + sys.exit(main()) + + def sphinx_get_doctree(app, doctree, docname): logging.debug("Got doctree") scan_doctree(doctree,docname) - def sphinx_build_finished(app, exception): env = app.builder.env OUTPUT_DIR = app.config.rst2code_output_dir process_blocks() clean_output_dir() write_files() - def setup(app): app.add_config_value("rst2code_output_dir", "./src", "env") app.add_config_value("rst2code_max_iterations", 10, "env") app.connect('doctree-resolved',sphinx_get_doctree) app.connect('build-finished', sphinx_build_finished) - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file diff --git a/rst2code/rst2doc.initial.py b/rst2code/rst2doc.initial.py new file mode 100644 index 0000000..aca3b79 --- /dev/null +++ b/rst2code/rst2doc.initial.py @@ -0,0 +1,231 @@ +__author__ = 'jmbarbier' +#!python +# SEE: http://fr.slideshare.net/doughellmann/better-documentation-through-automation-creating-docutils-sphinx-extensions + +from docutils.core import publish_doctree +import re +import sys +import getopt +import sys +import argparse +import logging +import os +import shutil + +logging.basicConfig(level=logging.DEBUG) + +# GLOBALS +BLOCKS = {} +DEBUG = 0 +OUTPUT_DIR = "./src" +fileblocks = {} +codeblocks = {} +MAX_ITERATIONS = 10 + +# REGULAR EXPRESSIONS +macro = r"@@([@\w/.+! -]+)@@" +macro_in_code = "^([ ]*).*?@@([@\w/.+! -]+)@@.*$" + + +class CodeBlock(object): + def __init__(self): + self.name = "" + self.is_file_block = False + self.filename = "" + self.content = "" + self.language = "" + self.iterations = 0 + self.replacement_done = False + + def macro_replace_step(self, blocks): + logging.debug("Preparing blocks") + #logging.debug(self.content) + if self.replacement_done: + return 0 + + def macro_replace(match): + indent = "" + if match.group(1) is not None: + indent = match.group(1) + if len(indent)>0 and indent[0]=="\n": + if len(indent)>1: + indent=indent[1:] + else: + indent="" + logging.debug("INDENT:|%s|" % indent) + logging.debug("Replacing %s", match.group(2)) + if match.group(2) in BLOCKS.keys(): + # Found a macro : replace with it + out = "" + for b in BLOCKS[match.group(2)]: + for line in b.content.split("\n"): + out += indent+line+"\n" + logging.debug("OUT %s" % out) + return out + else: + # Macro not found, don't replace anything + logging.warning("Unknown macro - not replaced") + return match.group(0) + + if self.iterations > MAX_ITERATIONS: + logging.warning("Replacemement max iterations done") + return 0 + self.iterations += 1 + self.content, n = re.subn(macro_in_code, macro_replace, + self.content, flags=re.MULTILINE) + logging.debug("Replaced %s " % n) + if n==0: + self.replacement_done = True + return n + + + +def store_block(block): + """ Store block """ + logging.debug("Storing code block %s" % BLOCKS) + if block.name not in BLOCKS.keys(): + BLOCKS[block.name] = [] + BLOCKS[block.name].append(block) + + +def get_block(item, filename): + """ + Get block from doctree item + """ + logging.info("Getting code block") + block = CodeBlock() + block.filename = filename + for name in item.attributes['names']: + m1 = re.match(macro, name) + if m1 is not None: + # Got macro name from name attribute + block.name = m1.group(1) + block.content = item.astext() + block.is_file_block = block.name[0]=="/" + break + if block.name == "": + # Have to search in code : search in 1st line + m2 = re.match(macro_in_code, item.astext(), re.MULTILINE) + if m2 is not None: + block.name = m2.group(2) + block.content = "\n".join(item.astext().split("\n")[1:]) + block.is_file_block = block.name[0]=="/" + if block.name == "": + logging.info("Block is not macro block") + return False + else: + logging.debug("Have code block") + store_block(block) + return True + + +def scan_doctree(doctree, filename=""): + for item in doctree.traverse(): + if item.tagname == "literal_block": + get_block(item, filename) + + + +def scan_file(filename): + with open(filename, "r") as f: + lines = f.read() + doctree = publish_doctree(lines) + scan_doctree(doctree, filename) + +def clean_output_dir(): + for root, dirs, files in os.walk(OUTPUT_DIR): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + +def write_files(): + files = {} + for blocks in BLOCKS.values(): + for block in blocks: + if block.is_file_block: + if block.name not in files.keys(): + files[block.name] = [] + files[block.name].append(block.content) + + print("RST2CODE: Writing files : ",end="") + for filename in files: + destfile = os.path.abspath(os.path.join(OUTPUT_DIR + filename)) + d = os.path.dirname(destfile) + if not os.path.exists(d): + os.makedirs(d) + with open(destfile, "w") as f: + f.write("\n".join(files[filename])) + print(".", end="") + print("") + + +def process_blocks(): + logging.info("Generating code") + replaced = 1 + while replaced != 0: + replaced = 0 + for blocks in BLOCKS.values(): + for block in blocks: + result = block.macro_replace_step(BLOCKS) + replaced += result + + +class Usage(Exception): + def __init__(self, msg): + self.msg = msg + + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + try: + try: + parser = argparse.ArgumentParser("Write code from rst files") + parser.add_argument('outdir', metavar="OUTPUT_DIR", type=str, nargs=1, + help="Output directory base for code") + parser.add_argument('srcfiles', metavar="SRC_FILES", type=str, nargs="+", + help="Source files (.rst files)") + parser.add_argument('--debug', '-d') + args = parser.parse_args(argv) + + except Exception as msg: + raise Usage(msg) + + + # process arguments + DEBUG = args.debug + OUTPUT_DIR = args.outdir + SRC_FILES = args.srcfiles + + for filename in SRC_FILES: + scan_file(filename) + process_blocks() + return 0 + + except Usage as err: + print(sys.argv[0].split('/')[-1].split('\\')[-1] + ': ' \ + + str(err.msg) \ + , file=sys.stderr) + print ("for help use --help", file=sys.stderr) + return 2 + +def sphinx_get_doctree(app, doctree, docname): + logging.debug("Got doctree") + scan_doctree(doctree,docname) + +def sphinx_build_finished(app, exception): + env = app.builder.env + OUTPUT_DIR = app.config.rst2code_output_dir + process_blocks() + clean_output_dir() + write_files() + +def setup(app): + app.add_config_value("rst2code_output_dir", "./src", "env") + app.add_config_value("rst2code_max_iterations", 10, "env") + app.connect('doctree-resolved',sphinx_get_doctree) + app.connect('build-finished', sphinx_build_finished) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/source/cmdline.rst b/source/cmdline.rst new file mode 100644 index 0000000..1502e24 --- /dev/null +++ b/source/cmdline.rst @@ -0,0 +1,87 @@ +Command line +============ + +Command line interface +---------------------- + +For vanilla reStructuredText (that can directly be parsed with docutils), we have to implement +a command line interface (main() function), using argparse :: + + #@@imports@@ + import argparse + import sys + +then we define our main function, to parse command line arguments and options, and to do the actual processing +of input files :: + + #@@command_line_actions@@ + class Usage(Exception): + def __init__(self, msg): + self.msg = msg + + def main(argv=None): + if argv is None: + argv = sys.argv[1:] + try: + try: + parser = argparse.ArgumentParser("Write code from rst files") + parser.add_argument('outdir', metavar="OUTPUT_DIR", type=str, nargs=1, + help="Output directory base for code") + parser.add_argument('srcfiles', metavar="SRC_FILES", type=str, nargs="+", + help="Source files (.rst files)") + parser.add_argument('--debug', '-d') + args = parser.parse_args(argv) + + except Exception as msg: + raise Usage(msg) + + + # process arguments + DEBUG = args.debug + OUTPUT_DIR = args.outdir + SRC_FILES = args.srcfiles + + for filename in SRC_FILES: + scan_file(filename) + process_blocks() + return 0 + + except Usage as err: + print(sys.argv[0].split('/')[-1].split('\\')[-1] + ': ' \ + + str(err.msg) \ + , file=sys.stderr) + print ("for help use --help", file=sys.stderr) + return 2 + + if __name__ == "__main__": + sys.exit(main()) + + + +RST files scan +-------------- + +We use docutils for rst files scan :: + + #@@imports@@ + from docutils.core import publish_doctree + +We make docutils scan source rst files from command line, getting a doctree if no syntax error found, then we +check every item in doctree against literal_block type :: + + #@@input_file_analysis@@ + def scan_doctree(doctree, filename=""): + for item in doctree.traverse(): + if item.tagname == "literal_block": + get_block(item, filename) + + + + def scan_file(filename): + with open(filename, "r") as f: + lines = f.read() + doctree = publish_doctree(lines) + scan_doctree(doctree, filename) + + +Then each doctree literal_block item is analyzed to get its name (with getBlock). diff --git a/source/index.rst b/source/index.rst new file mode 100644 index 0000000..33f4c52 --- /dev/null +++ b/source/index.rst @@ -0,0 +1,264 @@ +rst2code source +=============== + +Written with rst2code :) + +Program structure +------------------ + +**rst2code** program structure is the following :: + + #@@/rst2code.py@@ + #@@python_headers@@ + #@@license@@ + + #@@imports@@ + + #@@globals@@ + #@@regular_expressions@@ + + #@@objects@@ + + #@@blocks_storage@@ + #@@block_processing@@ + #@@block_analysis@@ + + #@@file_operations@@ + #@@files_generator@@ + + #@@input_file_analysis@@ + + #@@command_line_actions@@ + + #@@sphinx_extension@@ + +Logging can be enabled for debug / trace purposes :: + + #@@imports@@ + import logging + logging.basicConfig(level=logging.WARNING) + +Code blocks +----------- + +Structure +~~~~~~~~~ + +A block is a named portion of code :: + + #@@objects@@ + class CodeBlock(object): + def __init__(self): + self.name = "" + self.is_file_block = False + self.filename = "" + self.content = "" + self.language = "" + self.iterations = 0 + self.replacement_done = False + + + #@@codeblock_actions@@ + +Fields are : + +- name : code block name (many code blocks can have the same name, in this case their content is appended). + +- is_file_block : True if block is outputted directly into af file + +- filename : source (.rst file) filename for this block (not used for now). + +- content : source content + +- language : source language (if defined, not used for now) + +- iterations : when a code block is processed to replace macros, inserted code can introduce new macros. rst2code makes + MAX_ITERATIONS maximum of macro replacing :: + + #@@globals@@ + MAX_ITERATIONS = 10 + +- replacement_done : macro replacement is over + +Block from doctree +~~~~~~~~~~~~~~~~~~ + +With docutils rst file parsing, we obtain a doctree; for every "literal_block" item (code, code-block, ...), +we have to search an identifier (in :name: option or in code first line) :: + + #@@block_analysis@@ + def get_block(item, filename): + """ + Get block from doctree item + """ + logging.info("Getting code block") + block = CodeBlock() + block.filename = filename + for name in item.attributes['names']: + m1 = re.match(macro, name) + if m1 is not None: + # Got macro name from name attribute + block.name = m1.group(1) + block.content = item.astext() + block.is_file_block = block.name[0]=="/" + break + if block.name == "": + # Have to search in code : search in 1st line + m2 = re.match(macro_in_code, item.astext(), re.MULTILINE) + if m2 is not None: + block.name = m2.group(2) + block.content = "\n".join(item.astext().split("\n")[1:]) + block.is_file_block = block.name[0]=="/" + if block.name == "": + logging.info("Block is not macro block") + return False + else: + logging.debug("Have code block") + store_block(block) + return True + + + + +Storage +~~~~~~~ + +Blocks are stored into a global dictionary named BLOCKS :: + + #@@globals@@ + BLOCKS = {} + +Each block is stored in an array :: + + #@@blocks_storage@@ + def store_block(block): + """ Store block """ + logging.debug("Storing code block %s" % BLOCKS) + if block.name not in BLOCKS.keys(): + BLOCKS[block.name] = [] + BLOCKS[block.name].append(block) + + +Transformation +~~~~~~~~~~~~~~ + +Regular expressions are used to search macro name in code blocks :: + + #@@regular_expressions@@ + macro = r"@@([@\w/.+! -]+)@@" + macro_in_code = "^([ ]*).*?@@([@\w/.+! -]+)@@.*$" + +(so we have to import re) :: + + #@@imports@@ + import re + +Code blocks content is searched for macro names, and each found macro is replaced by its content. If no macro is +found with this name, comment block is left untouched. + +If no macro name is found inside code, or if iterations are more than MAX_ITERATIONS, +then macro transformation returns False :: + + #@@codeblock_actions@@ + def macro_replace_step(self, blocks): + logging.debug("Preparing blocks") + logging.debug(self.content) + if self.replacement_done: + return 0 + + def macro_replace(match): + indent = "" + if match.group(1) is not None: + indent = match.group(1) + if len(indent)>0 and indent[0]=="\n": + if len(indent)>1: + indent=indent[1:] + else: + indent="" + logging.debug("INDENT:|%s|" % indent) + logging.debug("Replacing %s", match.group(2)) + if match.group(2) in BLOCKS.keys(): + # Found a macro : replace with it + out = "" + for b in BLOCKS[match.group(2)]: + for line in b.content.split("\n"): + out += indent+line+"\n" + logging.debug("OUT %s" % out) + return out + else: + # Macro not found, don't replace anything + logging.warning("@@%s@@ : unknown macro - not replaced" % match.group(2)) + return match.group(0) + + if self.iterations > MAX_ITERATIONS: + logging.warning("Replacemement max iterations done") + return 0 + self.iterations += 1 + self.content, n = re.subn(macro_in_code, macro_replace, + self.content, flags=re.MULTILINE) + if n==0: + self.replacement_done = True + return n + + + +All blocks are transformed looping with this macro_replace_step above :: + + #@@block_processing@@ + def process_blocks(): + logging.info("Generating code") + replaced = 1 + while replaced != 0: + replaced = 0 + for blocks in BLOCKS.values(): + for block in blocks: + result = block.macro_replace_step(BLOCKS) + replaced += result + +Files output +------------ + +Output directory is defined as OUTPUT_DIR global :: + + #@@globals@@ + OUTPUT_DIR = "./src" + +using shutil.rmtree and os.walk :: + + #@@imports@@ + import shutil, os + +it can be cleaned :: + + #@@file_operations@@ + def clean_output_dir(): + for root, dirs, files in os.walk(OUTPUT_DIR): + for f in files: + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + + +All blocks are scanned, file block contents are concatened then written to file :: + + #@@files_generator@@ + def write_files(): + files = {} + for blocks in BLOCKS.values(): + for block in blocks: + if block.is_file_block: + if block.name not in files.keys(): + files[block.name] = [] + files[block.name].append(block.content) + + print("RST2CODE: Writing files : ",end="") + for filename in files: + destfile = os.path.abspath(os.path.join(OUTPUT_DIR + filename)) + d = os.path.dirname(destfile) + if not os.path.exists(d): + os.makedirs(d) + with open(destfile, "w") as f: + f.write("\n".join(files[filename])) + print(".", end="") + print("") + diff --git a/source/misc.rst b/source/misc.rst new file mode 100644 index 0000000..a200611 --- /dev/null +++ b/source/misc.rst @@ -0,0 +1,28 @@ +Miscelleanous data +================== + +We need to include a license :: + + @@license@@ + # rst2code : reStructuredText to code literal programming + # Copyright (C) 2013 JM Barbier + # + # This program is free software: you can redistribute it and/or modify + # it under the terms of the GNU General Public License as published by + # the Free Software Foundation, either version 3 of the License, or + # (at your option) any later version. + # + # This program is distributed in the hope that it will be useful, + # but WITHOUT ANY WARRANTY; without even the implied warranty of + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + # GNU General Public License for more details. + # + # You should have received a copy of the GNU General Public License + # along with this program. If not, see . + + +And a python header for UTF-8 encoding :: + + @@python_header@@ + # encoding=UTF-8 + diff --git a/source/sphinx.rst b/source/sphinx.rst new file mode 100644 index 0000000..3bff092 --- /dev/null +++ b/source/sphinx.rst @@ -0,0 +1,32 @@ +Sphinx extension +================ + + +Source file processing can be done with sphinx, we just need to respond to "doctree-resolved" event that +gives us processed doctree :: + + #@@sphinx_extension@@ + def sphinx_get_doctree(app, doctree, docname): + logging.debug("Got doctree") + scan_doctree(doctree,docname) + +We wait for "build-finished" event to create source code files (SEE? maybe we should do it before ?) :: + + #@@sphinx_extension@@ + def sphinx_build_finished(app, exception): + env = app.builder.env + OUTPUT_DIR = app.config.rst2code_output_dir + process_blocks() + clean_output_dir() + write_files() + + +To use rst2code within sphinx, [configuration]... So we need a "setup" function :: + + #@@sphinx_extension@@ + def setup(app): + app.add_config_value("rst2code_output_dir", "./src", "env") + app.add_config_value("rst2code_max_iterations", 10, "env") + app.connect('doctree-resolved',sphinx_get_doctree) + app.connect('build-finished', sphinx_build_finished) +