Merge pull request #4 from sarnold/pre-commit

readme and pre-commit updates

.gitignore

@@ -5,11 +5,10 @@ __pycache__/
 
 # generated/user files
 src/ymltoxml/_version.py
+.ymltoxml.yaml
 in.*
 out.*
 munch/
-.*.yaml
-*.xml
 
 # C extensions
 *.so

.pep8speaks.yml

@@ -0,0 +1,15 @@
+scanner:
+    diff_only: True  # If False, the entire file touched by the Pull Request is scanned for errors. If True, only the diff is scanned.
+    linter: flake8  # Other option is pycodestyle
+
+no_blank_comment: False  # If True, no comment is made on PR without any errors.
+descending_issues_order: True  # If True, PEP 8 issues in message will be displayed in descending order of line numbers in the file
+
+pycodestyle:  # Same as scanner.linter value. Other option is flake8
+    max-line-length: 110  # Default is 79 in PEP 8
+
+flake8:
+    max-line-length: 90  # Default is 79 in PEP 8
+    exclude:
+        - tests
+        - docs

.pre-commit-config.yaml

@@ -0,0 +1,130 @@
+# To install the git pre-commit hook run:
+#   pre-commit install
+# To update the pre-commit hooks run:
+#   pre-commit install-hooks
+exclude: '^(.tox/|docs/|^setup.py$)'
+repos:
+  - repo: meta
+    hooks:
+      - id: check-useless-excludes
+      - id: check-hooks-apply
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      # - id: check-symlinks
+      - id: debug-statements
+      - id: requirements-txt-fixer
+      - id: fix-encoding-pragma
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: check-toml
+      - id: check-yaml
+        exclude: '(conda/meta.yaml|.pep8speaks.yml)'
+
+  # use ffffff (black fork) for single quote normalization
+  # (otherwise switch to black for double quotes)
+  - repo: https://github.com/grktsh/ffffff
+    rev: v2020.8.31
+    hooks:
+      - id: ffffff
+        name: "Format code (ffffff)"
+        files: src/ymltoxml/ymltoxml.py
+        language_version: python3
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: [--settings-path=pyproject.toml]
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.961
+    hooks:
+    - id: mypy
+      additional_dependencies:
+        - importlib_metadata
+        - importlib_resources
+        - munch
+        - ruamel.yaml
+      args:
+        - --follow-imports=normal
+        - --install-types
+        - --non-interactive
+        - --ignore-missing-imports
+      files: src/ymltoxml/ymltoxml.py
+
+  - repo: "https://github.com/asottile/blacken-docs"
+    rev: "v1.12.1"
+    hooks:
+      - id: "blacken-docs"
+        name: "Format docs (blacken-docs)"
+        args: ["-l", "64"]
+        additional_dependencies:
+          - "black==21.9b0"
+
+  - repo: https://github.com/PyCQA/doc8
+    rev: 0.11.2
+    hooks:
+      - id: doc8
+        args:
+          - '--max-line-length=90'
+          - '--ignore=D001'
+
+  - repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.9.0
+    hooks:
+      - id: rst-backticks
+        # exclude: ChangeLog\.rst$
+      - id: rst-directive-colons
+      - id: rst-inline-touching-normal
+
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        files: src/ymltoxml/ymltoxml.py
+        args:
+          - --in-place
+          - --remove-all-unused-imports
+          - --remove-duplicate-keys
+          - --remove-unused-variables
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8
+        files: src/ymltoxml/ymltoxml.py
+        additional_dependencies: ["flake8-bugbear"]
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.4
+    hooks:
+      - id: bandit
+        args: ["-ll", "-q"]
+        files: src/ymltoxml/ymltoxml.py
+
+#  - repo: https://github.com/lovesegfault/beautysh
+#    rev: v6.2.1
+#    hooks:
+#      - id: beautysh
+
+ci:
+    autofix_commit_msg: |
+        [pre-commit.ci] auto fixes from pre-commit.com hooks
+
+        for more information, see https://pre-commit.ci
+    autofix_prs: false
+    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+    autoupdate_schedule: weekly
+    skip: []
+    submodules: false
+
+# re-running a pull request: you can trigger a re-run on a pull request by
+#    commenting pre-commit.ci run (must appear on a line by itself).
+# skipping push runs: skip a run by putting [skip ci], [ci skip],
+#    [skip pre-commit.ci], or [pre-commit.ci skip] in the commit message.

README.rst

@@ -1,10 +1,10 @@
-=========================================
- ymltoxml (and xmltoyml) for mavlink xml
-=========================================
+=========================
+ ymltoxml (and xmltoyml)
+=========================
 
 |ci| |wheels| |release| |badge|
 
-|pylint|
+|pre| |pylint|
 
 |tag| |license| |python|
 
@@ -13,12 +13,90 @@ preserving attributes and comments (with minor corrections).  The default
 file encoding for both types is UTF-8 without a BOM.  The main intent is
 to support YAML-based development of custom mavlink_ dialects.
 
+Quick Start
+===========
+
+Install with pip
+----------------
+
+This package is *not* yet published on PyPI, thus use one of the
+following to install the latest ymltoxml on any platform::
+
+  $ pip install -U -f https://github.com/sarnold/ymltoxml/releases/ ymltoxml
+
+or use this command to install a specific version::
+
+  $ pip install git+https://github.com/sarnold/ymltoxml.git@0.1.0
+
+The full package provides the ``ymltoxml.py`` executable as well as
+a reference configuration file that provides defaults for all values.
+
+If you'd rather work from the source repository, it supports the common
+idiom to install it on your system in a virtual env after cloning::
+
+  $ python3 -m venv env
+  $ source env/bin/activate
+  $ pip install .
+  $ ymltoxml --version
+  $ ymltoxml --dump-config
+  $ deactivate
+
+The alternative to python venv is the ``tox`` test driver.  If you have it
+installed already, see the example tox commands below.
+
+Usage
+-----
+
+The current version reads very minimal command options, and the only
+required command arguments are one or more files of a single type::
+
+  $ ymltoxml
+  Transform YAML to XML and XML to YAML.
+
+  Usage:
+      ymltoxml file1.yaml file2.yaml ...
+      ymltoxml file1.xml file2.xml ...
+
+  Each output file is named for the corresponding input file using
+  the output extension (more options coming soon).
+
+The main processing tweaks for yml/xml output formatting are specified
+in the default configuration file; if you need to change something, you
+can use your own config file in the working directory; note the local
+copy must be named ``.ymltoxml.yaml``.  To get a copy of the default
+configuration file, do::
+
+  $ cd path/to/work/dir/
+  $ ymltoxml --dump-config > .ymltoxml.yaml
+  $ $EDITOR .ymltoxml.yaml
+
+
+Features and limitations
+------------------------
+
+We only test on mavlink XML message definitions, so it probably *will not*
+work at all on arbitrarily complex XML files with namespaces, etc.  The
+current round-trip is not exact, due to the following:
+
+* missing encoding is added to version tag
+* leading/trailing whitespace in text elements and comments is not preserved
+* elements with self-closing tags are converted to full closing tags
+* empty elements on more than one line are not preserved
+
+For the files tested (eg, mavlink) the end result is cleaner/shinier XML.
+
 Local workflow
 ===============
 
-Tool requirements:
+This tool is intended to be part of larger workflow, ie, developing a
+custom mavlink message dialect and generating/deploying the resulting
+mavlink language interfaces.  To be more specific, for this example we
+use a mavlink-compatible component running on a micro-controller, thus
+the target language bindings are C and C++.
+
+Tool requirements for the full mavlink workflow:
 
-* initially just recent Python and Tox_
+* initially just recent pymavlink, Python, and Tox_
 
 Both mavlink and pymavlink require a (host) GCC toolchain for full builds,
 however, the basic workflow to generate mavlink library headers requires
@@ -78,6 +156,59 @@ To build/lint the api docs, use the following tox commands:
 * ``tox -e docs`` build the documentation using sphinx and the api-doc plugin
 * ``tox -e docs-lint`` build the docs and run the sphinx link checking
 
+Pre-commit
+----------
+
+This repo is now pre-commit_ enabled for python/rst source and file-type
+linting. The checks run automatically on commit and will fail the commit
+(if not clean) and perform simple file corrections.  For example, if the
+mypy check fails on commit, you must first fix any fatal errors for the
+commit to succeed. That said, pre-commit does nothing if you don't install
+it first (both the program itself and the hooks in your local repository
+copy).
+
+You will need to install pre-commit before contributing any changes;
+installing it using your system's package manager is recommended,
+otherwise install with pip into your usual virtual environment using
+something like::
+
+  $ sudo emerge pre-commit  --or--
+  $ pip install pre-commit
+
+then install it into the repo you just cloned::
+
+  $ git clone https://github.com/sarnold/ymltoxml
+  $ cd ymltoxml/
+  $ pre-commit install
+
+It's usually a good idea to update the hooks to the latest version::
+
+    $ pre-commit autoupdate
+
+Most (but not all) of the pre-commit checks will make corrections for you,
+however, some will only report errors, so these you will need to correct
+manually.
+
+Automatic-fix checks include ffffff, isort, autoflake, the yaml/xml format
+checks, and the miscellaneous file fixers. If any of these fail, you can
+review the changes with ``git diff`` and just add them to your commit and
+continue.
+
+If any of the mypy, bandit, or rst source checks fail, you will get a report,
+and you must fix any errors before you can continue adding/committing.
+
+To see a "replay" of any ``rst`` check errors, run::
+
+  $ pre-commit run rst-backticks -a
+  $ pre-commit run rst-directive-colons -a
+  $ pre-commit run rst-inline-touching-normal -a
+
+To run all ``pre-commit`` checks manually, try::
+
+  $ pre-commit run -a
+
+.. _pre-commit: https://pre-commit.com/index.html
+
 
 .. |ci| image:: https://github.com/sarnold/ymltoxml/actions/workflows/ci.yml/badge.svg
     :target: https://github.com/sarnold/ymltoxml/actions/workflows/ci.yml
@@ -110,3 +241,7 @@ To build/lint the api docs, use the following tox commands:
 .. |python| image:: https://img.shields.io/badge/python-3.6+-blue.svg
     :target: https://www.python.org/downloads/
     :alt: Python
+
+.. |pre| image:: https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white
+   :target: https://github.com/pre-commit/pre-commit
+   :alt: pre-commit

pyproject.toml

@@ -24,6 +24,23 @@ show_missing = true
 
 [tool.black]
 line-length = 90
+skip-string-normalization = true
+include = '\.py$'
+exclude = '''
+/(
+    \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | docs
+  | dist
+  | tests
+)/
+'''
 
 [tool.pycln]
 all = true

requirements.txt

@@ -1,4 +1,4 @@
-xmltodict
 munch
-ruamel.yaml
 PyYAML
+ruamel.yaml
+xmltodict

scripts/genxml.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from pathlib import Path
 
 import xmltodict

scripts/genyaml.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from pathlib import Path
 
 import xmltodict

src/ymltoxml/__init__.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from ._version import __version__
 
 version = __version__