Configuration system for CLI (qmk#6708)

* Rework how bin/qmk handles subcommands

* qmk config wip

* Code to show all configs

* Fully working `qmk config` command

* Mark some CLI arguments so they don't pollute the config file

* Fleshed out config support, nicer subcommand support

* sync with installable cli

* pyformat

* Add a test for subcommand_modules

* Documentation for the `qmk config` command

* split config_token on space so qmk config is more predictable

* Rework how subcommands are imported

* Document `arg_only`

* Document deleting from CLI

* Document how multiple operations work

* Add cli config to the doc index

* Add tests for the cli commands

* Make running the tests more reliable

* Be more selective about building all default keymaps

* Update new-keymap to fit the new subcommand style

* Add documentation about writing CLI scripts

* Document new-keyboard

* Update docs/cli_configuration.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Update docs/cli_development.md

Co-Authored-By: noroadsleft <18669334+noroadsleft@users.noreply.github.com>

* Address yan's comments.

* Apply suggestions from code review

suggestions from @noahfrederick

Co-Authored-By: Noah Frederick <code@noahfrederick.com>

* Apply suggestions from code review

Co-Authored-By: Noah Frederick <code@noahfrederick.com>

* Remove pip3 from the test runner

bin/qmk

@@ -4,23 +4,17 @@
 import os
 import subprocess
 import sys
-from glob import glob
-from time import strftime
-from importlib import import_module
 from importlib.util import find_spec
+from time import strftime
 
 # Add the QMK python libs to our path
 script_dir = os.path.dirname(os.path.realpath(__file__))
 qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
 python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
 sys.path.append(python_lib_dir)
 
-# Change to the root of our checkout
-os.environ['ORIG_CWD'] = os.getcwd()
-os.chdir(qmk_dir)
-
 # Make sure our modules have been setup
-with open('requirements.txt', 'r') as fd:
+with open(os.path.join(qmk_dir, 'requirements.txt'), 'r') as fd:
     for line in fd.readlines():
         line = line.strip().replace('<', '=').replace('>', '=')
 
@@ -32,72 +26,58 @@ with open('requirements.txt', 'r') as fd:
 
         module = line.split('=')[0] if '=' in line else line
         if not find_spec(module):
-            print('Your QMK build environment is not fully setup!\n')
-            print('Please run `./util/qmk_install.sh` to setup QMK.')
+            print('Could not find module %s!', module)
+            print('Please run `pip3 install -r requirements.txt` to install the python dependencies.')
             exit(255)
 
 # Figure out our version
+# TODO(skullydazed/anyone): Find a method that doesn't involve git. This is slow in docker and on windows.
 command = ['git', 'describe', '--abbrev=6', '--dirty', '--always', '--tags']
-result = subprocess.run(command, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+result = subprocess.run(command, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
 if result.returncode == 0:
-    os.environ['QMK_VERSION'] = 'QMK ' + result.stdout.strip()
+    os.environ['QMK_VERSION'] = result.stdout.strip()
 else:
-    os.environ['QMK_VERSION'] = 'QMK ' + strftime('%Y-%m-%d-%H:%M:%S')
+    os.environ['QMK_VERSION'] = 'nogit-' + strftime('%Y-%m-%d-%H:%M:%S') + '-dirty'
 
 # Setup the CLI
 import milc
-milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}Ψ{style_reset_all}'
 
-# If we were invoked as `qmk <cmd>` massage sys.argv into `qmk-<cmd>`.
-# This means we can't accept arguments to the qmk script itself.
-script_name = os.path.basename(sys.argv[0])
-if script_name == 'qmk':
-    if len(sys.argv) == 1:
-        milc.cli.log.error('No subcommand specified!\n')
-
-    if len(sys.argv) == 1 or sys.argv[1] in ['-h', '--help']:
-        milc.cli.echo('usage: qmk <subcommand> [...]')
-        milc.cli.echo('\nsubcommands:')
-        subcommands = glob(os.path.join(qmk_dir, 'bin', 'qmk-*'))
-        for subcommand in sorted(subcommands):
-            subcommand = os.path.basename(subcommand).split('-', 1)[1]
-            milc.cli.echo('\t%s', subcommand)
-        milc.cli.echo('\nqmk <subcommand> --help for more information')
-        exit(1)
+milc.EMOJI_LOGLEVELS['INFO'] = '{fg_blue}Ψ{style_reset_all}'
 
-    if sys.argv[1] in ['-V', '--version']:
-        milc.cli.echo(os.environ['QMK_VERSION'])
-        exit(0)
 
-    sys.argv[0] = script_name = '-'.join((script_name, sys.argv[1]))
-    del sys.argv[1]
+@milc.cli.entrypoint('QMK Helper Script')
+def qmk_main(cli):
+    """The function that gets run when no subcommand is provided.
+    """
+    cli.print_help()
 
-# Look for which module to import
-if script_name == 'qmk':
-    milc.cli.print_help()
-    exit(0)
-elif not script_name.startswith('qmk-'):
-    milc.cli.log.error('Invalid symlink, must start with "qmk-": %s', script_name)
-else:
-    subcommand = script_name.replace('-', '.').replace('_', '.').split('.')
-    subcommand.insert(1, 'cli')
-    subcommand = '.'.join(subcommand)
 
-    try:
-        import_module(subcommand)
-    except ModuleNotFoundError as e:
-        if e.__class__.__name__ != subcommand:
-            raise
+def main():
+    """Setup our environment and then call the CLI entrypoint.
+    """
+    # Change to the root of our checkout
+    os.environ['ORIG_CWD'] = os.getcwd()
+    os.chdir(qmk_dir)
 
-        milc.cli.log.error('Invalid subcommand! Could not import %s.', subcommand)
-        exit(1)
+    # Import the subcommands
+    import qmk.cli
 
-if __name__ == '__main__':
+    # Execute
     return_code = milc.cli()
+
     if return_code is False:
         exit(1)
-    elif return_code is not True and isinstance(return_code, int) and return_code < 256:
+
+    elif return_code is not True and isinstance(return_code, int):
+        if return_code < 0 or return_code > 255:
+            milc.cli.log.error('Invalid return_code: %d', return_code)
+            exit(255)
+
         exit(return_code)
-    else:
-        exit(0)
+
+    exit(0)
+
+
+if __name__ == '__main__':
+    main()

build_json.mk

@@ -23,5 +23,5 @@ endif
 
 # Generate the keymap.c
 ifneq ("$(KEYMAP_JSON)","")
-    _ = $(shell test -e $(KEYMAP_C) || bin/qmk-json-keymap $(KEYMAP_JSON) -o $(KEYMAP_C))
+    _ = $(shell test -e $(KEYMAP_C) || bin/qmk json-keymap $(KEYMAP_JSON) -o $(KEYMAP_C))
 endif

docs/_summary.md

@@ -9,6 +9,7 @@
 * [QMK Basics](README.md)
   * [QMK Introduction](getting_started_introduction.md)
   * [QMK CLI](cli.md)
+  * [QMK CLI Config](cli_configuration.md)
   * [Contributing to QMK](contributing.md)
   * [How to Use Github](getting_started_github.md)
   * [Getting Help](getting_started_getting_help.md)
@@ -48,7 +49,7 @@
   * [Useful Functions](ref_functions.md)
   * [Configurator Support](reference_configurator_support.md)
   * [info.json Format](reference_info_json.md)
-  * [Python Development](python_development.md)
+  * [Python CLI Development](cli_development.md)
 
 * [Features](features.md)
   * [Basic Keycodes](keycodes_basic.md)

docs/cli.md

@@ -4,22 +4,70 @@ This page describes how to setup and use the QMK CLI.
 
 # Overview
 
-The QMK CLI makes building and working with QMK keyboards easier. We have provided a number of commands to help you work with QMK:
+The QMK CLI makes building and working with QMK keyboards easier. We have provided a number of commands to simplify and streamline tasks such as obtaining and compiling the QMK firmware, creating keymaps, and more.
 
-* `qmk compile`
-* `qmk doctor`
+* [Global CLI](#global-cli)
+* [Local CLI](#local-cli)
+* [CLI Commands](#cli-commands)
 
-# Setup
+# Requirements
 
-Simply add the `qmk_firmware/bin` directory to your `PATH`. You can run the `qmk` commands from any directory.
+The CLI requires Python 3.5 or greater. We try to keep the number of requirements small but you will also need to install the packages listed in [`requirements.txt`](https://github.com/qmk/qmk_firmware/blob/master/requirements.txt).
+
+# Global CLI
+
+QMK provides an installable CLI that can be used to setup your QMK build environment, work with QMK, and which makes working with multiple copies of `qmk_firmware` easier. We recommend installing and updating this periodically.
+
+## Install Using Homebrew (macOS, some Linux)
+
+If you have installed [Homebrew](https://brew.sh) you can tap and install QMK:
+
+```
+brew tap qmk/qmk
+brew install qmk
+export QMK_HOME='~/qmk_firmware' # Optional, set the location for `qmk_firmware`
+qmk setup  # This will clone `qmk/qmk_firmware` and optionally set up your build environment
+```
+
+## Install Using easy_install or pip
+
+If your system is not listed above you can install QMK manually. First ensure that you have python 3.5 (or later) installed and have installed pip. Then install QMK with this command:
+
+```
+pip3 install qmk
+export QMK_HOME='~/qmk_firmware' # Optional, set the location for `qmk_firmware`
+qmk setup  # This will clone `qmk/qmk_firmware` and optionally set up your build environment
+```
+
+## Packaging For Other Operating Systems
+
+We are looking for people to create and maintain a `qmk` package for more operating systems. If you would like to create a package for your OS please follow these guidelines:
+
+* Follow best practices for your OS when they conflict with these guidelines
+    * Documment why in a comment when you do deviate
+* Install using a virtualenv
+* Instruct the user to set the environment variable `QMK_HOME` to have the firmware source checked out somewhere other than `~/qmk_firmware`.
+
+# Local CLI
+
+If you do not want to use the global CLI there is a local CLI bundled with `qmk_firmware`. You can find it in `qmk_firmware/bin/qmk`. You can run the `qmk` command from any directory and it will always operate on that copy of `qmk_firmware`.
+
+**Example**:
 
 ```
-export PATH=$PATH:$HOME/qmk_firmware/bin
+$ ~/qmk_firmware/bin/qmk hello
+Ψ Hello, World!
 ```
 
-You may want to add this to your `.profile`, `.bash_profile`, `.zsh_profile`, or other shell startup scripts.
+## Local CLI Limitations
 
-# Commands
+There are some limitations to the local CLI compared to the global CLI:
+
+* The local CLI does not support `qmk setup` or `qmk clone`
+* The local CLI always operates on the same `qmk_firmware` tree, even if you have multiple repositories cloned.
+* The local CLI does not run in a virtualenv, so it's possible that dependencies will conflict
+
+# CLI Commands
 
 ## `qmk compile`
 
@@ -46,3 +94,53 @@ This command formats C code using clang-format. Run it with no arguments to form
 ```
 qmk cformat [file1] [file2] [...] [fileN]
 ```
+
+## `qmk config`
+
+This command lets you configure the behavior of QMK. For the full `qmk config` documentation see [CLI Configuration](cli_configuration.md).
+
+**Usage**:
+
+```
+qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN]
+```
+
+## `qmk doctor`
+
+This command examines your environment and alerts you to potential build or flash problems.
+
+**Usage**:
+
+```
+qmk doctor
+```
+
+## `qmk new-keymap`
+
+This command creates a new keymap based on a keyboard's existing default keymap.
+
+**Usage**:
+
+```
+qmk new-keymap [-kb KEYBOARD] [-km KEYMAP]
+```
+
+## `qmk pyformat`
+
+This command formats python code in `qmk_firmware`.
+
+**Usage**:
+
+```
+qmk pyformat
+```
+
+## `qmk pytest`
+
+This command runs the python test suite. If you make changes to python code you should ensure this runs successfully.
+
+**Usage**:
+
+```
+qmk pytest
+```