make tests independent from each other (#317)

test/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

test/README.md

@@ -1,15 +1,37 @@
-## Integration tests
+# Integration tests
 
 This dir contains integration tests, the aim is to test the Command Line Interface and its output
 from a pure user point of view.
 
-### Installation
+## Installation
 
-    cd test
-    virtualenv --python=python3 venv
-    source venv/bin/activate
-    pip install -r requirements.txt
-
-### Running tests
+See also [Contributing][0].
 
-    pytest
+```shell
+cd test
+virtualenv --python=python3 venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+## Running tests
+
+To run all the tests:
+
+```shell
+pytest
+```
+
+To run specific modules:
+
+```shell
+pytest test_lib.py
+```
+
+To run very specific test functions:
+
+```shell
+pytest test_lib.py::test_list
+```
+
+[0]: ../CONTRIBUTING.md

test/common.py

@@ -0,0 +1,23 @@
+# This file is part of arduino-cli.
+
+# Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+
+# This software is released under the GNU General Public License version 3,
+# which covers the main part of arduino-cli.
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to modify or
+# otherwise use the software for commercial activities involving the Arduino
+# software without disclosing the source code of your own applications. To purchase
+# a commercial license, send an email to license@arduino.cc.
+import os
+
+
+def running_on_ci():
+    """
+    Returns whether the program is running on a CI environment
+    """
+    val = os.getenv("APPVEYOR") or os.getenv("DRONE")
+    return val is not None

test/conftest.py

@@ -18,29 +18,39 @@
 from invoke import run
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def data_dir(tmpdir_factory):
     """
     A tmp folder will be created before running
-    the tests and deleted at the end.
+    each test and deleted at the end, this way all the
+    tests work in isolation.
     """
-    return str(tmpdir_factory.mktemp('ArduinoTest'))
+    return str(tmpdir_factory.mktemp("ArduinoTest"))
 
 
 @pytest.fixture(scope="session")
-def run_command(data_dir):
+def downloads_dir(tmpdir_factory):
+    """
+    To save time and bandwidth, all the tests will access
+    the same download cache folder.
+    """
+    return str(tmpdir_factory.mktemp("ArduinoTest"))
+
+
+@pytest.fixture(scope="function")
+def run_command(data_dir, downloads_dir):
     """
     Provide a wrapper around invoke's `run` API so that every test
     will work in the same temporary folder.
 
     Useful reference:
         http://docs.pyinvoke.org/en/1.2/api/runners.html#invoke.runners.Result
     """
-    cli_path = os.path.join(pytest.config.rootdir, '..', 'arduino-cli')
+    cli_path = os.path.join(pytest.config.rootdir, "..", "arduino-cli")
     env = {
-            "ARDUINO_DATA_DIR": data_dir,
-	        "ARDUINO_DOWNLOADS_DIR": data_dir,
-	        "ARDUINO_SKETCHBOOK_DIR": data_dir
+        "ARDUINO_DATA_DIR": data_dir,
+        "ARDUINO_DOWNLOADS_DIR": downloads_dir,
+        "ARDUINO_SKETCHBOOK_DIR": data_dir,
     }
 
     def _run(cmd_string):

test/requirements.txt

@@ -15,6 +15,7 @@ pylint==2.3.1
 pyparsing==2.4.0
 pytest==5.0.1
 semver==2.8.1
+simplejson==3.16.0
 six==1.12.0
 typed-ast==1.4.0
 wcwidth==0.1.7

test/test_board.py

@@ -0,0 +1,41 @@
+# This file is part of arduino-cli.
+
+# Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+
+# This software is released under the GNU General Public License version 3,
+# which covers the main part of arduino-cli.
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to modify or
+# otherwise use the software for commercial activities involving the Arduino
+# software without disclosing the source code of your own applications. To purchase
+# a commercial license, send an email to license@arduino.cc.
+import pytest
+import simplejson as json
+
+from .common import running_on_ci
+
+
+@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
+def test_board_list(run_command):
+    result = run_command("core update-index")
+    assert result.ok
+    result = run_command("board list --format json")
+    assert result.ok
+    # check is a valid json and contains a list of ports
+    ports = json.loads(result.stdout).get("ports")
+    assert isinstance(ports, list)
+    for port in ports:
+        assert "protocol" in port
+        assert "protocol_label" in port
+
+
+@pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports")
+def test_board_listall(run_command):
+    assert run_command("core update-index")
+    result = run_command("board listall")
+    print(result.stderr, result.stdout)
+    assert result.ok
+    assert ["Board", "Name", "FQBN"] == result.stdout.splitlines()[0].strip().split()

test/test_lib.py

@@ -0,0 +1,109 @@
+# This file is part of arduino-cli.
+
+# Copyright 2019 ARDUINO SA (http://www.arduino.cc/)
+
+# This software is released under the GNU General Public License version 3,
+# which covers the main part of arduino-cli.
+# The terms of this license can be found at:
+# https://www.gnu.org/licenses/gpl-3.0.en.html
+
+# You can be released from the requirements of the above licenses by purchasing
+# a commercial license. Buying such a license is mandatory if you want to modify or
+# otherwise use the software for commercial activities involving the Arduino
+# software without disclosing the source code of your own applications. To purchase
+# a commercial license, send an email to license@arduino.cc.
+import pytest
+import simplejson as json
+
+
+def test_list(run_command):
+    # Init the environment explicitly
+    assert run_command("core update-index")
+
+    # When ouput is empty, nothing is printed out, no matter the output format
+    result = run_command("lib list")
+    assert result.ok
+    assert "" == result.stderr
+    assert "" == result.stdout
+    result = run_command("lib list --format json")
+    assert result.ok
+    assert "" == result.stderr
+    assert "" == result.stdout
+
+    # Install something we can list at a version older than latest
+    result = run_command("lib install ArduinoJson@6.11.0")
+    assert result.ok
+
+    # Look at the plain text output
+    result = run_command("lib list")
+    assert result.ok
+    assert "" == result.stderr
+    lines = result.stdout.strip().splitlines()
+    assert 2 == len(lines)
+    toks = lines[1].split("\t")
+    # be sure line contain the current version AND the available version
+    assert "" != toks[1]
+    assert "" != toks[2]
+
+    # Look at the JSON output
+    result = run_command("lib list --format json")
+    assert result.ok
+    assert "" == result.stderr
+    data = json.loads(result.stdout)
+    assert 1 == len(data)
+    # be sure data contains the available version
+    assert "" != data[0]["release"]["version"]
+
+
+def test_install(run_command):
+    libs = ['"AzureIoTProtocol_MQTT"', '"CMMC MQTT Connector"', '"WiFiNINA"']
+    # Should be safe to run install multiple times
+    assert run_command("lib install {}".format(" ".join(libs)))
+    assert run_command("lib install {}".format(" ".join(libs)))
+
+
+def test_update_index(run_command):
+    result = run_command("lib update-index")
+    assert result.ok
+    assert (
+        "Updating index: library_index.json downloaded"
+        == result.stdout.splitlines()[-1].strip()
+    )
+
+
+def test_remove(run_command):
+    libs = ['"AzureIoTProtocol_MQTT"', '"CMMC MQTT Connector"', '"WiFiNINA"']
+    assert run_command("lib install {}".format(" ".join(libs)))
+
+    result = run_command("lib uninstall {}".format(" ".join(libs)))
+    assert result.ok
+
+
+@pytest.mark.slow
+def test_search(run_command):
+    result = run_command("lib search")
+    assert result.ok
+    out_lines = result.stdout.splitlines()
+    # Create an array with just the name of the vars
+    libs = []
+    for line in out_lines:
+        if line.startswith("Name: "):
+            start = line.find('"') + 1
+            libs.append(line[start:-1])
+
+    expected = {"WiFi101", "WiFi101OTA", "Firebase Arduino based on WiFi101"}
+    assert expected == {lib for lib in libs if "WiFi101" in lib}
+
+    result = run_command("lib search --format json")
+    assert result.ok
+    libs_json = json.loads(result.stdout)
+    assert len(libs) == len(libs_json.get("libraries"))
+
+    result = run_command("lib search")
+    assert result.ok
+
+    # Search for a specific target
+    result = run_command("lib search ArduinoJson --format json")
+    assert result.ok
+    libs_json = json.loads(result.stdout)
+    assert 1 == len(libs_json.get("libraries"))