Merge pull request #6850 from drew2a/feature/ga_build_doc

Add GitHub actions for PR (documentation, pytest, coverage)

.github/workflows/coverage.yml

@@ -0,0 +1,47 @@
+name: Coverage
+
+on:
+  pull_request:
+
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: [ '3.9' ]
+        os: [ ubuntu-latest ]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ./requirements-test.txt
+
+      - name: Run Pytest with Coverage
+        run: |
+          coverage run --source=./src/tribler/core -p -m pytest ./src/tribler/core
+          coverage run --source=./src/tribler/core -p -m pytest ./src/tribler/core/components/tunnel/tests/test_full_session --tunneltests
+          coverage combine
+          coverage xml
+
+      - name: Code Coverage Summary Report
+        uses: irongut/CodeCoverageSummary@v1.2.0
+        with:
+          filename: coverage.xml
+          format: markdown
+          badge: true
+          output: console
+
+      - uses: actions/upload-artifact@v2
+        with:
+          name: coverage_report
+          path: ./coverage.xml
+
+

.github/workflows/documentation.yml

@@ -0,0 +1,24 @@
+name: Documentation (no publish)
+
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ./doc/requirements.txt
+
+      - name: Build documentation
+        run: |
+          python -m sphinx -T -E -b html -d _build/doctrees -D language=en ./doc _build/html
+

.github/workflows/pytest.yml

@@ -0,0 +1,43 @@
+name: Pytest
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  run:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python-version: ['3.8']
+        os: [macos-10.15, windows-latest, ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Libsodium
+        if: matrix.os == 'windows-latest'
+        run: |
+          C:\msys64\usr\bin\wget.exe -q https://download.libsodium.org/libsodium/releases/libsodium-1.0.17-msvc.zip
+          7z x libsodium-1.0.17-msvc.zip
+          copy ./x64/Release/v141/dynamic/libsodium.dll C:\Windows\system32\
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ./requirements-test.txt
+
+      - name: Run Pytest
+        run: |
+          pytest ./src/tribler/core
+
+      - name: Run Tunnels Tests
+        run: |
+          pytest ./src/tribler/core/components/tunnel/tests/test_full_session --tunneltests

.github/workflows/upload_coverage.yml

@@ -0,0 +1,27 @@
+# See: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+name: Upload Coverage
+
+on:
+  workflow_run:
+    workflows: [ 'Coverage' ]
+    types:
+      - completed
+
+jobs:
+  upload:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+
+    steps:
+      - name: Download artifact
+      - uses: actions/download-artifact@v3
+        with:
+          name: coverage_report
+
+      - name: Upload a Coverage Report
+        uses: paambaati/codeclimate-action@v3.0.0
+        env:
+          CC_TEST_REPORTER_ID: ${{secrets.CODECLIMATE_TEST_REPORTER_ID}}
+        with:
+          coverageLocations: |
+            ./coverage.xml:cobertura

doc/development/development_on_windows.rst

@@ -5,56 +5,14 @@ Introduction
 
 In this guide, all required dependencies of Tribler will be explained. It presents how to install these dependencies. Some dependencies have to be built from source whereas other dependencies can be installed using a .msi or .exe installer. The guide targets Windows 7 or higher, 64-bit systems, however, it is probably not very hard to install 32-bit packages.
 
-First, Python 3 should be installed. If you already have a Python version installed, please check whether this version is 64 bit before proceeding.
-
-.. code-block:: bash
-
-    python -c "import struct;print( 8 * struct.calcsize('P'))"
-
-This outputs whether your current installation is 32 or 64 bit.
-
-Python can be downloaded from the official `Python website <https://www.python.org/downloads/>`_. You should download the Windows x86-64 MSI Installer which is an executable. **During the setup, remember to add Python to the PATH variable to access Python from the command line. The option to add Python to the PATH variable is unchecked by default!** You can verify whether Python is installed correctly by typing ``python`` in the command line. If they are not working, verify whether the PATH variables are correctly set. Instructions on how to set path variable can be found `here <http://www.computerhope.com/issues/ch000549.htm>`__.
-
-In order to compile some of the dependencies of Tribler, you will need the Visual Studio installed which can be downloaded from `here <https://www.visualstudio.com/downloads/download-visual-studio-vs>`__ or `here <https://imagine.microsoft.com/en-us/Catalog/Product/101>`__. You should select the community edition. Visual Studio ships with a command line interface and all required tools that are used for building some of the Python packages. After the installation of Visual Studio, you should install the Visual C++ tools. This can be done from within Visual Studio by creating a new Visual C++ project. Visual Studio then gives an option to install the Visual C++ developer tools.
-
-In case importing one of the modules fail due to a DLL error, you can inspect if there are files missing by opening it with `Dependency Walker <www.dependencywalker.com>`_. It should show missing dependencies.
-
-libtorrent
-----------
-
-First, install Boost which can be downloaded from `SourceForge <http://sourceforge.net/projects/boost/files/boost-binaries/>`__. Make sure to select the latest version and choose the version is compatible with your version of Visual C++ tools (probably msvc-14).
-
-After installation, you should set an environment variable to let libtorrent know where Boost can be found. You can do this by going to Control Panel > System > Advanced > Environment Variables (more information about setting environment variables can be found `here <http://www.computerhope.com/issues/ch000549.htm>`__). Now add a variable named BOOST_ROOT and with the value of your Boost location. The default installation location for the Boost libraries is ``C:\\local\\boost_<BOOST VERSION>`` where ``<BOOST VERSION>`` indicates the installed Boost version.
-
-Next, you should build Boost.build. You can do this by opening the Visual Studio command prompt and navigating to your Boost libraries. Navigate to ``tools\\build`` and execute ``bootstrap.bat``. This will create the ``b2.exe`` file. In order to invoke ``b2`` from anywhere in your command line, you should add the Boost directory to your user PATH environment variable. After modifying your PATH, you should reopen your command prompt.
-
-Now, download the libtorrent source code from `GitHub <https://github.com/arvidn/libtorrent/releases>`__ and extract it. It is advised to compile version 1.0.8. Note that you if you have a 32-bit system, you can download the ``.msi`` installer so you do not have to compile libtorrent yourself. Open the Developer Command Prompt shipped with Visual Studio (not the regular command prompt) and navigate to the location where you extracted the libtorrent source. In the directory where the libtorrent source code is located, navigate to ``bindings\\python`` and build libtorrent by executing the following command (this takes a while so make sure to grab a coffee while waiting):
-
-.. code-block:: bash
-
-    b2 boost=source libtorrent-link=static address-model=64
-
-This command will build a static libtorrent 64-bit debug binary. You can also build a release binary by appending ``release`` to the command given above. After the build has been completed, the resulting ``libtorrent.pyd`` can be found in ``LIBTORRENT_SOURCE\\bindings\\python\\bin\\msvc-14\\debug\\address-model-64\\boost-source\\link-static\\`` where ``LIBTORRENT_SOURCE`` indicates the directory with the libtorrent source files. Copy ``libtorrent.pyd`` to your site-packages location (the default location is ``C:\\Python37\\Lib\\site-packages``)
-
-After successfully copying the ``libtorrent.pyd`` file either compiled or from the repository, you can check if the installation was successful:
-
-.. code-block:: bash
-
-    python -c "import libtorrent" # this should work without any error
-
-libsodium
----------
-
-Libsodium is required for the ``libnacl`` library, used for cryptographic operations. Libsodium can be download as precompiled binary from `their website <https://download.libsodium.org/libsodium/releases/>`__. Download the latest version, built with msvc. Extract the archive to any location on your machine. Next, you should add the location of the dynamic library to your ``PATH`` variables (either as system variable or as user variable). These library files can be found in ``LIBSODIUM_ROOT\\x64\\Release\\v142\\dynamic\\`` where ``LIBSODIUM_ROOT`` is the location of your extracted libsodium files. After modifying your PATH, you should reopen your command prompt. You test whether Python is able to load ``libsodium.dll`` by executing:
-
-.. code-block:: bash
-
-    python -c "import ctypes; ctypes.cdll.LoadLibrary('libsodium')"
-
-Note that this might fail on Python 3.8, since directories have to be explicitly whitelisted to load DLLs from them. You can either copy the ``libsodium.dll`` to your ``System32`` directory or by whitelisting that directory using ``os.add_dll_directory`` when running Tribler.
+Prerequisites
+------------
 
+* `Python 3.8 <https://www.python.org/downloads/release/python-3813/>`_
+* `OpenSSL <https://community.chocolatey.org/packages?q=openssl>`_
+* `Libsodium <https://github.com/Tribler/py-ipv8/blob/master/doc/preliminaries/install_libsodium.rst>`_
 
-Additional Packages
+Python Packages
 -------------------
 
 There are some additional packages which should be installed. They can easily be installed using pip:

pytest.ini

@@ -8,7 +8,6 @@ markers =
     tunneltest:Slow tests for tunnels. Skipped by default, use --tunneltests option to enable them
     enable_https:Use HTTPS instead of HTTP in marked tests
     api_key:Used by rest_manager fixture to inject api_key value
-    no_parallel:Run tests that marked as "no_parallel".
 
 filterwarnings =
     ignore:Passing field metadata as a keyword arg is deprecated:DeprecationWarning:marshmallow

requirements-test.txt

@@ -3,12 +3,11 @@
 pytest==6.2.5
 pytest-aiohttp==0.3.0
 pytest-asyncio==0.16.0
-pytest-cov==3.0.0
 pytest-mock==3.6.1
 pytest-randomly==3.10.2
 pytest-timeout==2.0.1
 pytest-xdist==2.4.0
 pytest-freezegun==0.4.2
 freezegun==1.1.0
-
 asynctest==0.13.0
+coverage==6.3.2

src/tribler/core/components/bandwidth_accounting/tests/test_bandwidth_accounting_component.py

@@ -9,7 +9,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_bandwidth_accounting_component(tribler_config):
     components = [KeyComponent(), Ipv8Component(), BandwidthAccountingComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/gigachannel/tests/test_gigachannel_component.py

@@ -11,7 +11,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_giga_channel_component(tribler_config):
     tribler_config.ipv8.enabled = True
     tribler_config.libtorrent.enabled = True

src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py

@@ -13,7 +13,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_gigachannel_manager_component(tribler_config):
     components = [Ipv8Component(), TagComponent(), SocksServersComponent(), KeyComponent(), MetadataStoreComponent(),
                   LibtorrentComponent(), GigachannelManagerComponent()]

src/tribler/core/components/ipv8/tests/test_ipv8_component.py

@@ -7,7 +7,6 @@
 
 # pylint: disable=protected-access
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_ipv8_component(tribler_config):
     async with Session(tribler_config, [KeyComponent(), Ipv8Component()]).start():
         comp = Ipv8Component.instance()
@@ -20,7 +19,6 @@ async def test_ipv8_component(tribler_config):
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_ipv8_component_dht_disabled(tribler_config):
     tribler_config.ipv8.enabled = True
     tribler_config.dht.enabled = True
@@ -30,7 +28,6 @@ async def test_ipv8_component_dht_disabled(tribler_config):
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_ipv8_component_discovery_community_enabled(tribler_config):
     tribler_config.ipv8.enabled = True
     tribler_config.gui_test_mode = False

src/tribler/core/components/libtorrent/tests/test_libtorrent_component.py

@@ -8,7 +8,6 @@
 
 # pylint: disable=protected-access
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_libtorrent_component(tribler_config):
     components = [KeyComponent(), SocksServersComponent(), LibtorrentComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/metadata_store/tests/test_metadata_store_component.py

@@ -10,7 +10,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_metadata_store_component(tribler_config):
     components = [TagComponent(), Ipv8Component(), KeyComponent(), MetadataStoreComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/popularity/tests/test_popularity_component.py

@@ -14,7 +14,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_popularity_component(tribler_config):
     components = [SocksServersComponent(), LibtorrentComponent(), TorrentCheckerComponent(), TagComponent(),
                   MetadataStoreComponent(), KeyComponent(), Ipv8Component(), PopularityComponent()]

src/tribler/core/components/restapi/tests/test_restapi_component.py

@@ -19,7 +19,6 @@
 
 # pylint: disable=protected-access, not-callable, redefined-outer-name
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_rest_component(tribler_config):
     components = [KeyComponent(), RESTComponent(), Ipv8Component(), LibtorrentComponent(), ResourceMonitorComponent(),
                   BandwidthAccountingComponent(), GigaChannelComponent(), TagComponent(), SocksServersComponent(),
@@ -55,6 +54,7 @@ def rest_component():
     component.root_endpoint = MagicMock()
     return component
 
+
 @pytest.mark.asyncio
 async def test_maybe_add_check_args(rest_component, endpoint_cls):
     # test that in case `*args` in `maybe_add` function contains `NoneComponent` instance

src/tribler/core/components/socks_servers/tests/test_socks_servers_component.py

@@ -6,7 +6,6 @@
 
 # pylint: disable=protected-access
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_socks_servers_component(tribler_config):
     components = [SocksServersComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/tag/tests/test_tag_component.py

@@ -10,7 +10,6 @@
 
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_tag_component(tribler_config):
     components = [MetadataStoreComponent(), KeyComponent(), Ipv8Component(), TagComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/torrent_checker/tests/test_torrent_checker_component.py

@@ -12,7 +12,6 @@
 
 # pylint: disable=protected-access
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_torrent_checker_component(tribler_config):
     components = [SocksServersComponent(), LibtorrentComponent(), KeyComponent(),
                   Ipv8Component(), TagComponent(), MetadataStoreComponent(), TorrentCheckerComponent()]

src/tribler/core/components/tunnel/tests/test_tunnel_component.py

@@ -9,7 +9,6 @@
 # pylint: disable=protected-access
 
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_tunnels_component(tribler_config):
     components = [Ipv8Component(), KeyComponent(), TunnelsComponent()]
     async with Session(tribler_config, components).start():

src/tribler/core/components/watch_folder/tests/test_watch_folder_component.py

@@ -9,7 +9,6 @@
 
 # pylint: disable=protected-access
 @pytest.mark.asyncio
-@pytest.mark.no_parallel
 async def test_watch_folder_component(tribler_config):
     components = [KeyComponent(), SocksServersComponent(), LibtorrentComponent(), WatchFolderComponent()]
     async with Session(tribler_config, components).start():