diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
new file mode 100644
index 000000000..f3ca0a374
--- /dev/null
+++ b/.github/workflows/python-package.yml
@@ -0,0 +1,33 @@
+name: Python package
+
+on:
+- push
+# - pull_request
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+    # Use "python -m pytest" instead of "pytest" to fix imports
+    - name: Test metrics
+      run: |
+        python -m pytest tests/metrics
+    - name: Test evaluation_setting
+      run: |
+        python -m pytest tests/evaluation_setting
diff --git a/tests/metrics/test_loss_metrics.py b/tests/metrics/test_loss_metrics.py
index 399e03ff8..aaf63333c 100644
--- a/tests/metrics/test_loss_metrics.py
+++ b/tests/metrics/test_loss_metrics.py
@@ -21,7 +21,7 @@ def get_result(name, case=0):
                 getattr(TestCases, f'preds_{case}'))
 
 
-class TestLossMetrics(unittest.TestCases):
+class TestLossMetrics(unittest.TestCase):
     def test_auc(self):
         name = 'auc'
         self.assertEqual(get_result(name, case=0), 0)