diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
new file mode 100644
index 00000000..b50784bc
--- /dev/null
+++ b/.github/workflows/pytest.yaml
@@ -0,0 +1,57 @@
+
+name: Pytest
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+on:
+  push:
+    branches:
+      - main
+      - "test-ci/**"
+    paths-ignore:
+      - 'docs/**'
+      - 'README.md'
+      - 'mkdocs.yml'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - 'docs/**'
+      - 'README.md'
+      - 'mkdocs.yml'
+
+# Not possible to test windows capability:
+# https://github.com/orgs/community/discussions/25491
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - name: Install uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install dependencies
+        run: |
+            uv pip install --python ${Python_ROOT_DIR} '.'
+      - name: Install dev dependencies
+        run: |
+            uv pip install --python ${Python_ROOT_DIR} pytest pytest-cov
+      - name: Run pytest
+        uses: sjvrijn/pytest-last-failed@v2
+        with:
+          pytest-args: '--exitfirst --cov'
+      - name: Explicitly convert coverage to xml
+        run: coverage xml
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v4.0.1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: princeton-nlp/SWE-bench
\ No newline at end of file
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 00000000..f63deeea
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,16 @@
+# Configuration for codecov
+coverage:
+  status:
+    project:
+      default:
+        # If we get < 45% coverage, codecov is gonna mark it a failure
+        target: 45%
+        threshold: null
+    patch:
+      default:
+        # Codecov won't mark it as a failure if a patch is not covered well
+        informational: true
+github_checks:
+  # Don't mark lines that aren't covered
+  annotations: false
+
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 00000000..b295fb1b
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,17 @@
+import subprocess
+
+
+def test_smoke_test():
+    cmd = ["python", "-m", "swebench.harness.run_evaluation", "--help"]
+    result = subprocess.run(cmd, capture_output=True)
+    print(result.stdout)
+    print(result.stderr)
+    assert result.returncode == 0
+
+
+def test_one_instance():
+    cmd = ["python", "-m", "swebench.harness.run_evaluation", "--predictions_path",  "gold", "--max_workers", "1", "--instance_ids", "sympy__sympy-20590", "--run_id", "validate-gold"]
+    result = subprocess.run(cmd, capture_output=True)
+    print(result.stdout)
+    print(result.stderr)
+    assert result.returncode == 0
\ No newline at end of file