diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000000..c765e471155
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,7 @@
+[run]
+branch = True
+
+[paths]
+source = 
+    torchvision
+    /**/site-packages/torchvision
diff --git a/.gitignore b/.gitignore
index 2abd33a8556..c269b275ac2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,5 @@ torchvision.egg-info/
 */**/*~
 *~
 docs/build
+.coverage
+htmlcov
diff --git a/.travis.yml b/.travis.yml
index 15b2f3b14ab..77c4b0b1bba 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ matrix:
       python: "2.7"
       install: pip install flake8
       script: flake8
+      after_success: []
     - python: "2.7"
       env: IMAGE_BACKEND=Pillow-SIMD
     - python: "2.7"
@@ -13,7 +14,7 @@ matrix:
       env: IMAGE_BACKEND=Pillow-SIMD
     - python: "3.5"
 
-install:
+before_install:
   - sudo apt-get update
   - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh;
   - bash miniconda.sh -b -p $HOME/miniconda
@@ -26,11 +27,34 @@ install:
 
   - conda create -q -n test-environment python=$TRAVIS_PYTHON_VERSION pytorch scipy -c pytorch
   - source activate test-environment
-  - python setup.py install
-  - pip install --upgrade pytest
-  - if [[ "$IMAGE_BACKEND" == "Pillow-SIMD" ]]; then
-      pip uninstall -y pillow && CC="cc -march=native" pip install --force-reinstall pillow-simd;
+  - |
+    if [[ "$IMAGE_BACKEND" == "Pillow-SIMD" ]]; then
+      pip uninstall -y pillow && CC="cc -march=native" pip install --force-reinstall pillow-simd
     fi
+  - pip install pytest pytest-cov codecov
+
+
+install:
+  # Using pip instead of setup.py ensures we install a non-compressed version of the package
+  # (as opposed to an egg), which is necessary to collect coverage.
+  # We still get the benefit of testing an installed version over the
+  # test version to iron out installation file-inclusion bugs but can
+  # also collect coverage.
+  - pip install .
+  # Move to home dir, otherwise we'll end up with the path to the
+  # package in $PWD rather than the installed v
+  - |
+    cd $HOME
+    export TV_INSTALL_PATH="$(python -c 'import os; import torchvision; print(os.path.dirname(os.path.abspath(torchvision.__file__)))')"
+    echo "$TV_INSTALL_PATH"
+    cd -
+
 script:
-  - pytest test/
+  - pytest --cov-config .coveragerc --cov torchvision --cov $TV_INSTALL_PATH test
 
+after_success:
+  # Necessary to run coverage combine to rewrite paths from
+  # /travis/env/path/site-packages/torchvision to actual path
+  - coverage combine .coverage
+  - coverage report
+  - codecov
diff --git a/README.rst b/README.rst
index 3f2be535d7a..c35b100ad02 100644
--- a/README.rst
+++ b/README.rst
@@ -4,6 +4,9 @@ torchvision
 .. image:: https://travis-ci.org/pytorch/vision.svg?branch=master
     :target: https://travis-ci.org/pytorch/vision
 
+.. image:: https://codecov.io/gh/pytorch/vision/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/pytorch/vision
+
 The torchvision package consists of popular datasets, model architectures, and common image transformations for computer vision.
 
 Installation