Merge pull request #243 from freedomofpress/241-wheels-repro-against-vc

Tests wheel reproducibility against version control

.circleci/config.yml

@@ -226,7 +226,7 @@ jobs:
           command: |
             make install-deps
             make lint-desktop-files
-            virtualenv -p python3 .venv
+            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
             pip install -r test-requirements.txt
             sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
@@ -241,7 +241,7 @@ jobs:
           name: install test requirements and run tests
           command: |
             make install-deps
-            virtualenv -p python3 .venv
+            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
             pip install -r test-requirements.txt
             sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
@@ -256,7 +256,7 @@ jobs:
           name: install test requirements and run tests
           command: |
             make install-deps
-            virtualenv -p python3 .venv
+            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
             pip install -r test-requirements.txt
             # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.

tests/test_reproducible_wheels.py

@@ -17,6 +17,7 @@ def get_repo_root():
     return top_level
 
 
+@pytest.mark.skip(reason="Comparing to version control is sufficient")
 @pytest.mark.parametrize("repo_name", REPOS_WITH_WHEELS)
 def test_wheel_builds_are_reproducible(repo_name):
     """
@@ -45,3 +46,15 @@ def test_wheel_builds_are_reproducible(repo_name):
     ]
     repo_root = get_repo_root()
     subprocess.check_call(cmd, env=cmd_env, cwd=repo_root)
+
+
+@pytest.mark.parametrize("repo_name", REPOS_WITH_WHEELS)
+def test_wheel_builds_match_version_control(repo_name):
+    repo_url = f"https://github.com/freedomofpress/{repo_name}"
+    build_cmd = f"./scripts/build-sync-wheels -p {repo_url} --clobber".split()
+    subprocess.check_call(build_cmd)
+    # Check for modified files (won't catch new, untracked files)
+    subprocess.check_call("git diff --exit-code".split())
+    # Check for new, untracked files. Test will fail if working copy is dirty
+    # in any way, so mostly useful in CI.
+    assert subprocess.check_output("git status --porcelain".split()) == b""