diff --git a/komodo/check_unused_package.py b/komodo/check_unused_package.py
index 6f3858cb..5be15428 100644
--- a/komodo/check_unused_package.py
+++ b/komodo/check_unused_package.py
@@ -20,41 +20,69 @@ class CheckResult:
     exitcode: int
 
 
+class NoSuchPackageStatus(ValueError):
+    pass
+
+
 def check_for_unused_package(
     release_file: ReleaseFile,
     package_status: dict[str, Any],
     repository: RepositoryFile,
     builtin_python_versions: dict[str, str],
 ) -> CheckResult:
-    public_and_plugin_packages = [
-        (pkg, version)
-        for pkg, version in release_file.content.items()
-        if package_status[pkg]["visibility"] in ("public", "private-plugin")
-    ]
-    python_version = release_file.content["python"]
-    full_python_version = builtin_python_versions[python_version]
-
-    dependencies = PypiDependencies(
-        release_file.content, release_file.content, python_version=full_python_version
-    )
-    for name, version in release_file.content.items():
-        metadata = repository.content.get(name, {}).get(version, {})
-        if metadata.get("source") != "pypi":
-            dependencies.add_user_specified(name, metadata.get("depends", []))
-    unused_private_packages = {
-        pkg
-        for pkg in release_file.content
-        if package_status[pkg]["visibility"] == "private"
-    }.difference(dependencies.used_packages(public_and_plugin_packages))
-    if unused_private_packages:
-        return CheckResult(
-            message=f"The following {len(unused_private_packages)} private packages are not dependencies of any public or private-plugin packages:"
-            + ", ".join(sorted(unused_private_packages))
-            + "If you have added or removed any packages check that the dependencies in repository.yml are correct.",
-            exitcode=1,
+    def get_visibility(pkg):
+        try:
+            status = package_status[pkg]
+        except KeyError as err:
+            raise NoSuchPackageStatus(
+                f"Could not find {pkg} in the package status file"
+            ) from err
+        try:
+            return status["visibility"]
+        except KeyError as err:
+            raise NoSuchPackageStatus(
+                f"{pkg} does not have visibility field in package status file"
+            ) from err
+
+    try:
+        public_and_plugin_packages = [
+            (pkg, version)
+            for pkg, version in release_file.content.items()
+            if get_visibility(pkg) in ("public", "private-plugin")
+        ]
+
+        python_version = release_file.content["python"]
+        full_python_version = builtin_python_versions[python_version]
+
+        dependencies = PypiDependencies(
+            release_file.content,
+            release_file.content,
+            python_version=full_python_version,
         )
-    else:
-        return CheckResult(message="Everything seems fine.", exitcode=0)
+        for name, version in release_file.content.items():
+            try:
+                metadata = repository.content[name][version]
+            except KeyError:
+                return CheckResult(
+                    message=f"Could not find package-version: {name}:{version} in the repository file",
+                    exitcode=3,
+                )
+            if metadata.get("source") != "pypi":
+                dependencies.add_user_specified(name, metadata.get("depends", []))
+        unused_private_packages = {
+            pkg for pkg in release_file.content if get_visibility(pkg) == "private"
+        }.difference(dependencies.used_packages(public_and_plugin_packages))
+        if unused_private_packages:
+            return CheckResult(
+                message=f"The following {len(unused_private_packages)} private packages are not dependencies of any public or private-plugin packages:"
+                + ", ".join(sorted(unused_private_packages))
+                + "If you have added or removed any packages check that the dependencies in repository.yml are correct.",
+                exitcode=1,
+            )
+        else:
+            return CheckResult(message="Everything seems fine.", exitcode=0)
+    except NoSuchPackageStatus as err:
+        return CheckResult(message=str(err), exitcode=2)
 
 
 def main():
diff --git a/tests/test_check_unused_package.py b/tests/test_check_unused_package.py
index d770e3a4..d2951614 100644
--- a/tests/test_check_unused_package.py
+++ b/tests/test_check_unused_package.py
@@ -41,7 +41,7 @@
                 },
             },
             "package_f": {
-                "1.0": {
+                "1.0.0": {
                     "make": "pip",
                     "maintainer": "scout",
                     "depends": ["package_c"],
@@ -82,6 +82,7 @@
 def test_check_unused_package(repo, release, package_status):
     package_status["python"] = {"visibility": "public"}
     release["python"] = "3.8-builtin"
+    repo["python"] = {"3.8-builtin": {"maintainer": "me", "make": "sh"}}
 
     repo = RepositoryFile.from_dictionary(value=repo)
     release = ReleaseFile.from_dictionary(value=release)
@@ -93,3 +94,70 @@ def test_check_unused_package(repo, release, package_status):
     )
     assert result.exitcode == 1
     assert "The following 1" in result.message and "package_f" in result.message
+
+
+def has_unused_packages(repo, release, package_status):
+    package_status["python"] = {"visibility": "public"}
+    release["python"] = "3.8-builtin"
+    repo["python"] = {"3.8-builtin": {"maintainer": "me", "make": "sh"}}
+
+    return check_for_unused_package(
+        release_file=ReleaseFile.from_dictionary(release),
+        package_status=package_status,
+        repository=RepositoryFile.from_dictionary(repo),
+        builtin_python_versions={"3.8-builtin": "3.8.6"},
+    )
+
+
+def test_empty_release_has_no_unused_packages():
+    assert has_unused_packages({}, {}, {}).exitcode == 0
+
+
+def test_missing_package_status_gives_error_code_2():
+    assert has_unused_packages({}, {"package": "version"}, {}).exitcode == 2
+
+
+def test_missing_visibility_gives_error_code_2():
+    assert (
+        has_unused_packages({}, {"package": "version"}, {"package": {}}).exitcode == 2
+    )
+
+
+def test_missing_repo_gives_error_code_3():
+    assert (
+        has_unused_packages(
+            {}, {"package": "version"}, {"package": {"visibility": "public"}}
+        ).exitcode
+        == 3
+    )
+
+
+def test_private_unused_package_gives_error_code_1():
+    assert (
+        has_unused_packages(
+            {
+                "package": {
+                    "1.0": {"source": "github", "make": "pip", "maintainer": "me"}
+                }
+            },
+            {"package": "1.0"},
+            {"package": {"visibility": "private"}},
+        ).exitcode
+        == 1
+    )
+
+
+@pytest.mark.parametrize("public_type", ("public", "private-plugin"))
+def test_public_packages_are_used(public_type):
+    assert (
+        has_unused_packages(
+            {
+                "package": {
+                    "1.0": {"source": "github", "make": "pip", "maintainer": "me"}
+                }
+            },
+            {"package": "1.0"},
+            {"package": {"visibility": public_type}},
+        ).exitcode
+        == 0
+    )