Fix #105: Include checks about normandy-recipes-capabilities (#136)

checks/normandy/remotesettings_recipes.py

@@ -1,35 +1,75 @@
 """
-The recipes in the Remote Settings collection should match the Normandy API.
-The lists of missing and extraneous recipes are returned.
+The recipes in the Remote Settings collection should match the Normandy API. The
+collection of recipes with capabilities should contain all baseline recipes.
+
+The lists of missing and extraneous recipes are returned, as well the list of
+inconsistencies between the baseline and capabilities collections.
 """
 from poucave.typings import CheckResult
 from poucave.utils import fetch_json
 
-NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1"
-REMOTESETTINGS_URL = "{server}/buckets/main/collections/normandy-recipes/records"
+NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1&only_baseline_capabilities={baseline_only}"
+REMOTESETTINGS_URL = "{server}/buckets/main/collections/{cid}/records"
 
 
-async def run(normandy_server: str, remotesettings_server: str) -> CheckResult:
-    # Recipes from source of truth.
-    normandy_url = NORMANDY_URL.format(server=normandy_server)
-    normandy_recipes = await fetch_json(normandy_url)
+def compare_recipes_lists(a, b):
+    """
+    Return list of recipes present in `a` and missing in `b`, and present in `b` and missing in `a`.
+    """
+    a_by_id = {r["recipe"]["id"]: r["recipe"] for r in a}
+    b_by_id = {r["recipe"]["id"]: r["recipe"] for r in b}
+    missing = []
+    for rid, r in a_by_id.items():
+        r_in_b = b_by_id.pop(rid, None)
+        if r_in_b is None:
+            missing.append({"id": r["id"], "name": r["name"]})
+    extras = [{"id": r["id"], "name": r["name"]} for r in b_by_id.values()]
+    return missing, extras
 
-    # Recipes published on Remote Settings.
-    remotesettings_url = REMOTESETTINGS_URL.format(server=remotesettings_server)
-    body = await fetch_json(remotesettings_url)
-    remotesettings_recipes = body["data"]
 
-    remotesettings_by_id = {
-        r["recipe"]["id"]: r["recipe"] for r in remotesettings_recipes
-    }
-    normandy_by_id = {r["recipe"]["id"]: r["recipe"] for r in normandy_recipes}
+async def run(normandy_server: str, remotesettings_server: str) -> CheckResult:
+    # Baseline recipes from source of truth.
+    normandy_url_baseline = NORMANDY_URL.format(server=normandy_server, baseline_only=1)
+    normandy_recipes_baseline = await fetch_json(normandy_url_baseline)
+    # Recipes with capabilities
+    normandy_url_caps = NORMANDY_URL.format(server=normandy_server, baseline_only=0)
+    normandy_recipes_caps = await fetch_json(normandy_url_caps)
 
-    missing = []
-    for rid, r in normandy_by_id.items():
-        published = remotesettings_by_id.pop(rid, None)
-        if published is None:
-            missing.append({"id": r["id"], "name": r["name"]})
-    extras = [{"id": r["id"], "name": r["name"]} for r in remotesettings_by_id.values()]
+    # Baseline recipes published on Remote Settings.
+    rs_recipes_baseline_url = REMOTESETTINGS_URL.format(
+        server=remotesettings_server, cid="normandy-recipes"
+    )
+    rs_recipes_baseline = (await fetch_json(rs_recipes_baseline_url))["data"]
+    # Recipes with advanced capabilities.
+    rs_recipes_caps_urls = REMOTESETTINGS_URL.format(
+        server=remotesettings_server, cid="normandy-recipes-capabilities"
+    )
+    rs_recipes_caps = (await fetch_json(rs_recipes_caps_urls))["data"]
 
-    ok = (len(missing) + len(extras)) == 0
-    return ok, {"missing": missing, "extras": extras}
+    # Make sure the baseline recipes are all listed in the baseline collection
+    missing_baseline, extras_baseline = compare_recipes_lists(
+        normandy_recipes_baseline, rs_recipes_baseline
+    )
+    # Make sure the baseline recipes are all listed in the baseline collection
+    missing_caps, extras_caps = compare_recipes_lists(
+        normandy_recipes_caps, rs_recipes_caps
+    )
+    # Make sure the baseline recipes are all listed in the capabilities collection.
+    inconsistent, _ = compare_recipes_lists(rs_recipes_baseline, rs_recipes_caps)
+
+    ok = (
+        len(missing_baseline)
+        + len(missing_caps)
+        + len(extras_baseline)
+        + len(extras_caps)
+        + len(inconsistent)
+    ) == 0
+    data = {
+        "baseline": {"missing": missing_baseline, "extras": extras_baseline},
+        "capabilities": {
+            "missing": missing_caps,
+            "extras": extras_caps,
+            "inconsistent": inconsistent,
+        },
+    }
+    return ok, data

tests/checks/normandy/test_remotesettings_recipes.py

@@ -2,9 +2,13 @@
 
 NORMANDY_SERVER = "http://n"
 REMOTESETTINGS_SERVER = "http://rs/v1"
-REMOTESETTINGS_URL = (
+REMOTESETTINGS_BASELINE_URL = (
     REMOTESETTINGS_SERVER + "/buckets/main/collections/normandy-recipes/records"
 )
+REMOTESETTINGS_CAPABILITIES_URL = (
+    REMOTESETTINGS_SERVER
+    + "/buckets/main/collections/normandy-recipes-capabilities/records"
+)
 
 NORMANDY_RECIPE = {
     "signature": {
@@ -41,32 +45,70 @@
     },
 }
 
+REMOTESETTINGS_RECIPE_WITH_CAPS = {
+    "id": "314",
+    "recipe": {
+        "id": 314,
+        "name": f"With caps",
+        "capabilities": ["action.preference-experiment"],
+    },
+}
+
 
 async def test_positive(mock_aioresponses):
     mock_aioresponses.get(
-        NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[NORMANDY_RECIPE]
+        NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0),
+        payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS],
+    )
+    mock_aioresponses.get(
+        NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1),
+        payload=[NORMANDY_RECIPE],
+    )
+    mock_aioresponses.get(
+        REMOTESETTINGS_BASELINE_URL, payload={"data": [REMOTESETTINGS_RECIPE]}
+    )
+    mock_aioresponses.get(
+        REMOTESETTINGS_CAPABILITIES_URL,
+        payload={"data": [REMOTESETTINGS_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS]},
     )
-    mock_aioresponses.get(REMOTESETTINGS_URL, payload={"data": [REMOTESETTINGS_RECIPE]})
 
     status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER)
 
     assert status is True
-    assert data == {"missing": [], "extras": []}
+    assert data == {
+        "baseline": {"missing": [], "extras": []},
+        "capabilities": {"missing": [], "extras": [], "inconsistent": []},
+    }
 
 
 async def test_negative(mock_aioresponses):
     mock_aioresponses.get(
-        NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[NORMANDY_RECIPE]
+        NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0),
+        payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS],
+    )
+    mock_aioresponses.get(
+        NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1),
+        payload=[NORMANDY_RECIPE],
     )
     mock_aioresponses.get(
-        REMOTESETTINGS_URL,
+        REMOTESETTINGS_BASELINE_URL,
         payload={"data": [{"id": "42", "recipe": {"id": 42, "name": "Extra"}}]},
     )
-
+    mock_aioresponses.get(
+        REMOTESETTINGS_CAPABILITIES_URL,
+        payload={"data": [REMOTESETTINGS_RECIPE_WITH_CAPS]},
+    )
     status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER)
 
     assert status is False
     assert data == {
-        "missing": [{"id": 829, "name": "Mobile Browser usage"}],
-        "extras": [{"id": 42, "name": "Extra"}],
+        "baseline": {
+            "missing": [{"id": 829, "name": "Mobile Browser usage"}],
+            "extras": [{"id": 42, "name": "Extra"}],
+        },
+        "capabilities": {
+            "missing": [{"id": 829, "name": "Mobile Browser usage"}],
+            "extras": [],
+            "inconsistent": [{"id": 42, "name": "Extra"}],
+        },
     }