diff --git a/checks/normandy/remotesettings_recipes.py b/checks/normandy/remotesettings_recipes.py index 3b095c46..423609b0 100644 --- a/checks/normandy/remotesettings_recipes.py +++ b/checks/normandy/remotesettings_recipes.py @@ -1,35 +1,75 @@ """ -The recipes in the Remote Settings collection should match the Normandy API. -The lists of missing and extraneous recipes are returned. +The recipes in the Remote Settings collection should match the Normandy API. The +collection of recipes with capabilities should contain all baseline recipes. + +The lists of missing and extraneous recipes are returned, as well the list of +inconsistencies between the baseline and capabilities collections. """ from poucave.typings import CheckResult from poucave.utils import fetch_json -NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1" -REMOTESETTINGS_URL = "{server}/buckets/main/collections/normandy-recipes/records" +NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1&only_baseline_capabilities={baseline_only}" +REMOTESETTINGS_URL = "{server}/buckets/main/collections/{cid}/records" -async def run(normandy_server: str, remotesettings_server: str) -> CheckResult: - # Recipes from source of truth. - normandy_url = NORMANDY_URL.format(server=normandy_server) - normandy_recipes = await fetch_json(normandy_url) +def compare_recipes_lists(a, b): + """ + Return list of recipes present in `a` and missing in `b`, and present in `b` and missing in `a`. + """ + a_by_id = {r["recipe"]["id"]: r["recipe"] for r in a} + b_by_id = {r["recipe"]["id"]: r["recipe"] for r in b} + missing = [] + for rid, r in a_by_id.items(): + r_in_b = b_by_id.pop(rid, None) + if r_in_b is None: + missing.append({"id": r["id"], "name": r["name"]}) + extras = [{"id": r["id"], "name": r["name"]} for r in b_by_id.values()] + return missing, extras - # Recipes published on Remote Settings. - remotesettings_url = REMOTESETTINGS_URL.format(server=remotesettings_server) - body = await fetch_json(remotesettings_url) - remotesettings_recipes = body["data"] - remotesettings_by_id = { - r["recipe"]["id"]: r["recipe"] for r in remotesettings_recipes - } - normandy_by_id = {r["recipe"]["id"]: r["recipe"] for r in normandy_recipes} +async def run(normandy_server: str, remotesettings_server: str) -> CheckResult: + # Baseline recipes from source of truth. + normandy_url_baseline = NORMANDY_URL.format(server=normandy_server, baseline_only=1) + normandy_recipes_baseline = await fetch_json(normandy_url_baseline) + # Recipes with capabilities + normandy_url_caps = NORMANDY_URL.format(server=normandy_server, baseline_only=0) + normandy_recipes_caps = await fetch_json(normandy_url_caps) - missing = [] - for rid, r in normandy_by_id.items(): - published = remotesettings_by_id.pop(rid, None) - if published is None: - missing.append({"id": r["id"], "name": r["name"]}) - extras = [{"id": r["id"], "name": r["name"]} for r in remotesettings_by_id.values()] + # Baseline recipes published on Remote Settings. + rs_recipes_baseline_url = REMOTESETTINGS_URL.format( + server=remotesettings_server, cid="normandy-recipes" + ) + rs_recipes_baseline = (await fetch_json(rs_recipes_baseline_url))["data"] + # Recipes with advanced capabilities. + rs_recipes_caps_urls = REMOTESETTINGS_URL.format( + server=remotesettings_server, cid="normandy-recipes-capabilities" + ) + rs_recipes_caps = (await fetch_json(rs_recipes_caps_urls))["data"] - ok = (len(missing) + len(extras)) == 0 - return ok, {"missing": missing, "extras": extras} + # Make sure the baseline recipes are all listed in the baseline collection + missing_baseline, extras_baseline = compare_recipes_lists( + normandy_recipes_baseline, rs_recipes_baseline + ) + # Make sure the baseline recipes are all listed in the baseline collection + missing_caps, extras_caps = compare_recipes_lists( + normandy_recipes_caps, rs_recipes_caps + ) + # Make sure the baseline recipes are all listed in the capabilities collection. + inconsistent, _ = compare_recipes_lists(rs_recipes_baseline, rs_recipes_caps) + + ok = ( + len(missing_baseline) + + len(missing_caps) + + len(extras_baseline) + + len(extras_caps) + + len(inconsistent) + ) == 0 + data = { + "baseline": {"missing": missing_baseline, "extras": extras_baseline}, + "capabilities": { + "missing": missing_caps, + "extras": extras_caps, + "inconsistent": inconsistent, + }, + } + return ok, data diff --git a/tests/checks/normandy/test_remotesettings_recipes.py b/tests/checks/normandy/test_remotesettings_recipes.py index 61454a50..0d9b16cc 100644 --- a/tests/checks/normandy/test_remotesettings_recipes.py +++ b/tests/checks/normandy/test_remotesettings_recipes.py @@ -2,9 +2,13 @@ NORMANDY_SERVER = "http://n" REMOTESETTINGS_SERVER = "http://rs/v1" -REMOTESETTINGS_URL = ( +REMOTESETTINGS_BASELINE_URL = ( REMOTESETTINGS_SERVER + "/buckets/main/collections/normandy-recipes/records" ) +REMOTESETTINGS_CAPABILITIES_URL = ( + REMOTESETTINGS_SERVER + + "/buckets/main/collections/normandy-recipes-capabilities/records" +) NORMANDY_RECIPE = { "signature": { @@ -41,32 +45,70 @@ }, } +REMOTESETTINGS_RECIPE_WITH_CAPS = { + "id": "314", + "recipe": { + "id": 314, + "name": f"With caps", + "capabilities": ["action.preference-experiment"], + }, +} + async def test_positive(mock_aioresponses): mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[NORMANDY_RECIPE] + NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0), + payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS], + ) + mock_aioresponses.get( + NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1), + payload=[NORMANDY_RECIPE], + ) + mock_aioresponses.get( + REMOTESETTINGS_BASELINE_URL, payload={"data": [REMOTESETTINGS_RECIPE]} + ) + mock_aioresponses.get( + REMOTESETTINGS_CAPABILITIES_URL, + payload={"data": [REMOTESETTINGS_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS]}, ) - mock_aioresponses.get(REMOTESETTINGS_URL, payload={"data": [REMOTESETTINGS_RECIPE]}) status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER) assert status is True - assert data == {"missing": [], "extras": []} + assert data == { + "baseline": {"missing": [], "extras": []}, + "capabilities": {"missing": [], "extras": [], "inconsistent": []}, + } async def test_negative(mock_aioresponses): mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), payload=[NORMANDY_RECIPE] + NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0), + payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS], + ) + mock_aioresponses.get( + NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1), + payload=[NORMANDY_RECIPE], ) mock_aioresponses.get( - REMOTESETTINGS_URL, + REMOTESETTINGS_BASELINE_URL, payload={"data": [{"id": "42", "recipe": {"id": 42, "name": "Extra"}}]}, ) - + mock_aioresponses.get( + REMOTESETTINGS_CAPABILITIES_URL, + payload={"data": [REMOTESETTINGS_RECIPE_WITH_CAPS]}, + ) status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER) assert status is False assert data == { - "missing": [{"id": 829, "name": "Mobile Browser usage"}], - "extras": [{"id": 42, "name": "Extra"}], + "baseline": { + "missing": [{"id": 829, "name": "Mobile Browser usage"}], + "extras": [{"id": 42, "name": "Extra"}], + }, + "capabilities": { + "missing": [{"id": 829, "name": "Mobile Browser usage"}], + "extras": [], + "inconsistent": [{"id": 42, "name": "Extra"}], + }, }