diff --git a/.secrets.baseline b/.secrets.baseline index 95fe16fa..33e2538f 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -26,6 +26,9 @@ { "name": "GitHubTokenDetector" }, + { + "name": "GitLabTokenDetector" + }, { "name": "HexHighEntropyString", "limit": 3.0 @@ -36,6 +39,9 @@ { "name": "IbmCosHmacDetector" }, + { + "name": "IPPublicDetector" + }, { "name": "JwtTokenDetector" }, @@ -49,9 +55,15 @@ { "name": "NpmDetector" }, + { + "name": "OpenAIDetector" + }, { "name": "PrivateKeyDetector" }, + { + "name": "PypiTokenDetector" + }, { "name": "SendGridDetector" }, @@ -67,6 +79,9 @@ { "name": "StripeDetector" }, + { + "name": "TelegramBotTokenDetector" + }, { "name": "TwilioKeyDetector" } @@ -117,22 +132,6 @@ "line_number": 20 } ], - "tests/checks/normandy/test_remotesettings_recipes.py": [ - { - "type": "Base64 High Entropy String", - "filename": "tests/checks/normandy/test_remotesettings_recipes.py", - "hashed_secret": "1b24669be1077560589e7b74bbbf88472583e419", - "is_verified": false, - "line_number": 21 - }, - { - "type": "Base64 High Entropy String", - "filename": "tests/checks/normandy/test_remotesettings_recipes.py", - "hashed_secret": "bd704e6b3af0748aecb03c9bd0f3b79d06e65bc3", - "is_verified": false, - "line_number": 23 - } - ], "tests/checks/remotesettings/test_attachments_integrity.py": [ { "type": "Hex High Entropy String", @@ -159,5 +158,5 @@ } ] }, - "generated_at": "2024-04-18T11:22:16Z" + "generated_at": "2024-10-18T09:21:04Z" } diff --git a/checks/normandy/jexl_error_rate.py b/checks/normandy/jexl_error_rate.py deleted file mode 100644 index 98edd426..00000000 --- a/checks/normandy/jexl_error_rate.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -The percentage of JEXL filter expressions errors in Normandy should be under the specified -maximum. - -The error rate percentage is returned. The min/max timestamps give the datetime range of the -obtained dataset. -""" - -from collections import Counter, defaultdict -from typing import Dict, List, Tuple - -from telescope.typings import CheckResult, Datetime - -from .uptake_error_rate import fetch_normandy_uptake - - -EXPOSED_PARAMETERS = ["max_error_percentage"] -DEFAULT_PLOT = ".error_rate" - - -async def run( - max_error_percentage: float, - channels: List[str] = [], - period_hours: int = 6, - period_sampling_seconds: int = 600, -) -> CheckResult: - rows = await fetch_normandy_uptake( - channels=channels, - period_hours=period_hours, - period_sampling_seconds=period_sampling_seconds, - ) - - min_timestamp = min(r["min_timestamp"] for r in rows) - max_timestamp = max(r["max_timestamp"] for r in rows) - - # The query returns statuses by periods (eg. 10min). - # First, agregate totals by period and status. - periods: Dict[Tuple[Datetime, Datetime], Counter] = defaultdict(Counter) - for row in rows: - period: Tuple[Datetime, Datetime] = (row["min_timestamp"], row["max_timestamp"]) - status = row["status"] - periods[period][status] += row["total"] - - # Then, keep the period with highest error rate. - max_error_rate = 0.0 - for period, all_statuses in periods.items(): - total = sum(all_statuses.values()) - classify_errors = all_statuses.get("content_error", 0) - error_rate = classify_errors * 100.0 / total - max_error_rate = max(max_error_rate, error_rate) - # If this period is over threshold, show it in check result. - if max_error_rate > max_error_percentage: - min_timestamp, max_timestamp = period - - data = { - "error_rate": round(max_error_rate, 2), - "min_timestamp": min_timestamp.isoformat(), - "max_timestamp": max_timestamp.isoformat(), - } - """ - { - "error_rate": 2.11, - "min_timestamp": "2019-09-19T03:47:42.773", - "max_timestamp": "2019-09-19T09:43:26.083" - } - """ - return error_rate <= max_error_percentage, data diff --git a/checks/normandy/recipe_signatures.py b/checks/normandy/recipe_signatures.py deleted file mode 100644 index b559e63c..00000000 --- a/checks/normandy/recipe_signatures.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Signatures should be valid for each published recipe. - -The list of failing recipes is returned. -""" - -import json -import logging -import random -from typing import Optional - -from autograph_utils import MemoryCache, SignatureVerifier, decode_mozilla_hash - -from telescope.typings import CheckResult -from telescope.utils import ClientSession, fetch_json - - -RECIPES_URL = ( - "{server}/buckets/main/collections/{collection}/changeset?_expected={expected}" -) - - -logger = logging.getLogger(__name__) - - -async def validate_signature(verifier, recipe): - signature_payload = recipe["signature"] - x5u = signature_payload["x5u"] - signature = signature_payload["signature"] - attributes = recipe["recipe"] - data = json.dumps(attributes, sort_keys=True, separators=(",", ":")).encode("utf8") - return await verifier.verify(data, signature, x5u) - - -async def run( - server: str, collection: str, root_hash: Optional[str] = None -) -> CheckResult: - """Fetch recipes from Remote Settings and verify that each attached signature - is verified with the related recipe attributes. - - :param server: URL of Remote Settings server. - :param collection: Collection id to obtain recipes from (eg. ``"normandy-recipes"``. - :param root_hash: The expected hash for the first certificate in a chain. - """ - root_hash_bytes: Optional[bytes] = ( - decode_mozilla_hash(root_hash) if root_hash else None - ) - expected = random.randint(999999000000, 999999999999) - resp = await fetch_json( - RECIPES_URL.format(server=server, collection=collection, expected=expected) - ) - recipes = resp["changes"] - - cache = MemoryCache() - - errors = {} - - async with ClientSession() as session: - verifier = SignatureVerifier(session, cache, root_hash_bytes) - - for recipe in recipes: - try: - await validate_signature(verifier, recipe) - except Exception as e: - errors[recipe["id"]] = repr(e) - - return len(errors) == 0, errors diff --git a/checks/normandy/remotesettings_recipes.py b/checks/normandy/remotesettings_recipes.py deleted file mode 100644 index 9453d6ea..00000000 --- a/checks/normandy/remotesettings_recipes.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -The recipes in the Remote Settings collection should match the Normandy API. The -collection of recipes with capabilities should contain all baseline recipes. - -The lists of missing and extraneous recipes are returned for the baseline and -capabilities collections. -""" - -import random - -from telescope.typings import CheckResult -from telescope.utils import fetch_json - - -NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1&only_baseline_capabilities={baseline_only}" -REMOTESETTINGS_URL = ( - "{server}/buckets/main/collections/{cid}/changeset?_expected={expected}" -) - - -def compare_recipes_lists(a, b): - """ - Return list of recipes present in `a` and missing in `b`, and present in `b` and missing in `a`. - """ - a_by_id = {r["recipe"]["id"]: r["recipe"] for r in a} - b_by_id = {r["recipe"]["id"]: r["recipe"] for r in b} - missing = [] - for rid, r in a_by_id.items(): - r_in_b = b_by_id.pop(rid, None) - if r_in_b is None: - missing.append({"id": r["id"], "name": r["name"]}) - extras = [{"id": r["id"], "name": r["name"]} for r in b_by_id.values()] - return missing, extras - - -async def run(normandy_server: str, remotesettings_server: str) -> CheckResult: - # Baseline recipes from source of truth. - normandy_url_baseline = NORMANDY_URL.format(server=normandy_server, baseline_only=1) - normandy_recipes_baseline = await fetch_json(normandy_url_baseline) - # Recipes with capabilities - normandy_url_caps = NORMANDY_URL.format(server=normandy_server, baseline_only=0) - normandy_recipes_caps = await fetch_json(normandy_url_caps) - - # Baseline recipes published on Remote Settings. - rs_recipes_baseline_url = REMOTESETTINGS_URL.format( - server=remotesettings_server, - cid="normandy-recipes", - expected=random.randint(999999000000, 999999999999), - ) - rs_recipes_baseline = (await fetch_json(rs_recipes_baseline_url))["changes"] - # Recipes with advanced capabilities. - rs_recipes_caps_urls = REMOTESETTINGS_URL.format( - server=remotesettings_server, - cid="normandy-recipes-capabilities", - expected=random.randint(999999000000, 999999999999), - ) - rs_recipes_caps = (await fetch_json(rs_recipes_caps_urls))["changes"] - - # Make sure the baseline recipes are all listed in the baseline collection - missing_baseline, extras_baseline = compare_recipes_lists( - normandy_recipes_baseline, rs_recipes_baseline - ) - # Make sure the baseline recipes are all listed in the baseline collection - missing_caps, extras_caps = compare_recipes_lists( - normandy_recipes_caps, rs_recipes_caps - ) - - ok = ( - len(missing_baseline) - + len(missing_caps) - + len(extras_baseline) - + len(extras_caps) - ) == 0 - data = { - "baseline": {"missing": missing_baseline, "extras": extras_baseline}, - "capabilities": {"missing": missing_caps, "extras": extras_caps}, - } - return ok, data diff --git a/checks/normandy/reported_recipes.py b/checks/normandy/reported_recipes.py deleted file mode 100644 index 98c155f5..00000000 --- a/checks/normandy/reported_recipes.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Recipes available on the server should match the recipes clients are reporting -Uptake Telemetry about. - -The list of recipes for which no event was received is returned. The min/max -timestamps give the datetime range of the obtained dataset. -""" - -import logging -from collections import defaultdict -from typing import Dict, List - -from telescope.typings import CheckResult -from telescope.utils import fetch_json - -from .uptake_error_rate import fetch_normandy_uptake - - -EXPOSED_PARAMETERS = ["server", "lag_margin", "channels"] - -NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1" - -RFC_3339 = "%Y-%m-%dT%H:%M:%S.%fZ" - - -logger = logging.getLogger(__name__) - - -async def run( - server: str, - lag_margin: int = 600, - channels: List[str] = [], - period_hours: int = 6, - period_sampling_seconds: int = 600, -) -> CheckResult: - rows = await fetch_normandy_uptake( - channels=channels, - period_hours=period_hours, - period_sampling_seconds=period_sampling_seconds, - ) - - min_timestamp = min(r["min_timestamp"] for r in rows) - max_timestamp = max(r["max_timestamp"] for r in rows) - - count_by_id: Dict[int, int] = defaultdict(int) - for row in rows: - try: - rid = int(row["source"].split("/")[-1]) - except ValueError: - # The query also returns action and runner uptake. - continue - count_by_id[rid] += row["total"] - - # Recipes from source of truth. - normandy_url = NORMANDY_URL.format(server=server) - normandy_recipes = await fetch_json(normandy_url) - - reported_recipes_ids = set(count_by_id.keys()) - - normandy_recipes_ids = set(r["recipe"]["id"] for r in normandy_recipes) - missing = normandy_recipes_ids - reported_recipes_ids - - data = { - "min_timestamp": min_timestamp.isoformat(), - "max_timestamp": max_timestamp.isoformat(), - "missing": sorted(missing), - } - return len(missing) == 0, data diff --git a/checks/normandy/uptake_error_rate.py b/checks/normandy/uptake_error_rate.py deleted file mode 100644 index 01699275..00000000 --- a/checks/normandy/uptake_error_rate.py +++ /dev/null @@ -1,302 +0,0 @@ -""" -The percentage of reported errors in Uptake Telemetry should be under the specified -maximum. Error rate is computed for each period (of 10min by default). - -For each recipe whose error rate is above the maximum, the total number of events -for each status is returned. The min/max timestamps give the datetime range of the -obtained dataset. -""" - -import re -from collections import Counter, defaultdict -from typing import Dict, List, Tuple, Union - -from telescope.typings import CheckResult -from telescope.utils import csv_quoted, fetch_bigquery, fetch_json - - -EXPOSED_PARAMETERS = ["max_error_percentage", "min_total_events"] -DEFAULT_PLOT = ".max_rate" - -EVENTS_TELEMETRY_QUERY = r""" --- This query returns the total of events received per recipe and status. - --- The events table receives data every 5 minutes. - -WITH event_uptake_telemetry AS ( - SELECT - normalized_channel, - timestamp AS submission_timestamp, - UNIX_SECONDS(timestamp) AS epoch, - (CASE WHEN session_start_time > timestamp THEN timestamp ELSE session_start_time END) AS client_timestamp, - event_string_value, - event_map_values, - event_category, - event_object - FROM - `moz-fx-data-shared-prod.telemetry_derived.events_live` - WHERE - timestamp > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL {period_hours} HOUR) - {channel_condition} -), -uptake_telemetry AS ( - SELECT - submission_timestamp, - normalized_channel, - client_timestamp, - event_string_value AS status, - `moz-fx-data-shared-prod`.udf.get_key(event_map_values, "source") AS source, - epoch - MOD(epoch, {period_sampling_seconds}) AS period - FROM - event_uptake_telemetry - WHERE event_category = 'uptake.remotecontent.result' - AND event_object = 'normandy' - -- Sanity check for client timestamps - AND client_timestamp > TIMESTAMP_SUB(submission_timestamp, INTERVAL 1 DAY) -) -SELECT - -- Min/Max timestamps of this period - PARSE_TIMESTAMP('%s', CAST(period AS STRING)) AS min_timestamp, - PARSE_TIMESTAMP('%s', CAST(period + {period_sampling_seconds} AS STRING)) AS max_timestamp, - normalized_channel AS channel, - source, - status, - COUNT(*) AS total -FROM uptake_telemetry -WHERE source LIKE 'normandy/%' -GROUP BY period, normalized_channel, source, status -ORDER BY period, normalized_channel, source, status -""" - -NORMANDY_URL = "{server}/api/v1/recipe/signed/?enabled=1" - -# Normandy uses the Uptake telemetry statuses in a specific way. -# See https://searchfox.org/mozilla-central/rev/4218cb868d8deed13e902718ba2595d85e12b86b/toolkit/components/normandy/lib/Uptake.jsm#23-43 -UPTAKE_STATUSES = { - "recipe_action_disabled": "custom_1_error", - "recipe_didnt_match_filter": "backoff", - "recipe_execution_error": "apply_error", - "recipe_filter_broken": "content_error", - "recipe_invalid_action": "download_error", - "runner_invalid_signature": "signature_error", - "action_pre_execution_error": "custom_1_error", - "action_post_execution_error": "custom_2_error", -} - -# Invert status dict {("recipe", "custom_1_error"): "recipe_action_disabled", ...} -NORMANDY_STATUSES = {(k.split("_")[0], v): k for k, v in UPTAKE_STATUSES.items()} - - -async def fetch_normandy_uptake( - channels: List[str], period_hours: int, period_sampling_seconds: int -): - # Filter by channel if parameter is specified. - channel_condition = ( - f"AND LOWER(normalized_channel) IN ({csv_quoted(channels)})" if channels else "" - ) - return await fetch_bigquery( - EVENTS_TELEMETRY_QUERY.format( - period_hours=period_hours, - channel_condition=channel_condition, - period_sampling_seconds=period_sampling_seconds, - ) - ) - - -def sort_dict_desc(d, key): - return dict(sorted(d.items(), key=key, reverse=True)) - - -async def run( - max_error_percentage: Union[float, Dict], - server: str, - min_total_events: int = 20, - ignore_status: List[str] = [], - sources: List[str] = [], - channels: List[str] = [], - period_hours: int = 6, - period_sampling_seconds: int = 600, -) -> CheckResult: - if not isinstance(max_error_percentage, dict): - max_error_percentage = {"default": max_error_percentage} - # max_error_percentage["default"] is mandatory. - max_error_percentage.setdefault("with_telemetry", max_error_percentage["default"]) - max_error_percentage.setdefault( - "with_classify_client", max_error_percentage["default"] - ) - - # By default, only look at recipes. - if len(sources) == 0: - sources = ["recipe"] - - sources_re = [re.compile(s) for s in sources] - - # Ignored statuses are specified using the Normandy ones. - ignored_status = [UPTAKE_STATUSES.get(s, s) for s in ignore_status] - - # Fetch list of enabled recipes from Normandy server. - normandy_url = NORMANDY_URL.format(server=server) - normandy_recipes = await fetch_json(normandy_url) - enabled_recipes_by_ids = { - str(r["recipe"]["id"]): r["recipe"] for r in normandy_recipes - } - enabled_recipe_ids = enabled_recipes_by_ids.keys() - - rows = await fetch_normandy_uptake( - channels=channels, - period_hours=period_hours, - period_sampling_seconds=period_sampling_seconds, - ) - - min_timestamp = min(r["min_timestamp"] for r in rows) - max_timestamp = max(r["max_timestamp"] for r in rows) - - # We will store reported events by period, by collection, - # by version, and by status. - # { - # ('2020-01-17T07:50:00', '2020-01-17T08:00:00'): { - # 'recipes/113': { - # 'success': 4699, - # 'sync_error': 39 - # }, - # ... - # } - # } - periods: Dict[Tuple[str, str], Dict] = {} - for row in rows: - # Check if the source matches the selected ones. - source = row["source"].replace("normandy/", "") - if not any(s.match(source) for s in sources_re): - continue - - period: Tuple[str, str] = ( - row["min_timestamp"].isoformat(), - row["max_timestamp"].isoformat(), - ) - periods.setdefault(period, defaultdict(Counter)) - - status = row["status"] - if "recipe" in source: - # Make sure this recipe is enabled, otherwise ignore. - rid = row["source"].split("/")[-1] - if rid not in enabled_recipe_ids: - continue - # In Firefox 67, `custom_2_error` was used instead of `backoff`. - if status == "custom_2_error": - status = "backoff" - - periods[period][source][status] += row["total"] - - error_rates: Dict[str, Dict] = {} - min_rate = None - max_rate = None - for (min_period, max_period), by_collection in periods.items(): - # Compute error rate by period. - # This allows us to prevent error rate to be "spread" over the overall datetime - # range of events (eg. a spike of errors during 10min over 2H). - for source, all_statuses in by_collection.items(): - total_statuses = sum(total for status, total in all_statuses.items()) - - # Ignore uptake Telemetry of a certain recipe if the total of collected - # events is too small. - if total_statuses < min_total_events: - continue - - # Show overridden status in check output. - source_type = source.split("/")[0] - statuses = { - NORMANDY_STATUSES.get((source_type, status), status): total - for status, total in all_statuses.items() - if status not in ignored_status - } - ignored = { - NORMANDY_STATUSES.get((source_type, status), status): total - for status, total in all_statuses.items() - if status in ignored_status - } - total_errors = sum( - total - for status, total in statuses.items() - if UPTAKE_STATUSES.get(status, status).endswith("_error") - ) - error_rate = round(total_errors * 100 / total_statuses, 2) - - if min_rate is None: - min_rate = max_rate = error_rate - else: - min_rate = min(min_rate, error_rate) - max_rate = max(max_rate, error_rate) - - # If error rate for this period is below threshold, or lower than one reported - # in another period, then we ignore it. - other_period_rate = error_rates.get(source, {"error_rate": 0.0})[ - "error_rate" - ] - - details = {} - max_percentage = max_error_percentage["default"] - if "recipe" in source: - rid = source.split("/")[-1] - recipe = enabled_recipes_by_ids[rid] - with_telemetry = "normandy.telemetry" in recipe["filter_expression"] - with_classify_client = "normandy.country" in recipe["filter_expression"] - details["name"] = recipe["name"] - details["with_telemetry"] = with_telemetry - details["with_classify_client"] = with_classify_client - if with_telemetry: - max_percentage = max_error_percentage["with_telemetry"] - # If recipe has both Telemetry and Classify Client, keep highest threshold. - if with_classify_client: - max_percentage = max( - max_percentage, max_error_percentage["with_classify_client"] - ) - - if error_rate < max_percentage or error_rate < other_period_rate: - continue - - error_rates[source] = { - "error_rate": error_rate, - **details, - "statuses": sort_dict_desc(statuses, key=lambda item: item[1]), - "ignored": sort_dict_desc(ignored, key=lambda item: item[1]), - "min_timestamp": min_period, - "max_timestamp": max_period, - } - - sort_by_rate = sort_dict_desc(error_rates, key=lambda item: item[1]["error_rate"]) - - data = { - "sources": sort_by_rate, - "min_rate": min_rate, - "max_rate": max_rate, - "min_timestamp": min_timestamp.isoformat(), - "max_timestamp": max_timestamp.isoformat(), - } - """ - { - "sources": { - "recipes/123": { - "error_rate": 60.4, - "name": "Disable OS auth", - "with_classify_client": true, - "with_telemetry": false, - "statuses": { - "recipe_execution_error": 56, - "success": 35, - "action_post_execution_error": 5 - }, - "ignored": { - "recipe_didnt_match_filter": 5 - }, - "min_timestamp": "2020-01-17T08:10:00", - "max_timestamp": "2020-01-17T08:20:00", - }, - ... - }, - "min_rate": 2.1, - "max_rate": 60.4, - "min_timestamp": "2020-01-17T08:00:00", - "max_timestamp": "2020-01-17T10:00:00" - } - """ - return len(sort_by_rate) == 0, data diff --git a/tests/checks/normandy/test_normandy_jexl_error_rate.py b/tests/checks/normandy/test_normandy_jexl_error_rate.py deleted file mode 100644 index 9be5fcfd..00000000 --- a/tests/checks/normandy/test_normandy_jexl_error_rate.py +++ /dev/null @@ -1,74 +0,0 @@ -from datetime import datetime - -from checks.normandy.jexl_error_rate import run -from tests.utils import patch_async - - -MODULE = "checks.normandy.jexl_error_rate" - -FAKE_ROWS = [ - { - "status": "success", - "source": "normandy/recipe/123", - "channel": "release", - "total": 1800, - "min_timestamp": datetime.fromisoformat("2019-09-16T02:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T06:24:58.741"), - }, - { - "status": "success", - "source": "normandy/recipe/123", - "channel": "release", - "total": 9000, - "min_timestamp": datetime.fromisoformat("2019-09-16T03:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T05:24:58.741"), - }, - { - "status": "content_error", - "source": "normandy/recipe/123", - "channel": "release", - "total": 1000, - "min_timestamp": datetime.fromisoformat("2019-09-16T03:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T05:24:58.741"), - }, - { - "status": "success", - "source": "normandy/recipe/456", - "channel": "release", - "total": 900, - "min_timestamp": datetime.fromisoformat("2019-09-16T01:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T07:24:58.741"), - }, - { - "status": "content_error", - "source": "normandy/recipe/456", - "channel": "release", - "total": 100, - "min_timestamp": datetime.fromisoformat("2019-09-16T01:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T07:24:58.741"), - }, -] - - -async def test_positive(): - with patch_async(f"{MODULE}.fetch_normandy_uptake", return_value=FAKE_ROWS): - status, data = await run(max_error_percentage=100.0, channels=["release"]) - - assert status is True - assert data == { - "error_rate": 10.0, - "min_timestamp": "2019-09-16T01:36:12.348000", - "max_timestamp": "2019-09-16T07:24:58.741000", - } - - -async def test_negative(): - with patch_async(f"{MODULE}.fetch_normandy_uptake", return_value=FAKE_ROWS): - status, data = await run(max_error_percentage=1.0, channels=["release"]) - - assert status is False - assert data == { - "error_rate": 10.0, - "min_timestamp": "2019-09-16T01:36:12.348000", - "max_timestamp": "2019-09-16T07:24:58.741000", - } diff --git a/tests/checks/normandy/test_normandy_uptake_error_rate.py b/tests/checks/normandy/test_normandy_uptake_error_rate.py deleted file mode 100644 index 5d916473..00000000 --- a/tests/checks/normandy/test_normandy_uptake_error_rate.py +++ /dev/null @@ -1,382 +0,0 @@ -from datetime import datetime - -from checks.normandy.uptake_error_rate import NORMANDY_URL, run -from tests.utils import patch_async - - -NORMANDY_SERVER = "http://normandy" -MODULE = "checks.normandy.uptake_error_rate" - -FAKE_ROWS = [ - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "success", - "source": "normandy/recipe/456", - "channel": "release", - "total": 20000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "success", - "source": "normandy/recipe/123", - "channel": "release", - "total": 20000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "apply_error", # recipe_execution_error - "source": "normandy/recipe/123", - "channel": "release", - "total": 10000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "download_error", # recipe_invalid_action - "source": "normandy/recipe/123", - "channel": "release", - "total": 5000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "backoff", # recipe_didnt_match_filter - "source": "normandy/recipe/123", - "channel": "release", - "total": 4000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "custom_2_error", # recipe_didnt_match_filter in Fx 67 - "source": "normandy/recipe/123", - "channel": "release", - "total": 1000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:50:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T01:00:00"), - "status": "success", - "source": "normandy/recipe/123", - "channel": "release", - "total": 1000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:50:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T01:00:00"), - "status": "apply_error", # recipe_execution_error - "source": "normandy/recipe/123", - "channel": "release", - "total": 500, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "success", - "source": "normandy/action/AddonStudyAction", - "channel": "release", - "total": 9000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "custom_2_error", - "source": "normandy/action/AddonStudyAction", - "channel": "release", - "total": 1000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "success", - "source": "normandy/runner", - "channel": "release", - "total": 2000, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:30:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T00:40:00"), - "status": "server_error", - "source": "normandy/runner", - "channel": "release", - "total": 500, - }, - { - "min_timestamp": datetime.fromisoformat("2019-09-16T00:50:00"), - "max_timestamp": datetime.fromisoformat("2019-09-16T01:00:00"), - "status": "success", - "source": "normandy/runner", - "channel": "release", - "total": 1000, - }, -] - -RECIPE = { - "id": 123, - "name": "un dos tres", - "filter_expression": "", -} - - -async def test_positive(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=100.0, - channels=["release"], - ) - - assert status is True - assert data == { - "sources": {}, - "min_rate": 33.33, - "max_rate": 37.5, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_negative(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - channels=["release"], - ) - - assert status is False - assert data == { - "sources": { - "recipe/123": { - "error_rate": 37.5, - "name": "un dos tres", - "with_telemetry": False, - "with_classify_client": False, - "statuses": { - "success": 20000, - "recipe_didnt_match_filter": 5000, - "recipe_execution_error": 10000, - "recipe_invalid_action": 5000, - }, - "ignored": {}, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T00:40:00", - } - }, - "min_rate": 33.33, - "max_rate": 37.5, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_ignore_status(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - ignore_status=["recipe_execution_error", "recipe_invalid_action"], - channels=["release"], - ) - - assert status is True - assert data == { - "sources": {}, - "min_rate": 0.0, - "max_rate": 0.0, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_ignore_disabled_recipes(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": {**RECIPE, "id": 456}}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - channels=["release"], - ) - - assert status is True - assert data == { - "sources": {}, - "min_rate": 0.0, - "max_rate": 0.0, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_min_total_events(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - min_total_events=40001, - channels=["release"], - ) - - assert status is True - assert data == { - "sources": {}, - "min_rate": None, - "max_rate": None, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_filter_on_action_uptake(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - sources=["action"], - server=NORMANDY_SERVER, - max_error_percentage=10, - channels=["release"], - ) - - assert status is False - assert data == { - "sources": { - "action/AddonStudyAction": { - "error_rate": 10.0, - "statuses": {"success": 9000, "action_post_execution_error": 1000}, - "ignored": {}, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T00:40:00", - } - }, - "min_rate": 10.0, - "max_rate": 10.0, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_filter_on_runner_uptake(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": RECIPE}], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - sources=["runner"], - server=NORMANDY_SERVER, - max_error_percentage=0.1, - channels=["release"], - ) - - assert status is False - assert data == { - "sources": { - "runner": { - "error_rate": 20.0, - "statuses": {"success": 2000, "server_error": 500}, - "ignored": {}, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T00:40:00", - } - }, - "min_rate": 0.0, - "max_rate": 20.0, - "min_timestamp": "2019-09-16T00:30:00", - "max_timestamp": "2019-09-16T01:00:00", - } - - -async def test_error_rate_with_classify(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[ - {"recipe": {**RECIPE, "filter_expression": '(normandy.country in ["US"])'}} - ], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - ) - - assert status is False - assert data["sources"]["recipe/123"]["with_classify_client"] - - -async def test_error_rate_with_telemetry(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[ - { - "recipe": { - **RECIPE, - "filter_expression": "(normandy.telemetry.main.sum > 0)", - } - } - ], - ) - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=0.1, - ) - - assert status is False - assert data["sources"]["recipe/123"]["with_telemetry"] - - -async def test_error_rate_with_classifyclient_and_telemetry(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[ - { - "recipe": { - **RECIPE, - "filter_expression": ( - '(normandy.country in ["US"]) &&' - "(normandy.telemetry.main.sum > 0)" - ), - } - } - ], - ) - max_error_percentage = { - "default": 0.1, - "with_classify_client": 20, - "with_telemetry": 30, - } - with patch_async(f"{MODULE}.fetch_bigquery", return_value=FAKE_ROWS): - status, data = await run( - server=NORMANDY_SERVER, - max_error_percentage=max_error_percentage, - ) - - assert status is False - assert data["sources"]["recipe/123"]["error_rate"] == 37.5 - assert data["sources"]["recipe/123"]["with_telemetry"] - assert data["sources"]["recipe/123"]["with_classify_client"] diff --git a/tests/checks/normandy/test_recipe_signatures.py b/tests/checks/normandy/test_recipe_signatures.py deleted file mode 100644 index 40570ce0..00000000 --- a/tests/checks/normandy/test_recipe_signatures.py +++ /dev/null @@ -1,120 +0,0 @@ -from unittest import mock - -import autograph_utils -import pytest - -from checks.normandy.recipe_signatures import run, validate_signature -from telescope.utils import ClientSession -from tests.utils import patch_async - - -MODULE = "checks.normandy.recipe_signatures" -CHANGESET_URL = "/buckets/{}/collections/{}/changeset?_expected={}" - -CERT = """-----BEGIN CERTIFICATE----- -MIIDBTCCAougAwIBAgIIFcbkDrCrHAkwCgYIKoZIzj0EAwMwgaMxCzAJBgNVBAYT -AlVTMRwwGgYDVQQKExNNb3ppbGxhIENvcnBvcmF0aW9uMS8wLQYDVQQLEyZNb3pp -bGxhIEFNTyBQcm9kdWN0aW9uIFNpZ25pbmcgU2VydmljZTFFMEMGA1UEAww8Q29u -dGVudCBTaWduaW5nIEludGVybWVkaWF0ZS9lbWFpbEFkZHJlc3M9Zm94c2VjQG1v -emlsbGEuY29tMB4XDTE5MDgyMzIyNDQzMVoXDTE5MTExMTIyNDQzMVowgakxCzAJ -BgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQHEw1Nb3VudGFp -biBWaWV3MRwwGgYDVQQKExNNb3ppbGxhIENvcnBvcmF0aW9uMRcwFQYDVQQLEw5D -bG91ZCBTZXJ2aWNlczE2MDQGA1UEAxMtcGlubmluZy1wcmVsb2FkLmNvbnRlbnQt -c2lnbmF0dXJlLm1vemlsbGEub3JnMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEX6Zd -vZ32rj9rDdRInp0kckbMtAdxOQxJ7EVAEZB2KOLUyotQL6A/9YWrMB4Msb4hfvxj -Nw05CS5/J4qUmsTkKLXQskjRe9x96uOXxprWiVwR4OLYagkJJR7YG1mTXmFzo4GD -MIGAMA4GA1UdDwEB/wQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDAzAfBgNVHSME -GDAWgBSgHUoXT4zCKzVF8WPx2nBwp8744TA4BgNVHREEMTAvgi1waW5uaW5nLXBy -ZWxvYWQuY29udGVudC1zaWduYXR1cmUubW96aWxsYS5vcmcwCgYIKoZIzj0EAwMD -aAAwZQIxAOi2Eusi6MtEPOARiU+kZIi1vPnzTI71cA2ZIpzZ9aYg740eoJml8Guz -3oC6yXiIDAIwSy4Eylf+/nSMA73DUclcCjZc2yfRYIogII+krXBxoLkbPJcGaitx -qvRy6gQ1oC/z ------END CERTIFICATE----- -""" - - -@pytest.fixture() -def mock_randint(): - with mock.patch("checks.normandy.remotesettings_recipes.random.randint") as mocked: - yield mocked - - -async def test_positive(mock_aioresponses, mock_randint): - server_url = "http://fake.local/v1" - mock_randint.return_value = 314 - changeset_url = server_url + CHANGESET_URL.format("main", "normandy-recipes", 314) - mock_aioresponses.get( - changeset_url, - payload={ - "changes": [ - { - "id": "12", - "last_modified": 42, - "signature": {"signature": "abc", "x5u": "http://fake-x5u-url"}, - "recipe": {"id": 12}, - } - ] - }, - ) - - with patch_async(f"{MODULE}.validate_signature", return_value=True): - status, data = await run(server_url, "normandy-recipes") - - assert status is True - assert data == {} - - -async def test_negative(mock_aioresponses, mock_randint): - server_url = "http://fake.local/v1" - mock_randint.return_value = 314 - changeset_url = server_url + CHANGESET_URL.format("main", "normandy-recipes", 314) - mock_aioresponses.get( - changeset_url, - payload={ - "changes": [ - { - "id": "12", - "last_modified": 42, - "signature": {"signature": "abc", "x5u": "http://fake-x5u-url"}, - "recipe": {"id": 12}, - } - ] - }, - ) - - with patch_async(f"{MODULE}.validate_signature", side_effect=ValueError("boom")): - status, data = await run(server_url, "normandy-recipes") - - assert status is False - assert data == {"12": "ValueError('boom')"} - - -async def test_invalid_x5u(mock_aioresponses): - x5u = "http://fake-x5u-url" - mock_aioresponses.get(x5u, body=CERT) - cache = autograph_utils.MemoryCache() - async with ClientSession() as session: - verifier = autograph_utils.SignatureVerifier(session, cache, root_hash=None) - - recipe = { - "signature": {"signature": "abc", "x5u": x5u}, - "recipe": {"id": 12}, - } - - with pytest.raises(autograph_utils.BadCertificate): - await validate_signature(verifier, recipe) - - -async def test_invalid_signature(): - verifier = mock.MagicMock() - verifier.verify.side_effect = autograph_utils.BadSignature - - recipe = { - "signature": {"signature": "abc", "x5u": "http://fake-x5u-url"}, - "recipe": {"id": 12}, - } - - with pytest.raises(autograph_utils.BadSignature): - await validate_signature(verifier, recipe) - - verifier.verify.assert_called_with(b'{"id":12}', "abc", "http://fake-x5u-url") diff --git a/tests/checks/normandy/test_remotesettings_recipes.py b/tests/checks/normandy/test_remotesettings_recipes.py deleted file mode 100644 index 682b66e6..00000000 --- a/tests/checks/normandy/test_remotesettings_recipes.py +++ /dev/null @@ -1,127 +0,0 @@ -from typing import Any, Dict -from unittest import mock - -from checks.normandy.remotesettings_recipes import NORMANDY_URL, run - - -NORMANDY_SERVER = "http://n" -REMOTESETTINGS_SERVER = "http://rs/v1" -REMOTESETTINGS_BASELINE_URL = ( - REMOTESETTINGS_SERVER - + "/buckets/main/collections/normandy-recipes/changeset?_expected=42" -) -REMOTESETTINGS_CAPABILITIES_URL = ( - REMOTESETTINGS_SERVER - + "/buckets/main/collections/normandy-recipes-capabilities/changeset?_expected=42" -) - -NORMANDY_RECIPE: Dict[str, Dict[str, Any]] = { - "signature": { - "timestamp": "2019-08-16T21:14:28.651337Z", - "signature": "ZQyCVZEltEzmTH1lavnlzY_BiR-hMSNGp2DrqQRhlbnoRy5wBjpvSu8o4DVb2VSUUo5tUMGvC0fFCvedw7XH9y2CIUZl6xQo8x8KJe75RPZr8zLEuoG8LhzWpOnx1Fuz", - "x5u": "https://content-signature-2.cdn.mozilla.net/chains/normandy.content-signature.mozilla.org-2019-10-02-18-15-02.chain?cachebust=2017-06-13-21-06", - "public_key": "MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEwFEtpUsPaLeS0npnCdyHjYDAvRT1cKeaJdRoOXQLuucfVir4DgGGW4FoM7WsddmcVllXMlGHsPeXr01atWe42wHILl1su0ZaFJDtJ42G5gGYd4nH0S6PTGo97/ux3oO0", - }, - "recipe": { - "id": 829, - "name": "Mobile Browser usage", - "revision_id": "2687", - "action": "show-heartbeat", - "arguments": { - "repeatOption": "once", - "surveyId": "hb-mobile-browser-usage", - "message": "Please help make Firefox better by taking this short survey", - "learnMoreMessage": "Learn More", - "learnMoreUrl": "https://support.mozilla.org/en-US/kb/rate-your-firefox-experience-heartbeat", - "engagementButtonLabel": "Take Survey", - "thanksMessage": "Thanks", - "postAnswerUrl": "https://qsurvey.mozilla.com/s3/3ea2dbbe74d5", - "includeTelemetryUUID": False, - }, - "filter_expression": '(normandy.locale in ["en-US","en-AU","en-GB","en-CA","en-NZ","en-ZA"]) && (normandy.country in ["US"]) && (normandy.channel in ["release"]) && (["global-v1",normandy.userId]|bucketSample(2135,10,10000)) && (!normandy.firstrun)', - }, -} - -REMOTESETTINGS_RECIPE = { - "id": str(NORMANDY_RECIPE["recipe"]["id"]), - "recipe": { - "id": NORMANDY_RECIPE["recipe"]["id"], - "name": NORMANDY_RECIPE["recipe"]["name"], - }, -} - -REMOTESETTINGS_RECIPE_WITH_CAPS = { - "id": "314", - "recipe": { - "id": 314, - "name": "With caps", - "capabilities": ["action.preference-experiment"], - }, -} - - -async def test_positive(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0), - payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS], - ) - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1), - payload=[NORMANDY_RECIPE], - ) - mock_aioresponses.get( - REMOTESETTINGS_CAPABILITIES_URL, - payload={"changes": [REMOTESETTINGS_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS]}, - ) - mock_aioresponses.get( - REMOTESETTINGS_BASELINE_URL, payload={"changes": [REMOTESETTINGS_RECIPE]} - ) - - with mock.patch( - "checks.normandy.remotesettings_recipes.random.randint", return_value=42 - ): - status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER) - - assert status is True - assert data == { - "baseline": {"missing": [], "extras": []}, - "capabilities": {"missing": [], "extras": []}, - } - - -async def test_negative(mock_aioresponses): - RECIPE_42 = {"id": "42", "recipe": {"id": 42, "name": "Extra"}} - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=0), - payload=[NORMANDY_RECIPE, REMOTESETTINGS_RECIPE_WITH_CAPS], - ) - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER, baseline_only=1), - payload=[NORMANDY_RECIPE], - ) - mock_aioresponses.get( - REMOTESETTINGS_CAPABILITIES_URL, - payload={"changes": [REMOTESETTINGS_RECIPE_WITH_CAPS]}, - ) - mock_aioresponses.get( - REMOTESETTINGS_BASELINE_URL, - payload={"changes": [RECIPE_42]}, - ) - - with mock.patch( - "checks.normandy.remotesettings_recipes.random.randint", return_value=42 - ): - status, data = await run(NORMANDY_SERVER, REMOTESETTINGS_SERVER) - - assert status is False - assert data == { - "baseline": { - "missing": [_d(REMOTESETTINGS_RECIPE)], - "extras": [_d(RECIPE_42)], - }, - "capabilities": {"missing": [_d(REMOTESETTINGS_RECIPE)], "extras": []}, - } - - -def _d(d): - return {k: v for k, v in d["recipe"].items() if k in ("id", "name")} diff --git a/tests/checks/normandy/test_reported_recipes.py b/tests/checks/normandy/test_reported_recipes.py deleted file mode 100644 index 919d1969..00000000 --- a/tests/checks/normandy/test_reported_recipes.py +++ /dev/null @@ -1,89 +0,0 @@ -from datetime import datetime - -from checks.normandy.reported_recipes import NORMANDY_URL, run -from tests.utils import patch_async - - -NORMANDY_SERVER = "http://normandy" -MODULE = "checks.normandy.reported_recipes" - -FAKE_ROWS = [ - { - "status": "success", - "source": "normandy/recipe/123", - "channel": "release", - "total": 20000, - "min_timestamp": datetime.fromisoformat("2019-09-16T02:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T06:24:58.741"), - }, - { - "status": "backoff", - "source": "normandy/recipe/456", - "channel": "beta", - "total": 15000, - "min_timestamp": datetime.fromisoformat("2019-09-16T03:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T05:24:58.741"), - }, - { - "status": "apply_error", - "source": "normandy/recipe/123", - "channel": "release", - "total": 5000, - "min_timestamp": datetime.fromisoformat("2019-09-16T01:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T07:24:58.741"), - }, - { - "status": "custom_2_error", - "source": "normandy/recipe/111", - "channel": "release", - "total": 999, - "min_timestamp": datetime.fromisoformat("2019-09-16T03:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T05:24:58.741"), - }, - { - "status": "success", - "source": "normandy/runner", - "channel": "release", - "total": 42, - "min_timestamp": datetime.fromisoformat("2019-09-16T03:36:12.348"), - "max_timestamp": datetime.fromisoformat("2019-09-16T05:24:58.741"), - }, -] - - -async def test_positive(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[{"recipe": {"id": 123}}, {"recipe": {"id": 456}}], - ) - - with patch_async(f"{MODULE}.fetch_normandy_uptake", return_value=FAKE_ROWS): - status, data = await run(server=NORMANDY_SERVER) - - assert status is True - assert data == { - "missing": [], - "min_timestamp": "2019-09-16T01:36:12.348000", - "max_timestamp": "2019-09-16T07:24:58.741000", - } - - -async def test_negative(mock_aioresponses): - mock_aioresponses.get( - NORMANDY_URL.format(server=NORMANDY_SERVER), - payload=[ - {"recipe": {"id": 123}}, - {"recipe": {"id": 456}}, - {"recipe": {"id": 789}}, - ], - ) - - with patch_async(f"{MODULE}.fetch_normandy_uptake", return_value=FAKE_ROWS): - status, data = await run(server=NORMANDY_SERVER) - - assert status is False - assert data == { - "missing": [789], - "min_timestamp": "2019-09-16T01:36:12.348000", - "max_timestamp": "2019-09-16T07:24:58.741000", - }